andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Performance tuning

Browse Source
This commit is contained in:
Andrea Cavalli 2021-05-18 01:10:30 +02:00
parent f8377c3e63
commit c9f64195e7
1 changed files with 25 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -199,7 +199,6 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
var options = new Options(); var options = new Options();
options.setCreateIfMissing(true); options.setCreateIfMissing(true);
options.setCompactionStyle(CompactionStyle.LEVEL); options.setCompactionStyle(CompactionStyle.LEVEL);
options.setLevelCompactionDynamicLevelBytes(true);
options.setTargetFileSizeBase(64 * 1024 * 1024); // 64MiB sst file options.setTargetFileSizeBase(64 * 1024 * 1024); // 64MiB sst file
options.setTargetFileSizeMultiplier(2); // Each level is 2 times the previous level options.setTargetFileSizeMultiplier(2); // Each level is 2 times the previous level
options.setCompressionPerLevel(List.of(CompressionType.NO_COMPRESSION, options.setCompressionPerLevel(List.of(CompressionType.NO_COMPRESSION,
@ -210,7 +209,6 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
options.setManualWalFlush(false); options.setManualWalFlush(false);
options.setMinWriteBufferNumberToMerge(3); options.setMinWriteBufferNumberToMerge(3);
options.setMaxWriteBufferNumber(4); options.setMaxWriteBufferNumber(4);
options.setWalTtlSeconds(30); // flush wal after 30 seconds
options.setAvoidFlushDuringShutdown(false); // Flush all WALs during shutdown options.setAvoidFlushDuringShutdown(false); // Flush all WALs during shutdown
options.setAvoidFlushDuringRecovery(false); // Flush all WALs during startup options.setAvoidFlushDuringRecovery(false); // Flush all WALs during startup
options.setWalRecoveryMode(crashIfWalError options.setWalRecoveryMode(crashIfWalError
@ -219,8 +217,6 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
options.setDeleteObsoleteFilesPeriodMicros(20 * 1000000); // 20 seconds options.setDeleteObsoleteFilesPeriodMicros(20 * 1000000); // 20 seconds
options.setPreserveDeletes(false); options.setPreserveDeletes(false);
options.setKeepLogFileNum(10); options.setKeepLogFileNum(10);
options.setAllowMmapReads(true);
options.setAllowMmapWrites(true);
options.setAllowFAllocate(true); options.setAllowFAllocate(true);
options.setRateLimiter(new RateLimiter(10L * 1024L * 1024L)); // 10MiB/s max compaction write speed options.setRateLimiter(new RateLimiter(10L * 1024L * 1024L)); // 10MiB/s max compaction write speed
options.setDbPaths(List.of(new DbPath(databasesDirPath.resolve(path.getFileName() + "_hot"), options.setDbPaths(List.of(new DbPath(databasesDirPath.resolve(path.getFileName() + "_hot"),
@ -238,20 +234,31 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
if (lowMemory) { if (lowMemory) {
// LOW MEMORY // LOW MEMORY
options options
.setBytesPerSync(512 * 1024) // 512KiB .setLevelCompactionDynamicLevelBytes(false)
.setWalBytesPerSync(1024 * 1024) .setAllowMmapReads(false)
.setAllowMmapWrites(false)
.setBytesPerSync(0) // default
.setWalBytesPerSync(0) // default
.setIncreaseParallelism(1) .setIncreaseParallelism(1)
.setMaxOpenFiles(15) .setMaxOpenFiles(15)
.optimizeLevelStyleCompaction(1024 * 1024) // 1MiB of ram will be used for level style compaction .optimizeLevelStyleCompaction(1024 * 1024) // 1MiB of ram will be used for level style compaction
.setWriteBufferSize(1024 * 1024) // 1MB .setWriteBufferSize(1024 * 1024) // 1MB
.setWalSizeLimitMB(16) // 16MB .setWalTtlSeconds(0)
.setMaxTotalWalSize(1024L * 1024L * 1024L) // 1GiB max wal directory size .setWalSizeLimitMB(0) // 16MB
.setMaxTotalWalSize(0) // automatic
;
tableOptions
.setBlockCache(new LRUCache(8L * 1024L * 1024L)) // 8MiB
.setCacheIndexAndFilterBlocks(false)
.setPinL0FilterAndIndexBlocksInCache(false)
; ;
tableOptions.setBlockCache(new LRUCache(8L * 1024L * 1024L)); // 8MiB
options.setWriteBufferManager(new WriteBufferManager(8L * 1024L * 1024L, new LRUCache(8L * 1024L * 1024L))); // 8MiB options.setWriteBufferManager(new WriteBufferManager(8L * 1024L * 1024L, new LRUCache(8L * 1024L * 1024L))); // 8MiB
} else { } else {
// HIGH MEMORY // HIGH MEMORY
options options
.setLevelCompactionDynamicLevelBytes(true)
.setAllowMmapReads(true)
.setAllowMmapWrites(true)
.setAllowConcurrentMemtableWrite(true) .setAllowConcurrentMemtableWrite(true)
.setEnableWriteThreadAdaptiveYield(true) .setEnableWriteThreadAdaptiveYield(true)
.setIncreaseParallelism(Runtime.getRuntime().availableProcessors()) .setIncreaseParallelism(Runtime.getRuntime().availableProcessors())
@ -261,19 +268,22 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.optimizeLevelStyleCompaction( .optimizeLevelStyleCompaction(
128 * 1024 * 1024) // 128MiB of ram will be used for level style compaction 128 * 1024 * 1024) // 128MiB of ram will be used for level style compaction
.setWriteBufferSize(64 * 1024 * 1024) // 64MB .setWriteBufferSize(64 * 1024 * 1024) // 64MB
.setWalTtlSeconds(30) // flush wal after 30 seconds
.setWalSizeLimitMB(1024) // 1024MB .setWalSizeLimitMB(1024) // 1024MB
.setMaxTotalWalSize(2L * 1024L * 1024L * 1024L) // 2GiB max wal directory size .setMaxTotalWalSize(2L * 1024L * 1024L * 1024L) // 2GiB max wal directory size
; ;
tableOptions.setBlockCache(new LRUCache(128L * 1024L * 1024L)); // 128MiB tableOptions
.setBlockCache(new LRUCache(128L * 1024L * 1024L)) // 128MiB
.setCacheIndexAndFilterBlocks(true)
.setPinL0FilterAndIndexBlocksInCache(true)
;
final BloomFilter bloomFilter = new BloomFilter(10, false);
tableOptions.setOptimizeFiltersForMemory(true);
tableOptions.setFilterPolicy(bloomFilter);
options.setWriteBufferManager(new WriteBufferManager(256L * 1024L * 1024L, new LRUCache(128L * 1024L * 1024L))); // 128MiB options.setWriteBufferManager(new WriteBufferManager(256L * 1024L * 1024L, new LRUCache(128L * 1024L * 1024L))); // 128MiB
} }
final BloomFilter bloomFilter = new BloomFilter(10, false);
tableOptions.setOptimizeFiltersForMemory(true);
tableOptions.setFilterPolicy(bloomFilter);
tableOptions.setBlockSize(16 * 1024); // 16MiB tableOptions.setBlockSize(16 * 1024); // 16MiB
tableOptions.setCacheIndexAndFilterBlocks(true);
tableOptions.setPinL0FilterAndIndexBlocksInCache(true);
options.setTableFormatConfig(tableOptions); options.setTableFormatConfig(tableOptions);
options.setCompactionPriority(CompactionPriority.MinOverlappingRatio); options.setCompactionPriority(CompactionPriority.MinOverlappingRatio);