andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Explain the hard-coded rocksdb settings

Browse Source
This commit is contained in:
Andrea Cavalli 2022-04-04 11:16:20 +02:00
parent 32dc615f88
commit c711bbc5ad
5 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -330,7 +330,7 @@
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>7.0.3</version>
<version>7.0.4</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>

1
src/main/data-generator/quic-rpc.yaml
View File

@ -235,6 +235,7 @@ versions:
memtableMemoryBudgetBytes: -long
blockCache: -long
setCacheIndexAndFilterBlocks: -boolean
spinning: boolean
DatabaseVolume:
data:
volumePath: Path

33
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -143,12 +143,6 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
for (Column column : columns) {
var columnOptions = new ColumnFamilyOptions();
columnOptions.setMaxBytesForLevelBase(256 * SizeUnit.MB);
columnOptions.setMaxBytesForLevelMultiplier(10);
columnOptions.setLevelCompactionDynamicLevelBytes(true);
columnOptions.setLevel0FileNumCompactionTrigger(2);
columnOptions.setLevel0SlowdownWritesTrigger(20);
columnOptions.setLevel0StopWritesTrigger(36);
//noinspection ConstantConditions
if (databaseOptions.memtableMemoryBudgetBytes() != null) {
@ -158,6 +152,20 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.orElse(databaseOptions.lowMemory() ? 16L * SizeUnit.MB : 128L * SizeUnit.MB));
}
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
columnOptions.setMaxBytesForLevelBase(256 * SizeUnit.MB);
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
columnOptions.setMaxBytesForLevelMultiplier(10);
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
columnOptions.setLevelCompactionDynamicLevelBytes(true);
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
columnOptions.setLevel0FileNumCompactionTrigger(2);
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
columnOptions.setLevel0SlowdownWritesTrigger(20);
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
columnOptions.setLevel0StopWritesTrigger(36);
if (!databaseOptions.levels().isEmpty()) {
var firstLevelOptions = getRocksLevelOptions(databaseOptions.levels().get(0));
columnOptions.setCompressionType(firstLevelOptions.compressionType);
@ -198,11 +206,16 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
tableOptions.setOptimizeFiltersForMemory(true);
tableOptions.setVerifyCompression(false);
}
boolean cacheIndexAndFilterBlocks = databaseOptions.setCacheIndexAndFilterBlocks().orElse(true);
if (databaseOptions.spinning()) {
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
cacheIndexAndFilterBlocks = true;
}
tableOptions
.setPinTopLevelIndexAndFilter(true)
.setPinL0FilterAndIndexBlocksInCache(true)
.setCacheIndexAndFilterBlocksWithHighPriority(true)
.setCacheIndexAndFilterBlocks(databaseOptions.setCacheIndexAndFilterBlocks().orElse(true))
.setCacheIndexAndFilterBlocks(cacheIndexAndFilterBlocks)
.setPartitionFilters(true)
.setIndexType(IndexType.kTwoLevelIndexSearch)
.setFormatVersion(5)
@ -211,11 +224,11 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.setBlockCacheCompressed(optionsWithCache.compressedCache())
.setBlockCache(optionsWithCache.standardCache())
// Spinning disks: 64KiB to 256KiB (also 512KiB). SSDs: 16KiB
.setBlockSize(256 * 1024);
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
.setBlockSize((databaseOptions.spinning() ? 256 : 16) * SizeUnit.KB);
//columnOptions.setLevelCompactionDynamicLevelBytes(true);
columnOptions.setTableFormatConfig(tableOptions);
columnOptions.setCompactionPriority(CompactionPriority.OldestSmallestSeqFirst);
columnOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
descriptors.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions));
}

3
src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java
View File

@ -96,7 +96,8 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
Nullableint.of(-1),
Nullablelong.empty(),
Nullablelong.empty(),
Nullableboolean.empty()
Nullableboolean.empty(),
false
)
),
conn.getLuceneIndex("testluceneindex1",

3
src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java
View File

@ -67,7 +67,8 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
Nullableint.of(-1),
Nullablelong.empty(),
Nullablelong.empty(),
Nullableboolean.empty()
Nullableboolean.empty(),
false
)
),
conn.getLuceneIndex("testluceneindex1",