Explain the hard-coded rocksdb settings
This commit is contained in:
parent
32dc615f88
commit
c711bbc5ad
2
pom.xml
2
pom.xml
|
@ -330,7 +330,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.rocksdb</groupId>
|
<groupId>org.rocksdb</groupId>
|
||||||
<artifactId>rocksdbjni</artifactId>
|
<artifactId>rocksdbjni</artifactId>
|
||||||
<version>7.0.3</version>
|
<version>7.0.4</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.lucene</groupId>
|
<groupId>org.apache.lucene</groupId>
|
||||||
|
|
|
@ -235,6 +235,7 @@ versions:
|
||||||
memtableMemoryBudgetBytes: -long
|
memtableMemoryBudgetBytes: -long
|
||||||
blockCache: -long
|
blockCache: -long
|
||||||
setCacheIndexAndFilterBlocks: -boolean
|
setCacheIndexAndFilterBlocks: -boolean
|
||||||
|
spinning: boolean
|
||||||
DatabaseVolume:
|
DatabaseVolume:
|
||||||
data:
|
data:
|
||||||
volumePath: Path
|
volumePath: Path
|
||||||
|
|
|
@ -143,12 +143,6 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
|
descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
|
||||||
for (Column column : columns) {
|
for (Column column : columns) {
|
||||||
var columnOptions = new ColumnFamilyOptions();
|
var columnOptions = new ColumnFamilyOptions();
|
||||||
columnOptions.setMaxBytesForLevelBase(256 * SizeUnit.MB);
|
|
||||||
columnOptions.setMaxBytesForLevelMultiplier(10);
|
|
||||||
columnOptions.setLevelCompactionDynamicLevelBytes(true);
|
|
||||||
columnOptions.setLevel0FileNumCompactionTrigger(2);
|
|
||||||
columnOptions.setLevel0SlowdownWritesTrigger(20);
|
|
||||||
columnOptions.setLevel0StopWritesTrigger(36);
|
|
||||||
|
|
||||||
//noinspection ConstantConditions
|
//noinspection ConstantConditions
|
||||||
if (databaseOptions.memtableMemoryBudgetBytes() != null) {
|
if (databaseOptions.memtableMemoryBudgetBytes() != null) {
|
||||||
|
@ -158,6 +152,20 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
.orElse(databaseOptions.lowMemory() ? 16L * SizeUnit.MB : 128L * SizeUnit.MB));
|
.orElse(databaseOptions.lowMemory() ? 16L * SizeUnit.MB : 128L * SizeUnit.MB));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
|
||||||
|
columnOptions.setMaxBytesForLevelBase(256 * SizeUnit.MB);
|
||||||
|
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
|
||||||
|
columnOptions.setMaxBytesForLevelMultiplier(10);
|
||||||
|
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
|
||||||
|
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
|
||||||
|
columnOptions.setLevelCompactionDynamicLevelBytes(true);
|
||||||
|
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
|
||||||
|
columnOptions.setLevel0FileNumCompactionTrigger(2);
|
||||||
|
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
|
||||||
|
columnOptions.setLevel0SlowdownWritesTrigger(20);
|
||||||
|
// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
|
||||||
|
columnOptions.setLevel0StopWritesTrigger(36);
|
||||||
|
|
||||||
if (!databaseOptions.levels().isEmpty()) {
|
if (!databaseOptions.levels().isEmpty()) {
|
||||||
var firstLevelOptions = getRocksLevelOptions(databaseOptions.levels().get(0));
|
var firstLevelOptions = getRocksLevelOptions(databaseOptions.levels().get(0));
|
||||||
columnOptions.setCompressionType(firstLevelOptions.compressionType);
|
columnOptions.setCompressionType(firstLevelOptions.compressionType);
|
||||||
|
@ -198,11 +206,16 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
tableOptions.setOptimizeFiltersForMemory(true);
|
tableOptions.setOptimizeFiltersForMemory(true);
|
||||||
tableOptions.setVerifyCompression(false);
|
tableOptions.setVerifyCompression(false);
|
||||||
}
|
}
|
||||||
|
boolean cacheIndexAndFilterBlocks = databaseOptions.setCacheIndexAndFilterBlocks().orElse(true);
|
||||||
|
if (databaseOptions.spinning()) {
|
||||||
|
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
|
||||||
|
cacheIndexAndFilterBlocks = true;
|
||||||
|
}
|
||||||
tableOptions
|
tableOptions
|
||||||
.setPinTopLevelIndexAndFilter(true)
|
.setPinTopLevelIndexAndFilter(true)
|
||||||
.setPinL0FilterAndIndexBlocksInCache(true)
|
.setPinL0FilterAndIndexBlocksInCache(true)
|
||||||
.setCacheIndexAndFilterBlocksWithHighPriority(true)
|
.setCacheIndexAndFilterBlocksWithHighPriority(true)
|
||||||
.setCacheIndexAndFilterBlocks(databaseOptions.setCacheIndexAndFilterBlocks().orElse(true))
|
.setCacheIndexAndFilterBlocks(cacheIndexAndFilterBlocks)
|
||||||
.setPartitionFilters(true)
|
.setPartitionFilters(true)
|
||||||
.setIndexType(IndexType.kTwoLevelIndexSearch)
|
.setIndexType(IndexType.kTwoLevelIndexSearch)
|
||||||
.setFormatVersion(5)
|
.setFormatVersion(5)
|
||||||
|
@ -211,11 +224,11 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
.setBlockCacheCompressed(optionsWithCache.compressedCache())
|
.setBlockCacheCompressed(optionsWithCache.compressedCache())
|
||||||
.setBlockCache(optionsWithCache.standardCache())
|
.setBlockCache(optionsWithCache.standardCache())
|
||||||
// Spinning disks: 64KiB to 256KiB (also 512KiB). SSDs: 16KiB
|
// Spinning disks: 64KiB to 256KiB (also 512KiB). SSDs: 16KiB
|
||||||
.setBlockSize(256 * 1024);
|
// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
|
||||||
|
.setBlockSize((databaseOptions.spinning() ? 256 : 16) * SizeUnit.KB);
|
||||||
|
|
||||||
//columnOptions.setLevelCompactionDynamicLevelBytes(true);
|
|
||||||
columnOptions.setTableFormatConfig(tableOptions);
|
columnOptions.setTableFormatConfig(tableOptions);
|
||||||
columnOptions.setCompactionPriority(CompactionPriority.OldestSmallestSeqFirst);
|
columnOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
|
||||||
|
|
||||||
descriptors.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions));
|
descriptors.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions));
|
||||||
}
|
}
|
||||||
|
|
|
@ -96,7 +96,8 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||||
Nullableint.of(-1),
|
Nullableint.of(-1),
|
||||||
Nullablelong.empty(),
|
Nullablelong.empty(),
|
||||||
Nullablelong.empty(),
|
Nullablelong.empty(),
|
||||||
Nullableboolean.empty()
|
Nullableboolean.empty(),
|
||||||
|
false
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
conn.getLuceneIndex("testluceneindex1",
|
conn.getLuceneIndex("testluceneindex1",
|
||||||
|
|
|
@ -67,7 +67,8 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||||
Nullableint.of(-1),
|
Nullableint.of(-1),
|
||||||
Nullablelong.empty(),
|
Nullablelong.empty(),
|
||||||
Nullablelong.empty(),
|
Nullablelong.empty(),
|
||||||
Nullableboolean.empty()
|
Nullableboolean.empty(),
|
||||||
|
false
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
conn.getLuceneIndex("testluceneindex1",
|
conn.getLuceneIndex("testluceneindex1",
|
||||||
|
|
Loading…
Reference in New Issue
Block a user