From 95d436860f4073e8c8e13ee7896cfabaca88c2fa Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Fri, 8 Apr 2022 14:32:47 +0200 Subject: [PATCH] Update options --- src/main/data-generator/quic-rpc.yaml | 29 ++++++- .../client/DefaultDatabaseOptions.java | 62 +++++++++++++ .../disk/LLLocalKeyValueDatabase.java | 87 ++++++++++++------- .../dbengine/LocalTemporaryDbGenerator.java | 20 +---- .../dbengine/MemoryTemporaryDbGenerator.java | 18 +--- 5 files changed, 149 insertions(+), 67 deletions(-) create mode 100644 src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java diff --git a/src/main/data-generator/quic-rpc.yaml b/src/main/data-generator/quic-rpc.yaml index 983f4dc..8363999 100644 --- a/src/main/data-generator/quic-rpc.yaml +++ b/src/main/data-generator/quic-rpc.yaml @@ -13,6 +13,12 @@ interfacesData: extendInterfaces: [RPCEvent] ServerBoundResponse: extendInterfaces: [RPCEvent] + ColumnOptions: + commonGetters: + levels: DatabaseLevel[] + memtableMemoryBudgetBytes: -long + cacheIndexAndFilterBlocks: -boolean + filter: -Filter # versions must have only numbers, lowercase letters, dots, dashes. Maximum: 99.999.9999 versions: 0.0.0: @@ -84,6 +90,10 @@ versions: Filter: [ BloomFilter ] + ColumnOptions: [ + DefaultColumnOptions, + NamedColumnOptions + ] customTypes: Path: javaClass: java.nio.file.Path @@ -226,7 +236,6 @@ versions: DatabaseOptions: data: volumes: DatabaseVolume[] - levels: DatabaseLevel[] extraFlags: StringMap absoluteConsistency: boolean lowMemory: boolean @@ -235,10 +244,24 @@ versions: allowNettyDirect: boolean optimistic: boolean maxOpenFiles: -int - memtableMemoryBudgetBytes: -long blockCache: -long - cacheIndexAndFilterBlocks: -boolean spinning: boolean + defaultColumnOptions: DefaultColumnOptions + columnOptions: NamedColumnOptions[] + # Remember to update ColumnOptions common getters + DefaultColumnOptions: + data: + levels: DatabaseLevel[] + memtableMemoryBudgetBytes: -long + cacheIndexAndFilterBlocks: -boolean + filter: -Filter + # Remember to update ColumnOptions common getters + NamedColumnOptions: + data: + columnName: String + levels: DatabaseLevel[] + memtableMemoryBudgetBytes: -long + cacheIndexAndFilterBlocks: -boolean filter: -Filter BloomFilter: data: diff --git a/src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java b/src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java new file mode 100644 index 0000000..7da6db8 --- /dev/null +++ b/src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java @@ -0,0 +1,62 @@ +package it.cavallium.dbengine.client; + +import it.cavallium.data.generator.nativedata.Nullableboolean; +import it.cavallium.data.generator.nativedata.Nullableint; +import it.cavallium.data.generator.nativedata.Nullablelong; +import it.cavallium.dbengine.rpc.current.data.DatabaseOptions; +import it.cavallium.dbengine.rpc.current.data.DatabaseOptionsBuilder; +import it.cavallium.dbengine.rpc.current.data.DefaultColumnOptions; +import it.cavallium.dbengine.rpc.current.data.DefaultColumnOptionsBuilder; +import it.cavallium.dbengine.rpc.current.data.NamedColumnOptions; +import it.cavallium.dbengine.rpc.current.data.NamedColumnOptionsBuilder; +import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.rocksdb.RocksDB; + +public class DefaultDatabaseOptions { + + public static DefaultColumnOptions DEFAULT_DEFAULT_COLUMN_OPTIONS = new DefaultColumnOptions( + Collections.emptyList(), + Nullablelong.empty(), + Nullableboolean.empty(), + NullableFilter.empty() + ); + + public static NamedColumnOptions DEFAULT_NAMED_COLUMN_OPTIONS = new NamedColumnOptions( + new String(RocksDB.DEFAULT_COLUMN_FAMILY, StandardCharsets.UTF_8), + Collections.emptyList(), + Nullablelong.empty(), + Nullableboolean.empty(), + NullableFilter.empty() + ); + + public static DatabaseOptions DEFAULT_DATABASE_OPTIONS = new DatabaseOptions(List.of(), + Map.of(), + false, + false, + false, + false, + true, + true, + Nullableint.empty(), + Nullablelong.empty(), + false, + DEFAULT_DEFAULT_COLUMN_OPTIONS, + List.of() + ); + + public static DatabaseOptionsBuilder builder() { + return DatabaseOptionsBuilder.builder(DEFAULT_DATABASE_OPTIONS); + } + + public static DefaultColumnOptionsBuilder defaultColumnOptionsBuilder() { + return DefaultColumnOptionsBuilder.builder(DEFAULT_DEFAULT_COLUMN_OPTIONS); + } + + public static NamedColumnOptionsBuilder namedColumnOptionsBuilder() { + return NamedColumnOptionsBuilder.builder(DEFAULT_NAMED_COLUMN_OPTIONS); + } +} diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index 5a75a7f..0f9558e 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -16,9 +16,11 @@ import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.UpdateMode; import it.cavallium.dbengine.rpc.current.data.Column; +import it.cavallium.dbengine.rpc.current.data.ColumnOptions; import it.cavallium.dbengine.rpc.current.data.DatabaseLevel; import it.cavallium.dbengine.rpc.current.data.DatabaseOptions; import it.cavallium.dbengine.rpc.current.data.DatabaseVolume; +import it.cavallium.dbengine.rpc.current.data.NamedColumnOptions; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -39,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -144,50 +147,70 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { List descriptors = new ArrayList<>(); descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + + // Check column names validity + for (NamedColumnOptions columnOption : databaseOptions.columnOptions()) { + if (columns.stream().map(Column::name).noneMatch(columnName -> columnName.equals(columnOption.columnName()))) { + throw new IllegalArgumentException( + "Column " + columnOption.columnName() + " does not exist. Available columns: " + columns + .stream() + .map(Column::name) + .collect(Collectors.joining(", ", "[", "]"))); + } + } + for (Column column : columns) { - var columnOptions = new ColumnFamilyOptions(); + var columnFamilyOptions = new ColumnFamilyOptions(); + + var columnOptions = databaseOptions + .columnOptions() + .stream() + .filter(opts -> opts.columnName().equals(column.name())) + .findFirst() + .map(opts -> (ColumnOptions) opts) + .orElse(databaseOptions.defaultColumnOptions()); //noinspection ConstantConditions - if (databaseOptions.memtableMemoryBudgetBytes() != null) { + if (columnOptions.memtableMemoryBudgetBytes() != null) { // about 512MB of ram will be used for level style compaction - columnOptions.optimizeLevelStyleCompaction(databaseOptions.memtableMemoryBudgetBytes().orElse( + columnFamilyOptions.optimizeLevelStyleCompaction(columnOptions.memtableMemoryBudgetBytes().orElse( databaseOptions.lowMemory() ? (DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET / 4) : DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET)); } // https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html - columnOptions.setMaxBytesForLevelBase((databaseOptions.spinning() ? 512 : 256) * SizeUnit.MB); + columnFamilyOptions.setMaxBytesForLevelBase((databaseOptions.spinning() ? 512 : 256) * SizeUnit.MB); // https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html - columnOptions.setMaxBytesForLevelMultiplier(10); + columnFamilyOptions.setMaxBytesForLevelMultiplier(10); // https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html // https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks - columnOptions.setLevelCompactionDynamicLevelBytes(true); + columnFamilyOptions.setLevelCompactionDynamicLevelBytes(true); // https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html - columnOptions.setLevel0FileNumCompactionTrigger(2); + columnFamilyOptions.setLevel0FileNumCompactionTrigger(2); // https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html - columnOptions.setLevel0SlowdownWritesTrigger(20); + columnFamilyOptions.setLevel0SlowdownWritesTrigger(20); // https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html - columnOptions.setLevel0StopWritesTrigger(36); + columnFamilyOptions.setLevel0StopWritesTrigger(36); - if (!databaseOptions.levels().isEmpty()) { - var firstLevelOptions = getRocksLevelOptions(databaseOptions.levels().get(0)); - columnOptions.setCompressionType(firstLevelOptions.compressionType); - columnOptions.setCompressionOptions(firstLevelOptions.compressionOptions); + if (!columnOptions.levels().isEmpty()) { + var firstLevelOptions = getRocksLevelOptions(columnOptions.levels().get(0)); + columnFamilyOptions.setCompressionType(firstLevelOptions.compressionType); + columnFamilyOptions.setCompressionOptions(firstLevelOptions.compressionOptions); - var lastLevelOptions = getRocksLevelOptions(databaseOptions + var lastLevelOptions = getRocksLevelOptions(columnOptions .levels() - .get(databaseOptions.levels().size() - 1)); - columnOptions.setBottommostCompressionType(lastLevelOptions.compressionType); - columnOptions.setBottommostCompressionOptions(lastLevelOptions.compressionOptions); + .get(columnOptions.levels().size() - 1)); + columnFamilyOptions.setBottommostCompressionType(lastLevelOptions.compressionType); + columnFamilyOptions.setBottommostCompressionOptions(lastLevelOptions.compressionOptions); - columnOptions.setCompressionPerLevel(databaseOptions + columnFamilyOptions.setCompressionPerLevel(columnOptions .levels() .stream() .map(v -> v.compression().getType()) .toList()); } else { - columnOptions.setNumLevels(7); + columnFamilyOptions.setNumLevels(7); List compressionTypes = new ArrayList<>(7); for (int i = 0; i < 7; i++) { if (i < 2) { @@ -196,20 +219,20 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { compressionTypes.add(CompressionType.LZ4_COMPRESSION); } } - columnOptions.setBottommostCompressionType(CompressionType.LZ4_COMPRESSION); - columnOptions.setBottommostCompressionOptions(new CompressionOptions() + columnFamilyOptions.setBottommostCompressionType(CompressionType.LZ4_COMPRESSION); + columnFamilyOptions.setBottommostCompressionOptions(new CompressionOptions() .setEnabled(true) .setMaxDictBytes(32768)); - columnOptions.setCompressionPerLevel(compressionTypes); + columnFamilyOptions.setCompressionPerLevel(compressionTypes); } final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); if (!databaseOptions.lowMemory()) { tableOptions.setOptimizeFiltersForMemory(true); - tableOptions.setVerifyCompression(false); } - if (databaseOptions.filter().isPresent()) { - var filterOptions = databaseOptions.filter().get(); + tableOptions.setVerifyCompression(false); + if (columnOptions.filter().isPresent()) { + var filterOptions = columnOptions.filter().get(); if (filterOptions instanceof it.cavallium.dbengine.rpc.current.data.BloomFilter bloomFilterOptions) { // If OptimizeFiltersForHits == true: memory size = bitsPerKey * (totalKeys * 0.1) @@ -218,7 +241,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { tableOptions.setFilterPolicy(bloomFilter); } } - boolean cacheIndexAndFilterBlocks = databaseOptions.cacheIndexAndFilterBlocks() + boolean cacheIndexAndFilterBlocks = columnOptions.cacheIndexAndFilterBlocks() // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters .orElse(true); if (databaseOptions.spinning()) { @@ -243,17 +266,17 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { // https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks .setBlockSize((databaseOptions.spinning() ? 256 : 16) * SizeUnit.KB); - columnOptions.setTableFormatConfig(tableOptions); - columnOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio); - if (databaseOptions.filter().isPresent()) { - var filterOptions = databaseOptions.filter().get(); + columnFamilyOptions.setTableFormatConfig(tableOptions); + columnFamilyOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio); + if (columnOptions.filter().isPresent()) { + var filterOptions = columnOptions.filter().get(); if (filterOptions instanceof it.cavallium.dbengine.rpc.current.data.BloomFilter bloomFilterOptions) { boolean optimizeForHits = bloomFilterOptions.optimizeForHits() // https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks // https://github.com/EighteenZi/rocksdb_wiki/blob/master/RocksDB-Tuning-Guide.md#throughput-gap-between-random-read-vs-sequential-read-is-much-higher-in-spinning-disks-suggestions= .orElse(databaseOptions.spinning()); - columnOptions.setOptimizeFiltersForHits(optimizeForHits); + columnFamilyOptions.setOptimizeFiltersForHits(optimizeForHits); } } @@ -265,7 +288,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { // // (but the default value is 1, which means that the maximum sstable of each level is the same). // columnOptions.setTargetFileSizeMultiplier(1); - descriptors.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions)); + descriptors.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnFamilyOptions)); } // Get databases directory path diff --git a/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java b/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java index ccdf940..fe83310 100644 --- a/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java +++ b/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java @@ -2,6 +2,7 @@ package it.cavallium.dbengine; import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; +import static it.cavallium.dbengine.client.DefaultDatabaseOptions.DEFAULT_DATABASE_OPTIONS; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import it.cavallium.data.generator.nativedata.Nullableboolean; @@ -10,6 +11,7 @@ import it.cavallium.data.generator.nativedata.Nullableint; import it.cavallium.data.generator.nativedata.Nullablelong; import it.cavallium.dbengine.DbTestUtils.TempDb; import it.cavallium.dbengine.DbTestUtils.TestAllocator; +import it.cavallium.dbengine.client.DefaultDatabaseOptions; import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.database.ColumnUtils; @@ -21,6 +23,7 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer; import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity; import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory; import it.cavallium.dbengine.rpc.current.data.DatabaseOptions; +import it.cavallium.dbengine.rpc.current.data.DatabaseOptionsBuilder; import it.cavallium.dbengine.rpc.current.data.LuceneOptions; import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter; import java.io.IOException; @@ -85,22 +88,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator { return Mono.zip( conn.getDatabase("testdb", List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")), - new DatabaseOptions(List.of(), - List.of(), - Map.of(), - true, - false, - false, - true, - canUseNettyDirect, - false, - Nullableint.of(-1), - Nullablelong.empty(), - Nullablelong.empty(), - Nullableboolean.empty(), - false, - NullableFilter.empty() - ) + DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build() ), conn.getLuceneIndex("testluceneindex1", LuceneUtils.singleStructure(), diff --git a/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java b/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java index 4a889ef..f72814a 100644 --- a/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java +++ b/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java @@ -9,6 +9,7 @@ import it.cavallium.data.generator.nativedata.Nullableint; import it.cavallium.data.generator.nativedata.Nullablelong; import it.cavallium.dbengine.DbTestUtils.TempDb; import it.cavallium.dbengine.DbTestUtils.TestAllocator; +import it.cavallium.dbengine.client.DefaultDatabaseOptions; import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.database.ColumnUtils; @@ -56,22 +57,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator { .zip( conn.getDatabase("testdb", List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")), - DatabaseOptions.of(List.of(), - List.of(), - Map.of(), - true, - false, - false, - true, - canUseNettyDirect, - true, - Nullableint.of(-1), - Nullablelong.empty(), - Nullablelong.empty(), - Nullableboolean.empty(), - false, - NullableFilter.empty() - ) + DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build() ), conn.getLuceneIndex("testluceneindex1", LuceneUtils.singleStructure(),