Update options

src/main/data-generator/quic-rpc.yaml

@@ -13,6 +13,12 @@ interfacesData:
     extendInterfaces: [RPCEvent]
   ServerBoundResponse:
     extendInterfaces: [RPCEvent]
+  ColumnOptions:
+    commonGetters:
+      levels: DatabaseLevel[]
+      memtableMemoryBudgetBytes: -long
+      cacheIndexAndFilterBlocks: -boolean
+      filter: -Filter
 # versions must have only numbers, lowercase letters, dots, dashes. Maximum: 99.999.9999
 versions:
   0.0.0:
@@ -84,6 +90,10 @@ versions:
       Filter: [
         BloomFilter
       ]
+      ColumnOptions: [
+        DefaultColumnOptions,
+        NamedColumnOptions
+      ]
     customTypes:
       Path:
         javaClass: java.nio.file.Path
@@ -226,7 +236,6 @@ versions:
       DatabaseOptions:
         data:
           volumes: DatabaseVolume[]
-          levels: DatabaseLevel[]
           extraFlags: StringMap
           absoluteConsistency: boolean
           lowMemory: boolean
@@ -235,10 +244,24 @@ versions:
           allowNettyDirect: boolean
           optimistic: boolean
           maxOpenFiles: -int
-          memtableMemoryBudgetBytes: -long
           blockCache: -long
-          cacheIndexAndFilterBlocks: -boolean
           spinning: boolean
+          defaultColumnOptions: DefaultColumnOptions
+          columnOptions: NamedColumnOptions[]
+      # Remember to update ColumnOptions common getters
+      DefaultColumnOptions:
+        data:
+          levels: DatabaseLevel[]
+          memtableMemoryBudgetBytes: -long
+          cacheIndexAndFilterBlocks: -boolean
+          filter: -Filter
+      # Remember to update ColumnOptions common getters
+      NamedColumnOptions:
+        data:
+          columnName: String
+          levels: DatabaseLevel[]
+          memtableMemoryBudgetBytes: -long
+          cacheIndexAndFilterBlocks: -boolean
           filter: -Filter
       BloomFilter:
         data:

src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java

@@ -0,0 +1,62 @@
+package it.cavallium.dbengine.client;
+
+import it.cavallium.data.generator.nativedata.Nullableboolean;
+import it.cavallium.data.generator.nativedata.Nullableint;
+import it.cavallium.data.generator.nativedata.Nullablelong;
+import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
+import it.cavallium.dbengine.rpc.current.data.DatabaseOptionsBuilder;
+import it.cavallium.dbengine.rpc.current.data.DefaultColumnOptions;
+import it.cavallium.dbengine.rpc.current.data.DefaultColumnOptionsBuilder;
+import it.cavallium.dbengine.rpc.current.data.NamedColumnOptions;
+import it.cavallium.dbengine.rpc.current.data.NamedColumnOptionsBuilder;
+import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.rocksdb.RocksDB;
+
+public class DefaultDatabaseOptions {
+
+	public static DefaultColumnOptions DEFAULT_DEFAULT_COLUMN_OPTIONS = new DefaultColumnOptions(
+			Collections.emptyList(),
+			Nullablelong.empty(),
+			Nullableboolean.empty(),
+			NullableFilter.empty()
+	);
+
+	public static NamedColumnOptions DEFAULT_NAMED_COLUMN_OPTIONS = new NamedColumnOptions(
+			new String(RocksDB.DEFAULT_COLUMN_FAMILY, StandardCharsets.UTF_8),
+			Collections.emptyList(),
+			Nullablelong.empty(),
+			Nullableboolean.empty(),
+			NullableFilter.empty()
+	);
+
+	public static DatabaseOptions DEFAULT_DATABASE_OPTIONS = new DatabaseOptions(List.of(),
+			Map.of(),
+			false,
+			false,
+			false,
+			false,
+			true,
+			true,
+			Nullableint.empty(),
+			Nullablelong.empty(),
+			false,
+			DEFAULT_DEFAULT_COLUMN_OPTIONS,
+			List.of()
+	);
+
+	public static DatabaseOptionsBuilder builder() {
+		return DatabaseOptionsBuilder.builder(DEFAULT_DATABASE_OPTIONS);
+	}
+
+	public static DefaultColumnOptionsBuilder defaultColumnOptionsBuilder() {
+		return DefaultColumnOptionsBuilder.builder(DEFAULT_DEFAULT_COLUMN_OPTIONS);
+	}
+
+	public static NamedColumnOptionsBuilder namedColumnOptionsBuilder() {
+		return NamedColumnOptionsBuilder.builder(DEFAULT_NAMED_COLUMN_OPTIONS);
+	}
+}

src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java

@@ -16,9 +16,11 @@ import it.cavallium.dbengine.database.LLSnapshot;
 import it.cavallium.dbengine.database.LLUtils;
 import it.cavallium.dbengine.database.UpdateMode;
 import it.cavallium.dbengine.rpc.current.data.Column;
+import it.cavallium.dbengine.rpc.current.data.ColumnOptions;
 import it.cavallium.dbengine.rpc.current.data.DatabaseLevel;
 import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
 import it.cavallium.dbengine.rpc.current.data.DatabaseVolume;
+import it.cavallium.dbengine.rpc.current.data.NamedColumnOptions;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -39,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
 import org.apache.commons.lang3.time.StopWatch;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -144,50 +147,70 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 			List<ColumnFamilyDescriptor> descriptors = new ArrayList<>();
 
 			descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
+
+			// Check column names validity
+			for (NamedColumnOptions columnOption : databaseOptions.columnOptions()) {
+				if (columns.stream().map(Column::name).noneMatch(columnName -> columnName.equals(columnOption.columnName()))) {
+					throw new IllegalArgumentException(
+							"Column " + columnOption.columnName() + " does not exist. Available columns: " + columns
+									.stream()
+									.map(Column::name)
+									.collect(Collectors.joining(", ", "[", "]")));
+				}
+			}
+
 			for (Column column : columns) {
-				var columnOptions = new ColumnFamilyOptions();
+				var columnFamilyOptions = new ColumnFamilyOptions();
+
+				var columnOptions = databaseOptions
+						.columnOptions()
+						.stream()
+						.filter(opts -> opts.columnName().equals(column.name()))
+						.findFirst()
+						.map(opts -> (ColumnOptions) opts)
+						.orElse(databaseOptions.defaultColumnOptions());
 
 				//noinspection ConstantConditions
-				if (databaseOptions.memtableMemoryBudgetBytes() != null) {
+				if (columnOptions.memtableMemoryBudgetBytes() != null) {
 					// about 512MB of ram will be used for level style compaction
-					columnOptions.optimizeLevelStyleCompaction(databaseOptions.memtableMemoryBudgetBytes().orElse(
+					columnFamilyOptions.optimizeLevelStyleCompaction(columnOptions.memtableMemoryBudgetBytes().orElse(
 							databaseOptions.lowMemory()
 									? (DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET / 4)
 									: DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET));
 				}
 
 				// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
-				columnOptions.setMaxBytesForLevelBase((databaseOptions.spinning() ? 512 : 256) * SizeUnit.MB);
+				columnFamilyOptions.setMaxBytesForLevelBase((databaseOptions.spinning() ? 512 : 256) * SizeUnit.MB);
 				// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
-				columnOptions.setMaxBytesForLevelMultiplier(10);
+				columnFamilyOptions.setMaxBytesForLevelMultiplier(10);
 				// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
 				// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
-				columnOptions.setLevelCompactionDynamicLevelBytes(true);
+				columnFamilyOptions.setLevelCompactionDynamicLevelBytes(true);
 				// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
-				columnOptions.setLevel0FileNumCompactionTrigger(2);
+				columnFamilyOptions.setLevel0FileNumCompactionTrigger(2);
 				// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
-				columnOptions.setLevel0SlowdownWritesTrigger(20);
+				columnFamilyOptions.setLevel0SlowdownWritesTrigger(20);
 				// https://www.arangodb.com/docs/stable/programs-arangod-rocksdb.html
-				columnOptions.setLevel0StopWritesTrigger(36);
+				columnFamilyOptions.setLevel0StopWritesTrigger(36);
 
-				if (!databaseOptions.levels().isEmpty()) {
+				if (!columnOptions.levels().isEmpty()) {
-					var firstLevelOptions = getRocksLevelOptions(databaseOptions.levels().get(0));
+					var firstLevelOptions = getRocksLevelOptions(columnOptions.levels().get(0));
-					columnOptions.setCompressionType(firstLevelOptions.compressionType);
+					columnFamilyOptions.setCompressionType(firstLevelOptions.compressionType);
-					columnOptions.setCompressionOptions(firstLevelOptions.compressionOptions);
+					columnFamilyOptions.setCompressionOptions(firstLevelOptions.compressionOptions);
 
-					var lastLevelOptions = getRocksLevelOptions(databaseOptions
+					var lastLevelOptions = getRocksLevelOptions(columnOptions
 							.levels()
-							.get(databaseOptions.levels().size() - 1));
+							.get(columnOptions.levels().size() - 1));
-					columnOptions.setBottommostCompressionType(lastLevelOptions.compressionType);
+					columnFamilyOptions.setBottommostCompressionType(lastLevelOptions.compressionType);
-					columnOptions.setBottommostCompressionOptions(lastLevelOptions.compressionOptions);
+					columnFamilyOptions.setBottommostCompressionOptions(lastLevelOptions.compressionOptions);
 
-					columnOptions.setCompressionPerLevel(databaseOptions
+					columnFamilyOptions.setCompressionPerLevel(columnOptions
 							.levels()
 							.stream()
 							.map(v -> v.compression().getType())
 							.toList());
 				} else {
-					columnOptions.setNumLevels(7);
+					columnFamilyOptions.setNumLevels(7);
 					List<CompressionType> compressionTypes = new ArrayList<>(7);
 					for (int i = 0; i < 7; i++) {
 						if (i < 2) {
@@ -196,20 +219,20 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 							compressionTypes.add(CompressionType.LZ4_COMPRESSION);
 						}
 					}
-					columnOptions.setBottommostCompressionType(CompressionType.LZ4_COMPRESSION);
+					columnFamilyOptions.setBottommostCompressionType(CompressionType.LZ4_COMPRESSION);
-					columnOptions.setBottommostCompressionOptions(new CompressionOptions()
+					columnFamilyOptions.setBottommostCompressionOptions(new CompressionOptions()
 							.setEnabled(true)
 							.setMaxDictBytes(32768));
-					columnOptions.setCompressionPerLevel(compressionTypes);
+					columnFamilyOptions.setCompressionPerLevel(compressionTypes);
 				}
 
 				final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
 				if (!databaseOptions.lowMemory()) {
 					tableOptions.setOptimizeFiltersForMemory(true);
-					tableOptions.setVerifyCompression(false);
 				}
-				if (databaseOptions.filter().isPresent()) {
+				tableOptions.setVerifyCompression(false);
-					var filterOptions = databaseOptions.filter().get();
+				if (columnOptions.filter().isPresent()) {
+					var filterOptions = columnOptions.filter().get();
 
 					if (filterOptions instanceof it.cavallium.dbengine.rpc.current.data.BloomFilter bloomFilterOptions) {
 						// If OptimizeFiltersForHits == true: memory size = bitsPerKey * (totalKeys * 0.1)
@@ -218,7 +241,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 						tableOptions.setFilterPolicy(bloomFilter);
 					}
 				}
-				boolean cacheIndexAndFilterBlocks = databaseOptions.cacheIndexAndFilterBlocks()
+				boolean cacheIndexAndFilterBlocks = columnOptions.cacheIndexAndFilterBlocks()
 						// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
 						.orElse(true);
 				if (databaseOptions.spinning()) {
@@ -243,17 +266,17 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 						// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
 						.setBlockSize((databaseOptions.spinning() ? 256 : 16) * SizeUnit.KB);
 
-				columnOptions.setTableFormatConfig(tableOptions);
+				columnFamilyOptions.setTableFormatConfig(tableOptions);
-				columnOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
+				columnFamilyOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
-				if (databaseOptions.filter().isPresent()) {
+				if (columnOptions.filter().isPresent()) {
-					var filterOptions = databaseOptions.filter().get();
+					var filterOptions = columnOptions.filter().get();
 
 					if (filterOptions instanceof it.cavallium.dbengine.rpc.current.data.BloomFilter bloomFilterOptions) {
 						boolean optimizeForHits = bloomFilterOptions.optimizeForHits()
 								// https://github.com/facebook/rocksdb/wiki/Tuning-RocksDB-on-Spinning-Disks
 								// https://github.com/EighteenZi/rocksdb_wiki/blob/master/RocksDB-Tuning-Guide.md#throughput-gap-between-random-read-vs-sequential-read-is-much-higher-in-spinning-disks-suggestions=
 								.orElse(databaseOptions.spinning());
-						columnOptions.setOptimizeFiltersForHits(optimizeForHits);
+						columnFamilyOptions.setOptimizeFiltersForHits(optimizeForHits);
 					}
 				}
 
@@ -265,7 +288,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 				// // (but the default value is 1, which means that the maximum sstable of each level is the same).
 				// columnOptions.setTargetFileSizeMultiplier(1);
 
-				descriptors.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions));
+				descriptors.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnFamilyOptions));
 			}
 
 			// Get databases directory path

src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java

@@ -2,6 +2,7 @@ package it.cavallium.dbengine;
 
 import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
 import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
+import static it.cavallium.dbengine.client.DefaultDatabaseOptions.DEFAULT_DATABASE_OPTIONS;
 
 import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import it.cavallium.data.generator.nativedata.Nullableboolean;
@@ -10,6 +11,7 @@ import it.cavallium.data.generator.nativedata.Nullableint;
 import it.cavallium.data.generator.nativedata.Nullablelong;
 import it.cavallium.dbengine.DbTestUtils.TempDb;
 import it.cavallium.dbengine.DbTestUtils.TestAllocator;
+import it.cavallium.dbengine.client.DefaultDatabaseOptions;
 import it.cavallium.dbengine.client.IndicizerAnalyzers;
 import it.cavallium.dbengine.client.IndicizerSimilarities;
 import it.cavallium.dbengine.database.ColumnUtils;
@@ -21,6 +23,7 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
 import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
 import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
 import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
+import it.cavallium.dbengine.rpc.current.data.DatabaseOptionsBuilder;
 import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
 import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter;
 import java.io.IOException;
@@ -85,22 +88,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
 						return Mono.zip(
 										conn.getDatabase("testdb",
 												List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
-												new DatabaseOptions(List.of(),
+												DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build()
-														List.of(),
-														Map.of(),
-														true,
-														false,
-														false,
-														true,
-														canUseNettyDirect,
-														false,
-														Nullableint.of(-1),
-														Nullablelong.empty(),
-														Nullablelong.empty(),
-														Nullableboolean.empty(),
-														false,
-														NullableFilter.empty()
-												)
 										),
 										conn.getLuceneIndex("testluceneindex1",
 												LuceneUtils.singleStructure(),

src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java

@@ -9,6 +9,7 @@ import it.cavallium.data.generator.nativedata.Nullableint;
 import it.cavallium.data.generator.nativedata.Nullablelong;
 import it.cavallium.dbengine.DbTestUtils.TempDb;
 import it.cavallium.dbengine.DbTestUtils.TestAllocator;
+import it.cavallium.dbengine.client.DefaultDatabaseOptions;
 import it.cavallium.dbengine.client.IndicizerAnalyzers;
 import it.cavallium.dbengine.client.IndicizerSimilarities;
 import it.cavallium.dbengine.database.ColumnUtils;
@@ -56,22 +57,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
 							.zip(
 									conn.getDatabase("testdb",
 											List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
-											DatabaseOptions.of(List.of(),
+											DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build()
-													List.of(),
-													Map.of(),
-													true,
-													false,
-													false,
-													true,
-													canUseNettyDirect,
-													true,
-													Nullableint.of(-1),
-													Nullablelong.empty(),
-													Nullablelong.empty(),
-													Nullableboolean.empty(),
-													false,
-													NullableFilter.empty()
-											)
 									),
 									conn.getLuceneIndex("testluceneindex1",
 											LuceneUtils.singleStructure(),