From 1b150dcbafb603ac5718b27107736ae5c13cecc6 Mon Sep 17 00:00:00 2001
From: Andrea Cavalli <andrea@cavallium.it>
Date: Mon, 11 Apr 2022 20:04:27 +0200
Subject: [PATCH] configurable partition filters

---
 src/main/data-generator/quic-rpc.yaml          |  3 +++
 .../client/DefaultDatabaseOptions.java         |  2 ++
 .../database/disk/LLLocalKeyValueDatabase.java | 18 +++++++++++-------
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/main/data-generator/quic-rpc.yaml b/src/main/data-generator/quic-rpc.yaml
index 9622e91..3b3f434 100644
--- a/src/main/data-generator/quic-rpc.yaml
+++ b/src/main/data-generator/quic-rpc.yaml
@@ -18,6 +18,7 @@ interfacesData:
       levels: DatabaseLevel[]
       memtableMemoryBudgetBytes: -long
       cacheIndexAndFilterBlocks: -boolean
+      partitionFilters: -boolean
       filter: -Filter
       blockSize: -int
 # versions must have only numbers, lowercase letters, dots, dashes. Maximum: 99.999.9999
@@ -257,6 +258,7 @@ versions:
           levels: DatabaseLevel[]
           memtableMemoryBudgetBytes: -long
           cacheIndexAndFilterBlocks: -boolean
+          partitionFilters: -boolean
           filter: -Filter
           blockSize: -int
       # Remember to update ColumnOptions common getters
@@ -266,6 +268,7 @@ versions:
           levels: DatabaseLevel[]
           memtableMemoryBudgetBytes: -long
           cacheIndexAndFilterBlocks: -boolean
+          partitionFilters: -boolean
           filter: -Filter
           blockSize: -int
       BloomFilter:
diff --git a/src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java b/src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java
index ca71352..217df66 100644
--- a/src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java
+++ b/src/main/java/it/cavallium/dbengine/client/DefaultDatabaseOptions.java
@@ -22,6 +22,7 @@ public class DefaultDatabaseOptions {
 			Collections.emptyList(),
 			Nullablelong.empty(),
 			Nullableboolean.empty(),
+			Nullableboolean.empty(),
 			NullableFilter.empty(),
 			Nullableint.empty()
 	);
@@ -31,6 +32,7 @@ public class DefaultDatabaseOptions {
 			Collections.emptyList(),
 			Nullablelong.empty(),
 			Nullableboolean.empty(),
+			Nullableboolean.empty(),
 			NullableFilter.empty(),
 			Nullableint.empty()
 	);
diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
index af6bb82..4844778 100644
--- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
+++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
@@ -269,7 +269,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 						.setCacheIndexAndFilterBlocksWithHighPriority(true)
 						.setCacheIndexAndFilterBlocks(cacheIndexAndFilterBlocks)
 						// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
-						.setPartitionFilters(true)
+						.setPartitionFilters(columnOptions.partitionFilters().orElse(false))
 						// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
 						.setIndexType(IndexType.kTwoLevelIndexSearch)
 						//todo: replace with kxxhash3
@@ -512,9 +512,13 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 				logger.debug("Failed to release snapshot " + id, ex2);
 			}
 		});
-		db.closeE();
-		compressedCache.close();
-		standardCache.close();
+		db.close();
+		if (compressedCache != null) {
+			compressedCache.close();
+		}
+		if (standardCache != null) {
+			standardCache.close();
+		}
 	}
 
 	private void flushDb(RocksDB db, List<ColumnFamilyHandle> handles) throws RocksDBException {
@@ -664,9 +668,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 					.setBytesPerSync(64 * SizeUnit.KB)
 					.setWalBytesPerSync(64 * SizeUnit.KB)
 
-					.setWalTtlSeconds(0)
-					.setWalSizeLimitMB(0)
-					.setMaxTotalWalSize(80 * SizeUnit.MB) // 80MiB max wal directory size
+					.setWalTtlSeconds(30) // flush wal after 30 seconds
+					.setWalSizeLimitMB(1024) // 1024MB
+					.setMaxTotalWalSize(2L * SizeUnit.GB) // 2GiB max wal directory size
 			;
 			// DO NOT USE ClockCache! IT'S BROKEN!
 			blockCache = new LRUCache(writeBufferManagerSize + databaseOptions.blockCache().orElse( 512 * SizeUnit.MB));