From 7891b0b9e081067cdb8d50a1c7f045eacbec4a29 Mon Sep 17 00:00:00 2001
From: Andrea Cavalli <andrea@cavallium.it>
Date: Wed, 30 Mar 2022 23:44:55 +0200
Subject: [PATCH] Optimizations

---
 .../database/disk/LLLocalKeyValueDatabase.java    | 15 ++++++++-------
 .../database/disk/LLLocalLuceneIndex.java         |  9 ++++++++-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
index f4cf11f..77b5bb3 100644
--- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
+++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
@@ -193,9 +193,10 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 
 				final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
 				if (!databaseOptions.lowMemory()) {
-					final BloomFilter bloomFilter = new BloomFilter(3, false);
+					final BloomFilter bloomFilter = new BloomFilter(10);
 					tableOptions.setFilterPolicy(bloomFilter);
 					tableOptions.setOptimizeFiltersForMemory(true);
+					tableOptions.setVerifyCompression(false);
 				}
 				tableOptions
 						.setPinTopLevelIndexAndFilter(true)
@@ -209,7 +210,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 						.setChecksumType(ChecksumType.kxxHash)
 						.setBlockCacheCompressed(optionsWithCache.compressedCache())
 						.setBlockCache(optionsWithCache.standardCache())
-						.setBlockSize(512 * 1024); // 512KiB
+						.setBlockSize(16 * 1024); // 16KiB
 
 				//columnOptions.setLevelCompactionDynamicLevelBytes(true);
 				columnOptions.setTableFormatConfig(tableOptions);
@@ -483,7 +484,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 					.setIncreaseParallelism(1)
 					.setDbWriteBufferSize(8 * SizeUnit.MB)
 					.setWalTtlSeconds(0)
-					.setWalSizeLimitMB(0) // 16MB
+					.setWalSizeLimitMB(0)
 					.setMaxTotalWalSize(0) // automatic
 			;
 			blockCache = new ClockCache(databaseOptions.blockCache().orElse( 8L * SizeUnit.MB) / 2, -1, true);
@@ -501,14 +502,13 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 		} else {
 			// HIGH MEMORY
 			options
-					.setIncreaseParallelism(Runtime.getRuntime().availableProcessors())
 					.setDbWriteBufferSize(64 * SizeUnit.MB)
 					.setBytesPerSync(64 * SizeUnit.KB)
 					.setWalBytesPerSync(64 * SizeUnit.KB)
 
-					.setWalTtlSeconds(30) // flush wal after 30 seconds
-					.setWalSizeLimitMB(1024) // 1024MB
-					.setMaxTotalWalSize(2L * SizeUnit.GB) // 2GiB max wal directory size
+					.setWalTtlSeconds(0)
+					.setWalSizeLimitMB(0)
+					.setMaxTotalWalSize(80 * SizeUnit.MB) // 80MiB max wal directory size
 			;
 			blockCache = new ClockCache(databaseOptions.blockCache().orElse( 512 * SizeUnit.MB) / 2);
 			compressedCache = null;
@@ -522,6 +522,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
 						.setWritableFileMaxBufferSize(4 * 1024 * 1024)
 				;
 			}
+			options.setIncreaseParallelism(Runtime.getRuntime().availableProcessors());
 		}
 
 		options.setWriteBufferManager(new WriteBufferManager(256L * 1024L * 1024L, blockCache));
diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
index 2b62353..c3dc463 100644
--- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
+++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
@@ -40,6 +40,7 @@ import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
 import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
 import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
 import java.io.IOException;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map.Entry;
@@ -85,7 +86,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
 	 * There is only a single thread globally to not overwhelm the disk with
 	 * concurrent commits or concurrent refreshes.
 	 */
-	private static final Scheduler luceneHeavyTasksScheduler = uninterruptibleScheduler(Schedulers.single(Schedulers.boundedElastic()));
+	private static final Scheduler luceneHeavyTasksScheduler = uninterruptibleScheduler(Schedulers.newBoundedElastic(
+			DEFAULT_BOUNDED_ELASTIC_SIZE,
+			DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
+			"heavy-tasks",
+			Math.toIntExact(Duration.ofHours(1).toSeconds()),
+			true
+	));
 	private static final Scheduler bulkScheduler = uninterruptibleScheduler(Schedulers.boundedElastic());
 
 	static {