andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Optimizations

Browse Source
This commit is contained in:
Andrea Cavalli 2022-03-30 23:44:55 +02:00
parent cb83c17811
commit 7891b0b9e0
2 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -193,9 +193,10 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
if (!databaseOptions.lowMemory()) {
final BloomFilter bloomFilter = new BloomFilter(3, false);
final BloomFilter bloomFilter = new BloomFilter(10);
tableOptions.setFilterPolicy(bloomFilter);
tableOptions.setOptimizeFiltersForMemory(true);
tableOptions.setVerifyCompression(false);
}
tableOptions
.setPinTopLevelIndexAndFilter(true)
@ -209,7 +210,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.setChecksumType(ChecksumType.kxxHash)
.setBlockCacheCompressed(optionsWithCache.compressedCache())
.setBlockCache(optionsWithCache.standardCache())
.setBlockSize(512 * 1024); // 512KiB
.setBlockSize(16 * 1024); // 16KiB
//columnOptions.setLevelCompactionDynamicLevelBytes(true);
columnOptions.setTableFormatConfig(tableOptions);
@ -483,7 +484,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.setIncreaseParallelism(1)
.setDbWriteBufferSize(8 * SizeUnit.MB)
.setWalTtlSeconds(0)
.setWalSizeLimitMB(0) // 16MB
.setWalSizeLimitMB(0)
.setMaxTotalWalSize(0) // automatic
;
blockCache = new ClockCache(databaseOptions.blockCache().orElse( 8L * SizeUnit.MB) / 2, -1, true);
@ -501,14 +502,13 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
} else {
// HIGH MEMORY
options
.setIncreaseParallelism(Runtime.getRuntime().availableProcessors())
.setDbWriteBufferSize(64 * SizeUnit.MB)
.setBytesPerSync(64 * SizeUnit.KB)
.setWalBytesPerSync(64 * SizeUnit.KB)
.setWalTtlSeconds(30) // flush wal after 30 seconds
.setWalSizeLimitMB(1024) // 1024MB
.setMaxTotalWalSize(2L * SizeUnit.GB) // 2GiB max wal directory size
.setWalTtlSeconds(0)
.setWalSizeLimitMB(0)
.setMaxTotalWalSize(80 * SizeUnit.MB) // 80MiB max wal directory size
;
blockCache = new ClockCache(databaseOptions.blockCache().orElse( 512 * SizeUnit.MB) / 2);
compressedCache = null;
@ -522,6 +522,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.setWritableFileMaxBufferSize(4 * 1024 * 1024)
;
}
options.setIncreaseParallelism(Runtime.getRuntime().availableProcessors());
}
options.setWriteBufferManager(new WriteBufferManager(256L * 1024L * 1024L, blockCache));

9
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -40,6 +40,7 @@ import it.cavallium.dbengine.rpc.current.data.IndicizerAnalyzers;
import it.cavallium.dbengine.rpc.current.data.IndicizerSimilarities;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
@ -85,7 +86,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
* There is only a single thread globally to not overwhelm the disk with
* concurrent commits or concurrent refreshes.
*/
private static final Scheduler luceneHeavyTasksScheduler = uninterruptibleScheduler(Schedulers.single(Schedulers.boundedElastic()));
private static final Scheduler luceneHeavyTasksScheduler = uninterruptibleScheduler(Schedulers.newBoundedElastic(
DEFAULT_BOUNDED_ELASTIC_SIZE,
DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
"heavy-tasks",
Math.toIntExact(Duration.ofHours(1).toSeconds()),
true
));
private static final Scheduler bulkScheduler = uninterruptibleScheduler(Schedulers.boundedElastic());
static {