From 372c45220c618d924b929e1f44c2ef6b03a8331f Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Tue, 22 Mar 2022 00:23:32 +0100 Subject: [PATCH] Update rocksdb options, don't delete logs manually --- .../collections/DatabaseStageMap.java | 3 +++ .../database/disk/LLLocalDictionary.java | 6 ++++-- .../disk/LLLocalKeyValueDatabase.java | 20 ++++++++++++++----- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageMap.java b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageMap.java index 69e9bcb..b4d93ec 100644 --- a/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageMap.java +++ b/src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageMap.java @@ -11,14 +11,17 @@ import it.cavallium.dbengine.database.serialization.SerializationFunction; import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap; import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps; +import java.time.Duration; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.function.Function; +import java.util.logging.Level; import org.jetbrains.annotations.Nullable; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import reactor.core.publisher.SignalType; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java index da784eb..291aeaf 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java @@ -31,6 +31,7 @@ import it.cavallium.dbengine.database.serialization.SerializationFunction; import it.cavallium.dbengine.rpc.current.data.DatabaseOptions; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -79,7 +80,7 @@ public class LLLocalDictionary implements LLDictionary { static final ReadOptions EMPTY_READ_OPTIONS = new UnreleasableReadOptions(new UnmodifiableReadOptions()); static final WriteOptions EMPTY_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions()); static final WriteOptions BATCH_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions()); - static final boolean PREFER_SEEK_TO_FIRST = true; + static final boolean PREFER_SEEK_TO_FIRST = false; /** * It used to be false, * now it's true to avoid crashes during iterations on completely corrupted files @@ -255,8 +256,9 @@ public class LLLocalDictionary implements LLDictionary { boolean existsAlmostCertainly) { return keyMono .publishOn(dbScheduler) - .handle((keySend, sink) -> { + .>handle((keySend, sink) -> { try (var key = keySend.receive()) { + logger.trace(MARKER_ROCKSDB, "Reading {}", () -> toStringSafe(key)); try { var readOptions = requireNonNullElse(resolveSnapshot(snapshot), EMPTY_READ_OPTIONS); Buffer result; diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index 8e48b25..a3a2524 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -74,6 +74,8 @@ import reactor.core.scheduler.Schedulers; public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { + private static final boolean DELETE_LOG_FILES = false; + static { RocksDB.loadLibrary(); LLUtils.initHooks(); @@ -147,8 +149,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { .add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); for (Column column : columns) { var columnOptions = new ColumnFamilyOptions(); - columnOptions.setTargetFileSizeBase(16 * SizeUnit.MB); - columnOptions.setTargetFileSizeMultiplier(2); + columnOptions.setTargetFileSizeBase(64 * SizeUnit.MB); + columnOptions.setTargetFileSizeMultiplier(1); //noinspection ConstantConditions if (databaseOptions.memtableMemoryBudgetBytes() != null) { @@ -229,8 +231,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { .setBlockCache(optionsWithCache.standardCache()) .setBlockSize(16 * 1024); // 16KiB + //columnOptions.setLevelCompactionDynamicLevelBytes(true); columnOptions.setTableFormatConfig(tableOptions); - columnOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio); + columnOptions.setCompactionPriority(CompactionPriority.OldestSmallestSeqFirst); descriptors .add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions)); @@ -448,11 +451,15 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { // the Options class contains a set of configurable DB options // that determines the behaviour of the database. var options = new DBOptions(); + options.setMaxSubcompactions(2); + options.setMaxFileOpeningThreads(16); + options.setNewTableReaderForCompactionInputs(true); options.setParanoidChecks(false); options.setCreateIfMissing(true); + options.setSkipStatsUpdateOnDbOpen(true); options.setCreateMissingColumnFamilies(true); - options.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL); - options.setAvoidFlushDuringShutdown(true); // Flush all WALs during shutdown + options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL); + options.setAvoidFlushDuringShutdown(false); // Flush all WALs during shutdown options.setAvoidFlushDuringRecovery(true); // Flush all WALs during startup options.setWalRecoveryMode(databaseOptions.absoluteConsistency() ? WALRecoveryMode.AbsoluteConsistency @@ -741,6 +748,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { */ @SuppressWarnings("unused") private void deleteUnusedOldLogFiles() { + if (!DELETE_LOG_FILES) { + return; + } Path basePath = dbPath; try { Files