andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update rocksdb options, don't delete logs manually

Browse Source
This commit is contained in:
Andrea Cavalli 2022-03-22 00:23:32 +01:00
parent 5f6dfac1da
commit 372c45220c
3 changed files with 22 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/it/cavallium/dbengine/database/collections/DatabaseStageMap.java
View File

@ -11,14 +11,17 @@ import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps;
import java.time.Duration;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.logging.Level;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;

6
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

@ -31,6 +31,7 @@ import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -79,7 +80,7 @@ public class LLLocalDictionary implements LLDictionary {
static final ReadOptions EMPTY_READ_OPTIONS = new UnreleasableReadOptions(new UnmodifiableReadOptions());
static final WriteOptions EMPTY_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions());
static final WriteOptions BATCH_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions());
static final boolean PREFER_SEEK_TO_FIRST = true;
static final boolean PREFER_SEEK_TO_FIRST = false;
/**
* It used to be false,
* now it's true to avoid crashes during iterations on completely corrupted files
@ -255,8 +256,9 @@ public class LLLocalDictionary implements LLDictionary {
boolean existsAlmostCertainly) {
return keyMono
.publishOn(dbScheduler)
.handle((keySend, sink) -> {
.<Send<Buffer>>handle((keySend, sink) -> {
try (var key = keySend.receive()) {
logger.trace(MARKER_ROCKSDB, "Reading {}", () -> toStringSafe(key));
try {
var readOptions = requireNonNullElse(resolveSnapshot(snapshot), EMPTY_READ_OPTIONS);
Buffer result;

20
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -74,6 +74,8 @@ import reactor.core.scheduler.Schedulers;
public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
private static final boolean DELETE_LOG_FILES = false;
static {
RocksDB.loadLibrary();
LLUtils.initHooks();
@ -147,8 +149,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
for (Column column : columns) {
var columnOptions = new ColumnFamilyOptions();
columnOptions.setTargetFileSizeBase(16 * SizeUnit.MB);
columnOptions.setTargetFileSizeMultiplier(2);
columnOptions.setTargetFileSizeBase(64 * SizeUnit.MB);
columnOptions.setTargetFileSizeMultiplier(1);
//noinspection ConstantConditions
if (databaseOptions.memtableMemoryBudgetBytes() != null) {
@ -229,8 +231,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
.setBlockCache(optionsWithCache.standardCache())
.setBlockSize(16 * 1024); // 16KiB
//columnOptions.setLevelCompactionDynamicLevelBytes(true);
columnOptions.setTableFormatConfig(tableOptions);
columnOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
columnOptions.setCompactionPriority(CompactionPriority.OldestSmallestSeqFirst);
descriptors
.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions));
@ -448,11 +451,15 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
// the Options class contains a set of configurable DB options
// that determines the behaviour of the database.
var options = new DBOptions();
options.setMaxSubcompactions(2);
options.setMaxFileOpeningThreads(16);
options.setNewTableReaderForCompactionInputs(true);
options.setParanoidChecks(false);
options.setCreateIfMissing(true);
options.setSkipStatsUpdateOnDbOpen(true);
options.setCreateMissingColumnFamilies(true);
options.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
options.setAvoidFlushDuringShutdown(true); // Flush all WALs during shutdown
options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
options.setAvoidFlushDuringShutdown(false); // Flush all WALs during shutdown
options.setAvoidFlushDuringRecovery(true); // Flush all WALs during startup
options.setWalRecoveryMode(databaseOptions.absoluteConsistency()
? WALRecoveryMode.AbsoluteConsistency
@ -741,6 +748,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
*/
@SuppressWarnings("unused")
private void deleteUnusedOldLogFiles() {
if (!DELETE_LOG_FILES) {
return;
}
Path basePath = dbPath;
try {
Files