Update rocksdb options, don't delete logs manually
This commit is contained in:
parent
5f6dfac1da
commit
372c45220c
|
@ -11,14 +11,17 @@ import it.cavallium.dbengine.database.serialization.SerializationFunction;
|
||||||
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
|
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
|
||||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps;
|
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps;
|
||||||
|
import java.time.Duration;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
|
import java.util.logging.Level;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import reactor.core.publisher.Flux;
|
import reactor.core.publisher.Flux;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
|
import reactor.core.publisher.SignalType;
|
||||||
import reactor.util.function.Tuple2;
|
import reactor.util.function.Tuple2;
|
||||||
import reactor.util.function.Tuples;
|
import reactor.util.function.Tuples;
|
||||||
|
|
||||||
|
|
|
@ -31,6 +31,7 @@ import it.cavallium.dbengine.database.serialization.SerializationFunction;
|
||||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
|
import java.time.Duration;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -79,7 +80,7 @@ public class LLLocalDictionary implements LLDictionary {
|
||||||
static final ReadOptions EMPTY_READ_OPTIONS = new UnreleasableReadOptions(new UnmodifiableReadOptions());
|
static final ReadOptions EMPTY_READ_OPTIONS = new UnreleasableReadOptions(new UnmodifiableReadOptions());
|
||||||
static final WriteOptions EMPTY_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions());
|
static final WriteOptions EMPTY_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions());
|
||||||
static final WriteOptions BATCH_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions());
|
static final WriteOptions BATCH_WRITE_OPTIONS = new UnreleasableWriteOptions(new UnmodifiableWriteOptions());
|
||||||
static final boolean PREFER_SEEK_TO_FIRST = true;
|
static final boolean PREFER_SEEK_TO_FIRST = false;
|
||||||
/**
|
/**
|
||||||
* It used to be false,
|
* It used to be false,
|
||||||
* now it's true to avoid crashes during iterations on completely corrupted files
|
* now it's true to avoid crashes during iterations on completely corrupted files
|
||||||
|
@ -255,8 +256,9 @@ public class LLLocalDictionary implements LLDictionary {
|
||||||
boolean existsAlmostCertainly) {
|
boolean existsAlmostCertainly) {
|
||||||
return keyMono
|
return keyMono
|
||||||
.publishOn(dbScheduler)
|
.publishOn(dbScheduler)
|
||||||
.handle((keySend, sink) -> {
|
.<Send<Buffer>>handle((keySend, sink) -> {
|
||||||
try (var key = keySend.receive()) {
|
try (var key = keySend.receive()) {
|
||||||
|
logger.trace(MARKER_ROCKSDB, "Reading {}", () -> toStringSafe(key));
|
||||||
try {
|
try {
|
||||||
var readOptions = requireNonNullElse(resolveSnapshot(snapshot), EMPTY_READ_OPTIONS);
|
var readOptions = requireNonNullElse(resolveSnapshot(snapshot), EMPTY_READ_OPTIONS);
|
||||||
Buffer result;
|
Buffer result;
|
||||||
|
|
|
@ -74,6 +74,8 @@ import reactor.core.scheduler.Schedulers;
|
||||||
|
|
||||||
public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
|
|
||||||
|
private static final boolean DELETE_LOG_FILES = false;
|
||||||
|
|
||||||
static {
|
static {
|
||||||
RocksDB.loadLibrary();
|
RocksDB.loadLibrary();
|
||||||
LLUtils.initHooks();
|
LLUtils.initHooks();
|
||||||
|
@ -147,8 +149,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
|
.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
|
||||||
for (Column column : columns) {
|
for (Column column : columns) {
|
||||||
var columnOptions = new ColumnFamilyOptions();
|
var columnOptions = new ColumnFamilyOptions();
|
||||||
columnOptions.setTargetFileSizeBase(16 * SizeUnit.MB);
|
columnOptions.setTargetFileSizeBase(64 * SizeUnit.MB);
|
||||||
columnOptions.setTargetFileSizeMultiplier(2);
|
columnOptions.setTargetFileSizeMultiplier(1);
|
||||||
|
|
||||||
//noinspection ConstantConditions
|
//noinspection ConstantConditions
|
||||||
if (databaseOptions.memtableMemoryBudgetBytes() != null) {
|
if (databaseOptions.memtableMemoryBudgetBytes() != null) {
|
||||||
|
@ -229,8 +231,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
.setBlockCache(optionsWithCache.standardCache())
|
.setBlockCache(optionsWithCache.standardCache())
|
||||||
.setBlockSize(16 * 1024); // 16KiB
|
.setBlockSize(16 * 1024); // 16KiB
|
||||||
|
|
||||||
|
//columnOptions.setLevelCompactionDynamicLevelBytes(true);
|
||||||
columnOptions.setTableFormatConfig(tableOptions);
|
columnOptions.setTableFormatConfig(tableOptions);
|
||||||
columnOptions.setCompactionPriority(CompactionPriority.MinOverlappingRatio);
|
columnOptions.setCompactionPriority(CompactionPriority.OldestSmallestSeqFirst);
|
||||||
|
|
||||||
descriptors
|
descriptors
|
||||||
.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions));
|
.add(new ColumnFamilyDescriptor(column.name().getBytes(StandardCharsets.US_ASCII), columnOptions));
|
||||||
|
@ -448,11 +451,15 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
// the Options class contains a set of configurable DB options
|
// the Options class contains a set of configurable DB options
|
||||||
// that determines the behaviour of the database.
|
// that determines the behaviour of the database.
|
||||||
var options = new DBOptions();
|
var options = new DBOptions();
|
||||||
|
options.setMaxSubcompactions(2);
|
||||||
|
options.setMaxFileOpeningThreads(16);
|
||||||
|
options.setNewTableReaderForCompactionInputs(true);
|
||||||
options.setParanoidChecks(false);
|
options.setParanoidChecks(false);
|
||||||
options.setCreateIfMissing(true);
|
options.setCreateIfMissing(true);
|
||||||
|
options.setSkipStatsUpdateOnDbOpen(true);
|
||||||
options.setCreateMissingColumnFamilies(true);
|
options.setCreateMissingColumnFamilies(true);
|
||||||
options.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
|
options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
|
||||||
options.setAvoidFlushDuringShutdown(true); // Flush all WALs during shutdown
|
options.setAvoidFlushDuringShutdown(false); // Flush all WALs during shutdown
|
||||||
options.setAvoidFlushDuringRecovery(true); // Flush all WALs during startup
|
options.setAvoidFlushDuringRecovery(true); // Flush all WALs during startup
|
||||||
options.setWalRecoveryMode(databaseOptions.absoluteConsistency()
|
options.setWalRecoveryMode(databaseOptions.absoluteConsistency()
|
||||||
? WALRecoveryMode.AbsoluteConsistency
|
? WALRecoveryMode.AbsoluteConsistency
|
||||||
|
@ -741,6 +748,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
private void deleteUnusedOldLogFiles() {
|
private void deleteUnusedOldLogFiles() {
|
||||||
|
if (!DELETE_LOG_FILES) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
Path basePath = dbPath;
|
Path basePath = dbPath;
|
||||||
try {
|
try {
|
||||||
Files
|
Files
|
||||||
|
|
Loading…
Reference in New Issue
Block a user