From aa04a64c34e617a54653a29800d028af67ef8e0d Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Mon, 27 Dec 2021 17:34:44 +0100 Subject: [PATCH] Fix logging, configurable compression --- pom.xml | 2 +- .../dbengine/client/Compression.java | 23 +++++++++++++++++++ .../dbengine/client/DatabaseVolume.java | 2 +- .../database/disk/LLLocalDictionary.java | 11 +++------ .../disk/LLLocalKeyValueDatabase.java | 7 +++--- 5 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 src/main/java/it/cavallium/dbengine/client/Compression.java diff --git a/pom.xml b/pom.xml index 4522732..96812a9 100644 --- a/pom.xml +++ b/pom.xml @@ -361,7 +361,7 @@ org.rocksdb rocksdbjni - 6.26.1 + 6.27.3 org.apache.lucene diff --git a/src/main/java/it/cavallium/dbengine/client/Compression.java b/src/main/java/it/cavallium/dbengine/client/Compression.java new file mode 100644 index 0000000..ef35342 --- /dev/null +++ b/src/main/java/it/cavallium/dbengine/client/Compression.java @@ -0,0 +1,23 @@ +package it.cavallium.dbengine.client; + +import org.rocksdb.CompressionType; + +public enum Compression { + PLAIN(CompressionType.NO_COMPRESSION), + SNAPPY(CompressionType.SNAPPY_COMPRESSION), + LZ4(CompressionType.LZ4_COMPRESSION), + LZ4_HC(CompressionType.LZ4HC_COMPRESSION), + ZSTD(CompressionType.ZSTD_COMPRESSION), + ZLIB(CompressionType.ZLIB_COMPRESSION), + BZLIB2(CompressionType.BZLIB2_COMPRESSION); + + private final CompressionType type; + + Compression(CompressionType compressionType) { + this.type = compressionType; + } + + public CompressionType getType() { + return type; + } +} diff --git a/src/main/java/it/cavallium/dbengine/client/DatabaseVolume.java b/src/main/java/it/cavallium/dbengine/client/DatabaseVolume.java index 072f1c8..39611d4 100644 --- a/src/main/java/it/cavallium/dbengine/client/DatabaseVolume.java +++ b/src/main/java/it/cavallium/dbengine/client/DatabaseVolume.java @@ -11,4 +11,4 @@ import java.nio.file.Path; * Target size can be exceeded if all the volumes are full */ @RecordBuilder -public record DatabaseVolume(Path volumePath, long targetSizeBytes) {} +public record DatabaseVolume(Path volumePath, long targetSizeBytes, Compression compression) {} diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java index 5f6d8e2..37e30e6 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java @@ -1,6 +1,5 @@ package it.cavallium.dbengine.database.disk; -import static io.net5.buffer.Unpooled.wrappedBuffer; import static io.net5.buffer.api.StandardAllocationTypes.OFF_HEAP; import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static it.cavallium.dbengine.database.LLUtils.asReadOnlyDirect; @@ -42,12 +41,11 @@ import java.util.concurrent.Callable; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinTask; import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.util.Supplier; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.rocksdb.AbstractSlice; @@ -319,11 +317,8 @@ public class LLLocalDictionary implements LLDictionary { try (var value = entry.getValue().receive()) { assert key.isAccessible(); assert value.isAccessible(); - logger.trace(MARKER_ROCKSDB, - "Writing {}: {}", - (Supplier) () -> toStringSafe(key), - (Supplier) () -> toStringSafe(value) - ); + var varargs = new Supplier[]{() -> toStringSafe(key), () -> toStringSafe(value)}; + logger.trace(MARKER_ROCKSDB, "Writing {}: {}", varargs); db.put(EMPTY_WRITE_OPTIONS, key, value); sink.complete(); } diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index 302608b..8b880d1 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -297,10 +297,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { options.setCompactionStyle(CompactionStyle.LEVEL); options.setTargetFileSizeBase(64 * 1024 * 1024); // 64MiB sst file options.setTargetFileSizeMultiplier(2); // Each level is 2 times the previous level - options.setCompressionPerLevel(List.of(CompressionType.NO_COMPRESSION, - CompressionType.SNAPPY_COMPRESSION, - CompressionType.SNAPPY_COMPRESSION - )); + if (!databaseOptions.volumes().isEmpty()) { + options.setCompressionPerLevel(databaseOptions.volumes().stream().map(v -> v.compression().getType()).toList()); + } options.setManualWalFlush(false); options.setMinWriteBufferNumberToMerge(3); options.setMaxWriteBufferNumber(4);