andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix logging, configurable compression

Browse Source
This commit is contained in:
Andrea Cavalli 2021-12-27 17:34:44 +01:00
parent 582813b6c7
commit aa04a64c34
5 changed files with 31 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -361,7 +361,7 @@
<dependency> <dependency>
<groupId>org.rocksdb</groupId> <groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId> <artifactId>rocksdbjni</artifactId>
<version>6.26.1</version> <version>6.27.3</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.lucene</groupId> <groupId>org.apache.lucene</groupId>

23
src/main/java/it/cavallium/dbengine/client/Compression.java Normal file
View File

@ -0,0 +1,23 @@
package it.cavallium.dbengine.client;
import org.rocksdb.CompressionType;
public enum Compression {
PLAIN(CompressionType.NO_COMPRESSION),
SNAPPY(CompressionType.SNAPPY_COMPRESSION),
LZ4(CompressionType.LZ4_COMPRESSION),
LZ4_HC(CompressionType.LZ4HC_COMPRESSION),
ZSTD(CompressionType.ZSTD_COMPRESSION),
ZLIB(CompressionType.ZLIB_COMPRESSION),
BZLIB2(CompressionType.BZLIB2_COMPRESSION);
private final CompressionType type;
Compression(CompressionType compressionType) {
this.type = compressionType;
}
public CompressionType getType() {
return type;
}
}

2
src/main/java/it/cavallium/dbengine/client/DatabaseVolume.java
View File

@ -11,4 +11,4 @@ import java.nio.file.Path;
* Target size can be exceeded if all the volumes are full * Target size can be exceeded if all the volumes are full
*/ */
@RecordBuilder @RecordBuilder
public record DatabaseVolume(Path volumePath, long targetSizeBytes) {} public record DatabaseVolume(Path volumePath, long targetSizeBytes, Compression compression) {}

11
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

@ -1,6 +1,5 @@
package it.cavallium.dbengine.database.disk; package it.cavallium.dbengine.database.disk;
import static io.net5.buffer.Unpooled.wrappedBuffer;
import static io.net5.buffer.api.StandardAllocationTypes.OFF_HEAP; import static io.net5.buffer.api.StandardAllocationTypes.OFF_HEAP;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static it.cavallium.dbengine.database.LLUtils.asReadOnlyDirect; import static it.cavallium.dbengine.database.LLUtils.asReadOnlyDirect;
@ -42,12 +41,11 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask; import java.util.concurrent.ForkJoinTask;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.util.Supplier;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.rocksdb.AbstractSlice; import org.rocksdb.AbstractSlice;
@ -319,11 +317,8 @@ public class LLLocalDictionary implements LLDictionary {
try (var value = entry.getValue().receive()) { try (var value = entry.getValue().receive()) {
assert key.isAccessible(); assert key.isAccessible();
assert value.isAccessible(); assert value.isAccessible();
logger.trace(MARKER_ROCKSDB, var varargs = new Supplier<?>[]{() -> toStringSafe(key), () -> toStringSafe(value)};
"Writing {}: {}", logger.trace(MARKER_ROCKSDB, "Writing {}: {}", varargs);
(Supplier<String>) () -> toStringSafe(key),
(Supplier<String>) () -> toStringSafe(value)
);
db.put(EMPTY_WRITE_OPTIONS, key, value); db.put(EMPTY_WRITE_OPTIONS, key, value);
sink.complete(); sink.complete();
} }

7
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -297,10 +297,9 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
options.setCompactionStyle(CompactionStyle.LEVEL); options.setCompactionStyle(CompactionStyle.LEVEL);
options.setTargetFileSizeBase(64 * 1024 * 1024); // 64MiB sst file options.setTargetFileSizeBase(64 * 1024 * 1024); // 64MiB sst file
options.setTargetFileSizeMultiplier(2); // Each level is 2 times the previous level options.setTargetFileSizeMultiplier(2); // Each level is 2 times the previous level
options.setCompressionPerLevel(List.of(CompressionType.NO_COMPRESSION, if (!databaseOptions.volumes().isEmpty()) {
CompressionType.SNAPPY_COMPRESSION, options.setCompressionPerLevel(databaseOptions.volumes().stream().map(v -> v.compression().getType()).toList());
CompressionType.SNAPPY_COMPRESSION }
));
options.setManualWalFlush(false); options.setManualWalFlush(false);
options.setMinWriteBufferNumberToMerge(3); options.setMinWriteBufferNumberToMerge(3);
options.setMaxWriteBufferNumber(4); options.setMaxWriteBufferNumber(4);