andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Major bugfix

This commit is contained in:
Andrea Cavalli 2023-05-22 23:08:37 +02:00
parent 93fc28101a
commit df946146a1
9 changed files with 89 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/it/cavallium/dbengine/client/MemoryStats.java
View File

@ -2,4 +2,4 @@ package it.cavallium.dbengine.client;
public record MemoryStats(long estimateTableReadersMem, long sizeAllMemTables,
long curSizeAllMemTables, long estimateNumKeys, long blockCacheUsage,
long blockCachePinnedUsage) {}
long blockCachePinnedUsage, long liveVersions) {}

3
src/main/java/it/cavallium/dbengine/database/LLKeyValueDatabase.java
View File

@ -6,13 +6,14 @@ import io.micrometer.core.instrument.MeterRegistry;
import it.cavallium.dbengine.client.IBackuppable;
import it.cavallium.dbengine.database.collections.DatabaseInt;
import it.cavallium.dbengine.database.collections.DatabaseLong;
import java.io.Closeable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import org.jetbrains.annotations.Nullable;
import org.rocksdb.RocksDBException;
public interface LLKeyValueDatabase extends LLSnapshottable, LLKeyValueDatabaseStructure, DatabaseProperties,
IBackuppable, DatabaseOperations {
IBackuppable, DatabaseOperations, Closeable {
LLSingleton getSingleton(byte[] singletonListColumnName, byte[] name, byte @Nullable [] defaultValue);

4
src/main/java/it/cavallium/dbengine/database/LLUtils.java
View File

@ -678,6 +678,10 @@ public class LLUtils {
return result;
}
public static Buf wrapNullable(byte[] array) {
return array != null ? Buf.wrap(array) : null;
}
private static class FakeBytesRefBuilder extends BytesRefBuilder {
private final LLTerm term;

4
src/main/java/it/cavallium/dbengine/database/disk/AbstractRocksDBColumn.java
View File

@ -235,9 +235,7 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
* This method should not modify or move the writerIndex/readerIndex of the key
*/
static void setIterateBound(LLReadOptions readOpts, IterateBound boundType, Buf key) {
requireNonNull(key);
LLSlice slice;
slice = LLSlice.of(requireNonNull(LLUtils.asArray(key)));
byte[] slice = key != null ? requireNonNull(LLUtils.asArray(key)) : null;
if (boundType == IterateBound.LOWER) {
readOpts.setIterateLowerBound(slice);
} else {

31
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

@ -5,6 +5,7 @@ import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
import static it.cavallium.dbengine.database.LLUtils.mapList;
import static it.cavallium.dbengine.database.LLUtils.toStringSafe;
import static it.cavallium.dbengine.database.LLUtils.wrapNullable;
import static it.cavallium.dbengine.database.disk.UpdateAtomicResultMode.DELTA;
import static it.cavallium.dbengine.utils.StreamUtils.ROCKSDB_POOL;
import static it.cavallium.dbengine.utils.StreamUtils.collectOn;
@ -1092,26 +1093,11 @@ public class LLLocalDictionary implements LLDictionary {
)).map(range -> {
long partialCount = 0;
try (var rangeReadOpts = readOpts.copy()) {
LLSlice sliceBegin;
if (range.getKey() != null) {
sliceBegin = LLSlice.of(range.getKey());
} else {
sliceBegin = null;
}
LLSlice sliceEnd;
if (range.getValue() != null) {
sliceEnd = LLSlice.of(range.getValue());
} else {
sliceEnd = null;
}
try {
if (sliceBegin != null) {
rangeReadOpts.setIterateLowerBound(sliceBegin);
}
if (sliceEnd != null) {
rangeReadOpts.setIterateUpperBound(sliceEnd);
}
try (var rocksIterator = db.newIterator(rangeReadOpts, null, null)) {
try (var rocksIterator = db.newIterator(rangeReadOpts,
wrapNullable(range.getKey()),
wrapNullable(range.getValue())
)) {
rocksIterator.seekToFirst();
while (rocksIterator.isValid()) {
partialCount++;
@ -1121,13 +1107,6 @@ public class LLLocalDictionary implements LLDictionary {
}
} catch (RocksDBException ex) {
throw new CompletionException(new IOException("Failed to get size", ex));
} finally {
if (sliceBegin != null) {
sliceBegin.close();
}
if (sliceEnd != null) {
sliceEnd.close();
}
}
}
}), fastSummingLong());

6
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -56,6 +56,7 @@ import java.util.stream.Stream;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.rocksdb.AbstractImmutableNativeReference;
import org.rocksdb.BlockBasedTableConfig;
@ -135,7 +136,7 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
private volatile boolean closed = false;
@SuppressWarnings("SwitchStatementWithTooFewBranches")
public LLLocalKeyValueDatabase(MeterRegistry meterRegistry,
public LLLocalKeyValueDatabase(@NotNull MeterRegistry meterRegistry,
String name,
boolean inMemory,
@Nullable Path path,
@ -1263,7 +1264,8 @@ public class LLLocalKeyValueDatabase extends Backuppable implements LLKeyValueDa
db.getAggregatedLongProperty("rocksdb.cur-size-all-mem-tables"),
db.getAggregatedLongProperty("rocksdb.estimate-num-keys"),
db.getAggregatedLongProperty("rocksdb.block-cache-usage") / this.handles.size(),
db.getAggregatedLongProperty("rocksdb.block-cache-pinned-usage") / this.handles.size()
db.getAggregatedLongProperty("rocksdb.block-cache-pinned-usage") / this.handles.size(),
db.getAggregatedLongProperty("rocksdb.num-live-versions") / this.handles.size()
);
} catch (RocksDBException e) {
throw new DBException("Failed to read memory stats", e);

86
src/main/java/it/cavallium/dbengine/database/disk/rocksdb/LLReadOptions.java
View File

@ -11,9 +11,34 @@ public final class LLReadOptions extends SimpleResource {
private final ReadOptions val;
private LLSlice itLowerBoundRef;
private boolean itLowerBoundRefOwned;
private LLSlice itUpperBoundRef;
private boolean itUpperBoundRefOwned;
private Snapshot snapshot;
public LLReadOptions copy() {
if (this.val.timestamp() != null) {
throw new IllegalStateException("Unsupported copy of timestamped read options");
}
if (this.val.iterStartTs() != null) {
throw new IllegalStateException("Unsupported copy of read options with non-null iterStartTs property");
}
ReadOptions newVal = new ReadOptions(this.val);
var ro = new LLReadOptions(newVal);
if (this.val.iterateLowerBound() != null) {
ro.setIterateLowerBound(this.val.iterateLowerBound().data());
}
if (this.val.iterateUpperBound() != null) {
ro.setIterateUpperBound(this.val.iterateUpperBound().data());
}
ro.snapshot = this.snapshot;
return ro;
}
public LLReadOptions(ReadOptions val) {
super(val::close);
this.val = val;
@ -23,30 +48,59 @@ public final class LLReadOptions extends SimpleResource {
this(new ReadOptions());
}
public LLReadOptions copy() {
var ro = new LLReadOptions(new ReadOptions(this.val));
ro.itUpperBoundRef = this.itUpperBoundRef;
ro.itLowerBoundRef = this.itLowerBoundRef;
ro.snapshot = this.snapshot;
return ro;
}
@Override
protected void onClose() {
val.close();
itLowerBoundRef = null;
itUpperBoundRef = null;
if (this.itUpperBoundRefOwned && this.itUpperBoundRef != null) {
this.itUpperBoundRef.close();
}
this.itUpperBoundRef = null;
if (this.itLowerBoundRefOwned && this.itLowerBoundRef != null) {
this.itLowerBoundRef.close();
}
this.itLowerBoundRef = null;
snapshot = null;
}
public void setIterateLowerBound(LLSlice slice) {
val.setIterateLowerBound(slice.getSliceUnsafe());
itLowerBoundRef = slice;
public void setIterateLowerBound(byte[] slice) {
setIterateLowerBound(slice != null ? LLSlice.copyOf(slice) : null, true);
}
/**
* @param move if true, the read options will close the slice when they are closed
*/
public void setIterateLowerBound(LLSlice slice, boolean move) {
val.setIterateLowerBound(slice != null ? slice.getSliceUnsafe() : null);
// Close the previous owned value, if present
if (this.itLowerBoundRefOwned && this.itLowerBoundRef != null) {
this.itLowerBoundRef.close();
}
this.itLowerBoundRef = slice;
this.itLowerBoundRefOwned = move;
}
public void setIterateUpperBound(LLSlice slice) {
val.setIterateUpperBound(slice.getSliceUnsafe());
itUpperBoundRef = slice;
public void setIterateUpperBound(byte[] slice) {
setIterateUpperBound(slice != null ? LLSlice.copyOf(slice) : null, true);
}
/**
* @param move if true, the read options will close the slice when they are closed
*/
public void setIterateUpperBound(LLSlice slice, boolean move) {
val.setIterateUpperBound(slice != null ? slice.getSliceUnsafe() : null);
// Close the previous owned value, if present
if (this.itUpperBoundRefOwned && this.itUpperBoundRef != null) {
this.itUpperBoundRef.close();
}
this.itUpperBoundRef = slice;
this.itUpperBoundRefOwned = move;
}
public long readaheadSize() {

2
src/main/java/it/cavallium/dbengine/database/disk/rocksdb/LLSlice.java
View File

@ -13,7 +13,7 @@ public final class LLSlice extends SimpleResource {
this.val = val;
}
public static LLSlice of(byte[] data) {
public static LLSlice copyOf(byte[] data) {
return new LLSlice(new Slice(data));
}

2
src/main/java/it/cavallium/dbengine/database/memory/LLMemoryKeyValueDatabase.java
View File

@ -78,7 +78,7 @@ public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase {
@Override
public MemoryStats getMemoryStats() {
return new MemoryStats(0, 0, 0, 0, 0, 0);
return new MemoryStats(0, 0, 0, 0, 0, 0, 1);
}
@Override