andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Bugfixes

Browse Source
This commit is contained in:
Andrea Cavalli 2021-12-13 01:57:37 +01:00
parent 907561d93c
commit d58d696ca4
8 changed files with 55 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/it/cavallium/dbengine/client/query/ClientQueryParams.java
View File

@ -8,6 +8,7 @@ import it.cavallium.dbengine.client.query.current.data.NoSort;
import it.cavallium.dbengine.client.query.current.data.Query; import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.client.query.current.data.QueryParams; import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.QueryParamsBuilder; import it.cavallium.dbengine.client.query.current.data.QueryParamsBuilder;
import java.time.Duration;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
@ -18,7 +19,8 @@ public final record ClientQueryParams(@Nullable CompositeSnapshot snapshot,
long limit, long limit,
@Nullable Float minCompetitiveScore, @Nullable Float minCompetitiveScore,
@Nullable Sort sort, @Nullable Sort sort,
boolean computePreciseHitsCount) { boolean computePreciseHitsCount,
@NotNull Duration timeout) {
public static ClientQueryParamsBuilder builder() { public static ClientQueryParamsBuilder builder() {
return ClientQueryParamsBuilder return ClientQueryParamsBuilder
@ -28,6 +30,7 @@ public final record ClientQueryParams(@Nullable CompositeSnapshot snapshot,
.limit(Long.MAX_VALUE) .limit(Long.MAX_VALUE)
.minCompetitiveScore(null) .minCompetitiveScore(null)
.sort(null) .sort(null)
.timeout(Duration.ofSeconds(10))
.computePreciseHitsCount(true); .computePreciseHitsCount(true);
} }
@ -44,6 +47,7 @@ public final record ClientQueryParams(@Nullable CompositeSnapshot snapshot,
.offset(offset()) .offset(offset())
.limit(limit()) .limit(limit())
.computePreciseHitsCount(computePreciseHitsCount()) .computePreciseHitsCount(computePreciseHitsCount())
.timeoutMilliseconds(timeout.toMillis())
.build(); .build();
} }
} }

5
src/main/java/it/cavallium/dbengine/database/LLUtils.java
View File

@ -437,10 +437,9 @@ public class LLUtils {
/** /**
* @return null if size is equal to RocksDB.NOT_FOUND * @return null if size is equal to RocksDB.NOT_FOUND
*/ */
@SuppressWarnings("ConstantConditions")
@Nullable @Nullable
public static Buffer readNullableDirectNioBuffer(BufferAllocator alloc, ToIntFunction<ByteBuffer> reader) { public static Buffer readNullableDirectNioBuffer(BufferAllocator alloc, ToIntFunction<ByteBuffer> reader) {
var directBuffer = LLUtils.allocateShared(INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES); var directBuffer = allocateShared(INITIAL_DIRECT_READ_BYTE_BUF_SIZE_BYTES);
assert directBuffer.readerOffset() == 0; assert directBuffer.readerOffset() == 0;
assert directBuffer.writerOffset() == 0; assert directBuffer.writerOffset() == 0;
var directBufferWriter = ((WritableComponent) directBuffer).writableBuffer(); var directBufferWriter = ((WritableComponent) directBuffer).writableBuffer();
@ -464,7 +463,7 @@ public class LLUtils {
directBufferWriter = ((WritableComponent) directBuffer).writableBuffer(); directBufferWriter = ((WritableComponent) directBuffer).writableBuffer();
assert directBufferWriter.position() == 0; assert directBufferWriter.position() == 0;
assert directBufferWriter.isDirect(); assert directBufferWriter.isDirect();
reader.applyAsInt(directBufferWriter); reader.applyAsInt(directBufferWriter.position(0));
return directBuffer.writerOffset(trueSize); return directBuffer.writerOffset(trueSize);
} }
} catch (Throwable t) { } catch (Throwable t) {

8
src/main/java/it/cavallium/dbengine/database/disk/AbstractRocksDBColumn.java
View File

@ -110,13 +110,13 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
if (nettyDirect) { if (nettyDirect) {
// Get the key nio buffer to pass to RocksDB // Get the key nio buffer to pass to RocksDB
ByteBuffer keyNioBuffer = LLUtils.asReadOnlyDirect(key); ByteBuffer keyNioBuffer = LLUtils.asReadOnlyDirect(key);
assert keyNioBuffer.isDirect();
boolean mustCloseKey; boolean mustCloseKey;
if (keyNioBuffer == null) { if (keyNioBuffer == null) {
mustCloseKey = true; mustCloseKey = true;
// If the nio buffer is not available, copy the netty buffer into a new direct buffer // If the nio buffer is not available, copy the netty buffer into a new direct buffer
keyNioBuffer = LLUtils.copyToNewDirectBuffer(key); keyNioBuffer = LLUtils.copyToNewDirectBuffer(key);
} else { } else {
assert keyNioBuffer.isDirect();
mustCloseKey = false; mustCloseKey = false;
} }
assert keyNioBuffer.limit() == key.readableBytes(); assert keyNioBuffer.limit() == key.readableBytes();
@ -144,7 +144,7 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
assert keyMayExistValueLength == 0; assert keyMayExistValueLength == 0;
resultWritable.clear(); resultWritable.clear();
// real data size // real data size
size = db.get(cfh, readOptions, keyNioBuffer, resultWritable); size = db.get(cfh, readOptions, keyNioBuffer.position(0), resultWritable);
if (size == RocksDB.NOT_FOUND) { if (size == RocksDB.NOT_FOUND) {
resultBuffer.close(); resultBuffer.close();
return null; return null;
@ -164,7 +164,7 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
assert resultBuffer.readerOffset() == 0; assert resultBuffer.readerOffset() == 0;
assert resultBuffer.writerOffset() == 0; assert resultBuffer.writerOffset() == 0;
size = db.get(cfh, readOptions, keyNioBuffer, resultWritable); size = db.get(cfh, readOptions, keyNioBuffer.position(0), resultWritable);
if (size == RocksDB.NOT_FOUND) { if (size == RocksDB.NOT_FOUND) {
resultBuffer.close(); resultBuffer.close();
return null; return null;
@ -301,7 +301,7 @@ public sealed abstract class AbstractRocksDBColumn<T extends RocksDB> implements
} }
try { try {
if (db.keyMayExist(cfh, keyNioBuffer)) { if (db.keyMayExist(cfh, keyNioBuffer)) {
int size = db.get(cfh, readOptions, keyNioBuffer, LLUtils.EMPTY_BYTE_BUFFER); int size = db.get(cfh, readOptions, keyNioBuffer.position(0), LLUtils.EMPTY_BYTE_BUFFER);
return size != RocksDB.NOT_FOUND; return size != RocksDB.NOT_FOUND;
} else { } else {
return false; return false;

24
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine.database.disk;
import static io.net5.buffer.Unpooled.wrappedBuffer; import static io.net5.buffer.Unpooled.wrappedBuffer;
import static io.net5.buffer.api.StandardAllocationTypes.OFF_HEAP; import static io.net5.buffer.api.StandardAllocationTypes.OFF_HEAP;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB; import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static it.cavallium.dbengine.database.LLUtils.asReadOnlyDirect;
import static it.cavallium.dbengine.database.LLUtils.fromByteArray; import static it.cavallium.dbengine.database.LLUtils.fromByteArray;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;
import static java.util.Objects.requireNonNullElse; import static java.util.Objects.requireNonNullElse;
@ -258,7 +259,7 @@ public class LLLocalDictionary implements LLDictionary {
readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED); readOpts.setVerifyChecksums(VERIFY_CHECKSUMS_WHEN_NOT_NEEDED);
readOpts.setFillCache(false); readOpts.setFillCache(false);
if (range.hasMin()) { if (range.hasMin()) {
var rangeMinInternalByteBuffer = LLUtils.asReadOnlyDirect(range.getMinUnsafe()); var rangeMinInternalByteBuffer = asReadOnlyDirect(range.getMinUnsafe());
if (nettyDirect && rangeMinInternalByteBuffer != null) { if (nettyDirect && rangeMinInternalByteBuffer != null) {
readOpts.setIterateLowerBound(slice1 = new DirectSlice(rangeMinInternalByteBuffer, readOpts.setIterateLowerBound(slice1 = new DirectSlice(rangeMinInternalByteBuffer,
range.getMinUnsafe().readableBytes())); range.getMinUnsafe().readableBytes()));
@ -267,7 +268,7 @@ public class LLLocalDictionary implements LLDictionary {
} }
} }
if (range.hasMax()) { if (range.hasMax()) {
var rangeMaxInternalByteBuffer = LLUtils.asReadOnlyDirect(range.getMaxUnsafe()); var rangeMaxInternalByteBuffer = asReadOnlyDirect(range.getMaxUnsafe());
if (nettyDirect && rangeMaxInternalByteBuffer != null) { if (nettyDirect && rangeMaxInternalByteBuffer != null) {
readOpts.setIterateUpperBound(slice2 = new DirectSlice(rangeMaxInternalByteBuffer, readOpts.setIterateUpperBound(slice2 = new DirectSlice(rangeMaxInternalByteBuffer,
range.getMaxUnsafe().readableBytes())); range.getMaxUnsafe().readableBytes()));
@ -277,7 +278,7 @@ public class LLLocalDictionary implements LLDictionary {
} }
try (RocksIterator rocksIterator = db.newIterator(readOpts)) { try (RocksIterator rocksIterator = db.newIterator(readOpts)) {
if (!LLLocalDictionary.PREFER_SEEK_TO_FIRST && range.hasMin()) { if (!LLLocalDictionary.PREFER_SEEK_TO_FIRST && range.hasMin()) {
var rangeMinInternalByteBuffer = LLUtils.asReadOnlyDirect(range.getMinUnsafe()); var rangeMinInternalByteBuffer = asReadOnlyDirect(range.getMinUnsafe());
if (nettyDirect && rangeMinInternalByteBuffer != null) { if (nettyDirect && rangeMinInternalByteBuffer != null) {
rocksIterator.seek(rangeMinInternalByteBuffer); rocksIterator.seek(rangeMinInternalByteBuffer);
} else { } else {
@ -828,7 +829,7 @@ public class LLLocalDictionary implements LLDictionary {
} }
ro.setVerifyChecksums(true); ro.setVerifyChecksums(true);
var rocksIteratorTuple = getRocksIterator(alloc, var rocksIteratorTuple = getRocksIterator(alloc,
nettyDirect, ro, range.send(), db nettyDirect, ro, range, db
); );
try { try {
try (var rocksIterator = rocksIteratorTuple.getT1()) { try (var rocksIterator = rocksIteratorTuple.getT1()) {
@ -1193,9 +1194,11 @@ public class LLLocalDictionary implements LLDictionary {
private static SafeCloseable rocksIterSeekTo(boolean allowNettyDirect, private static SafeCloseable rocksIterSeekTo(boolean allowNettyDirect,
RocksIterator rocksIterator, Buffer key) { RocksIterator rocksIterator, Buffer key) {
ByteBuffer keyInternalByteBuffer; ByteBuffer keyInternalByteBuffer;
if (allowNettyDirect && (keyInternalByteBuffer = LLUtils.asReadOnlyDirect(key)) != null) { if (allowNettyDirect && (keyInternalByteBuffer = asReadOnlyDirect(key)) != null) {
assert keyInternalByteBuffer.position() == 0;
rocksIterator.seek(keyInternalByteBuffer); rocksIterator.seek(keyInternalByteBuffer);
return null; // This is useful to retain the key buffer in memory and avoid deallocations
return key::isAccessible;
} else { } else {
rocksIterator.seek(LLUtils.toArray(key)); rocksIterator.seek(LLUtils.toArray(key));
return null; return null;
@ -1208,7 +1211,8 @@ public class LLLocalDictionary implements LLDictionary {
AbstractSlice<?> slice; AbstractSlice<?> slice;
ByteBuffer keyInternalByteBuffer; ByteBuffer keyInternalByteBuffer;
if (allowNettyDirect && LLLocalDictionary.USE_DIRECT_BUFFER_BOUNDS if (allowNettyDirect && LLLocalDictionary.USE_DIRECT_BUFFER_BOUNDS
&& (keyInternalByteBuffer = LLUtils.asReadOnlyDirect(key)) != null) { && (keyInternalByteBuffer = asReadOnlyDirect(key)) != null) {
assert keyInternalByteBuffer.position() == 0;
slice = new DirectSlice(keyInternalByteBuffer, key.readableBytes()); slice = new DirectSlice(keyInternalByteBuffer, key.readableBytes());
assert slice.size() == key.readableBytes(); assert slice.size() == key.readableBytes();
assert slice.compare(new Slice(LLUtils.toArray(key))) == 0; assert slice.compare(new Slice(LLUtils.toArray(key))) == 0;
@ -1217,7 +1221,7 @@ public class LLLocalDictionary implements LLDictionary {
} else { } else {
readOpts.setIterateUpperBound(slice); readOpts.setIterateUpperBound(slice);
} }
return new ReleasableSliceImpl(slice, null, null); return new ReleasableSliceImpl(slice, null, key);
} else { } else {
slice = new Slice(requireNonNull(LLUtils.toArray(key))); slice = new Slice(requireNonNull(LLUtils.toArray(key)));
if (boundType == IterateBound.LOWER) { if (boundType == IterateBound.LOWER) {
@ -1679,9 +1683,8 @@ public class LLLocalDictionary implements LLDictionary {
public static Tuple4<RocksIterator, ReleasableSlice, ReleasableSlice, SafeCloseable> getRocksIterator(BufferAllocator alloc, public static Tuple4<RocksIterator, ReleasableSlice, ReleasableSlice, SafeCloseable> getRocksIterator(BufferAllocator alloc,
boolean allowNettyDirect, boolean allowNettyDirect,
ReadOptions readOptions, ReadOptions readOptions,
Send<LLRange> rangeToReceive, LLRange range,
RocksDBColumn db) { RocksDBColumn db) {
try (var range = rangeToReceive.receive()) {
if (Schedulers.isInNonBlockingThread()) { if (Schedulers.isInNonBlockingThread()) {
throw new UnsupportedOperationException("Called getRocksIterator in a nonblocking thread"); throw new UnsupportedOperationException("Called getRocksIterator in a nonblocking thread");
} }
@ -1708,5 +1711,4 @@ public class LLLocalDictionary implements LLDictionary {
} }
return Tuples.of(rocksIterator, sliceMin, sliceMax, seekTo); return Tuples.of(rocksIterator, sliceMin, sliceMax, seekTo);
} }
}
} }

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalGroupedReactiveRocksIterator.java
View File

@ -91,7 +91,7 @@ public abstract class LLLocalGroupedReactiveRocksIterator<T> extends
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace(MARKER_ROCKSDB, "Range {} started", LLUtils.toStringSafe(range)); logger.trace(MARKER_ROCKSDB, "Range {} started", LLUtils.toStringSafe(range));
} }
return LLLocalDictionary.getRocksIterator(db.getAllocator(), allowNettyDirect, readOptions, range.copy().send(), db); return LLLocalDictionary.getRocksIterator(db.getAllocator(), allowNettyDirect, readOptions, range, db);
}, (tuple, sink) -> { }, (tuple, sink) -> {
try { try {
var rocksIterator = tuple.getT1(); var rocksIterator = tuple.getT1();

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyPrefixReactiveRocksIterator.java
View File

@ -93,7 +93,7 @@ public class LLLocalKeyPrefixReactiveRocksIterator extends
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace(MARKER_ROCKSDB, "Range {} started", LLUtils.toStringSafe(range)); logger.trace(MARKER_ROCKSDB, "Range {} started", LLUtils.toStringSafe(range));
} }
return LLLocalDictionary.getRocksIterator(db.getAllocator(), allowNettyDirect, readOptions, rangeSend, db); return LLLocalDictionary.getRocksIterator(db.getAllocator(), allowNettyDirect, readOptions, range, db);
}, (tuple, sink) -> { }, (tuple, sink) -> {
try { try {
var rocksIterator = tuple.getT1(); var rocksIterator = tuple.getT1();

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalReactiveRocksIterator.java
View File

@ -83,7 +83,7 @@ public abstract class LLLocalReactiveRocksIterator<T> extends
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace(MARKER_ROCKSDB, "Range {} started", LLUtils.toStringSafe(range)); logger.trace(MARKER_ROCKSDB, "Range {} started", LLUtils.toStringSafe(range));
} }
return getRocksIterator(db.getAllocator(), allowNettyDirect, readOptions, range.copy().send(), db); return getRocksIterator(db.getAllocator(), allowNettyDirect, readOptions, range, db);
}, (tuple, sink) -> { }, (tuple, sink) -> {
try { try {
var rocksIterator = tuple.getT1(); var rocksIterator = tuple.getT1();

8
src/main/java/org/rocksdb/CappedWriteBatch.java
View File

@ -1,5 +1,6 @@
package org.rocksdb; package org.rocksdb;
import static it.cavallium.dbengine.database.LLUtils.asReadOnlyDirect;
import static it.cavallium.dbengine.database.LLUtils.isDirect; import static it.cavallium.dbengine.database.LLUtils.isDirect;
import io.net5.buffer.api.Buffer; import io.net5.buffer.api.Buffer;
@ -111,8 +112,8 @@ public class CappedWriteBatch extends WriteBatch {
ByteBuffer keyNioBuffer; ByteBuffer keyNioBuffer;
ByteBuffer valueNioBuffer; ByteBuffer valueNioBuffer;
if (USE_FAST_DIRECT_BUFFERS if (USE_FAST_DIRECT_BUFFERS
&& (keyNioBuffer = LLUtils.asReadOnlyDirect(key)) != null && (keyNioBuffer = asReadOnlyDirect(key)) != null
&& (valueNioBuffer = LLUtils.asReadOnlyDirect(value)) != null) { && (valueNioBuffer = asReadOnlyDirect(value)) != null) {
buffersToRelease.add(value); buffersToRelease.add(value);
buffersToRelease.add(key); buffersToRelease.add(key);
@ -171,7 +172,7 @@ public class CappedWriteBatch extends WriteBatch {
public synchronized void delete(ColumnFamilyHandle columnFamilyHandle, Send<Buffer> keyToReceive) throws RocksDBException { public synchronized void delete(ColumnFamilyHandle columnFamilyHandle, Send<Buffer> keyToReceive) throws RocksDBException {
var key = keyToReceive.receive(); var key = keyToReceive.receive();
ByteBuffer keyNioBuffer; ByteBuffer keyNioBuffer;
if (USE_FAST_DIRECT_BUFFERS && (keyNioBuffer = LLUtils.asReadOnlyDirect(key)) != null) { if (USE_FAST_DIRECT_BUFFERS && (keyNioBuffer = asReadOnlyDirect(key)) != null) {
buffersToRelease.add(key); buffersToRelease.add(key);
removeDirect(nativeHandle_, removeDirect(nativeHandle_,
keyNioBuffer, keyNioBuffer,
@ -179,7 +180,6 @@ public class CappedWriteBatch extends WriteBatch {
keyNioBuffer.remaining(), keyNioBuffer.remaining(),
columnFamilyHandle.nativeHandle_ columnFamilyHandle.nativeHandle_
); );
keyNioBuffer.position(keyNioBuffer.limit());
} else { } else {
try { try {
super.delete(columnFamilyHandle, LLUtils.toArray(key)); super.delete(columnFamilyHandle, LLUtils.toArray(key));