andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix map tests

Browse Source
This commit is contained in:
Andrea Cavalli 2021-09-02 17:15:40 +02:00
parent ff7823656e
commit 3091b81d34
27 changed files with 277 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/it/cavallium/dbengine/client/DatabaseOptions.java
View File

@ -14,4 +14,11 @@ public record DatabaseOptions(Map<String, String> extraFlags,
boolean allowMemoryMapping,
boolean allowNettyDirect,
boolean useNettyDirect,
int maxOpenFiles) {}
int maxOpenFiles) {
public DatabaseOptions {
if (useNettyDirect && !allowNettyDirect) {
throw new IllegalArgumentException("If allowNettyDirect is false, you must also set useNettyDirect to false");
}
}
}

11
src/main/java/it/cavallium/dbengine/client/MappedSerializer.java
View File

@ -6,21 +6,22 @@ import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import org.jetbrains.annotations.NotNull;
public class MappedSerializer<A, B> implements Serializer<B, Send<Buffer>> {
public class MappedSerializer<A, B> implements Serializer<B> {
private final Serializer<A, Send<Buffer>> serializer;
private final Serializer<A> serializer;
private final Mapper<A, B> keyMapper;
public MappedSerializer(Serializer<A, Send<Buffer>> serializer,
public MappedSerializer(Serializer<A> serializer,
Mapper<A, B> keyMapper) {
this.serializer = serializer;
this.keyMapper = keyMapper;
}
@Override
public @NotNull B deserialize(@NotNull Send<Buffer> serialized) throws SerializationException {
public @NotNull DeserializationResult<B> deserialize(@NotNull Send<Buffer> serialized) throws SerializationException {
try (serialized) {
return keyMapper.map(serializer.deserialize(serialized));
var deserialized = serializer.deserialize(serialized);
return new DeserializationResult<>(keyMapper.map(deserialized.deserializedData()), deserialized.bytesRead());
}
}

11
src/main/java/it/cavallium/dbengine/client/MappedSerializerFixedLength.java
View File

@ -6,21 +6,22 @@ import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import org.jetbrains.annotations.NotNull;
public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryLength<B, Send<Buffer>> {
public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryLength<B> {
private final SerializerFixedBinaryLength<A, Send<Buffer>> fixedLengthSerializer;
private final SerializerFixedBinaryLength<A> fixedLengthSerializer;
private final Mapper<A, B> keyMapper;
public MappedSerializerFixedLength(SerializerFixedBinaryLength<A, Send<Buffer>> fixedLengthSerializer,
public MappedSerializerFixedLength(SerializerFixedBinaryLength<A> fixedLengthSerializer,
Mapper<A, B> keyMapper) {
this.fixedLengthSerializer = fixedLengthSerializer;
this.keyMapper = keyMapper;
}
@Override
public @NotNull B deserialize(@NotNull Send<Buffer> serialized) throws SerializationException {
public @NotNull DeserializationResult<B> deserialize(@NotNull Send<Buffer> serialized) throws SerializationException {
try (serialized) {
return keyMapper.map(fixedLengthSerializer.deserialize(serialized));
var deserialized = fixedLengthSerializer.deserialize(serialized);
return new DeserializationResult<>(keyMapper.map(deserialized.deserializedData()), deserialized.bytesRead());
}
}

4
src/main/java/it/cavallium/dbengine/database/LLDelta.java
View File

@ -117,10 +117,10 @@ public class LLDelta extends ResourceSupport<LLDelta, LLDelta> {
@Override
public void drop(LLDelta obj) {
if (obj.previous != null) {
if (obj.previous != null && obj.previous.isAccessible()) {
obj.previous.close();
}
if (obj.current != null) {
if (obj.current != null && obj.current.isAccessible()) {
obj.current.close();
}
delegate.drop(obj);

10
src/main/java/it/cavallium/dbengine/database/LLEntry.java
View File

@ -16,9 +16,9 @@ public class LLEntry extends ResourceSupport<LLEntry, LLEntry> {
private LLEntry(Send<Buffer> key, Send<Buffer> value, Drop<LLEntry> drop) {
super(new LLEntry.CloseOnDrop(drop));
assert isAllAccessible();
this.key = key.receive().makeReadOnly();
this.value = value.receive().makeReadOnly();
assert isAllAccessible();
}
private boolean isAllAccessible() {
@ -119,8 +119,12 @@ public class LLEntry extends ResourceSupport<LLEntry, LLEntry> {
@Override
public void drop(LLEntry obj) {
obj.key.close();
obj.value.close();
if (obj.key.isAccessible()) {
obj.key.close();
}
if (obj.value.isAccessible()) {
obj.value.close();
}
delegate.drop(obj);
}
}

135
src/main/java/it/cavallium/dbengine/database/LLUtils.java
View File

@ -199,7 +199,7 @@ public class LLUtils {
b.append('[');
int i = 0;
while(true) {
while (true) {
b.append(key.getByte(startIndex + i));
if (i == iLimit) {
b.append("…");
@ -237,10 +237,9 @@ public class LLUtils {
/**
* Returns {@code true} if and only if the two specified buffers are
* identical to each other for {@code length} bytes starting at {@code aStartIndex}
* index for the {@code a} buffer and {@code bStartIndex} index for the {@code b} buffer.
* A more compact way to express this is:
* Returns {@code true} if and only if the two specified buffers are identical to each other for {@code length} bytes
* starting at {@code aStartIndex} index for the {@code a} buffer and {@code bStartIndex} index for the {@code b}
* buffer. A more compact way to express this is:
* <p>
* {@code a[aStartIndex : aStartIndex + length] == b[bStartIndex : bStartIndex + length]}
*/
@ -273,8 +272,9 @@ public class LLUtils {
}
public static int hashCode(Buffer buf) {
if (buf == null)
if (buf == null) {
return 0;
}
int result = 1;
var cur = buf.openCursor();
@ -287,7 +287,6 @@ public class LLUtils {
}
/**
*
* @return null if size is equal to RocksDB.NOT_FOUND
*/
@SuppressWarnings("ConstantConditions")
@ -366,12 +365,21 @@ public class LLUtils {
@NotNull
public static ByteBuffer obtainDirect(Buffer buffer) {
assert buffer.isAccessible();
if (buffer.readableBytes() == 0) {
return EMPTY_BYTE_BUFFER;
if (!PlatformDependent.hasUnsafe()) {
throw new UnsupportedOperationException("Please enable unsafe support or disable netty direct buffers",
PlatformDependent.getUnsafeUnavailabilityCause()
);
}
if (!PlatformDependent.hasDirectBufferNoCleanerConstructor()) {
throw new UnsupportedOperationException("Please enable unsafe support or disable netty direct buffers:"
+ " DirectBufferNoCleanerConstructor is not available");
}
assert buffer.isAccessible();
long nativeAddress;
if ((nativeAddress = buffer.nativeAddress()) == 0) {
if (buffer.capacity() == 0) {
return EMPTY_BYTE_BUFFER;
}
throw new IllegalStateException("Buffer is not direct");
}
return PlatformDependent.directBuffer(nativeAddress, buffer.capacity());
@ -395,7 +403,7 @@ public class LLUtils {
}
public static Send<Buffer> compositeBuffer(BufferAllocator alloc, Send<Buffer> buffer) {
try (var composite = buffer.receive().compact()) {
try (var composite = buffer.receive()) {
return composite.send();
}
}
@ -403,18 +411,21 @@ public class LLUtils {
public static Send<Buffer> compositeBuffer(BufferAllocator alloc, Send<Buffer> buffer1, Send<Buffer> buffer2) {
try (buffer1) {
try (buffer2) {
try (var composite = CompositeBuffer.compose(alloc, buffer1, buffer2).compact()) {
try (var composite = CompositeBuffer.compose(alloc, buffer1, buffer2)) {
return composite.send();
}
}
}
}
public static Send<Buffer> compositeBuffer(BufferAllocator alloc, Send<Buffer> buffer1, Send<Buffer> buffer2, Send<Buffer> buffer3) {
public static Send<Buffer> compositeBuffer(BufferAllocator alloc,
Send<Buffer> buffer1,
Send<Buffer> buffer2,
Send<Buffer> buffer3) {
try (buffer1) {
try (buffer2) {
try (buffer3) {
try (var composite = CompositeBuffer.compose(alloc, buffer1, buffer2, buffer3).compact()) {
try (var composite = CompositeBuffer.compose(alloc, buffer1, buffer2, buffer3)) {
return composite.send();
}
}
@ -431,7 +442,7 @@ public class LLUtils {
case 2 -> compositeBuffer(alloc, buffers[0], buffers[1]);
case 3 -> compositeBuffer(alloc, buffers[0], buffers[1], buffers[2]);
default -> {
try (var composite = CompositeBuffer.compose(alloc, buffers).compact()) {
try (var composite = CompositeBuffer.compose(alloc, buffers)) {
yield composite.send();
}
}
@ -568,30 +579,29 @@ public class LLUtils {
}
public static <T> Mono<T> handleDiscard(Mono<T> mono) {
return mono
.doOnDiscard(Object.class, obj -> {
if (obj instanceof SafeCloseable o) {
discardRefCounted(o);
} else if (obj instanceof Entry o) {
discardEntry(o);
} else if (obj instanceof Collection o) {
discardCollection(o);
} else if (obj instanceof Tuple3 o) {
discardTuple3(o);
} else if (obj instanceof Tuple2 o) {
discardTuple2(o);
} else if (obj instanceof LLEntry o) {
discardLLEntry(o);
} else if (obj instanceof LLRange o) {
discardLLRange(o);
} else if (obj instanceof Delta o) {
discardDelta(o);
} else if (obj instanceof Send o) {
discardSend(o);
} else if (obj instanceof Map o) {
discardMap(o);
}
});
return mono.doOnDiscard(Object.class, obj -> {
if (obj instanceof SafeCloseable o) {
discardRefCounted(o);
} else if (obj instanceof Entry o) {
discardEntry(o);
} else if (obj instanceof Collection o) {
discardCollection(o);
} else if (obj instanceof Tuple3 o) {
discardTuple3(o);
} else if (obj instanceof Tuple2 o) {
discardTuple2(o);
} else if (obj instanceof LLEntry o) {
discardLLEntry(o);
} else if (obj instanceof LLRange o) {
discardLLRange(o);
} else if (obj instanceof Delta o) {
discardDelta(o);
} else if (obj instanceof Send o) {
discardSend(o);
} else if (obj instanceof Map o) {
discardMap(o);
}
});
// todo: check if the single object discard hook is more performant
/*
.doOnDiscard(SafeCloseable.class, LLUtils::discardRefCounted)
@ -609,30 +619,29 @@ public class LLUtils {
}
public static <T> Flux<T> handleDiscard(Flux<T> mono) {
return mono
.doOnDiscard(Object.class, obj -> {
if (obj instanceof SafeCloseable o) {
discardRefCounted(o);
} else if (obj instanceof Entry o) {
discardEntry(o);
} else if (obj instanceof Collection o) {
discardCollection(o);
} else if (obj instanceof Tuple3 o) {
discardTuple3(o);
} else if (obj instanceof Tuple2 o) {
discardTuple2(o);
} else if (obj instanceof LLEntry o) {
discardLLEntry(o);
} else if (obj instanceof LLRange o) {
discardLLRange(o);
} else if (obj instanceof Delta o) {
discardDelta(o);
} else if (obj instanceof Send o) {
discardSend(o);
} else if (obj instanceof Map o) {
discardMap(o);
}
});
return mono.doOnDiscard(Object.class, obj -> {
if (obj instanceof SafeCloseable o) {
discardRefCounted(o);
} else if (obj instanceof Entry o) {
discardEntry(o);
} else if (obj instanceof Collection o) {
discardCollection(o);
} else if (obj instanceof Tuple3 o) {
discardTuple3(o);
} else if (obj instanceof Tuple2 o) {
discardTuple2(o);
} else if (obj instanceof LLEntry o) {
discardLLEntry(o);
} else if (obj instanceof LLRange o) {
discardLLRange(o);
} else if (obj instanceof Delta o) {
discardDelta(o);
} else if (obj instanceof Send o) {
discardSend(o);
} else if (obj instanceof Map o) {
discardMap(o);
}
});
// todo: check if the single object discard hook is more performant
/*
.doOnDiscard(SafeCloseable.class, LLUtils::discardRefCounted)

8
src/main/java/it/cavallium/dbengine/database/collections/DatabaseEmpty.java
View File

@ -5,6 +5,7 @@ import io.netty.buffer.api.BufferAllocator;
import io.netty.buffer.api.Send;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.Serializer.DeserializationResult;
import java.util.function.Function;
import org.jetbrains.annotations.NotNull;
@ -12,13 +13,14 @@ public class DatabaseEmpty {
@SuppressWarnings({"unused", "InstantiationOfUtilityClass"})
public static final Nothing NOTHING = new Nothing();
public static final DeserializationResult<Nothing> NOTHING_RESULT = new DeserializationResult<>(NOTHING, 0);
public static Serializer<Nothing, Send<Buffer>> nothingSerializer(BufferAllocator bufferAllocator) {
public static Serializer<Nothing> nothingSerializer(BufferAllocator bufferAllocator) {
return new Serializer<>() {
@Override
public @NotNull Nothing deserialize(@NotNull Send<Buffer> serialized) {
public @NotNull DeserializationResult<Nothing> deserialize(@NotNull Send<Buffer> serialized) {
try (serialized) {
return NOTHING;
return NOTHING_RESULT;
}
}

54
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionary.java
View File

@ -5,6 +5,7 @@ import io.netty.buffer.api.Send;
import io.netty.buffer.api.internal.ResourceSupport;
import io.netty.util.ReferenceCounted;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.client.NoMapper;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.ExtraKeyOperationResult;
import it.cavallium.dbengine.database.LLDictionary;
@ -40,27 +41,27 @@ import reactor.util.function.Tuples;
*/
public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U, DatabaseStageEntry<U>> {
private final Serializer<U, Send<Buffer>> valueSerializer;
private final Serializer<U> valueSerializer;
protected DatabaseMapDictionary(LLDictionary dictionary,
Send<Buffer> prefixKey,
SerializerFixedBinaryLength<T, Send<Buffer>> keySuffixSerializer,
Serializer<U, Send<Buffer>> valueSerializer) {
SerializerFixedBinaryLength<T> keySuffixSerializer,
Serializer<U> valueSerializer) {
// Do not retain or release or use the prefixKey here
super(dictionary, prefixKey, keySuffixSerializer, new SubStageGetterSingle<>(valueSerializer), 0);
this.valueSerializer = valueSerializer;
}
public static <T, U> DatabaseMapDictionary<T, U> simple(LLDictionary dictionary,
SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer,
Serializer<U, Send<Buffer>> valueSerializer) {
SerializerFixedBinaryLength<T> keySerializer,
Serializer<U> valueSerializer) {
return new DatabaseMapDictionary<>(dictionary, dictionary.getAllocator().allocate(0).send(), keySerializer, valueSerializer);
}
public static <T, U> DatabaseMapDictionary<T, U> tail(LLDictionary dictionary,
Send<Buffer> prefixKey,
SerializerFixedBinaryLength<T, Send<Buffer>> keySuffixSerializer,
Serializer<U, Send<Buffer>> valueSerializer) {
SerializerFixedBinaryLength<T> keySuffixSerializer,
Serializer<U> valueSerializer) {
return new DatabaseMapDictionary<>(dictionary, prefixKey, keySuffixSerializer, valueSerializer);
}
@ -73,7 +74,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
private void deserializeValue(Send<Buffer> value, SynchronousSink<U> sink) {
try {
sink.next(valueSerializer.deserialize(value));
sink.next(valueSerializer.deserialize(value).deserializedData());
} catch (SerializationException ex) {
sink.error(ex);
}
@ -86,7 +87,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
.<Entry<T, U>>handle((entrySend, sink) -> {
try (var entry = entrySend.receive()) {
var key = deserializeSuffix(stripPrefix(entry.getKey()));
var value = valueSerializer.deserialize(entry.getValue());
var value = valueSerializer.deserialize(entry.getValue()).deserializedData();
sink.next(Map.entry(key, value));
} catch (SerializationException ex) {
sink.error(ex);
@ -133,18 +134,15 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
@Override
public Mono<DatabaseStageEntry<U>> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
return Mono
.fromCallable(() -> new DatabaseSingleMapped<>(
new DatabaseSingle<>(dictionary, toKey(serializeSuffix(keySuffix)), Serializer.noop())
, valueSerializer)
);
return Mono.fromCallable(() ->
new DatabaseSingle<>(dictionary, toKey(serializeSuffix(keySuffix)), valueSerializer));
}
@Override
public Mono<U> getValue(@Nullable CompositeSnapshot snapshot, T keySuffix, boolean existsAlmostCertainly) {
return dictionary
.get(resolveSnapshot(snapshot), Mono.fromCallable(() -> toKey(serializeSuffix(keySuffix))), existsAlmostCertainly)
.handle(this::deserializeValue);
.handle((value, sink) -> deserializeValue(value, sink));
}
@Override
@ -170,7 +168,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
var keyMono = Mono.fromCallable(() -> toKey(serializeSuffix(keySuffix)));
return dictionary
.update(keyMono, getSerializedUpdater(updater), updateReturnMode, existsAlmostCertainly)
.handle(this::deserializeValue);
.handle((value, sink) -> deserializeValue(value, sink));
}
@Override
@ -180,7 +178,9 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
var keyMono = Mono.fromCallable(() -> toKey(serializeSuffix(keySuffix)));
return dictionary
.updateAndGetDelta(keyMono, getSerializedUpdater(updater), existsAlmostCertainly)
.transform(mono -> LLUtils.mapLLDelta(mono, valueSerializer::deserialize));
.transform(mono -> LLUtils.mapLLDelta(mono,
serialized -> valueSerializer.deserialize(serialized).deserializedData()
));
}
public SerializationFunction<@Nullable Send<Buffer>, @Nullable Send<Buffer>> getSerializedUpdater(
@ -191,7 +191,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
if (oldSerialized == null) {
result = updater.apply(null);
} else {
result = updater.apply(valueSerializer.deserialize(oldSerialized));
result = updater.apply(valueSerializer.deserialize(oldSerialized).deserializedData());
}
if (result == null) {
return null;
@ -210,7 +210,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
if (oldSerialized == null) {
result = updater.apply(null, extra);
} else {
result = updater.apply(valueSerializer.deserialize(oldSerialized), extra);
result = updater.apply(valueSerializer.deserialize(oldSerialized).deserializedData(), extra);
}
if (result == null) {
return null;
@ -229,7 +229,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
.put(keyMono,
valueMono,
LLDictionaryResultType.PREVIOUS_VALUE)
.handle(this::deserializeValue);
.handle((value1, sink) -> deserializeValue(value1, sink));
}
@Override
@ -238,7 +238,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
var valueMono = Mono.fromCallable(() -> valueSerializer.serialize(value));
return dictionary
.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE)
.handle(this::deserializeValue)
.handle((Send<Buffer> value1, SynchronousSink<U> sink) -> deserializeValue(value1, sink))
.map(oldValue -> !Objects.equals(oldValue, value))
.defaultIfEmpty(value != null);
}
@ -257,7 +257,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
var keyMono = Mono.fromCallable(() -> toKey(serializeSuffix(keySuffix)));
return dictionary
.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE)
.handle(this::deserializeValue);
.handle((value, sink) -> deserializeValue(value, sink));
}
@Override
@ -285,7 +285,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
Optional<U> valueOpt;
if (entry.getT3().isPresent()) {
try (var buf = entry.getT3().get()) {
valueOpt = Optional.of(valueSerializer.deserialize(buf));
valueOpt = Optional.of(valueSerializer.deserialize(buf).deserializedData());
}
} else {
valueOpt = Optional.empty();
@ -363,10 +363,10 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
try (key) {
try (var keySuffixWithExt = stripPrefix(key).receive()) {
sink.next(Map.entry(deserializeSuffix(keySuffixWithExt.copy().send()),
new DatabaseSingleMapped<>(new DatabaseSingle<>(dictionary,
new DatabaseSingle<>(dictionary,
toKey(keySuffixWithExt.send()),
Serializer.noop()
), valueSerializer)
valueSerializer
)
));
}
} catch (SerializationException ex) {
@ -382,7 +382,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
.<Entry<T, U>>handle((serializedEntryToReceive, sink) -> {
try (var serializedEntry = serializedEntryToReceive.receive()) {
sink.next(Map.entry(deserializeSuffix(stripPrefix(serializedEntry.getKey())),
valueSerializer.deserialize(serializedEntry.getValue())));
valueSerializer.deserialize(serializedEntry.getValue()).deserializedData()));
} catch (SerializationException e) {
sink.error(e);
}

14
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java
View File

@ -32,7 +32,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
protected final LLDictionary dictionary;
private final BufferAllocator alloc;
protected final SubStageGetter<U, US> subStageGetter;
protected final SerializerFixedBinaryLength<T, Send<Buffer>> keySuffixSerializer;
protected final SerializerFixedBinaryLength<T> keySuffixSerializer;
protected final Buffer keyPrefix;
protected final int keyPrefixLength;
protected final int keySuffixLength;
@ -191,14 +191,14 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
*/
@Deprecated
public static <T, U> DatabaseMapDictionaryDeep<T, U, DatabaseStageEntry<U>> simple(LLDictionary dictionary,
SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer,
SerializerFixedBinaryLength<T> keySerializer,
SubStageGetterSingle<U> subStageGetter) {
return new DatabaseMapDictionaryDeep<>(dictionary, dictionary.getAllocator().allocate(0).send(),
keySerializer, subStageGetter, 0);
}
public static <T, U, US extends DatabaseStage<U>> DatabaseMapDictionaryDeep<T, U, US> deepTail(LLDictionary dictionary,
SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer,
SerializerFixedBinaryLength<T> keySerializer,
int keyExtLength,
SubStageGetter<U, US> subStageGetter) {
return new DatabaseMapDictionaryDeep<>(dictionary,
@ -211,7 +211,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
public static <T, U, US extends DatabaseStage<U>> DatabaseMapDictionaryDeep<T, U, US> deepIntermediate(LLDictionary dictionary,
Send<Buffer> prefixKey,
SerializerFixedBinaryLength<T, Send<Buffer>> keySuffixSerializer,
SerializerFixedBinaryLength<T> keySuffixSerializer,
SubStageGetter<U, US> subStageGetter,
int keyExtLength) {
return new DatabaseMapDictionaryDeep<>(dictionary, prefixKey, keySuffixSerializer, subStageGetter, keyExtLength);
@ -219,7 +219,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
protected DatabaseMapDictionaryDeep(LLDictionary dictionary,
Send<Buffer> prefixKeyToReceive,
SerializerFixedBinaryLength<T, Send<Buffer>> keySuffixSerializer,
SerializerFixedBinaryLength<T> keySuffixSerializer,
SubStageGetter<U, US> subStageGetter,
int keyExtLength) {
try (var prefixKey = prefixKeyToReceive.receive()) {
@ -436,7 +436,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
.doOnNext(Send::close)
.then();
} else {
return dictionary.setRange(LLUtils.lazyRetainRange(range), Flux.empty());
return dictionary.setRange(rangeMono, Flux.empty());
}
});
}
@ -447,7 +447,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
assert suffixKeyConsistency(keySuffix.readableBytes());
var result = keySuffixSerializer.deserialize(keySuffix.send());
assert keyPrefix.isAccessible();
return result;
return result.deserializedData();
}
}

6
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSetDictionary.java
View File

@ -17,12 +17,12 @@ public class DatabaseSetDictionary<T> extends DatabaseMapDictionary<T, Nothing>
protected DatabaseSetDictionary(LLDictionary dictionary,
Send<Buffer> prefixKey,
SerializerFixedBinaryLength<T, Send<Buffer>> keySuffixSerializer) {
SerializerFixedBinaryLength<T> keySuffixSerializer) {
super(dictionary, prefixKey, keySuffixSerializer, DatabaseEmpty.nothingSerializer(dictionary.getAllocator()));
}
public static <T> DatabaseSetDictionary<T> simple(LLDictionary dictionary,
SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer) {
SerializerFixedBinaryLength<T> keySerializer) {
try (var buf = dictionary.getAllocator().allocate(0)) {
return new DatabaseSetDictionary<>(dictionary, buf.send(), keySerializer);
}
@ -30,7 +30,7 @@ public class DatabaseSetDictionary<T> extends DatabaseMapDictionary<T, Nothing>
public static <T> DatabaseSetDictionary<T> tail(LLDictionary dictionary,
Send<Buffer> prefixKey,
SerializerFixedBinaryLength<T, Send<Buffer>> keySuffixSerializer) {
SerializerFixedBinaryLength<T> keySuffixSerializer) {
return new DatabaseSetDictionary<>(dictionary, prefixKey, keySuffixSerializer);
}

16
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingle.java
View File

@ -27,9 +27,9 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
private final LLDictionary dictionary;
private final Buffer key;
private final Mono<Send<Buffer>> keyMono;
private final Serializer<U, Send<Buffer>> serializer;
private final Serializer<U> serializer;
public DatabaseSingle(LLDictionary dictionary, Send<Buffer> key, Serializer<U, Send<Buffer>> serializer) {
public DatabaseSingle(LLDictionary dictionary, Send<Buffer> key, Serializer<U> serializer) {
try (key) {
this.dictionary = dictionary;
this.key = key.receive();
@ -48,7 +48,7 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
private void deserializeValue(Send<Buffer> value, SynchronousSink<U> sink) {
try {
sink.next(serializer.deserialize(value));
sink.next(serializer.deserialize(value).deserializedData());
} catch (SerializationException ex) {
sink.error(ex);
}
@ -74,7 +74,8 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
boolean existsAlmostCertainly) {
return dictionary
.update(keyMono, (oldValueSer) -> {
var result = updater.apply(oldValueSer == null ? null : serializer.deserialize(oldValueSer));
var result = updater.apply(
oldValueSer == null ? null : serializer.deserialize(oldValueSer).deserializedData());
if (result == null) {
return null;
} else {
@ -89,13 +90,16 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
boolean existsAlmostCertainly) {
return dictionary
.updateAndGetDelta(keyMono, (oldValueSer) -> {
var result = updater.apply(oldValueSer == null ? null : serializer.deserialize(oldValueSer));
var result = updater.apply(
oldValueSer == null ? null : serializer.deserialize(oldValueSer).deserializedData());
if (result == null) {
return null;
} else {
return serializer.serialize(result);
}
}, existsAlmostCertainly).transform(mono -> LLUtils.mapLLDelta(mono, serializer::deserialize));
}, existsAlmostCertainly).transform(mono -> LLUtils.mapLLDelta(mono,
serialized -> serializer.deserialize(serialized).deserializedData()
));
}
@Override

36
src/main/java/it/cavallium/dbengine/database/collections/DatabaseSingleMapped.java
View File

@ -1,15 +1,13 @@
package it.cavallium.dbengine.database.collections;
import io.netty.buffer.api.Buffer;
import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.client.Mapper;
import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.database.serialization.Serializer;
import java.util.function.Function;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@ -19,16 +17,16 @@ import reactor.core.publisher.SynchronousSink;
public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
private final DatabaseStageEntry<B> serializedSingle;
private final Serializer<A, B> serializer;
private final Mapper<A, B> mapper;
public DatabaseSingleMapped(DatabaseStageEntry<B> serializedSingle, Serializer<A, B> serializer) {
public DatabaseSingleMapped(DatabaseStageEntry<B> serializedSingle, Mapper<A, B> mapper) {
this.serializedSingle = serializedSingle;
this.serializer = serializer;
this.mapper = mapper;
}
private void deserializeSink(B value, SynchronousSink<A> sink) {
try {
sink.next(this.deserialize(value));
sink.next(this.unMap(value));
} catch (SerializationException ex) {
sink.error(ex);
}
@ -47,14 +45,14 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
@Override
public Mono<Void> set(A value) {
return Mono
.fromCallable(() -> serialize(value))
.fromCallable(() -> map(value))
.flatMap(serializedSingle::set);
}
@Override
public Mono<A> setAndGetPrevious(A value) {
return Mono
.fromCallable(() -> serialize(value))
.fromCallable(() -> map(value))
.flatMap(serializedSingle::setAndGetPrevious)
.handle(this::deserializeSink);
}
@ -62,7 +60,7 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
@Override
public Mono<Boolean> setAndGetChanged(A value) {
return Mono
.fromCallable(() -> serialize(value))
.fromCallable(() -> map(value))
.flatMap(serializedSingle::setAndGetChanged)
.single();
}
@ -72,11 +70,11 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly) {
return serializedSingle.update(oldValue -> {
var result = updater.apply(oldValue == null ? null : this.deserialize(oldValue));
var result = updater.apply(oldValue == null ? null : this.unMap(oldValue));
if (result == null) {
return null;
} else {
return this.serialize(result);
return this.map(result);
}
}, updateReturnMode, existsAlmostCertainly).handle(this::deserializeSink);
}
@ -85,13 +83,13 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
public Mono<Delta<A>> updateAndGetDelta(SerializationFunction<@Nullable A, @Nullable A> updater,
boolean existsAlmostCertainly) {
return serializedSingle.updateAndGetDelta(oldValue -> {
var result = updater.apply(oldValue == null ? null : this.deserialize(oldValue));
var result = updater.apply(oldValue == null ? null : this.unMap(oldValue));
if (result == null) {
return null;
} else {
return this.serialize(result);
return this.map(result);
}
}, existsAlmostCertainly).transform(mono -> LLUtils.mapDelta(mono, this::deserialize));
}, existsAlmostCertainly).transform(mono -> LLUtils.mapDelta(mono, this::unMap));
}
@Override
@ -140,12 +138,12 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
}
//todo: temporary wrapper. convert the whole class to buffers
private A deserialize(B bytes) throws SerializationException {
return serializer.deserialize(bytes);
private A unMap(B bytes) throws SerializationException {
return mapper.unmap(bytes);
}
//todo: temporary wrapper. convert the whole class to buffers
private B serialize(A bytes) throws SerializationException {
return serializer.serialize(bytes);
private B map(A bytes) throws SerializationException {
return mapper.map(bytes);
}
}

12
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterHashMap.java
View File

@ -17,15 +17,15 @@ import reactor.core.publisher.Mono;
public class SubStageGetterHashMap<T, U, TH> implements
SubStageGetter<Map<T, U>, DatabaseMapDictionaryHashed<T, U, TH>> {
private final Serializer<T, Send<Buffer>> keySerializer;
private final Serializer<U, Send<Buffer>> valueSerializer;
private final Serializer<T> keySerializer;
private final Serializer<U> valueSerializer;
private final Function<T, TH> keyHashFunction;
private final SerializerFixedBinaryLength<TH, Send<Buffer>> keyHashSerializer;
private final SerializerFixedBinaryLength<TH> keyHashSerializer;
public SubStageGetterHashMap(Serializer<T, Send<Buffer>> keySerializer,
Serializer<U, Send<Buffer>> valueSerializer,
public SubStageGetterHashMap(Serializer<T> keySerializer,
Serializer<U> valueSerializer,
Function<T, TH> keyHashFunction,
SerializerFixedBinaryLength<TH, Send<Buffer>> keyHashSerializer) {
SerializerFixedBinaryLength<TH> keyHashSerializer) {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
this.keyHashFunction = keyHashFunction;

8
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterHashSet.java
View File

@ -17,13 +17,13 @@ import reactor.core.publisher.Mono;
public class SubStageGetterHashSet<T, TH> implements
SubStageGetter<Map<T, Nothing>, DatabaseSetDictionaryHashed<T, TH>> {
private final Serializer<T, Send<Buffer>> keySerializer;
private final Serializer<T> keySerializer;
private final Function<T, TH> keyHashFunction;
private final SerializerFixedBinaryLength<TH, Send<Buffer>> keyHashSerializer;
private final SerializerFixedBinaryLength<TH> keyHashSerializer;
public SubStageGetterHashSet(Serializer<T, Send<Buffer>> keySerializer,
public SubStageGetterHashSet(Serializer<T> keySerializer,
Function<T, TH> keyHashFunction,
SerializerFixedBinaryLength<TH, Send<Buffer>> keyHashSerializer) {
SerializerFixedBinaryLength<TH> keyHashSerializer) {
this.keySerializer = keySerializer;
this.keyHashFunction = keyHashFunction;
this.keyHashSerializer = keyHashSerializer;

8
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterMap.java
View File

@ -15,11 +15,11 @@ import reactor.core.publisher.Mono;
public class SubStageGetterMap<T, U> implements SubStageGetter<Map<T, U>, DatabaseMapDictionary<T, U>> {
private final SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer;
private final Serializer<U, Send<Buffer>> valueSerializer;
private final SerializerFixedBinaryLength<T> keySerializer;
private final Serializer<U> valueSerializer;
public SubStageGetterMap(SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer,
Serializer<U, Send<Buffer>> valueSerializer) {
public SubStageGetterMap(SerializerFixedBinaryLength<T> keySerializer,
Serializer<U> valueSerializer) {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
}

4
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterMapDeep.java
View File

@ -16,11 +16,11 @@ public class SubStageGetterMapDeep<T, U, US extends DatabaseStage<U>> implements
SubStageGetter<Map<T, U>, DatabaseMapDictionaryDeep<T, U, US>> {
private final SubStageGetter<U, US> subStageGetter;
private final SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer;
private final SerializerFixedBinaryLength<T> keySerializer;
private final int keyExtLength;
public SubStageGetterMapDeep(SubStageGetter<U, US> subStageGetter,
SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer,
SerializerFixedBinaryLength<T> keySerializer,
int keyExtLength) {
this.subStageGetter = subStageGetter;
this.keySerializer = keySerializer;

4
src/main/java/it/cavallium/dbengine/database/collections/SubStageGetterSet.java
View File

@ -15,9 +15,9 @@ import reactor.core.publisher.Mono;
public class SubStageGetterSet<T> implements SubStageGetter<Map<T, Nothing>, DatabaseSetDictionary<T>> {
private final SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer;
private final SerializerFixedBinaryLength<T> keySerializer;
public SubStageGetterSet(SerializerFixedBinaryLength<T, Send<Buffer>> keySerializer) {
public SubStageGetterSet(SerializerFixedBinaryLength<T> keySerializer) {
this.keySerializer = keySerializer;
}

6
src/main/java/it/cavallium/dbengine/database/collections/ValueWithHashSerializer.java
View File

@ -29,7 +29,11 @@ class ValueWithHashSerializer<X, Y> implements Serializer<Entry<X, Y>> {
throws SerializationException {
try (var serialized = serializedToReceive.receive()) {
DeserializationResult<X> deserializedKey = keySuffixSerializer.deserialize(serialized.copy().send());
DeserializationResult<Y> deserializedValue = valueSerializer.deserialize(serialized.send());
DeserializationResult<Y> deserializedValue = valueSerializer.deserialize(serialized
.copy(serialized.readerOffset() + deserializedKey.bytesRead(),
serialized.readableBytes() - deserializedKey.bytesRead()
)
.send());
return new DeserializationResult<>(Map.entry(deserializedKey.deserializedData(),
deserializedValue.deserializedData()), deserializedKey.bytesRead() + deserializedValue.bytesRead());
}

11
src/main/java/it/cavallium/dbengine/database/collections/ValuesSetSerializer.java
View File

@ -27,13 +27,17 @@ class ValuesSetSerializer<X> implements Serializer<ObjectArraySet<X>> {
@Override
public @NotNull DeserializationResult<ObjectArraySet<X>> deserialize(@NotNull Send<Buffer> serializedToReceive) throws SerializationException {
try (var serialized = serializedToReceive.receive()) {
int initialReaderOffset = serialized.readerOffset();
int entriesLength = serialized.readInt();
ArrayList<X> deserializedElements = new ArrayList<>(entriesLength);
for (int i = 0; i < entriesLength; i++) {
X entry = entrySerializer.deserialize(serialized.copy().send());
deserializedElements.add(entry);
var deserializationResult = entrySerializer.deserialize(serialized
.copy(serialized.readerOffset(), serialized.readableBytes())
.send());
deserializedElements.add(deserializationResult.deserializedData());
serialized.readerOffset(serialized.readerOffset() + deserializationResult.bytesRead());
}
return new ObjectArraySet<>(deserializedElements);
return new DeserializationResult<>(new ObjectArraySet<>(deserializedElements), serialized.readerOffset() - initialReaderOffset);
}
}
@ -43,6 +47,7 @@ class ValuesSetSerializer<X> implements Serializer<ObjectArraySet<X>> {
output.writeInt(deserialized.size());
for (X entry : deserialized) {
try (Buffer serialized = entrySerializer.serialize(entry).receive()) {
output.ensureWritable(serialized.readableBytes());
output.writeBytes(serialized);
}
}

34
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

@ -376,7 +376,8 @@ public class LLLocalDictionary implements LLDictionary {
PlatformDependent.freeDirectBuffer(keyNioBuffer.byteBuffer());
}
} else {
try (ReadOptions validReadOptions = Objects.requireNonNullElse(readOptions, EMPTY_READ_OPTIONS)) {
ReadOptions validReadOptions = Objects.requireNonNullElse(readOptions, EMPTY_READ_OPTIONS);
try {
byte[] keyArray = LLUtils.toArray(key);
requireNonNull(keyArray);
Holder<byte[]> data = existsAlmostCertainly ? null : new Holder<>();
@ -394,6 +395,10 @@ public class LLLocalDictionary implements LLDictionary {
} else {
return null;
}
} finally {
if (!(validReadOptions instanceof UnreleasableReadOptions)) {
validReadOptions.close();
}
}
}
}
@ -440,9 +445,9 @@ public class LLLocalDictionary implements LLDictionary {
rangeSend -> {
try (var range = rangeSend.receive()) {
if (range.isSingle()) {
return this.containsKey(snapshot, LLUtils.lazyRetain((range.getSingle().receive())));
return this.containsKey(snapshot, Mono.fromCallable(range::getSingle));
} else {
return this.containsRange(snapshot, LLUtils.lazyRetainRange(range));
return this.containsRange(snapshot, rangeMono);
}
}
},
@ -794,7 +799,7 @@ public class LLLocalDictionary implements LLDictionary {
prevData = null;
}
} else {
var obtainedPrevData = dbGet(cfh, null, key.send(), existsAlmostCertainly);
var obtainedPrevData = dbGet(cfh, null, key.copy().send(), existsAlmostCertainly);
if (obtainedPrevData == null) {
prevData = null;
} else {
@ -852,6 +857,8 @@ public class LLLocalDictionary implements LLDictionary {
logger.trace("Writing {}: {}",
LLUtils.toStringSafe(key), LLUtils.toStringSafe(newData));
}
assert key.isAccessible();
assert newData.isAccessible();
dbPut(cfh, null, key.send(), newData.copy().send());
}
return LLDelta.of(
@ -1128,7 +1135,15 @@ public class LLLocalDictionary implements LLDictionary {
for (LLEntry entry : entriesWindow) {
var k = entry.getKey();
var v = entry.getValue();
batch.put(cfh, k, v);
if (databaseOptions.allowNettyDirect()) {
batch.put(cfh, k, v);
} else {
try (var key = k.receive()) {
try (var value = v.receive()) {
batch.put(cfh, LLUtils.toArray(key), LLUtils.toArray(value));
}
}
}
}
batch.writeToDbAndClose();
batch.close();
@ -1655,7 +1670,14 @@ public class LLLocalDictionary implements LLDictionary {
)) {
for (LLEntry entry : entriesList) {
assert entry.isAccessible();
batch.put(cfh, entry.getKey(), entry.getValue());
if (databaseOptions.allowNettyDirect()) {
batch.put(cfh, entry.getKey(), entry.getValue());
} else {
batch.put(cfh,
LLUtils.toArray(entry.getKeyUnsafe()),
LLUtils.toArray(entry.getValueUnsafe())
);
}
}
batch.writeToDbAndClose();
}

14
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -1,6 +1,7 @@
package it.cavallium.dbengine.database.disk;
import io.netty.buffer.api.BufferAllocator;
import io.netty.util.internal.PlatformDependent;
import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
@ -89,6 +90,19 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
DatabaseOptions databaseOptions) throws IOException {
this.name = name;
this.allocator = allocator;
if (databaseOptions.allowNettyDirect()) {
if (!PlatformDependent.hasUnsafe()) {
throw new UnsupportedOperationException("Please enable unsafe support or disable netty direct buffers",
PlatformDependent.getUnsafeUnavailabilityCause()
);
}
if (!PlatformDependent.hasDirectBufferNoCleanerConstructor()) {
throw new UnsupportedOperationException("Please enable unsafe support or disable netty direct buffers:"
+ " DirectBufferNoCleanerConstructor is not available");
}
}
Options rocksdbOptions = openRocksDb(path, databaseOptions);
try {
List<ColumnFamilyDescriptor> descriptors = new LinkedList<>();

14
src/main/java/it/cavallium/dbengine/database/disk/LLLocalReactiveRocksIterator.java
View File

@ -68,10 +68,20 @@ public abstract class LLLocalReactiveRocksIterator<T> {
var rocksIterator = tuple.getT1();
rocksIterator.status();
if (rocksIterator.isValid()) {
try (Buffer key = LLUtils.readDirectNioBuffer(alloc, rocksIterator::key).receive()) {
Buffer key;
if (allowNettyDirect) {
key = LLUtils.readDirectNioBuffer(alloc, rocksIterator::key).receive();
} else {
key = LLUtils.fromByteArray(alloc, rocksIterator.key());
}
try (key) {
Buffer value;
if (readValues) {
value = LLUtils.readDirectNioBuffer(alloc, rocksIterator::value).receive();
if (allowNettyDirect) {
value = LLUtils.readDirectNioBuffer(alloc, rocksIterator::value).receive();
} else {
value = LLUtils.fromByteArray(alloc, rocksIterator.value());
}
} else {
value = alloc.allocate(0);
}

6
src/main/java/it/cavallium/dbengine/database/serialization/BufferDataInput.java
View File

@ -12,9 +12,11 @@ import org.jetbrains.annotations.NotNull;
public class BufferDataInput implements DataInput, SafeCloseable {
private final Buffer buf;
private final int initialReaderOffset;
public BufferDataInput(Send<Buffer> bufferSend) {
this.buf = bufferSend.receive().makeReadOnly();
this.initialReaderOffset = buf.readerOffset();
}
@Override
@ -104,4 +106,8 @@ public class BufferDataInput implements DataInput, SafeCloseable {
public void close() {
buf.close();
}
public int getReadBytesCount() {
return buf.readerOffset() - initialReaderOffset;
}
}

8
src/main/java/it/cavallium/dbengine/database/serialization/CodecSerializer.java
View File

@ -8,7 +8,7 @@ import java.io.IOException;
import org.jetbrains.annotations.NotNull;
import org.warp.commonutils.error.IndexOutOfBoundsException;
public class CodecSerializer<A> implements Serializer<A, Send<Buffer>> {
public class CodecSerializer<A> implements Serializer<A> {
private final BufferAllocator allocator;
private final Codecs<A> deserializationCodecs;
@ -37,8 +37,8 @@ public class CodecSerializer<A> implements Serializer<A, Send<Buffer>> {
}
@Override
public @NotNull A deserialize(@NotNull Send<Buffer> serialized) {
try (var is = new BufferDataInput(serialized)) {
public @NotNull DeserializationResult<A> deserialize(@NotNull Send<Buffer> serializedToReceive) {
try (var is = new BufferDataInput(serializedToReceive)) {
int codecId;
if (microCodecs) {
codecId = is.readUnsignedByte();
@ -46,7 +46,7 @@ public class CodecSerializer<A> implements Serializer<A, Send<Buffer>> {
codecId = is.readInt();
}
var serializer = deserializationCodecs.getCodec(codecId);
return serializer.deserialize(is);
return new DeserializationResult<>(serializer.deserialize(is), is.getReadBytesCount());
} catch (IOException ex) {
// This shouldn't happen
throw new IOError(ex);

6
src/main/java/it/cavallium/dbengine/database/serialization/Serializer.java
View File

@ -39,11 +39,11 @@ public interface Serializer<A> {
@Override
public @NotNull DeserializationResult<String> deserialize(@NotNull Send<Buffer> serializedToReceive) {
try (Buffer serialized = serializedToReceive.receive()) {
assert serialized.isAccessible();
int length = serialized.readInt();
var readerOffset = serialized.readerOffset();
var readableBytes = serialized.readableBytes();
return new DeserializationResult<>(LLUtils.deserializeString(serialized.send(),
readerOffset, length, StandardCharsets.UTF_8), readableBytes);
readerOffset, length, StandardCharsets.UTF_8), Integer.BYTES + length);
}
}
@ -51,8 +51,10 @@ public interface Serializer<A> {
public @NotNull Send<Buffer> serialize(@NotNull String deserialized) {
var bytes = deserialized.getBytes(StandardCharsets.UTF_8);
try (Buffer buf = allocator.allocate(Integer.BYTES + bytes.length)) {
assert buf.isAccessible();
buf.writeInt(bytes.length);
buf.writeBytes(bytes);
assert buf.isAccessible();
return buf.send();
}
}

10
src/main/java/it/cavallium/dbengine/database/serialization/SerializerFixedBinaryLength.java
View File

@ -58,9 +58,8 @@ public interface SerializerFixedBinaryLength<A> extends Serializer<A> {
+ serialized.readableBytes() + " bytes instead");
}
var readerOffset = serialized.readerOffset();
var readableBytes = serialized.readableBytes();
return new DeserializationResult<>(LLUtils.deserializeString(serialized.send(),
readerOffset, length, StandardCharsets.UTF_8), readableBytes);
readerOffset, length, StandardCharsets.UTF_8), length);
}
}
@ -68,12 +67,14 @@ public interface SerializerFixedBinaryLength<A> extends Serializer<A> {
public @NotNull Send<Buffer> serialize(@NotNull String deserialized) throws SerializationException {
// UTF-8 uses max. 3 bytes per char, so calculate the worst case.
try (Buffer buf = allocator.allocate(LLUtils.utf8MaxBytes(deserialized))) {
assert buf.isAccessible();
buf.writeBytes(deserialized.getBytes(StandardCharsets.UTF_8));
if (buf.readableBytes() != getSerializedBinaryLength()) {
throw new SerializationException("Fixed serializer with " + getSerializedBinaryLength()
+ " bytes has tried to serialize an element with "
+ buf.readableBytes() + " bytes instead");
}
assert buf.isAccessible();
return buf.send();
}
}
@ -95,8 +96,7 @@ public interface SerializerFixedBinaryLength<A> extends Serializer<A> {
"Fixed serializer with " + getSerializedBinaryLength() + " bytes has tried to deserialize an element with "
+ serialized.readableBytes() + " bytes instead");
}
var readableBytes = serialized.readableBytes();
return new DeserializationResult<>(serialized.readInt(), readableBytes);
return new DeserializationResult<>(serialized.readInt(), Integer.BYTES);
}
}
@ -125,7 +125,7 @@ public interface SerializerFixedBinaryLength<A> extends Serializer<A> {
+ serialized.readableBytes() + " bytes instead");
}
var readableBytes = serialized.readableBytes();
return new DeserializationResult<>(serialized.readLong(), readableBytes);
return new DeserializationResult<>(serialized.readLong(), Long.BYTES);
}
}

8
src/test/java/it/cavallium/dbengine/DbTestUtils.java
View File

@ -99,7 +99,7 @@ public class DbTestUtils {
.flatMap(conn -> conn
.getDatabase("testdb",
List.of(Column.dictionary("testmap"), Column.special("ints"), Column.special("longs")),
new DatabaseOptions(Map.of(), true, false, true, false, true, true, true, -1)
new DatabaseOptions(Map.of(), true, false, true, false, true, false, false, -1)
)
.map(db -> new TempDb(alloc, conn, db, wrkspcPath))
);
@ -165,12 +165,10 @@ public class DbTestUtils {
}
@Override
public @NotNull Short deserialize(@NotNull Send<Buffer> serializedToReceive) {
public @NotNull DeserializationResult<Short> deserialize(@NotNull Send<Buffer> serializedToReceive) {
try (var serialized = serializedToReceive.receive()) {
var prevReaderIdx = serialized.readerOffset();
var val = serialized.readShort();
serialized.readerOffset(prevReaderIdx + Short.BYTES);
return val;
return new DeserializationResult<>(val, Short.BYTES);
}
}