Add checked serialization exception

This commit is contained in:
Andrea Cavalli 2021-08-22 21:23:22 +02:00
parent 4e71e42d32
commit 2a24570512
20 changed files with 375 additions and 169 deletions

View File

@ -1,6 +1,7 @@
package it.cavallium.dbengine.client; package it.cavallium.dbengine.client;
import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBuf;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer; import it.cavallium.dbengine.database.serialization.Serializer;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
@ -16,7 +17,7 @@ public class MappedSerializer<A, B> implements Serializer<B, ByteBuf> {
} }
@Override @Override
public @NotNull B deserialize(@NotNull ByteBuf serialized) { public @NotNull B deserialize(@NotNull ByteBuf serialized) throws SerializationException {
try { try {
return keyMapper.map(serializer.deserialize(serialized.retain())); return keyMapper.map(serializer.deserialize(serialized.retain()));
} finally { } finally {
@ -25,7 +26,7 @@ public class MappedSerializer<A, B> implements Serializer<B, ByteBuf> {
} }
@Override @Override
public @NotNull ByteBuf serialize(@NotNull B deserialized) { public @NotNull ByteBuf serialize(@NotNull B deserialized) throws SerializationException {
return serializer.serialize(keyMapper.unmap(deserialized)); return serializer.serialize(keyMapper.unmap(deserialized));
} }
} }

View File

@ -1,6 +1,7 @@
package it.cavallium.dbengine.client; package it.cavallium.dbengine.client;
import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBuf;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength; import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
@ -16,7 +17,7 @@ public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryL
} }
@Override @Override
public @NotNull B deserialize(@NotNull ByteBuf serialized) { public @NotNull B deserialize(@NotNull ByteBuf serialized) throws SerializationException {
try { try {
return keyMapper.map(fixedLengthSerializer.deserialize(serialized.retain())); return keyMapper.map(fixedLengthSerializer.deserialize(serialized.retain()));
} finally { } finally {
@ -25,7 +26,7 @@ public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryL
} }
@Override @Override
public @NotNull ByteBuf serialize(@NotNull B deserialized) { public @NotNull ByteBuf serialize(@NotNull B deserialized) throws SerializationException {
return fixedLengthSerializer.serialize(keyMapper.unmap(deserialized)); return fixedLengthSerializer.serialize(keyMapper.unmap(deserialized));
} }

View File

@ -3,6 +3,8 @@ package it.cavallium.dbengine.database;
import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.ByteBufAllocator;
import it.cavallium.dbengine.client.BadBlock; import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.database.serialization.BiSerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import java.util.List; import java.util.List;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Optional; import java.util.Optional;
@ -34,7 +36,7 @@ public interface LLDictionary extends LLKeyValueDatabaseStructure {
Mono<UpdateMode> getUpdateMode(); Mono<UpdateMode> getUpdateMode();
default Mono<ByteBuf> update(Mono<ByteBuf> key, default Mono<ByteBuf> update(Mono<ByteBuf> key,
Function<@Nullable ByteBuf, @Nullable ByteBuf> updater, SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> updater,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return this return this
@ -43,17 +45,17 @@ public interface LLDictionary extends LLKeyValueDatabaseStructure {
} }
default Mono<ByteBuf> update(Mono<ByteBuf> key, default Mono<ByteBuf> update(Mono<ByteBuf> key,
Function<@Nullable ByteBuf, @Nullable ByteBuf> updater, SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> updater,
UpdateReturnMode returnMode) { UpdateReturnMode returnMode) {
return update(key, updater, returnMode, false); return update(key, updater, returnMode, false);
} }
Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> key, Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> key,
Function<@Nullable ByteBuf, @Nullable ByteBuf> updater, SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> updater,
boolean existsAlmostCertainly); boolean existsAlmostCertainly);
default Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> key, default Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> key,
Function<@Nullable ByteBuf, @Nullable ByteBuf> updater) { SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> updater) {
return updateAndGetDelta(key, updater, false); return updateAndGetDelta(key, updater, false);
} }
@ -72,7 +74,7 @@ public interface LLDictionary extends LLKeyValueDatabaseStructure {
Flux<Entry<ByteBuf, ByteBuf>> putMulti(Flux<Entry<ByteBuf, ByteBuf>> entries, boolean getOldValues); Flux<Entry<ByteBuf, ByteBuf>> putMulti(Flux<Entry<ByteBuf, ByteBuf>> entries, boolean getOldValues);
<X> Flux<ExtraKeyOperationResult<ByteBuf, X>> updateMulti(Flux<Tuple2<ByteBuf, X>> entries, <X> Flux<ExtraKeyOperationResult<ByteBuf, X>> updateMulti(Flux<Tuple2<ByteBuf, X>> entries,
BiFunction<ByteBuf, X, ByteBuf> updateFunction); BiSerializationFunction<ByteBuf, X, ByteBuf> updateFunction);
Flux<Entry<ByteBuf, ByteBuf>> getRange(@Nullable LLSnapshot snapshot, Mono<LLRange> range, boolean existsAlmostCertainly); Flux<Entry<ByteBuf, ByteBuf>> getRange(@Nullable LLSnapshot snapshot, Mono<LLRange> range, boolean existsAlmostCertainly);

View File

@ -7,17 +7,22 @@ import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufUtil; import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.CompositeByteBuf;
import io.netty.buffer.Unpooled; import io.netty.buffer.Unpooled;
import io.netty.util.AbstractReferenceCounted;
import io.netty.util.IllegalReferenceCountException; import io.netty.util.IllegalReferenceCountException;
import io.netty.util.ReferenceCounted;
import it.cavallium.dbengine.database.disk.ReleasableSlice;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.lucene.RandomSortField; import it.cavallium.dbengine.lucene.RandomSortField;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.ToIntFunction; import java.util.function.ToIntFunction;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
@ -37,6 +42,8 @@ import org.apache.lucene.search.SortedNumericSortField;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.rocksdb.RocksDB; import org.rocksdb.RocksDB;
import org.warp.commonutils.functional.IOFunction;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
@SuppressWarnings("unused") @SuppressWarnings("unused")
@ -94,16 +101,12 @@ public class LLUtils {
} }
public static ScoreMode toScoreMode(LLScoreMode scoreMode) { public static ScoreMode toScoreMode(LLScoreMode scoreMode) {
switch (scoreMode) { return switch (scoreMode) {
case COMPLETE: case COMPLETE -> ScoreMode.COMPLETE;
return ScoreMode.COMPLETE; case TOP_SCORES -> ScoreMode.TOP_SCORES;
case TOP_SCORES: case COMPLETE_NO_SCORES -> ScoreMode.COMPLETE_NO_SCORES;
return ScoreMode.TOP_SCORES; default -> throw new IllegalStateException("Unexpected value: " + scoreMode);
case COMPLETE_NO_SCORES: };
return ScoreMode.COMPLETE_NO_SCORES;
default:
throw new IllegalStateException("Unexpected value: " + scoreMode);
}
} }
public static Term toTerm(LLTerm term) { public static Term toTerm(LLTerm term) {
@ -143,25 +146,18 @@ public class LLUtils {
} }
private static IndexableField toField(LLItem item) { private static IndexableField toField(LLItem item) {
switch (item.getType()) { return switch (item.getType()) {
case IntPoint: case IntPoint -> new IntPoint(item.getName(), Ints.fromByteArray(item.getData()));
return new IntPoint(item.getName(), Ints.fromByteArray(item.getData())); case LongPoint -> new LongPoint(item.getName(), Longs.fromByteArray(item.getData()));
case LongPoint: case FloatPoint -> new FloatPoint(item.getName(), ByteBuffer.wrap(item.getData()).getFloat());
return new LongPoint(item.getName(), Longs.fromByteArray(item.getData())); case TextField -> new TextField(item.getName(), item.stringValue(), Field.Store.NO);
case FloatPoint: case TextFieldStored -> new TextField(item.getName(), item.stringValue(), Field.Store.YES);
return new FloatPoint(item.getName(), ByteBuffer.wrap(item.getData()).getFloat()); case SortedNumericDocValuesField -> new SortedNumericDocValuesField(item.getName(),
case TextField: Longs.fromByteArray(item.getData())
return new TextField(item.getName(), item.stringValue(), Field.Store.NO); );
case TextFieldStored: case StringField -> new StringField(item.getName(), item.stringValue(), Field.Store.NO);
return new TextField(item.getName(), item.stringValue(), Field.Store.YES); case StringFieldStored -> new StringField(item.getName(), item.stringValue(), Field.Store.YES);
case SortedNumericDocValuesField: };
return new SortedNumericDocValuesField(item.getName(), Longs.fromByteArray(item.getData()));
case StringField:
return new StringField(item.getName(), item.stringValue(), Field.Store.NO);
case StringFieldStored:
return new StringField(item.getName(), item.stringValue(), Field.Store.YES);
}
throw new UnsupportedOperationException("Unsupported field type");
} }
public static it.cavallium.dbengine.database.LLKeyScore toKeyScore(LLKeyScore hit) { public static it.cavallium.dbengine.database.LLKeyScore toKeyScore(LLKeyScore hit) {
@ -442,48 +438,50 @@ public class LLUtils {
public static <T> Mono<T> resolveDelta(Mono<Delta<T>> prev, UpdateReturnMode updateReturnMode) { public static <T> Mono<T> resolveDelta(Mono<Delta<T>> prev, UpdateReturnMode updateReturnMode) {
return prev.handle((delta, sink) -> { return prev.handle((delta, sink) -> {
switch (updateReturnMode) { switch (updateReturnMode) {
case GET_NEW_VALUE: case GET_NEW_VALUE -> {
var current = delta.current(); var current = delta.current();
if (current != null) { if (current != null) {
sink.next(current); sink.next(current);
} else { } else {
sink.complete(); sink.complete();
} }
break; }
case GET_OLD_VALUE: case GET_OLD_VALUE -> {
var previous = delta.previous(); var previous = delta.previous();
if (previous != null) { if (previous != null) {
sink.next(previous); sink.next(previous);
} else { } else {
sink.complete(); sink.complete();
} }
break; }
case NOTHING: case NOTHING -> sink.complete();
sink.complete(); default -> sink.error(new IllegalStateException());
break;
default:
sink.error(new IllegalStateException());
} }
}); });
} }
public static <T, U> Mono<Delta<U>> mapDelta(Mono<Delta<T>> mono, Function<@NotNull T, @Nullable U> mapper) { public static <T, U> Mono<Delta<U>> mapDelta(Mono<Delta<T>> mono,
return mono.map(delta -> { SerializationFunction<@NotNull T, @Nullable U> mapper) {
T prev = delta.previous(); return mono.handle((delta, sink) -> {
T curr = delta.current(); try {
U newPrev; T prev = delta.previous();
U newCurr; T curr = delta.current();
if (prev != null) { U newPrev;
newPrev = mapper.apply(prev); U newCurr;
} else { if (prev != null) {
newPrev = null; newPrev = mapper.apply(prev);
} else {
newPrev = null;
}
if (curr != null) {
newCurr = mapper.apply(curr);
} else {
newCurr = null;
}
sink.next(new Delta<>(newPrev, newCurr));
} catch (SerializationException ex) {
sink.error(ex);
} }
if (curr != null) {
newCurr = mapper.apply(curr);
} else {
newCurr = null;
}
return new Delta<>(newPrev, newCurr);
}); });
} }
@ -514,4 +512,44 @@ public class LLUtils {
return false; return false;
}); });
} }
public static <T> Mono<T> handleDiscard(Mono<T> mono) {
return mono.doOnDiscard(Map.Entry.class, e -> {
if (e.getKey() instanceof ByteBuf bb) {
if (bb.refCnt() > 0) {
bb.release();
}
}
if (e.getValue() instanceof ByteBuf bb) {
if (bb.refCnt() > 0) {
bb.release();
}
}
});
}
public static <T> Flux<T> handleDiscard(Flux<T> mono) {
return mono
.doOnDiscard(ReferenceCounted.class, LLUtils::discardRefCounted)
.doOnDiscard(Map.Entry.class, LLUtils::discardEntry);
}
private static void discardEntry(Map.Entry<?, ?> e) {
if (e.getKey() instanceof ByteBuf bb) {
if (bb.refCnt() > 0) {
bb.release();
}
}
if (e.getValue() instanceof ByteBuf bb) {
if (bb.refCnt() > 0) {
bb.release();
}
}
}
private static void discardRefCounted(ReferenceCounted referenceCounted) {
if (referenceCounted.refCnt() > 0) {
referenceCounted.release();
}
}
} }

View File

@ -10,6 +10,9 @@ import it.cavallium.dbengine.database.LLDictionaryResultType;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateMode; import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode; import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.BiSerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.database.serialization.Serializer; import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength; import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import java.util.Collections; import java.util.Collections;
@ -23,6 +26,7 @@ import java.util.function.Function;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.SynchronousSink;
import reactor.util.function.Tuple2; import reactor.util.function.Tuple2;
import reactor.util.function.Tuples; import reactor.util.function.Tuples;
@ -64,28 +68,48 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
} }
} }
private void deserializeValue(ByteBuf value, SynchronousSink<U> sink) {
try {
sink.next(valueSerializer.deserialize(value));
} catch (SerializationException ex) {
sink.error(ex);
}
}
@Override @Override
public Mono<Map<T, U>> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) { public Mono<Map<T, U>> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) {
return dictionary return dictionary
.getRange(resolveSnapshot(snapshot), rangeMono, existsAlmostCertainly) .getRange(resolveSnapshot(snapshot), rangeMono, existsAlmostCertainly)
.collectMap( .<Entry<T, U>>handle((entry, sink) -> {
entry -> deserializeSuffix(stripPrefix(entry.getKey(), false)), try {
entry -> valueSerializer.deserialize(entry.getValue()), var key = deserializeSuffix(stripPrefix(entry.getKey(), false));
HashMap::new) var value = valueSerializer.deserialize(entry.getValue());
sink.next(Map.entry(key, value));
} catch (SerializationException ex) {
sink.error(ex);
}
})
.collectMap(Entry::getKey, Entry::getValue, HashMap::new)
.filter(map -> !map.isEmpty()); .filter(map -> !map.isEmpty());
} }
@Override @Override
public Mono<Map<T, U>> setAndGetPrevious(Map<T, U> value) { public Mono<Map<T, U>> setAndGetPrevious(Map<T, U> value) {
return Mono.usingWhen( return this
Mono.just(true), .get(null, false)
b -> get(null, false), .concatWith(dictionary.setRange(rangeMono, Flux
b -> dictionary.setRange(rangeMono, Flux
.fromIterable(Collections.unmodifiableMap(value).entrySet()) .fromIterable(Collections.unmodifiableMap(value).entrySet())
.map(entry -> Map.entry(this.toKey(serializeSuffix(entry.getKey())), .handle((entry, sink) -> {
valueSerializer.serialize(entry.getValue()))) try {
) sink.next(Map.entry(this.toKey(serializeSuffix(entry.getKey())),
); valueSerializer.serialize(entry.getValue())));
} catch (SerializationException e) {
sink.error(e);
}
})
).then(Mono.empty()))
.singleOrEmpty()
.transform(LLUtils::handleDiscard);
} }
@Override @Override
@ -107,7 +131,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
@Override @Override
public Mono<DatabaseStageEntry<U>> at(@Nullable CompositeSnapshot snapshot, T keySuffix) { public Mono<DatabaseStageEntry<U>> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
return Mono return Mono
.fromSupplier(() -> new DatabaseSingleMapped<>( .fromCallable(() -> new DatabaseSingleMapped<>(
new DatabaseSingle<>(dictionary, toKey(serializeSuffix(keySuffix)), Serializer.noop()) new DatabaseSingle<>(dictionary, toKey(serializeSuffix(keySuffix)), Serializer.noop())
, valueSerializer) , valueSerializer)
); );
@ -120,7 +144,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
() -> toKey(serializeSuffix(keySuffix)), () -> toKey(serializeSuffix(keySuffix)),
keyBuf -> dictionary keyBuf -> dictionary
.get(resolveSnapshot(snapshot), LLUtils.lazyRetain(keyBuf), existsAlmostCertainly) .get(resolveSnapshot(snapshot), LLUtils.lazyRetain(keyBuf), existsAlmostCertainly)
.map(valueSerializer::deserialize), .handle(this::deserializeValue),
ReferenceCounted::release ReferenceCounted::release
); );
} }
@ -158,13 +182,13 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
public Mono<U> updateValue(T keySuffix, public Mono<U> updateValue(T keySuffix,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly, boolean existsAlmostCertainly,
Function<@Nullable U, @Nullable U> updater) { SerializationFunction<@Nullable U, @Nullable U> updater) {
return Mono return Mono
.using( .using(
() -> toKey(serializeSuffix(keySuffix)), () -> toKey(serializeSuffix(keySuffix)),
keyBuf -> dictionary keyBuf -> dictionary
.update(LLUtils.lazyRetain(keyBuf), getSerializedUpdater(updater), updateReturnMode, existsAlmostCertainly) .update(LLUtils.lazyRetain(keyBuf), getSerializedUpdater(updater), updateReturnMode, existsAlmostCertainly)
.map(valueSerializer::deserialize), .handle(this::deserializeValue),
ReferenceCounted::release ReferenceCounted::release
); );
} }
@ -172,7 +196,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
@Override @Override
public Mono<Delta<U>> updateValueAndGetDelta(T keySuffix, public Mono<Delta<U>> updateValueAndGetDelta(T keySuffix,
boolean existsAlmostCertainly, boolean existsAlmostCertainly,
Function<@Nullable U, @Nullable U> updater) { SerializationFunction<@Nullable U, @Nullable U> updater) {
return Mono return Mono
.using( .using(
() -> toKey(serializeSuffix(keySuffix)), () -> toKey(serializeSuffix(keySuffix)),
@ -183,10 +207,15 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
); );
} }
public Function<@Nullable ByteBuf, @Nullable ByteBuf> getSerializedUpdater(Function<@Nullable U, @Nullable U> updater) { public SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> getSerializedUpdater(SerializationFunction<@Nullable U, @Nullable U> updater) {
return oldSerialized -> { return oldSerialized -> {
try { try {
var result = updater.apply(oldSerialized == null ? null : valueSerializer.deserialize(oldSerialized.retain())); U result;
if (oldSerialized == null) {
result = updater.apply(null);
} else {
result = updater.apply(valueSerializer.deserialize(oldSerialized.retain()));
}
if (result == null) { if (result == null) {
return null; return null;
} else { } else {
@ -200,10 +229,16 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
}; };
} }
public <X> BiFunction<@Nullable ByteBuf, X, @Nullable ByteBuf> getSerializedUpdater(BiFunction<@Nullable U, X, @Nullable U> updater) { public <X> BiSerializationFunction<@Nullable ByteBuf, X, @Nullable ByteBuf> getSerializedUpdater(
BiSerializationFunction<@Nullable U, X, @Nullable U> updater) {
return (oldSerialized, extra) -> { return (oldSerialized, extra) -> {
try { try {
var result = updater.apply(oldSerialized == null ? null : valueSerializer.deserialize(oldSerialized.retain()), extra); U result;
if (oldSerialized == null) {
result = updater.apply(null, extra);
} else {
result = updater.apply(valueSerializer.deserialize(oldSerialized.retain()), extra);
}
if (result == null) { if (result == null) {
return null; return null;
} else { } else {
@ -231,7 +266,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
.put(LLUtils.lazyRetain(keyBuf), .put(LLUtils.lazyRetain(keyBuf),
LLUtils.lazyRetain(valueBuf), LLUtils.lazyRetain(valueBuf),
LLDictionaryResultType.PREVIOUS_VALUE) LLDictionaryResultType.PREVIOUS_VALUE)
.map(valueSerializer::deserialize), .handle(this::deserializeValue),
ReferenceCounted::release ReferenceCounted::release
), ),
ReferenceCounted::release ReferenceCounted::release
@ -255,7 +290,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
LLUtils.lazyRetain(valueBuf), LLUtils.lazyRetain(valueBuf),
LLDictionaryResultType.PREVIOUS_VALUE LLDictionaryResultType.PREVIOUS_VALUE
) )
.map(valueSerializer::deserialize) .handle(this::deserializeValue)
.map(oldValue -> !Objects.equals(oldValue, value)) .map(oldValue -> !Objects.equals(oldValue, value))
.defaultIfEmpty(value != null), .defaultIfEmpty(value != null),
ReferenceCounted::release ReferenceCounted::release
@ -286,7 +321,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
() -> toKey(serializeSuffix(keySuffix)), () -> toKey(serializeSuffix(keySuffix)),
keyBuf -> dictionary keyBuf -> dictionary
.remove(LLUtils.lazyRetain(keyBuf), LLDictionaryResultType.PREVIOUS_VALUE) .remove(LLUtils.lazyRetain(keyBuf), LLDictionaryResultType.PREVIOUS_VALUE)
.map(valueSerializer::deserialize), .handle(this::deserializeValue),
ReferenceCounted::release ReferenceCounted::release
); );
} }
@ -316,11 +351,19 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
})), existsAlmostCertainly) })), existsAlmostCertainly)
.flatMapSequential(entry -> { .flatMapSequential(entry -> {
entry.getT2().release(); entry.getT2().release();
return Mono.fromCallable(() -> Map.entry(entry.getT1(), entry.getT3().map(valueSerializer::deserialize))); return Mono.fromCallable(() -> {
Optional<U> valueOpt;
if (entry.getT3().isPresent()) {
valueOpt = Optional.of(valueSerializer.deserialize(entry.getT3().get()));
} else {
valueOpt = Optional.empty();
}
return Map.entry(entry.getT1(), valueOpt);
});
}); });
} }
private Entry<ByteBuf, ByteBuf> serializeEntry(T key, U value) { private Entry<ByteBuf, ByteBuf> serializeEntry(T key, U value) throws SerializationException {
ByteBuf serializedKey = toKey(serializeSuffix(key)); ByteBuf serializedKey = toKey(serializeSuffix(key));
try { try {
ByteBuf serializedValue = valueSerializer.serialize(value); ByteBuf serializedValue = valueSerializer.serialize(value);
@ -355,7 +398,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
@Override @Override
public <X> Flux<ExtraKeyOperationResult<T, X>> updateMulti(Flux<Tuple2<T, X>> entries, public <X> Flux<ExtraKeyOperationResult<T, X>> updateMulti(Flux<Tuple2<T, X>> entries,
BiFunction<@Nullable U, X, @Nullable U> updater) { BiSerializationFunction<@Nullable U, X, @Nullable U> updater) {
Flux<Tuple2<ByteBuf, X>> serializedEntries = entries Flux<Tuple2<ByteBuf, X>> serializedEntries = entries
.flatMap(entry -> Mono .flatMap(entry -> Mono
.fromCallable(() -> Tuples.of(serializeSuffix(entry.getT1()), entry.getT2())) .fromCallable(() -> Tuples.of(serializeSuffix(entry.getT1()), entry.getT2()))
@ -370,26 +413,34 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
}); });
var serializedUpdater = getSerializedUpdater(updater); var serializedUpdater = getSerializedUpdater(updater);
return dictionary.updateMulti(serializedEntries, serializedUpdater) return dictionary.updateMulti(serializedEntries, serializedUpdater)
.map(result -> new ExtraKeyOperationResult<>(deserializeSuffix(result.key()), .handle((result, sink) -> {
result.extra(), try {
result.changed() sink.next(new ExtraKeyOperationResult<>(deserializeSuffix(result.key()),
)); result.extra(),
result.changed()
));
} catch (SerializationException ex) {
sink.error(ex);
}
});
} }
@Override @Override
public Flux<Entry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot) { public Flux<Entry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot) {
return dictionary return dictionary
.getRangeKeys(resolveSnapshot(snapshot), rangeMono) .getRangeKeys(resolveSnapshot(snapshot), rangeMono)
.map(key -> { .handle((key, sink) -> {
ByteBuf keySuffixWithExt = stripPrefix(key.retain(), false); ByteBuf keySuffixWithExt = stripPrefix(key.retain(), false);
try { try {
try { try {
return Map.entry(deserializeSuffix(keySuffixWithExt.retainedSlice()), sink.next(Map.entry(deserializeSuffix(keySuffixWithExt.retainedSlice()),
new DatabaseSingleMapped<>(new DatabaseSingle<>(dictionary, new DatabaseSingleMapped<>(new DatabaseSingle<>(dictionary,
toKey(keySuffixWithExt.retainedSlice()), toKey(keySuffixWithExt.retainedSlice()),
Serializer.noop() Serializer.noop()
), valueSerializer) ), valueSerializer)
); ));
} catch (SerializationException ex) {
sink.error(ex);
} finally { } finally {
keySuffixWithExt.release(); keySuffixWithExt.release();
} }
@ -403,10 +454,16 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
public Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot) { public Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot) {
return dictionary return dictionary
.getRange(resolveSnapshot(snapshot), rangeMono) .getRange(resolveSnapshot(snapshot), rangeMono)
.map(serializedEntry -> Map.entry( .<Entry<T, U>>handle((serializedEntry, sink) -> {
deserializeSuffix(stripPrefix(serializedEntry.getKey(), false)), try {
valueSerializer.deserialize(serializedEntry.getValue()) sink.next(Map.entry(
)) deserializeSuffix(stripPrefix(serializedEntry.getKey(), false)),
valueSerializer.deserialize(serializedEntry.getValue())
));
} catch (SerializationException e) {
sink.error(e);
}
})
.doOnDiscard(Entry.class, uncastedEntry -> { .doOnDiscard(Entry.class, uncastedEntry -> {
if (uncastedEntry.getKey() instanceof ByteBuf byteBuf) { if (uncastedEntry.getKey() instanceof ByteBuf byteBuf) {
byteBuf.release(); byteBuf.release();
@ -419,17 +476,18 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
@Override @Override
public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) { public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) {
return Flux return Flux.usingWhen(
.usingWhen( Mono.just(true),
Mono.just(true), b -> getAllValues(null),
b -> getAllValues(null), b -> dictionary.setRange(rangeMono, entries.handle((entry, sink) -> {
b -> dictionary try {
.setRange(rangeMono, ByteBuf serializedValue = valueSerializer.serialize(entry.getValue());
entries.map(entry -> Map.entry(toKey(serializeSuffix(entry.getKey())), sink.next(Map.entry(toKey(serializeSuffix(entry.getKey())), serializedValue));
valueSerializer.serialize(entry.getValue()) } catch (SerializationException e) {
)) sink.error(e);
) }
); }))
);
} }
@Override @Override

View File

@ -13,6 +13,7 @@ import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateMode; import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.disk.LLLocalDictionary; import it.cavallium.dbengine.database.disk.LLLocalDictionary;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength; import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
@ -457,7 +458,14 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
LLUtils.lazyRetain(buffers.groupKeyWithoutExt), LLUtils.lazyRetain(buffers.groupKeyWithoutExt),
Flux.fromIterable(rangeKeys).map(ByteBuf::retain) Flux.fromIterable(rangeKeys).map(ByteBuf::retain)
) )
.map(us -> Map.entry(this.deserializeSuffix(buffers.groupSuffix.retain()), us)) .<Entry<T, US>>handle((us, sink) -> {
try {
var deserializedSuffix = this.deserializeSuffix(buffers.groupSuffix.retain());
sink.next(Map.entry(deserializedSuffix, us));
} catch (SerializationException ex) {
sink.error(ex);
}
})
), ),
buffers -> { buffers -> {
buffers.groupSuffix.release(); buffers.groupSuffix.release();
@ -494,7 +502,13 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
LLUtils.lazyRetain(groupKeyWithoutExt), LLUtils.lazyRetain(groupKeyWithoutExt),
Flux.empty() Flux.empty()
) )
.map(us -> Map.entry(this.deserializeSuffix(groupSuffix.retain()), us)), .<Entry<T, US>>handle((us, sink) -> {
try {
sink.next(Map.entry(this.deserializeSuffix(groupSuffix.retain()), us));
} catch (SerializationException ex) {
sink.error(ex);
}
}),
ReferenceCounted::release ReferenceCounted::release
) )
); );
@ -543,7 +557,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
} }
//todo: temporary wrapper. convert the whole class to buffers //todo: temporary wrapper. convert the whole class to buffers
protected T deserializeSuffix(ByteBuf keySuffix) { protected T deserializeSuffix(ByteBuf keySuffix) throws SerializationException {
try { try {
assert suffixKeyConsistency(keySuffix.readableBytes()); assert suffixKeyConsistency(keySuffix.readableBytes());
var result = keySuffixSerializer.deserialize(keySuffix.retain()); var result = keySuffixSerializer.deserialize(keySuffix.retain());
@ -555,7 +569,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
} }
//todo: temporary wrapper. convert the whole class to buffers //todo: temporary wrapper. convert the whole class to buffers
protected ByteBuf serializeSuffix(T keySuffix) { protected ByteBuf serializeSuffix(T keySuffix) throws SerializationException {
ByteBuf suffixData = keySuffixSerializer.serialize(keySuffix); ByteBuf suffixData = keySuffixSerializer.serialize(keySuffix);
assert suffixKeyConsistency(suffixData.readableBytes()); assert suffixKeyConsistency(suffixData.readableBytes());
assert keyPrefix.refCnt() > 0; assert keyPrefix.refCnt() > 0;

View File

@ -11,11 +11,14 @@ import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode; import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.database.serialization.Serializer; import it.cavallium.dbengine.database.serialization.Serializer;
import java.util.function.Function; import java.util.function.Function;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.SynchronousSink;
public class DatabaseSingle<U> implements DatabaseStageEntry<U> { public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
@ -43,11 +46,19 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
} }
} }
private void deserializeValue(ByteBuf value, SynchronousSink<U> sink) {
try {
sink.next(serializer.deserialize(value));
} catch (SerializationException ex) {
sink.error(ex);
}
}
@Override @Override
public Mono<U> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) { public Mono<U> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) {
return dictionary return dictionary
.get(resolveSnapshot(snapshot), keyMono, existsAlmostCertainly) .get(resolveSnapshot(snapshot), keyMono, existsAlmostCertainly)
.map(serializer::deserialize); .handle(this::deserializeValue);
} }
@Override @Override
@ -56,13 +67,13 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
.using(() -> serializer.serialize(value), .using(() -> serializer.serialize(value),
valueByteBuf -> dictionary valueByteBuf -> dictionary
.put(keyMono, LLUtils.lazyRetain(valueByteBuf), LLDictionaryResultType.PREVIOUS_VALUE) .put(keyMono, LLUtils.lazyRetain(valueByteBuf), LLDictionaryResultType.PREVIOUS_VALUE)
.map(serializer::deserialize), .handle(this::deserializeValue),
ReferenceCounted::release ReferenceCounted::release
); );
} }
@Override @Override
public Mono<U> update(Function<@Nullable U, @Nullable U> updater, public Mono<U> update(SerializationFunction<@Nullable U, @Nullable U> updater,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return dictionary return dictionary
@ -74,11 +85,11 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
return serializer.serialize(result); return serializer.serialize(result);
} }
}, updateReturnMode, existsAlmostCertainly) }, updateReturnMode, existsAlmostCertainly)
.map(serializer::deserialize); .handle(this::deserializeValue);
} }
@Override @Override
public Mono<Delta<U>> updateAndGetDelta(Function<@Nullable U, @Nullable U> updater, public Mono<Delta<U>> updateAndGetDelta(SerializationFunction<@Nullable U, @Nullable U> updater,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return dictionary return dictionary
.updateAndGetDelta(keyMono, (oldValueSer) -> { .updateAndGetDelta(keyMono, (oldValueSer) -> {
@ -95,7 +106,7 @@ public class DatabaseSingle<U> implements DatabaseStageEntry<U> {
public Mono<U> clearAndGetPrevious() { public Mono<U> clearAndGetPrevious() {
return dictionary return dictionary
.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE) .remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE)
.map(serializer::deserialize); .handle(this::deserializeValue);
} }
@Override @Override

View File

@ -6,6 +6,7 @@ import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.Delta; import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode; import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.unimi.dsi.fastutil.objects.ObjectArraySet; import it.unimi.dsi.fastutil.objects.ObjectArraySet;
import it.unimi.dsi.fastutil.objects.ObjectSets; import it.unimi.dsi.fastutil.objects.ObjectSets;
import java.util.HashSet; import java.util.HashSet;
@ -58,7 +59,7 @@ public class DatabaseSingleBucket<K, V, TH> implements DatabaseStageEntry<V> {
} }
@Override @Override
public Mono<V> update(Function<@Nullable V, @Nullable V> updater, public Mono<V> update(SerializationFunction<@Nullable V, @Nullable V> updater,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return bucketStage return bucketStage
@ -76,7 +77,7 @@ public class DatabaseSingleBucket<K, V, TH> implements DatabaseStageEntry<V> {
} }
@Override @Override
public Mono<Delta<V>> updateAndGetDelta(Function<@Nullable V, @Nullable V> updater, boolean existsAlmostCertainly) { public Mono<Delta<V>> updateAndGetDelta(SerializationFunction<@Nullable V, @Nullable V> updater, boolean existsAlmostCertainly) {
return bucketStage return bucketStage
.updateAndGetDelta(oldBucket -> { .updateAndGetDelta(oldBucket -> {
V oldValue = extractValue(oldBucket); V oldValue = extractValue(oldBucket);

View File

@ -1,15 +1,19 @@
package it.cavallium.dbengine.database.collections; package it.cavallium.dbengine.database.collections;
import io.netty.buffer.ByteBuf;
import it.cavallium.dbengine.client.BadBlock; import it.cavallium.dbengine.client.BadBlock;
import it.cavallium.dbengine.client.CompositeSnapshot; import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.Delta; import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode; import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.cavallium.dbengine.database.serialization.Serializer; import it.cavallium.dbengine.database.serialization.Serializer;
import java.util.function.Function; import java.util.function.Function;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.publisher.SynchronousSink;
@SuppressWarnings("unused") @SuppressWarnings("unused")
public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> { public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
@ -22,33 +26,49 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
this.serializer = serializer; this.serializer = serializer;
} }
private void deserializeSink(B value, SynchronousSink<A> sink) {
try {
sink.next(this.deserialize(value));
} catch (SerializationException ex) {
sink.error(ex);
}
}
@Override @Override
public Mono<A> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) { public Mono<A> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) {
return serializedSingle.get(snapshot, existsAlmostCertainly).map(this::deserialize); return serializedSingle.get(snapshot, existsAlmostCertainly).handle(this::deserializeSink);
} }
@Override @Override
public Mono<A> getOrDefault(@Nullable CompositeSnapshot snapshot, Mono<A> defaultValue) { public Mono<A> getOrDefault(@Nullable CompositeSnapshot snapshot, Mono<A> defaultValue) {
return serializedSingle.get(snapshot).map(this::deserialize).switchIfEmpty(defaultValue); return serializedSingle.get(snapshot).handle(this::deserializeSink).switchIfEmpty(defaultValue);
} }
@Override @Override
public Mono<Void> set(A value) { public Mono<Void> set(A value) {
return serializedSingle.set(serialize(value)); return Mono
.fromCallable(() -> serialize(value))
.flatMap(serializedSingle::set);
} }
@Override @Override
public Mono<A> setAndGetPrevious(A value) { public Mono<A> setAndGetPrevious(A value) {
return serializedSingle.setAndGetPrevious(serialize(value)).map(this::deserialize); return Mono
.fromCallable(() -> serialize(value))
.flatMap(serializedSingle::setAndGetPrevious)
.handle(this::deserializeSink);
} }
@Override @Override
public Mono<Boolean> setAndGetChanged(A value) { public Mono<Boolean> setAndGetChanged(A value) {
return serializedSingle.setAndGetChanged(serialize(value)).single(); return Mono
.fromCallable(() -> serialize(value))
.flatMap(serializedSingle::setAndGetChanged)
.single();
} }
@Override @Override
public Mono<A> update(Function<@Nullable A, @Nullable A> updater, public Mono<A> update(SerializationFunction<@Nullable A, @Nullable A> updater,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return serializedSingle.update(oldValue -> { return serializedSingle.update(oldValue -> {
@ -58,11 +78,11 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
} else { } else {
return this.serialize(result); return this.serialize(result);
} }
}, updateReturnMode, existsAlmostCertainly).map(this::deserialize); }, updateReturnMode, existsAlmostCertainly).handle(this::deserializeSink);
} }
@Override @Override
public Mono<Delta<A>> updateAndGetDelta(Function<@Nullable A, @Nullable A> updater, public Mono<Delta<A>> updateAndGetDelta(SerializationFunction<@Nullable A, @Nullable A> updater,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return serializedSingle.updateAndGetDelta(oldValue -> { return serializedSingle.updateAndGetDelta(oldValue -> {
var result = updater.apply(oldValue == null ? null : this.deserialize(oldValue)); var result = updater.apply(oldValue == null ? null : this.deserialize(oldValue));
@ -81,7 +101,7 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
@Override @Override
public Mono<A> clearAndGetPrevious() { public Mono<A> clearAndGetPrevious() {
return serializedSingle.clearAndGetPrevious().map(this::deserialize); return serializedSingle.clearAndGetPrevious().handle(this::deserializeSink);
} }
@Override @Override
@ -120,12 +140,12 @@ public class DatabaseSingleMapped<A, B> implements DatabaseStageEntry<A> {
} }
//todo: temporary wrapper. convert the whole class to buffers //todo: temporary wrapper. convert the whole class to buffers
private A deserialize(B bytes) { private A deserialize(B bytes) throws SerializationException {
return serializer.deserialize(bytes); return serializer.deserialize(bytes);
} }
//todo: temporary wrapper. convert the whole class to buffers //todo: temporary wrapper. convert the whole class to buffers
private B serialize(A bytes) { private B serialize(A bytes) throws SerializationException {
return serializer.serialize(bytes); return serializer.serialize(bytes);
} }
} }

View File

@ -5,6 +5,7 @@ import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.database.Delta; import it.cavallium.dbengine.database.Delta;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateReturnMode; import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import java.util.Objects; import java.util.Objects;
import java.util.function.Function; import java.util.function.Function;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
@ -44,7 +45,7 @@ public interface DatabaseStage<T> extends DatabaseStageWithEntry<T> {
.switchIfEmpty(Mono.fromSupplier(() -> value != null)); .switchIfEmpty(Mono.fromSupplier(() -> value != null));
} }
default Mono<T> update(Function<@Nullable T, @Nullable T> updater, default Mono<T> update(SerializationFunction<@Nullable T, @Nullable T> updater,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return this return this
@ -52,14 +53,14 @@ public interface DatabaseStage<T> extends DatabaseStageWithEntry<T> {
.transform(prev -> LLUtils.resolveDelta(prev, updateReturnMode)); .transform(prev -> LLUtils.resolveDelta(prev, updateReturnMode));
} }
default Mono<T> update(Function<@Nullable T, @Nullable T> updater, UpdateReturnMode updateReturnMode) { default Mono<T> update(SerializationFunction<@Nullable T, @Nullable T> updater, UpdateReturnMode updateReturnMode) {
return update(updater, updateReturnMode, false); return update(updater, updateReturnMode, false);
} }
Mono<Delta<T>> updateAndGetDelta(Function<@Nullable T, @Nullable T> updater, Mono<Delta<T>> updateAndGetDelta(SerializationFunction<@Nullable T, @Nullable T> updater,
boolean existsAlmostCertainly); boolean existsAlmostCertainly);
default Mono<Delta<T>> updateAndGetDelta(Function<@Nullable T, @Nullable T> updater) { default Mono<Delta<T>> updateAndGetDelta(SerializationFunction<@Nullable T, @Nullable T> updater) {
return updateAndGetDelta(updater, false); return updateAndGetDelta(updater, false);
} }

View File

@ -7,6 +7,9 @@ import it.cavallium.dbengine.database.KeyOperationResult;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateMode; import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode; import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.BiSerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
@ -58,7 +61,7 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
default Mono<U> updateValue(T key, default Mono<U> updateValue(T key,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly, boolean existsAlmostCertainly,
Function<@Nullable U, @Nullable U> updater) { SerializationFunction<@Nullable U, @Nullable U> updater) {
return Mono.usingWhen( return Mono.usingWhen(
this.at(null, key).single(), this.at(null, key).single(),
stage -> stage.update(updater, updateReturnMode, existsAlmostCertainly), stage -> stage.update(updater, updateReturnMode, existsAlmostCertainly),
@ -66,7 +69,8 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
); );
} }
default <X> Flux<ExtraKeyOperationResult<T, X>> updateMulti(Flux<Tuple2<T, X>> entries, BiFunction<@Nullable U, X, @Nullable U> updater) { default <X> Flux<ExtraKeyOperationResult<T, X>> updateMulti(Flux<Tuple2<T, X>> entries,
BiSerializationFunction<@Nullable U, X, @Nullable U> updater) {
return entries return entries
.flatMapSequential(entry -> this .flatMapSequential(entry -> this
.updateValue(entry.getT1(), prevValue -> updater.apply(prevValue, entry.getT2())) .updateValue(entry.getT1(), prevValue -> updater.apply(prevValue, entry.getT2()))
@ -74,21 +78,21 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
); );
} }
default Mono<U> updateValue(T key, UpdateReturnMode updateReturnMode, Function<@Nullable U, @Nullable U> updater) { default Mono<U> updateValue(T key, UpdateReturnMode updateReturnMode, SerializationFunction<@Nullable U, @Nullable U> updater) {
return updateValue(key, updateReturnMode, false, updater); return updateValue(key, updateReturnMode, false, updater);
} }
default Mono<Boolean> updateValue(T key, Function<@Nullable U, @Nullable U> updater) { default Mono<Boolean> updateValue(T key, SerializationFunction<@Nullable U, @Nullable U> updater) {
return updateValueAndGetDelta(key, false, updater).map(LLUtils::isDeltaChanged).single(); return updateValueAndGetDelta(key, false, updater).map(LLUtils::isDeltaChanged).single();
} }
default Mono<Boolean> updateValue(T key, boolean existsAlmostCertainly, Function<@Nullable U, @Nullable U> updater) { default Mono<Boolean> updateValue(T key, boolean existsAlmostCertainly, SerializationFunction<@Nullable U, @Nullable U> updater) {
return updateValueAndGetDelta(key, existsAlmostCertainly, updater).map(LLUtils::isDeltaChanged).single(); return updateValueAndGetDelta(key, existsAlmostCertainly, updater).map(LLUtils::isDeltaChanged).single();
} }
default Mono<Delta<U>> updateValueAndGetDelta(T key, default Mono<Delta<U>> updateValueAndGetDelta(T key,
boolean existsAlmostCertainly, boolean existsAlmostCertainly,
Function<@Nullable U, @Nullable U> updater) { SerializationFunction<@Nullable U, @Nullable U> updater) {
return Mono.usingWhen( return Mono.usingWhen(
this.at(null, key).single(), this.at(null, key).single(),
stage -> stage.updateAndGetDelta(updater, existsAlmostCertainly), stage -> stage.updateAndGetDelta(updater, existsAlmostCertainly),
@ -96,7 +100,7 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
); );
} }
default Mono<Delta<U>> updateValueAndGetDelta(T key, Function<@Nullable U, @Nullable U> updater) { default Mono<Delta<U>> updateValueAndGetDelta(T key, SerializationFunction<@Nullable U, @Nullable U> updater) {
return updateValueAndGetDelta(key, false, updater); return updateValueAndGetDelta(key, false, updater);
} }
@ -221,7 +225,7 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
} }
@Override @Override
default Mono<Delta<Map<T, U>>> updateAndGetDelta(Function<@Nullable Map<T, U>, @Nullable Map<T, U>> updater, default Mono<Delta<Map<T, U>>> updateAndGetDelta(SerializationFunction<@Nullable Map<T, U>, @Nullable Map<T, U>> updater,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return this return this
.getUpdateMode() .getUpdateMode()
@ -236,11 +240,15 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
if (v.isEmpty()) { if (v.isEmpty()) {
v = null; v = null;
} }
var result = updater.apply(v); try {
if (result != null && result.isEmpty()) { var result = updater.apply(v);
result = null; if (result != null && result.isEmpty()) {
result = null;
}
sink.next(Tuples.of(Optional.ofNullable(v), Optional.ofNullable(result)));
} catch (SerializationException ex) {
sink.error(ex);
} }
sink.next(Tuples.of(Optional.ofNullable(v), Optional.ofNullable(result)));
}) })
.flatMap(result -> Mono .flatMap(result -> Mono
.justOrEmpty(result.getT2()) .justOrEmpty(result.getT2())

View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine.database.collections;
import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.ByteBufAllocator;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer; import it.cavallium.dbengine.database.serialization.Serializer;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
@ -23,7 +24,7 @@ class ValueWithHashSerializer<X, Y> implements Serializer<Entry<X, Y>, ByteBuf>
} }
@Override @Override
public @NotNull Entry<X, Y> deserialize(@NotNull ByteBuf serialized) { public @NotNull Entry<X, Y> deserialize(@NotNull ByteBuf serialized) throws SerializationException {
try { try {
X deserializedKey = keySuffixSerializer.deserialize(serialized.retain()); X deserializedKey = keySuffixSerializer.deserialize(serialized.retain());
Y deserializedValue = valueSerializer.deserialize(serialized.retain()); Y deserializedValue = valueSerializer.deserialize(serialized.retain());
@ -34,7 +35,7 @@ class ValueWithHashSerializer<X, Y> implements Serializer<Entry<X, Y>, ByteBuf>
} }
@Override @Override
public @NotNull ByteBuf serialize(@NotNull Entry<X, Y> deserialized) { public @NotNull ByteBuf serialize(@NotNull Entry<X, Y> deserialized) throws SerializationException {
ByteBuf keySuffix = keySuffixSerializer.serialize(deserialized.getKey()); ByteBuf keySuffix = keySuffixSerializer.serialize(deserialized.getKey());
try { try {
ByteBuf value = valueSerializer.serialize(deserialized.getValue()); ByteBuf value = valueSerializer.serialize(deserialized.getValue());

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.collections;
import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.ByteBufAllocator;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer; import it.cavallium.dbengine.database.serialization.Serializer;
import it.unimi.dsi.fastutil.objects.ObjectArraySet; import it.unimi.dsi.fastutil.objects.ObjectArraySet;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
@ -23,7 +24,7 @@ class ValuesSetSerializer<X> implements Serializer<ObjectArraySet<X>, ByteBuf> {
} }
@Override @Override
public @NotNull ObjectArraySet<X> deserialize(@NotNull ByteBuf serialized) { public @NotNull ObjectArraySet<X> deserialize(@NotNull ByteBuf serialized) throws SerializationException {
try { try {
int entriesLength = serialized.readInt(); int entriesLength = serialized.readInt();
ArrayList<X> deserializedElements = new ArrayList<>(entriesLength); ArrayList<X> deserializedElements = new ArrayList<>(entriesLength);
@ -38,18 +39,18 @@ class ValuesSetSerializer<X> implements Serializer<ObjectArraySet<X>, ByteBuf> {
} }
@Override @Override
public @NotNull ByteBuf serialize(@NotNull ObjectArraySet<X> deserialized) { public @NotNull ByteBuf serialize(@NotNull ObjectArraySet<X> deserialized) throws SerializationException {
ByteBuf output = allocator.buffer(); ByteBuf output = allocator.buffer();
try { try {
output.writeInt(deserialized.size()); output.writeInt(deserialized.size());
deserialized.forEach((entry) -> { for (X entry : deserialized) {
ByteBuf serialized = entrySerializer.serialize(entry); ByteBuf serialized = entrySerializer.serialize(entry);
try { try {
output.writeBytes(serialized); output.writeBytes(serialized);
} finally { } finally {
serialized.release(); serialized.release();
} }
}); }
return output.retain(); return output.retain();
} finally { } finally {
output.release(); output.release();

View File

@ -18,6 +18,8 @@ import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.RepeatedElementList; import it.cavallium.dbengine.database.RepeatedElementList;
import it.cavallium.dbengine.database.UpdateMode; import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode; import it.cavallium.dbengine.database.UpdateReturnMode;
import it.cavallium.dbengine.database.serialization.BiSerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntArrayList;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
@ -567,7 +569,7 @@ public class LLLocalDictionary implements LLDictionary {
@SuppressWarnings("DuplicatedCode") @SuppressWarnings("DuplicatedCode")
@Override @Override
public Mono<ByteBuf> update(Mono<ByteBuf> keyMono, public Mono<ByteBuf> update(Mono<ByteBuf> keyMono,
Function<@Nullable ByteBuf, @Nullable ByteBuf> updater, SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> updater,
UpdateReturnMode updateReturnMode, UpdateReturnMode updateReturnMode,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return Mono.usingWhen(keyMono, return Mono.usingWhen(keyMono,
@ -700,7 +702,7 @@ public class LLLocalDictionary implements LLDictionary {
@SuppressWarnings("DuplicatedCode") @SuppressWarnings("DuplicatedCode")
@Override @Override
public Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> keyMono, public Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> keyMono,
Function<@Nullable ByteBuf, @Nullable ByteBuf> updater, SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> updater,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return Mono.usingWhen(keyMono, return Mono.usingWhen(keyMono,
key -> this.runOnDb(() -> { key -> this.runOnDb(() -> {
@ -1111,7 +1113,7 @@ public class LLLocalDictionary implements LLDictionary {
@Override @Override
public <X> Flux<ExtraKeyOperationResult<ByteBuf, X>> updateMulti(Flux<Tuple2<ByteBuf, X>> entries, public <X> Flux<ExtraKeyOperationResult<ByteBuf, X>> updateMulti(Flux<Tuple2<ByteBuf, X>> entries,
BiFunction<ByteBuf, X, ByteBuf> updateFunction) { BiSerializationFunction<ByteBuf, X, ByteBuf> updateFunction) {
return entries return entries
.buffer(Math.min(MULTI_GET_WINDOW, CAPPED_WRITE_BATCH_CAP)) .buffer(Math.min(MULTI_GET_WINDOW, CAPPED_WRITE_BATCH_CAP))
.flatMapSequential(ew -> Flux .flatMapSequential(ew -> Flux

View File

@ -11,6 +11,9 @@ import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateMode; import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.serialization.BiSerializationFunction;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.SerializationFunction;
import it.unimi.dsi.fastutil.bytes.ByteList; import it.unimi.dsi.fastutil.bytes.ByteList;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -165,7 +168,7 @@ public class LLMemoryDictionary implements LLDictionary {
@Override @Override
public Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> keyMono, public Mono<Delta<ByteBuf>> updateAndGetDelta(Mono<ByteBuf> keyMono,
Function<@Nullable ByteBuf, @Nullable ByteBuf> updater, SerializationFunction<@Nullable ByteBuf, @Nullable ByteBuf> updater,
boolean existsAlmostCertainly) { boolean existsAlmostCertainly) {
return Mono.usingWhen(keyMono, return Mono.usingWhen(keyMono,
key -> Mono.fromCallable(() -> { key -> Mono.fromCallable(() -> {
@ -174,7 +177,12 @@ public class LLMemoryDictionary implements LLDictionary {
if (old != null) { if (old != null) {
oldRef.set(kk(old)); oldRef.set(kk(old));
} }
var v = updater.apply(old != null ? kk(old) : null); ByteBuf v = null;
try {
v = updater.apply(old != null ? kk(old) : null);
} catch (SerializationException e) {
throw new IllegalStateException(e);
}
try { try {
return k(v); return k(v);
} finally { } finally {
@ -258,7 +266,7 @@ public class LLMemoryDictionary implements LLDictionary {
@Override @Override
public <X> Flux<ExtraKeyOperationResult<ByteBuf, X>> updateMulti(Flux<Tuple2<ByteBuf, X>> entries, public <X> Flux<ExtraKeyOperationResult<ByteBuf, X>> updateMulti(Flux<Tuple2<ByteBuf, X>> entries,
BiFunction<ByteBuf, X, ByteBuf> updateFunction) { BiSerializationFunction<ByteBuf, X, ByteBuf> updateFunction) {
return Flux.error(new UnsupportedOperationException("Not implemented")); return Flux.error(new UnsupportedOperationException("Not implemented"));
} }

View File

@ -0,0 +1,7 @@
package it.cavallium.dbengine.database.serialization;
@FunctionalInterface
public interface BiSerializationFunction<T1, T2, U> {
U apply(T1 argument1, T2 argument2) throws SerializationException;
}

View File

@ -0,0 +1,24 @@
package it.cavallium.dbengine.database.serialization;
import java.io.IOException;
import java.io.UncheckedIOException;
public class SerializationException extends Exception {
public SerializationException() {
super();
}
public SerializationException(String message) {
super(message);
}
public SerializationException(String message, Throwable cause) {
super(message, cause);
}
public SerializationException(Throwable cause) {
super(cause);
}
}

View File

@ -0,0 +1,9 @@
package it.cavallium.dbengine.database.serialization;
import org.jetbrains.annotations.NotNull;
@FunctionalInterface
public interface SerializationFunction<T, U> {
U apply(T argument) throws SerializationException;
}

View File

@ -9,9 +9,9 @@ import org.jetbrains.annotations.NotNull;
public interface Serializer<A, B> { public interface Serializer<A, B> {
@NotNull A deserialize(@NotNull B serialized); @NotNull A deserialize(@NotNull B serialized) throws SerializationException;
@NotNull B serialize(@NotNull A deserialized); @NotNull B serialize(@NotNull A deserialized) throws SerializationException;
Serializer<ByteBuf, ByteBuf> NOOP_SERIALIZER = new Serializer<>() { Serializer<ByteBuf, ByteBuf> NOOP_SERIALIZER = new Serializer<>() {
@Override @Override

View File

@ -8,7 +8,6 @@ import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.PooledByteBufAllocator; import io.netty.buffer.PooledByteBufAllocator;
import java.io.NotSerializableException; import java.io.NotSerializableException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import org.apache.commons.lang3.SerializationException;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
@SuppressWarnings("unused") @SuppressWarnings("unused")
@ -53,7 +52,7 @@ public interface SerializerFixedBinaryLength<A, B> extends Serializer<A, B> {
static SerializerFixedBinaryLength<String, ByteBuf> utf8(ByteBufAllocator allocator, int length) { static SerializerFixedBinaryLength<String, ByteBuf> utf8(ByteBufAllocator allocator, int length) {
return new SerializerFixedBinaryLength<>() { return new SerializerFixedBinaryLength<>() {
@Override @Override
public @NotNull String deserialize(@NotNull ByteBuf serialized) { public @NotNull String deserialize(@NotNull ByteBuf serialized) throws SerializationException {
try { try {
if (serialized.readableBytes() != getSerializedBinaryLength()) { if (serialized.readableBytes() != getSerializedBinaryLength()) {
throw new SerializationException( throw new SerializationException(
@ -69,7 +68,7 @@ public interface SerializerFixedBinaryLength<A, B> extends Serializer<A, B> {
} }
@Override @Override
public @NotNull ByteBuf serialize(@NotNull String deserialized) { public @NotNull ByteBuf serialize(@NotNull String deserialized) throws SerializationException {
// UTF-8 uses max. 3 bytes per char, so calculate the worst case. // UTF-8 uses max. 3 bytes per char, so calculate the worst case.
ByteBuf buf = allocator.buffer(ByteBufUtil.utf8MaxBytes(deserialized)); ByteBuf buf = allocator.buffer(ByteBufUtil.utf8MaxBytes(deserialized));
try { try {