2021-01-31 21:23:43 +01:00
|
|
|
package it.cavallium.dbengine.database.collections;
|
|
|
|
|
2023-05-24 01:44:06 +02:00
|
|
|
import static it.cavallium.dbengine.utils.StreamUtils.resourceStream;
|
|
|
|
|
2023-03-06 12:19:08 +01:00
|
|
|
import it.cavallium.buffer.Buf;
|
|
|
|
import it.cavallium.buffer.BufDataInput;
|
|
|
|
import it.cavallium.buffer.BufDataOutput;
|
2023-10-10 00:39:41 +02:00
|
|
|
import it.cavallium.dbengine.client.DbProgress;
|
2021-01-31 21:23:43 +01:00
|
|
|
import it.cavallium.dbengine.client.CompositeSnapshot;
|
2023-10-10 00:39:41 +02:00
|
|
|
import it.cavallium.dbengine.client.SSTVerificationProgress;
|
2021-01-31 21:23:43 +01:00
|
|
|
import it.cavallium.dbengine.database.LLDictionary;
|
2021-03-14 03:13:19 +01:00
|
|
|
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
2021-01-31 21:23:43 +01:00
|
|
|
import it.cavallium.dbengine.database.LLRange;
|
|
|
|
import it.cavallium.dbengine.database.LLSnapshot;
|
2021-04-30 19:15:04 +02:00
|
|
|
import it.cavallium.dbengine.database.LLUtils;
|
2022-05-20 23:59:56 +02:00
|
|
|
import it.cavallium.dbengine.database.SubStageEntry;
|
2021-05-02 19:18:15 +02:00
|
|
|
import it.cavallium.dbengine.database.UpdateMode;
|
2022-03-20 14:33:27 +01:00
|
|
|
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
|
2021-08-22 21:23:22 +02:00
|
|
|
import it.cavallium.dbengine.database.serialization.SerializationException;
|
2022-03-20 14:33:27 +01:00
|
|
|
import it.cavallium.dbengine.database.serialization.Serializer;
|
2021-02-02 19:40:37 +01:00
|
|
|
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
2021-12-18 18:16:56 +01:00
|
|
|
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
2021-01-31 21:23:43 +01:00
|
|
|
import java.util.Map.Entry;
|
2022-03-20 14:33:27 +01:00
|
|
|
import java.util.Optional;
|
|
|
|
import java.util.Set;
|
|
|
|
import java.util.concurrent.CompletionException;
|
2024-04-18 14:48:16 +02:00
|
|
|
import java.util.concurrent.ForkJoinPool;
|
2022-03-12 02:55:18 +01:00
|
|
|
import java.util.concurrent.atomic.AtomicLong;
|
2023-02-09 23:34:25 +01:00
|
|
|
import java.util.stream.Stream;
|
2022-03-12 02:55:18 +01:00
|
|
|
import org.apache.commons.lang3.function.TriFunction;
|
2021-12-17 01:48:49 +01:00
|
|
|
import org.apache.logging.log4j.LogManager;
|
|
|
|
import org.apache.logging.log4j.Logger;
|
2021-09-23 02:15:58 +02:00
|
|
|
import org.jetbrains.annotations.NotNull;
|
2021-01-31 21:23:43 +01:00
|
|
|
import org.jetbrains.annotations.Nullable;
|
2023-02-22 22:31:36 +01:00
|
|
|
import org.jetbrains.annotations.VisibleForTesting;
|
2021-05-02 19:18:15 +02:00
|
|
|
|
2021-08-22 18:20:05 +02:00
|
|
|
// todo: implement optimized methods (which?)
|
2023-02-09 23:34:25 +01:00
|
|
|
public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implements DatabaseStageMap<T, U, US> {
|
2021-01-31 21:23:43 +01:00
|
|
|
|
2022-03-11 17:59:46 +01:00
|
|
|
private static final Logger LOG = LogManager.getLogger(DatabaseMapDictionaryDeep.class);
|
2021-10-01 19:17:33 +02:00
|
|
|
|
2021-01-31 21:23:43 +01:00
|
|
|
protected final LLDictionary dictionary;
|
2022-03-12 02:55:18 +01:00
|
|
|
private final AtomicLong totalZeroBytesErrors = new AtomicLong();
|
2021-01-31 21:23:43 +01:00
|
|
|
protected final SubStageGetter<U, US> subStageGetter;
|
2021-09-02 17:15:40 +02:00
|
|
|
protected final SerializerFixedBinaryLength<T> keySuffixSerializer;
|
2021-04-30 19:15:04 +02:00
|
|
|
protected final int keyPrefixLength;
|
2021-01-31 21:23:43 +01:00
|
|
|
protected final int keySuffixLength;
|
|
|
|
protected final int keyExtLength;
|
2023-02-09 23:34:25 +01:00
|
|
|
protected final LLRange range;
|
2021-09-23 20:57:28 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
protected Buf keyPrefix;
|
2021-01-31 21:23:43 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private static void incrementPrefix(Buf modifiablePrefix, int prefixLength) {
|
|
|
|
assert modifiablePrefix.size() >= prefixLength;
|
|
|
|
final var originalKeyLength = modifiablePrefix.size();
|
2021-09-22 18:33:28 +02:00
|
|
|
boolean overflowed = true;
|
|
|
|
final int ff = 0xFF;
|
|
|
|
int writtenBytes = 0;
|
|
|
|
for (int i = prefixLength - 1; i >= 0; i--) {
|
2023-02-09 23:34:25 +01:00
|
|
|
int iByte = Byte.toUnsignedInt(modifiablePrefix.getByte(i));
|
2021-09-22 18:33:28 +02:00
|
|
|
if (iByte != ff) {
|
2023-02-09 23:34:25 +01:00
|
|
|
modifiablePrefix.set(i, (byte) (iByte + 1));
|
2021-09-22 18:33:28 +02:00
|
|
|
writtenBytes++;
|
|
|
|
overflowed = false;
|
|
|
|
break;
|
|
|
|
} else {
|
2023-02-09 23:34:25 +01:00
|
|
|
modifiablePrefix.set(i, (byte) 0x00);
|
2021-09-22 18:33:28 +02:00
|
|
|
writtenBytes++;
|
2021-03-13 19:01:36 +01:00
|
|
|
}
|
|
|
|
}
|
2021-09-22 18:33:28 +02:00
|
|
|
assert prefixLength - writtenBytes >= 0;
|
|
|
|
|
|
|
|
if (overflowed) {
|
2023-02-09 23:34:25 +01:00
|
|
|
modifiablePrefix.add((byte) 0);
|
2021-09-22 18:33:28 +02:00
|
|
|
for (int i = 0; i < originalKeyLength; i++) {
|
2023-02-09 23:34:25 +01:00
|
|
|
modifiablePrefix.set(i, (byte) 0xFF);
|
2021-09-22 18:33:28 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
modifiablePrefix.set(originalKeyLength, (byte) 0x00);
|
2021-09-22 18:33:28 +02:00
|
|
|
}
|
2021-03-13 19:01:36 +01:00
|
|
|
}
|
|
|
|
|
2023-02-22 22:31:36 +01:00
|
|
|
@VisibleForTesting
|
|
|
|
public static Buf firstRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
|
|
|
return createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
|
|
|
|
}
|
|
|
|
|
|
|
|
@VisibleForTesting
|
|
|
|
public static Buf nextRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
|
|
|
Buf modifiablePrefixKey = createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
|
|
|
|
incrementPrefix(modifiablePrefixKey, prefixLength);
|
2023-02-09 23:34:25 +01:00
|
|
|
return modifiablePrefixKey;
|
2022-03-16 22:41:51 +01:00
|
|
|
}
|
|
|
|
|
2023-02-22 22:31:36 +01:00
|
|
|
private static Buf createFullKeyWithEmptySuffixAndExt(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
2023-02-09 23:34:25 +01:00
|
|
|
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
|
2023-02-22 22:31:36 +01:00
|
|
|
if (prefixKey != null) {
|
|
|
|
modifiablePrefixKey.addAll(prefixKey);
|
|
|
|
}
|
|
|
|
assert prefixKey != null || prefixLength == 0 : "Prefix length is " + prefixLength + " but the prefix key is null";
|
2023-02-09 23:34:25 +01:00
|
|
|
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
|
|
|
|
return modifiablePrefixKey;
|
2022-03-16 22:41:51 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
/**
|
|
|
|
* @param modifiablePrefixKey This field content will be modified
|
|
|
|
*/
|
|
|
|
protected static void zeroFillKeySuffixAndExt(@NotNull Buf modifiablePrefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
2021-11-08 16:33:41 +01:00
|
|
|
//noinspection UnnecessaryLocalVariable
|
2023-02-09 23:34:25 +01:00
|
|
|
var result = modifiablePrefixKey;
|
|
|
|
var suffixLengthAndExtLength = suffixAndExtZeroes.size();
|
|
|
|
assert result.size() == prefixLength;
|
2022-03-16 22:41:51 +01:00
|
|
|
assert suffixLengthAndExtLength > 0 : "Suffix length + ext length is < 0: " + suffixLengthAndExtLength;
|
2023-02-22 22:31:36 +01:00
|
|
|
result.size(prefixLength);
|
2023-02-09 23:34:25 +01:00
|
|
|
modifiablePrefixKey.addAll(suffixAndExtZeroes);
|
|
|
|
assert modifiablePrefixKey.size() == prefixLength + suffixAndExtZeroes.size() : "Result buffer size is wrong";
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Use DatabaseMapDictionaryRange.simple instead
|
|
|
|
*/
|
|
|
|
@Deprecated
|
2021-02-02 19:40:37 +01:00
|
|
|
public static <T, U> DatabaseMapDictionaryDeep<T, U, DatabaseStageEntry<U>> simple(LLDictionary dictionary,
|
2022-07-19 23:45:39 +02:00
|
|
|
SerializerFixedBinaryLength<T> keySerializer, SubStageGetterSingle<U> subStageGetter) {
|
|
|
|
return new DatabaseMapDictionaryDeep<>(dictionary, null, keySerializer, subStageGetter, 0);
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
2021-09-23 20:57:28 +02:00
|
|
|
public static <T, U, US extends DatabaseStage<U>> DatabaseMapDictionaryDeep<T, U, US> deepTail(
|
|
|
|
LLDictionary dictionary, SerializerFixedBinaryLength<T> keySerializer, int keyExtLength,
|
2022-07-19 23:45:39 +02:00
|
|
|
SubStageGetter<U, US> subStageGetter) {
|
|
|
|
return new DatabaseMapDictionaryDeep<>(dictionary, null, keySerializer, subStageGetter, keyExtLength);
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
2021-09-23 20:57:28 +02:00
|
|
|
public static <T, U, US extends DatabaseStage<U>> DatabaseMapDictionaryDeep<T, U, US> deepIntermediate(
|
2023-02-09 23:34:25 +01:00
|
|
|
LLDictionary dictionary, Buf prefixKey, SerializerFixedBinaryLength<T> keySuffixSerializer,
|
2022-07-19 23:45:39 +02:00
|
|
|
SubStageGetter<U, US> subStageGetter, int keyExtLength) {
|
|
|
|
return new DatabaseMapDictionaryDeep<>(dictionary, prefixKey, keySuffixSerializer, subStageGetter, keyExtLength);
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
protected DatabaseMapDictionaryDeep(LLDictionary dictionary, @Nullable Buf prefixKey,
|
2022-07-19 23:45:39 +02:00
|
|
|
SerializerFixedBinaryLength<T> keySuffixSerializer, SubStageGetter<U, US> subStageGetter, int keyExtLength) {
|
2023-02-09 23:34:25 +01:00
|
|
|
this.dictionary = dictionary;
|
|
|
|
this.subStageGetter = subStageGetter;
|
|
|
|
this.keySuffixSerializer = keySuffixSerializer;
|
|
|
|
this.keyPrefixLength = prefixKey != null ? prefixKey.size() : 0;
|
|
|
|
this.keySuffixLength = keySuffixSerializer.getSerializedBinaryLength();
|
|
|
|
this.keyExtLength = keyExtLength;
|
|
|
|
var keySuffixAndExtZeroBuffer = Buf.createZeroes(keySuffixLength + keyExtLength);
|
|
|
|
assert keySuffixAndExtZeroBuffer.size() == keySuffixLength + keyExtLength :
|
|
|
|
"Key suffix and ext zero buffer readable length is not equal"
|
|
|
|
+ " to the key suffix length + key ext length. keySuffixAndExtZeroBuffer="
|
|
|
|
+ keySuffixAndExtZeroBuffer.size() + " keySuffixLength=" + keySuffixLength + " keyExtLength="
|
|
|
|
+ keyExtLength;
|
|
|
|
assert keySuffixAndExtZeroBuffer.size() > 0;
|
|
|
|
var firstKey = firstRangeKey(prefixKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
|
|
|
|
var nextRangeKey = nextRangeKey(prefixKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
|
|
|
|
assert keyPrefixLength == 0 || !LLUtils.equals(firstKey, nextRangeKey);
|
|
|
|
if (keyPrefixLength == 0) {
|
|
|
|
this.range = LLRange.all();
|
|
|
|
} else {
|
|
|
|
this.range = LLRange.of(firstKey, nextRangeKey);
|
2021-04-30 19:15:04 +02:00
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
assert subStageKeysConsistency(keyPrefixLength + keySuffixLength + keyExtLength);
|
2021-01-31 21:23:43 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
this.keyPrefix = prefixKey;
|
|
|
|
}
|
2021-09-23 20:57:28 +02:00
|
|
|
private DatabaseMapDictionaryDeep(LLDictionary dictionary,
|
|
|
|
SubStageGetter<U, US> subStageGetter,
|
|
|
|
SerializerFixedBinaryLength<T> keySuffixSerializer,
|
|
|
|
int keyPrefixLength,
|
|
|
|
int keySuffixLength,
|
|
|
|
int keyExtLength,
|
2023-02-09 23:34:25 +01:00
|
|
|
LLRange range,
|
|
|
|
Buf keyPrefix) {
|
2021-09-23 20:57:28 +02:00
|
|
|
this.dictionary = dictionary;
|
|
|
|
this.subStageGetter = subStageGetter;
|
|
|
|
this.keySuffixSerializer = keySuffixSerializer;
|
|
|
|
this.keyPrefixLength = keyPrefixLength;
|
|
|
|
this.keySuffixLength = keySuffixLength;
|
|
|
|
this.keyExtLength = keyExtLength;
|
2023-02-09 23:34:25 +01:00
|
|
|
this.range = range;
|
2021-09-23 20:57:28 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
this.keyPrefix = keyPrefix;
|
2021-09-23 20:57:28 +02:00
|
|
|
}
|
|
|
|
|
2021-01-31 21:23:43 +01:00
|
|
|
@SuppressWarnings("unused")
|
2021-10-19 00:22:05 +02:00
|
|
|
protected boolean suffixKeyLengthConsistency(int keySuffixLength) {
|
2023-02-22 22:31:36 +01:00
|
|
|
assert
|
|
|
|
this.keySuffixLength == keySuffixLength :
|
|
|
|
"Key suffix length is " + keySuffixLength + ", but it should be " + this.keySuffixLength + " bytes long";
|
|
|
|
//noinspection ConstantValue
|
2021-01-31 21:23:43 +01:00
|
|
|
return this.keySuffixLength == keySuffixLength;
|
|
|
|
}
|
|
|
|
|
|
|
|
@SuppressWarnings("unused")
|
|
|
|
protected boolean extKeyConsistency(int keyExtLength) {
|
|
|
|
return this.keyExtLength == keyExtLength;
|
|
|
|
}
|
|
|
|
|
|
|
|
@SuppressWarnings("unused")
|
|
|
|
protected boolean suffixAndExtKeyConsistency(int keySuffixAndExtLength) {
|
|
|
|
return this.keySuffixLength + this.keyExtLength == keySuffixAndExtLength;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2021-10-19 00:22:05 +02:00
|
|
|
* @return the prefix
|
2021-01-31 21:23:43 +01:00
|
|
|
*/
|
2023-02-09 23:34:25 +01:00
|
|
|
protected Buf prefixSubList(Buf key) {
|
|
|
|
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
|
|
|
|| key.size() == keyPrefixLength + keySuffixLength;
|
|
|
|
return key.subList(0, this.keyPrefixLength);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @return the suffix
|
|
|
|
*/
|
|
|
|
protected Buf suffixSubList(Buf key) {
|
|
|
|
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
|
|
|
|| key.size() == keyPrefixLength + keySuffixLength;
|
|
|
|
return key.subList(this.keyPrefixLength, keyPrefixLength + keySuffixLength);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @return the suffix
|
|
|
|
*/
|
|
|
|
protected Buf suffixAndExtSubList(Buf key) {
|
|
|
|
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
|
|
|
|| key.size() == keyPrefixLength + keySuffixLength;
|
|
|
|
return key.subList(this.keyPrefixLength, key.size());
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @return the ext
|
|
|
|
*/
|
|
|
|
protected Buf extSubList(Buf key) {
|
|
|
|
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
|
|
|
|| key.size() == keyPrefixLength + keySuffixLength;
|
|
|
|
return key.subList(this.keyPrefixLength + this.keySuffixLength, key.size());
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
protected LLSnapshot resolveSnapshot(@Nullable CompositeSnapshot snapshot) {
|
|
|
|
if (snapshot == null) {
|
|
|
|
return null;
|
|
|
|
} else {
|
|
|
|
return snapshot.getSnapshot(dictionary);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-24 16:43:07 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
|
|
|
return dictionary.sizeRange(resolveSnapshot(snapshot), range, fast);
|
2021-02-24 16:43:07 +01:00
|
|
|
}
|
|
|
|
|
2021-03-14 03:13:19 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
|
|
|
|
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), range, false);
|
2021-03-14 03:13:19 +01:00
|
|
|
}
|
|
|
|
|
2021-01-31 21:23:43 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public @NotNull US at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
|
|
|
|
BufDataOutput bufOutput = BufDataOutput.createLimited(keyPrefixLength + keySuffixLength + keyExtLength);
|
|
|
|
if (keyPrefix != null) {
|
|
|
|
bufOutput.writeBytes(keyPrefix);
|
|
|
|
}
|
|
|
|
serializeSuffixTo(keySuffix, bufOutput);
|
|
|
|
return this.subStageGetter.subStage(dictionary, snapshot, bufOutput.asList());
|
2021-05-02 19:18:15 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-09-12 20:14:56 +02:00
|
|
|
public UpdateMode getUpdateMode() {
|
2021-05-02 19:18:15 +02:00
|
|
|
return dictionary.getUpdateMode();
|
|
|
|
}
|
|
|
|
|
2021-06-26 02:35:33 +02:00
|
|
|
@Override
|
2023-10-10 00:39:41 +02:00
|
|
|
public Stream<DbProgress<SSTVerificationProgress>> verifyChecksum() {
|
2023-09-26 01:18:24 +02:00
|
|
|
return dictionary.verifyChecksum(range);
|
2021-06-26 02:35:33 +02:00
|
|
|
}
|
|
|
|
|
2021-01-31 21:23:43 +01:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public Stream<SubStageEntry<T, US>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
2021-09-02 21:14:26 +02:00
|
|
|
return dictionary
|
2023-02-09 23:34:25 +01:00
|
|
|
.getRangeKeyPrefixes(resolveSnapshot(snapshot), range, keyPrefixLength + keySuffixLength, smallRange)
|
|
|
|
.map(groupKeyWithoutExt -> {
|
|
|
|
T deserializedSuffix;
|
|
|
|
var splittedGroupSuffix = suffixSubList(groupKeyWithoutExt);
|
|
|
|
deserializedSuffix = this.deserializeSuffix(BufDataInput.create(splittedGroupSuffix));
|
|
|
|
return new SubStageEntry<>(deserializedSuffix,
|
|
|
|
this.subStageGetter.subStage(dictionary, snapshot, groupKeyWithoutExt));
|
|
|
|
});
|
2021-09-02 21:14:26 +02:00
|
|
|
}
|
|
|
|
|
2021-02-02 15:36:11 +01:00
|
|
|
private boolean subStageKeysConsistency(int totalKeyLength) {
|
|
|
|
if (subStageGetter instanceof SubStageGetterMapDeep) {
|
|
|
|
return totalKeyLength
|
2021-04-30 19:15:04 +02:00
|
|
|
== keyPrefixLength + keySuffixLength + ((SubStageGetterMapDeep<?, ?, ?>) subStageGetter).getKeyBinaryLength();
|
2021-02-02 15:36:11 +01:00
|
|
|
} else if (subStageGetter instanceof SubStageGetterMap) {
|
|
|
|
return totalKeyLength
|
2021-04-30 19:15:04 +02:00
|
|
|
== keyPrefixLength + keySuffixLength + ((SubStageGetterMap<?, ?>) subStageGetter).getKeyBinaryLength();
|
2021-02-02 15:36:11 +01:00
|
|
|
} else {
|
|
|
|
return true;
|
|
|
|
}
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-28 23:10:31 +01:00
|
|
|
public void setAllEntries(Stream<Entry<T, U>> entries) {
|
2023-02-09 23:34:25 +01:00
|
|
|
this.clear();
|
|
|
|
this.putMulti(entries);
|
2021-03-11 02:22:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-02-28 23:10:31 +01:00
|
|
|
public Stream<Entry<T, U>> setAllEntriesAndGetPrevious(Stream<Entry<T, U>> entries) {
|
2023-05-24 01:44:06 +02:00
|
|
|
return resourceStream(() -> this.getAllEntries(null, false), () -> setAllEntries(entries));
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
2024-04-18 14:48:16 +02:00
|
|
|
@Override
|
|
|
|
public ForkJoinPool getDbReadPool() {
|
|
|
|
return dictionary.getDbReadPool();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public ForkJoinPool getDbWritePool() {
|
|
|
|
return dictionary.getDbWritePool();
|
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
@Override
|
|
|
|
public void clear() {
|
|
|
|
if (range.isAll()) {
|
|
|
|
dictionary.clear();
|
|
|
|
} else if (range.isSingle()) {
|
|
|
|
dictionary.remove(range.getSingleUnsafe(), LLDictionaryResultType.VOID);
|
|
|
|
} else {
|
|
|
|
dictionary.setRange(range, Stream.empty(), false);
|
|
|
|
}
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
protected T deserializeSuffix(@NotNull BufDataInput keySuffix) throws SerializationException {
|
|
|
|
assert suffixKeyLengthConsistency(keySuffix.available());
|
|
|
|
return keySuffixSerializer.deserialize(keySuffix);
|
|
|
|
}
|
|
|
|
|
|
|
|
protected void serializeSuffixTo(T keySuffix, BufDataOutput output) throws SerializationException {
|
|
|
|
var beforeWriterOffset = output.size();
|
2023-02-22 22:31:36 +01:00
|
|
|
assert beforeWriterOffset == keyPrefixLength;
|
2023-05-22 23:33:58 +02:00
|
|
|
assert keySuffixSerializer.getSerializedBinaryLength() == keySuffixLength
|
|
|
|
: "Invalid key suffix serializer length: " + keySuffixSerializer.getSerializedBinaryLength()
|
|
|
|
+ ". Expected: " + keySuffixLength;
|
2021-10-19 00:22:05 +02:00
|
|
|
keySuffixSerializer.serialize(keySuffix, output);
|
2023-02-09 23:34:25 +01:00
|
|
|
var afterWriterOffset = output.size();
|
2022-05-21 22:41:48 +02:00
|
|
|
assert suffixKeyLengthConsistency(afterWriterOffset - beforeWriterOffset)
|
|
|
|
: "Invalid key suffix length: " + (afterWriterOffset - beforeWriterOffset) + ". Expected: " + keySuffixLength;
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|
2021-04-30 19:15:04 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
public static <K1, K2, V, R> Stream<R> getAllLeaves2(DatabaseMapDictionaryDeep<K1, Object2ObjectSortedMap<K2, V>, ? extends DatabaseStageMap<K2, V, DatabaseStageEntry<V>>> deepMap,
|
2022-03-12 02:55:18 +01:00
|
|
|
CompositeSnapshot snapshot,
|
2022-03-20 14:33:27 +01:00
|
|
|
TriFunction<K1, K2, V, R> merger,
|
2023-02-09 23:34:25 +01:00
|
|
|
@Nullable K1 savedProgressKey1) {
|
2022-03-20 14:33:27 +01:00
|
|
|
var keySuffix1Serializer = deepMap.keySuffixSerializer;
|
|
|
|
SerializerFixedBinaryLength<?> keySuffix2Serializer;
|
|
|
|
Serializer<?> valueSerializer;
|
|
|
|
boolean isHashed;
|
|
|
|
boolean isHashedSet;
|
|
|
|
if (deepMap.subStageGetter instanceof SubStageGetterMap subStageGetterMap) {
|
|
|
|
isHashed = false;
|
|
|
|
isHashedSet = false;
|
|
|
|
keySuffix2Serializer = subStageGetterMap.keySerializer;
|
|
|
|
valueSerializer = subStageGetterMap.valueSerializer;
|
|
|
|
} else if (deepMap.subStageGetter instanceof SubStageGetterHashMap subStageGetterHashMap) {
|
|
|
|
isHashed = true;
|
|
|
|
isHashedSet = false;
|
|
|
|
keySuffix2Serializer = subStageGetterHashMap.keyHashSerializer;
|
|
|
|
|
|
|
|
//noinspection unchecked
|
|
|
|
ValueWithHashSerializer<K2, V> valueWithHashSerializer = new ValueWithHashSerializer<>(
|
|
|
|
(Serializer<K2>) subStageGetterHashMap.keySerializer,
|
|
|
|
(Serializer<V>) subStageGetterHashMap.valueSerializer
|
|
|
|
);
|
|
|
|
valueSerializer = new ValuesSetSerializer<>(valueWithHashSerializer);
|
|
|
|
} else if (deepMap.subStageGetter instanceof SubStageGetterHashSet subStageGetterHashSet) {
|
|
|
|
isHashed = true;
|
|
|
|
isHashedSet = true;
|
|
|
|
keySuffix2Serializer = subStageGetterHashSet.keyHashSerializer;
|
|
|
|
|
|
|
|
//noinspection unchecked
|
|
|
|
valueSerializer = new ValuesSetSerializer<K2>(subStageGetterHashSet.keySerializer);
|
2022-03-12 02:55:18 +01:00
|
|
|
} else {
|
|
|
|
throw new IllegalArgumentException();
|
|
|
|
}
|
2022-03-20 14:33:27 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
var firstKey = Optional.ofNullable(savedProgressKey1);
|
|
|
|
var fullRange = deepMap.range;
|
2022-03-20 14:33:27 +01:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
|
|
|
|
LLRange range;
|
|
|
|
if (firstKey.isPresent()) {
|
|
|
|
var key1Buf = BufDataOutput.create(keySuffix1Serializer.getSerializedBinaryLength());
|
|
|
|
keySuffix1Serializer.serialize(firstKey.get(), key1Buf);
|
|
|
|
range = LLRange.of(key1Buf.asList(), fullRange.getMax());
|
|
|
|
} else {
|
|
|
|
range = fullRange;
|
|
|
|
}
|
|
|
|
|
|
|
|
return deepMap.dictionary.getRange(deepMap.resolveSnapshot(snapshot), range, false, false)
|
|
|
|
.flatMap(entry -> {
|
2022-03-20 14:33:27 +01:00
|
|
|
K1 key1 = null;
|
|
|
|
Object key2 = null;
|
2023-02-09 23:34:25 +01:00
|
|
|
try {
|
|
|
|
var keyBuf = entry.getKey();
|
|
|
|
var valueBuf = entry.getValue();
|
2022-03-20 14:33:27 +01:00
|
|
|
try {
|
|
|
|
assert keyBuf != null;
|
2023-02-09 23:34:25 +01:00
|
|
|
var suffix1And2 = BufDataInput.create(keyBuf.subList(deepMap.keyPrefixLength, deepMap.keyPrefixLength + deepMap.keySuffixLength + deepMap.keyExtLength));
|
|
|
|
key1 = keySuffix1Serializer.deserialize(suffix1And2);
|
|
|
|
key2 = keySuffix2Serializer.deserialize(suffix1And2);
|
2022-03-20 14:33:27 +01:00
|
|
|
assert valueBuf != null;
|
2023-02-09 23:34:25 +01:00
|
|
|
Object value = valueSerializer.deserialize(BufDataInput.create(valueBuf));
|
2022-03-20 14:33:27 +01:00
|
|
|
if (isHashedSet) {
|
|
|
|
//noinspection unchecked
|
|
|
|
Set<K2> set = (Set<K2>) value;
|
|
|
|
K1 finalKey1 = key1;
|
|
|
|
//noinspection unchecked
|
2023-02-09 23:34:25 +01:00
|
|
|
return set.stream().map(e -> merger.apply(finalKey1, e, (V) Nothing.INSTANCE));
|
2022-03-20 14:33:27 +01:00
|
|
|
} else if (isHashed) {
|
|
|
|
//noinspection unchecked
|
|
|
|
Set<Entry<K2, V>> set = (Set<Entry<K2, V>>) value;
|
|
|
|
K1 finalKey1 = key1;
|
2023-02-09 23:34:25 +01:00
|
|
|
return set.stream().map(e -> merger.apply(finalKey1, e.getKey(), e.getValue()));
|
2022-03-20 14:33:27 +01:00
|
|
|
} else {
|
|
|
|
//noinspection unchecked
|
2023-02-09 23:34:25 +01:00
|
|
|
return Stream.of(merger.apply(key1, (K2) key2, (V) value));
|
2022-03-20 14:33:27 +01:00
|
|
|
}
|
|
|
|
} catch (IndexOutOfBoundsException ex) {
|
|
|
|
var exMessage = ex.getMessage();
|
|
|
|
if (exMessage != null && exMessage.contains("read 0 to 0, write 0 to ")) {
|
|
|
|
var totalZeroBytesErrors = deepMap.totalZeroBytesErrors.incrementAndGet();
|
|
|
|
if (totalZeroBytesErrors < 512 || totalZeroBytesErrors % 10000 == 0) {
|
|
|
|
LOG.error("Unexpected zero-bytes value at " + deepMap.dictionary.getDatabaseName()
|
|
|
|
+ ":" + deepMap.dictionary.getColumnName()
|
|
|
|
+ ":[" + key1
|
|
|
|
+ ":" + key2
|
|
|
|
+ "](" + LLUtils.toStringSafe(keyBuf) + ") total=" + totalZeroBytesErrors);
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
return Stream.empty();
|
2022-03-20 14:33:27 +01:00
|
|
|
} else {
|
|
|
|
throw ex;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (SerializationException ex) {
|
|
|
|
throw new CompletionException(ex);
|
|
|
|
}
|
|
|
|
});
|
2022-03-12 02:55:18 +01:00
|
|
|
}
|
2021-01-31 21:23:43 +01:00
|
|
|
}
|