Fix sigsegv
This commit is contained in:
parent
cd15f8d23d
commit
59f9f01268
@ -489,6 +489,9 @@ public class LLUtils {
|
||||
}
|
||||
|
||||
public static Buf unmodifiableBytes(Buf previous) {
|
||||
if (previous == null) {
|
||||
return null;
|
||||
}
|
||||
previous.freeze();
|
||||
return previous;
|
||||
}
|
||||
|
@ -176,7 +176,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
|
||||
buf1.skipNBytes(keyPrefixLength);
|
||||
suffixAndExtKeyConsistency(buf1.available());
|
||||
|
||||
key = deserializeSuffix(serializedValue);
|
||||
key = deserializeSuffix(buf1);
|
||||
U value = valueSerializer.deserialize(serializedValue);
|
||||
deserializedEntry = Map.entry(key, value);
|
||||
return deserializedEntry;
|
||||
@ -295,6 +295,9 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
var valueMono = serializeValue(value);
|
||||
var valueBuf = dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE);
|
||||
if (valueBuf == null) {
|
||||
return null;
|
||||
}
|
||||
return deserializeValue(keySuffix, BufDataInput.create(valueBuf));
|
||||
}
|
||||
|
||||
|
@ -18,6 +18,7 @@ import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletionException;
|
||||
@ -28,6 +29,7 @@ import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.jetbrains.annotations.VisibleForTesting;
|
||||
|
||||
// todo: implement optimized methods (which?)
|
||||
public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implements DatabaseStageMap<T, U, US> {
|
||||
@ -74,18 +76,25 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
|
||||
}
|
||||
}
|
||||
|
||||
static Buf firstRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
|
||||
modifiablePrefixKey.addAll(prefixKey);
|
||||
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
|
||||
@VisibleForTesting
|
||||
public static Buf firstRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
return createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static Buf nextRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
Buf modifiablePrefixKey = createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
|
||||
incrementPrefix(modifiablePrefixKey, prefixLength);
|
||||
return modifiablePrefixKey;
|
||||
}
|
||||
|
||||
static Buf nextRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
private static Buf createFullKeyWithEmptySuffixAndExt(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
|
||||
modifiablePrefixKey.addAll(prefixKey);
|
||||
if (prefixKey != null) {
|
||||
modifiablePrefixKey.addAll(prefixKey);
|
||||
}
|
||||
assert prefixKey != null || prefixLength == 0 : "Prefix length is " + prefixLength + " but the prefix key is null";
|
||||
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
|
||||
incrementPrefix(modifiablePrefixKey, prefixLength);
|
||||
return modifiablePrefixKey;
|
||||
}
|
||||
|
||||
@ -98,7 +107,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
|
||||
var suffixLengthAndExtLength = suffixAndExtZeroes.size();
|
||||
assert result.size() == prefixLength;
|
||||
assert suffixLengthAndExtLength > 0 : "Suffix length + ext length is < 0: " + suffixLengthAndExtLength;
|
||||
result.size(prefixLength + suffixLengthAndExtLength);
|
||||
result.size(prefixLength);
|
||||
modifiablePrefixKey.addAll(suffixAndExtZeroes);
|
||||
assert modifiablePrefixKey.size() == prefixLength + suffixAndExtZeroes.size() : "Result buffer size is wrong";
|
||||
}
|
||||
@ -172,6 +181,10 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
protected boolean suffixKeyLengthConsistency(int keySuffixLength) {
|
||||
assert
|
||||
this.keySuffixLength == keySuffixLength :
|
||||
"Key suffix length is " + keySuffixLength + ", but it should be " + this.keySuffixLength + " bytes long";
|
||||
//noinspection ConstantValue
|
||||
return this.keySuffixLength == keySuffixLength;
|
||||
}
|
||||
|
||||
@ -313,8 +326,8 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
|
||||
}
|
||||
|
||||
protected void serializeSuffixTo(T keySuffix, BufDataOutput output) throws SerializationException {
|
||||
assert suffixKeyLengthConsistency(output.size());
|
||||
var beforeWriterOffset = output.size();
|
||||
assert beforeWriterOffset == keyPrefixLength;
|
||||
keySuffixSerializer.serialize(keySuffix, output);
|
||||
var afterWriterOffset = output.size();
|
||||
assert suffixKeyLengthConsistency(afterWriterOffset - beforeWriterOffset)
|
||||
|
@ -88,10 +88,9 @@ public final class DatabaseMapSingle<U> implements DatabaseStageEntry<U> {
|
||||
}
|
||||
|
||||
@Override
|
||||
public U update(SerializationFunction<@Nullable U, @Nullable U> updater,
|
||||
UpdateReturnMode updateReturnMode) {
|
||||
public U update(SerializationFunction<@Nullable U, @Nullable U> updater, UpdateReturnMode updateReturnMode) {
|
||||
Buf resultBytes = dictionary.update(key, this.createUpdater(updater), updateReturnMode);
|
||||
return deserializeValue(resultBytes);
|
||||
return resultBytes != null ? deserializeValue(resultBytes) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -39,6 +39,9 @@ public class DatabaseSingleton<U> implements DatabaseStageEntry<U> {
|
||||
}
|
||||
|
||||
private U deserializeValue(Buf value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return serializer.deserialize(BufDataInput.create(value));
|
||||
} catch (IndexOutOfBoundsException ex) {
|
||||
|
@ -203,10 +203,11 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
|
||||
|
||||
@Override
|
||||
default Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
|
||||
Object2ObjectSortedMap<T, U> map = this
|
||||
.getAllValues(snapshot, true)
|
||||
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
|
||||
return map.isEmpty() ? null : map;
|
||||
try (var stream = this.getAllValues(snapshot, true)) {
|
||||
Object2ObjectSortedMap<T, U> map = stream
|
||||
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
|
||||
return map.isEmpty() ? null : map;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -5,6 +5,7 @@ import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
|
||||
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
|
||||
import static it.cavallium.dbengine.database.LLUtils.toStringSafe;
|
||||
import static it.cavallium.dbengine.database.disk.UpdateAtomicResultMode.DELTA;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.batches;
|
||||
|
||||
@ -699,7 +700,7 @@ public class LLLocalDictionary implements LLDictionary {
|
||||
ro.close();
|
||||
throw new DBException("Failed to open rocksdb iterator", ex);
|
||||
}
|
||||
return Stream.generate(() -> {
|
||||
return streamWhileNonNull(() -> {
|
||||
if (!rocksIterator.isValid()) return null;
|
||||
Buf rawKey = null;
|
||||
try {
|
||||
@ -709,7 +710,7 @@ public class LLLocalDictionary implements LLDictionary {
|
||||
return new BadBlock(databaseName, ColumnUtils.special(columnName), rawKey, ex);
|
||||
}
|
||||
return null;
|
||||
}).takeWhile(x -> rocksIterator.isValid()).filter(Objects::nonNull).onClose(() -> {
|
||||
}).takeWhile(x -> rocksIterator.isValid()).onClose(() -> {
|
||||
rocksIterator.close();
|
||||
ro.close();
|
||||
});
|
||||
@ -1030,14 +1031,16 @@ public class LLLocalDictionary implements LLDictionary {
|
||||
if (USE_CURRENT_FASTSIZE_FOR_OLD_SNAPSHOTS || rocksdbSnapshot.snapshot() == null) {
|
||||
try {
|
||||
if (USE_NUM_ENTRIES_PRECISE_COUNTER) {
|
||||
return exactSizeAll(null);
|
||||
return getRocksDBNumEntries();
|
||||
}
|
||||
return db.getLongProperty("rocksdb.estimate-num-keys");
|
||||
} catch (RocksDBException e) {
|
||||
logger.error(MARKER_ROCKSDB, "Failed to get RocksDB estimated keys count property", e);
|
||||
return 0;
|
||||
}
|
||||
} else if (USE_NUM_ENTRIES_PRECISE_COUNTER || PARALLEL_EXACT_SIZE) {
|
||||
} else if (USE_NUM_ENTRIES_PRECISE_COUNTER && snapshot == null) {
|
||||
return getRocksDBNumEntries();
|
||||
} else if (PARALLEL_EXACT_SIZE) {
|
||||
return exactSizeAll(snapshot);
|
||||
} else {
|
||||
rocksdbSnapshot.setFillCache(false);
|
||||
@ -1057,17 +1060,18 @@ public class LLLocalDictionary implements LLDictionary {
|
||||
}
|
||||
}
|
||||
|
||||
private long getRocksDBNumEntries() {
|
||||
try {
|
||||
return db.getNumEntries();
|
||||
} catch (RocksDBException ex) {
|
||||
throw new IllegalStateException("Failed to read exact size", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private long exactSizeAll(@Nullable LLSnapshot snapshot) {
|
||||
if (LLUtils.isInNonBlockingThread()) {
|
||||
throw new UnsupportedOperationException("Called exactSizeAll in a nonblocking thread");
|
||||
}
|
||||
if (snapshot == null && USE_NUM_ENTRIES_PRECISE_COUNTER) {
|
||||
try {
|
||||
return db.getNumEntries();
|
||||
} catch (RocksDBException ex) {
|
||||
throw new IllegalStateException("Failed to read exact size", ex);
|
||||
}
|
||||
}
|
||||
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot), false, false, false)) {
|
||||
if (LLUtils.MANUAL_READAHEAD) {
|
||||
readOpts.setReadaheadSize(128 * 1024); // 128KiB
|
||||
|
@ -9,6 +9,7 @@ import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.disk.rocksdb.RocksIteratorObj;
|
||||
import it.cavallium.dbengine.utils.DBException;
|
||||
import it.cavallium.dbengine.utils.StreamUtils;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
@ -64,7 +65,7 @@ public abstract class LLLocalGroupedReactiveRocksIterator<T> {
|
||||
throw new DBException("Failed to iterate the range", e);
|
||||
}
|
||||
|
||||
return Stream.<List<T>>generate(() -> {
|
||||
return StreamUtils.<List<T>>streamWhileNonNull(() -> {
|
||||
try {
|
||||
ObjectArrayList<T> values = new ObjectArrayList<>();
|
||||
Buf firstGroupKey = null;
|
||||
@ -111,7 +112,7 @@ public abstract class LLLocalGroupedReactiveRocksIterator<T> {
|
||||
}
|
||||
throw new CompletionException(new DBException("Range failed", ex));
|
||||
}
|
||||
}).takeWhile(Objects::nonNull).onClose(() -> {
|
||||
}).onClose(() -> {
|
||||
rocksIterator.close();
|
||||
readOptions.close();
|
||||
});
|
||||
|
@ -3,12 +3,17 @@ package it.cavallium.dbengine.database.disk;
|
||||
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
|
||||
import static it.cavallium.dbengine.database.LLUtils.generateCustomReadOptions;
|
||||
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
||||
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.common.collect.Streams;
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.utils.DBException;
|
||||
import it.cavallium.dbengine.utils.StreamUtils;
|
||||
import java.io.IOException;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Stream;
|
||||
@ -52,7 +57,7 @@ public class LLLocalKeyPrefixReactiveRocksIterator {
|
||||
}
|
||||
var rocksIterator = db.newRocksIterator(readOptions, range, false);
|
||||
|
||||
return Stream.generate(() -> {
|
||||
return streamWhileNonNull(() -> {
|
||||
try {
|
||||
Buf firstGroupKey = null;
|
||||
while (rocksIterator.isValid()) {
|
||||
|
@ -1,6 +1,7 @@
|
||||
package it.cavallium.dbengine.database.disk;
|
||||
|
||||
import static it.cavallium.dbengine.database.LLUtils.generateCustomReadOptions;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
||||
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.database.LLEntry;
|
||||
@ -39,7 +40,7 @@ public final class LLLocalMigrationReactiveRocksIterator {
|
||||
} catch (RocksDBException e) {
|
||||
throw new DBException("Failed to open iterator", e);
|
||||
}
|
||||
return Stream.generate(() -> {
|
||||
return streamWhileNonNull(() -> {
|
||||
try {
|
||||
if (rocksIterator.isValid()) {
|
||||
var key = rocksIterator.keyBuf().copy();
|
||||
@ -52,7 +53,7 @@ public final class LLLocalMigrationReactiveRocksIterator {
|
||||
} catch (RocksDBException ex) {
|
||||
throw new CompletionException(new DBException("Failed to iterate", ex));
|
||||
}
|
||||
}).takeWhile(Objects::nonNull).onClose(() -> {
|
||||
}).onClose(() -> {
|
||||
rocksIterator.close();
|
||||
readOptions.close();
|
||||
});
|
||||
|
@ -3,6 +3,7 @@ package it.cavallium.dbengine.database.disk;
|
||||
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
|
||||
import static it.cavallium.dbengine.database.LLUtils.generateCustomReadOptions;
|
||||
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
||||
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
@ -59,7 +60,7 @@ public abstract class LLLocalReactiveRocksIterator<T> {
|
||||
throw new DBException("Failed to iterate the range", e);
|
||||
}
|
||||
|
||||
return Stream.generate(() -> {
|
||||
return streamWhileNonNull(() -> {
|
||||
try {
|
||||
if (rocksIterator.isValid()) {
|
||||
// Note that the underlying array is subject to changes!
|
||||
@ -100,7 +101,7 @@ public abstract class LLLocalReactiveRocksIterator<T> {
|
||||
}
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
}).takeWhile(Objects::nonNull).onClose(() -> {
|
||||
}).onClose(() -> {
|
||||
rocksIterator.close();
|
||||
readOptions.close();
|
||||
});
|
||||
|
@ -59,7 +59,7 @@ public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase {
|
||||
));
|
||||
var singleton = new LLMemorySingleton(dict, columnNameString, singletonName);
|
||||
Buf returnValue = singleton.get(null);
|
||||
if (returnValue == null) {
|
||||
if (returnValue == null && defaultValue != null) {
|
||||
singleton.set(Buf.wrap(defaultValue));
|
||||
}
|
||||
return singleton;
|
||||
|
@ -16,7 +16,7 @@ public class CheckIndexInput extends IndexInput {
|
||||
}
|
||||
|
||||
private static void checkThread() {
|
||||
assert LuceneUtils.isLuceneThread();
|
||||
warnLuceneThread();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -15,7 +15,7 @@ public class CheckIndexOutput extends IndexOutput {
|
||||
}
|
||||
|
||||
private static void checkThread() {
|
||||
assert LuceneUtils.isLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -49,7 +49,7 @@ public class CheckOutputDirectory extends Directory {
|
||||
|
||||
@Override
|
||||
public IndexOutput createOutput(String name, IOContext context) {
|
||||
LuceneUtils.checkLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
try {
|
||||
return new CheckIndexOutput(directory.createOutput(name, context));
|
||||
} catch (IOException e) {
|
||||
@ -59,7 +59,7 @@ public class CheckOutputDirectory extends Directory {
|
||||
|
||||
@Override
|
||||
public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) {
|
||||
LuceneUtils.checkLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
try {
|
||||
return new CheckIndexOutput(directory.createTempOutput(prefix, suffix, context));
|
||||
} catch (IOException e) {
|
||||
@ -69,7 +69,7 @@ public class CheckOutputDirectory extends Directory {
|
||||
|
||||
@Override
|
||||
public void sync(Collection<String> names) {
|
||||
LuceneUtils.checkLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
try {
|
||||
directory.sync(names);
|
||||
} catch (IOException e) {
|
||||
@ -79,7 +79,7 @@ public class CheckOutputDirectory extends Directory {
|
||||
|
||||
@Override
|
||||
public void syncMetaData() {
|
||||
LuceneUtils.checkLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
try {
|
||||
directory.syncMetaData();
|
||||
} catch (IOException e) {
|
||||
@ -89,7 +89,7 @@ public class CheckOutputDirectory extends Directory {
|
||||
|
||||
@Override
|
||||
public void rename(String source, String dest) {
|
||||
LuceneUtils.checkLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
try {
|
||||
directory.rename(source, dest);
|
||||
} catch (IOException e) {
|
||||
@ -99,7 +99,7 @@ public class CheckOutputDirectory extends Directory {
|
||||
|
||||
@Override
|
||||
public IndexInput openInput(String name, IOContext context) {
|
||||
LuceneUtils.checkLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
try {
|
||||
return new CheckIndexInput(directory.openInput(name, context));
|
||||
} catch (IOException e) {
|
||||
@ -109,7 +109,7 @@ public class CheckOutputDirectory extends Directory {
|
||||
|
||||
@Override
|
||||
public Lock obtainLock(String name) {
|
||||
LuceneUtils.checkLuceneThread();
|
||||
LuceneUtils.warnLuceneThread();
|
||||
try {
|
||||
return directory.obtainLock(name);
|
||||
} catch (IOException e) {
|
||||
|
@ -1,5 +1,7 @@
|
||||
package it.cavallium.dbengine.lucene.searcher;
|
||||
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
||||
|
||||
import java.io.IOException;
|
||||
import it.cavallium.dbengine.utils.DBException;
|
||||
import java.util.Iterator;
|
||||
@ -51,7 +53,7 @@ public class LuceneGenerator implements Supplier<ScoreDoc> {
|
||||
throw new IllegalArgumentException("Sorting is not allowed");
|
||||
}
|
||||
var lg = new LuceneGenerator(shard, localQueryParams, shardIndex);
|
||||
return Stream.generate(lg).takeWhile(Objects::nonNull);
|
||||
return streamWhileNonNull(lg);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -2,6 +2,7 @@ package it.cavallium.dbengine.lucene.searcher;
|
||||
|
||||
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS;
|
||||
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
||||
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import it.cavallium.dbengine.database.LLKeyScore;
|
||||
@ -130,14 +131,14 @@ public class PagedLocalSearcher implements LocalSearcher {
|
||||
LocalQueryParams queryParams, String keyFieldName, CurrentPageInfo secondPageInfo) {
|
||||
AtomicReference<CurrentPageInfo> pageInfo = new AtomicReference<>(secondPageInfo);
|
||||
Object lock = new Object();
|
||||
Stream<ScoreDoc> topFieldDocFlux = Stream.generate(() -> {
|
||||
Stream<ScoreDoc> topFieldDocFlux = streamWhileNonNull(() -> {
|
||||
synchronized (lock) {
|
||||
var currentPageInfo = pageInfo.getPlain();
|
||||
var result = searchPageSync(queryParams, indexSearchers, true, 0, currentPageInfo);
|
||||
pageInfo.setPlain(result.nextPageToIterate());
|
||||
return result.pageData();
|
||||
}
|
||||
}).takeWhile(Objects::nonNull).flatMap(pd -> Stream.of(pd.topDocs().scoreDocs));
|
||||
}).flatMap(pd -> Stream.of(pd.topDocs().scoreDocs));
|
||||
return LuceneUtils.convertHits(topFieldDocFlux, indexSearchers, keyFieldName);
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,7 @@
|
||||
package it.cavallium.dbengine.lucene.searcher;
|
||||
|
||||
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
||||
|
||||
import com.google.common.collect.Streams;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
@ -133,7 +134,7 @@ public class ScoredPagedMultiSearcher implements MultiSearcher {
|
||||
private Stream<LLKeyScore> searchOtherPages(List<IndexSearcher> indexSearchers,
|
||||
LocalQueryParams queryParams, String keyFieldName, CurrentPageInfo secondPageInfo) {
|
||||
AtomicReference<CurrentPageInfo> currentPageInfoRef = new AtomicReference<>(secondPageInfo);
|
||||
Stream<ScoreDoc> topFieldDocStream = Stream.generate(() -> {
|
||||
Stream<ScoreDoc> topFieldDocStream = streamWhileNonNull(() -> {
|
||||
var currentPageInfo = currentPageInfoRef.getPlain();
|
||||
if (currentPageInfo == null) return null;
|
||||
LOG.trace("Current page info: {}", currentPageInfo);
|
||||
@ -145,7 +146,7 @@ public class ScoredPagedMultiSearcher implements MultiSearcher {
|
||||
} else {
|
||||
return Arrays.asList(result.topDocs().scoreDocs);
|
||||
}
|
||||
}).takeWhile(Objects::nonNull).flatMap(Collection::stream);
|
||||
}).flatMap(Collection::stream);
|
||||
|
||||
return LuceneUtils.convertHits(topFieldDocStream, indexSearchers, keyFieldName);
|
||||
}
|
||||
|
@ -8,6 +8,7 @@ import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Spliterator;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
@ -48,6 +49,31 @@ public class StreamUtils {
|
||||
: StreamSupport.stream(new BatchSpliterator<>(stream.spliterator(), batchSize), stream.isParallel());
|
||||
}
|
||||
|
||||
@SuppressWarnings("UnstableApiUsage")
|
||||
public static <X> Stream<X> streamWhileNonNull(Supplier<X> supplier) {
|
||||
var it = new Iterator<X>() {
|
||||
|
||||
private boolean nextSet = false;
|
||||
private X next;
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (!nextSet) {
|
||||
next = supplier.get();
|
||||
nextSet = true;
|
||||
}
|
||||
return next != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public X next() {
|
||||
nextSet = false;
|
||||
return next;
|
||||
}
|
||||
};
|
||||
return Streams.stream(it);
|
||||
}
|
||||
|
||||
private record BatchSpliterator<E>(Spliterator<E> base, int batchSize) implements Spliterator<List<E>> {
|
||||
|
||||
@Override
|
||||
|
@ -19,6 +19,7 @@ module dbengine {
|
||||
exports it.cavallium.dbengine.utils;
|
||||
exports it.cavallium.dbengine.database.disk.rocksdb;
|
||||
exports it.cavallium.dbengine.buffers;
|
||||
exports it.cavallium.dbengine.lucene.hugepq.search;
|
||||
requires org.jetbrains.annotations;
|
||||
requires com.google.common;
|
||||
requires micrometer.core;
|
||||
|
@ -1,131 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.client.DefaultDatabaseOptions.DEFAULT_DATABASE_OPTIONS;
|
||||
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import it.cavallium.data.generator.nativedata.Nullableboolean;
|
||||
import it.cavallium.data.generator.nativedata.Nullabledouble;
|
||||
import it.cavallium.data.generator.nativedata.Nullableint;
|
||||
import it.cavallium.data.generator.nativedata.Nullablelong;
|
||||
import it.cavallium.dbengine.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
|
||||
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
||||
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
||||
import it.cavallium.dbengine.database.ColumnUtils;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
|
||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
|
||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptionsBuilder;
|
||||
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
|
||||
import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||
|
||||
private static final AtomicInteger dbId = new AtomicInteger(0);
|
||||
|
||||
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofSeconds(5),
|
||||
false,
|
||||
new ByteBuffersDirectory(),
|
||||
Nullableboolean.empty(),
|
||||
Nullabledouble.empty(),
|
||||
Nullableint.empty(),
|
||||
Nullableboolean.empty(),
|
||||
Nullableboolean.empty(),
|
||||
true,
|
||||
MAX_IN_MEMORY_RESULT_ENTRIES,
|
||||
LuceneUtils.getDefaultMergePolicy()
|
||||
);
|
||||
|
||||
@Override
|
||||
public Mono<TempDb> openTempDb(TestAllocator allocator) {
|
||||
boolean canUseNettyDirect = DbTestUtils.computeCanUseNettyDirect();
|
||||
return Mono.defer(() -> {
|
||||
var wrkspcPath = Path.of("/tmp/.cache/tempdb-" + dbId.incrementAndGet() + "/");
|
||||
return Mono
|
||||
.<LLKeyValueDatabase>fromCallable(() -> {
|
||||
if (Files.exists(wrkspcPath)) {
|
||||
Files.walk(wrkspcPath).sorted(Comparator.reverseOrder()).forEach(file -> {
|
||||
try {
|
||||
Files.delete(file);
|
||||
} catch (IOException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
Files.createDirectories(wrkspcPath);
|
||||
return null;
|
||||
})
|
||||
.subscribeOn(Schedulers.boundedElastic())
|
||||
.then(new LLLocalDatabaseConnection(allocator.allocator(),
|
||||
new SimpleMeterRegistry(),
|
||||
wrkspcPath,
|
||||
true,
|
||||
null
|
||||
).connect())
|
||||
.flatMap(conn -> {
|
||||
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
|
||||
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
|
||||
return Mono.zip(
|
||||
conn.getDatabase("testdb",
|
||||
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
|
||||
DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build()
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex1",
|
||||
LuceneUtils.singleStructure(),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex16",
|
||||
LuceneUtils.shardsStructure(3),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
Mono.just(searcher)
|
||||
)
|
||||
.map(tuple -> new TempDb(allocator, conn, tuple.getT1(), tuple.getT2(), tuple.getT3(), tuple.getT4(), wrkspcPath));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> closeTempDb(TempDb tempDb) {
|
||||
return tempDb.db().close().then(tempDb.connection().disconnect()).then(Mono.fromCallable(() -> {
|
||||
ensureNoLeaks(tempDb.allocator().allocator(), false, false);
|
||||
if (Files.exists(tempDb.path())) {
|
||||
Files.walk(tempDb.path()).sorted(Comparator.reverseOrder()).forEach(file -> {
|
||||
try {
|
||||
Files.delete(file);
|
||||
} catch (IOException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}).subscribeOn(Schedulers.boundedElastic())).then();
|
||||
}
|
||||
}
|
@ -1,87 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import it.cavallium.data.generator.nativedata.Nullableboolean;
|
||||
import it.cavallium.data.generator.nativedata.Nullabledouble;
|
||||
import it.cavallium.data.generator.nativedata.Nullableint;
|
||||
import it.cavallium.data.generator.nativedata.Nullablelong;
|
||||
import it.cavallium.dbengine.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
|
||||
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
||||
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
||||
import it.cavallium.dbengine.database.ColumnUtils;
|
||||
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
|
||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
|
||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
||||
import it.cavallium.dbengine.rpc.current.data.LuceneIndexStructure;
|
||||
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
|
||||
import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter;
|
||||
import it.unimi.dsi.fastutil.ints.IntList;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||
|
||||
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofSeconds(5),
|
||||
false,
|
||||
new ByteBuffersDirectory(),
|
||||
Nullableboolean.empty(),
|
||||
Nullabledouble.empty(),
|
||||
Nullableint.empty(),
|
||||
Nullableboolean.empty(),
|
||||
Nullableboolean.empty(),
|
||||
false,
|
||||
MAX_IN_MEMORY_RESULT_ENTRIES,
|
||||
LuceneUtils.getDefaultMergePolicy()
|
||||
);
|
||||
|
||||
@Override
|
||||
public Mono<TempDb> openTempDb(TestAllocator allocator) {
|
||||
boolean canUseNettyDirect = DbTestUtils.computeCanUseNettyDirect();
|
||||
return Mono
|
||||
.fromCallable(() -> new LLMemoryDatabaseConnection(allocator.allocator(), new SimpleMeterRegistry()))
|
||||
.flatMap(conn -> {
|
||||
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
|
||||
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
|
||||
return Mono
|
||||
.zip(
|
||||
conn.getDatabase("testdb",
|
||||
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
|
||||
DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build()
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex1",
|
||||
LuceneUtils.singleStructure(),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex16",
|
||||
LuceneUtils.shardsStructure(3),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
Mono.just(searcher)
|
||||
)
|
||||
.map(tuple -> new TempDb(allocator, conn, tuple.getT1(), tuple.getT2(), tuple.getT3(), tuple.getT4(), null));
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> closeTempDb(TempDb db) {
|
||||
return db.db().close();
|
||||
}
|
||||
}
|
@ -1,40 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
public class SyncUtils {
|
||||
|
||||
public static void run(Flux<?> publisher) {
|
||||
publisher.subscribeOn(Schedulers.immediate()).blockLast();
|
||||
}
|
||||
|
||||
public static void runVoid(Mono<Void> publisher) {
|
||||
publisher.then().subscribeOn(Schedulers.immediate()).block();
|
||||
}
|
||||
|
||||
public static <T> T run(Mono<T> publisher) {
|
||||
return publisher.subscribeOn(Schedulers.immediate()).block();
|
||||
}
|
||||
|
||||
public static <T> T run(boolean shouldFail, Mono<T> publisher) {
|
||||
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
|
||||
if (shouldFail) {
|
||||
return mono.onErrorResume(ex -> Mono.empty());
|
||||
} else {
|
||||
return mono;
|
||||
}
|
||||
}).block();
|
||||
}
|
||||
|
||||
public static void runVoid(boolean shouldFail, Mono<Void> publisher) {
|
||||
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
|
||||
if (shouldFail) {
|
||||
return mono.onErrorResume(ex -> Mono.empty());
|
||||
} else {
|
||||
return mono;
|
||||
}
|
||||
}).block();
|
||||
}
|
||||
}
|
@ -1,13 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import io.netty5.buffer.BufferAllocator;
|
||||
import it.cavallium.dbengine.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public interface TemporaryDbGenerator {
|
||||
|
||||
Mono<TempDb> openTempDb(TestAllocator allocator);
|
||||
|
||||
Mono<Void> closeTempDb(TempDb db);
|
||||
}
|
@ -1,62 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
|
||||
import io.netty5.buffer.Buffer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class TestAllocator {
|
||||
|
||||
|
||||
private DbTestUtils.TestAllocator allocator;
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
destroyAllocator(allocator);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoOp() {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldPass() {
|
||||
Buffer allocated = allocator.allocator().allocate(5000);
|
||||
allocated.close();
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldFail() {
|
||||
Buffer allocated = null;
|
||||
try {
|
||||
boolean failed;
|
||||
try {
|
||||
allocated = allocator.allocator().allocate(5000);
|
||||
ensureNoLeaks(allocator.allocator(), true, true);
|
||||
failed = false;
|
||||
} catch (Exception ex) {
|
||||
failed = true;
|
||||
}
|
||||
if (!failed) {
|
||||
Assertions.fail("A leak was not detected!");
|
||||
}
|
||||
} finally {
|
||||
if (allocated != null) {
|
||||
allocated.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,149 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static io.netty5.buffer.internal.InternalBufferUtils.allocatorClosedException;
|
||||
import static io.netty5.buffer.internal.InternalBufferUtils.assertValidBufferSize;
|
||||
import static io.netty5.buffer.internal.InternalBufferUtils.standardDrop;
|
||||
|
||||
import io.netty5.buffer.AllocationType;
|
||||
import io.netty5.buffer.AllocatorControl;
|
||||
import io.netty5.buffer.Buffer;
|
||||
import io.netty5.buffer.BufferAllocator;
|
||||
import io.netty5.buffer.Drop;
|
||||
import io.netty5.buffer.MemoryManager;
|
||||
import io.netty5.buffer.StandardAllocationTypes;
|
||||
import io.netty5.buffer.pool.PooledBufferAllocator;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public class TestAllocatorImpl implements BufferAllocator, AllocatorControl {
|
||||
|
||||
private final TestMemoryManager manager;
|
||||
private final AllocationType allocationType = StandardAllocationTypes.ON_HEAP;
|
||||
private volatile boolean closed;
|
||||
|
||||
private TestAllocatorImpl(TestMemoryManager testMemoryManager) {
|
||||
this.manager = testMemoryManager;
|
||||
}
|
||||
|
||||
public static TestAllocatorImpl create() {
|
||||
return new TestAllocatorImpl(new TestMemoryManager(MemoryManager.instance()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPooling() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AllocationType getAllocationType() {
|
||||
return allocationType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Buffer allocate(int size) {
|
||||
if (closed) {
|
||||
throw allocatorClosedException();
|
||||
}
|
||||
assertValidBufferSize(size);
|
||||
return manager.allocateShared(this, size, standardDrop(manager), allocationType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Supplier<Buffer> constBufferSupplier(byte[] bytes) {
|
||||
if (closed) {
|
||||
throw allocatorClosedException();
|
||||
}
|
||||
Buffer constantBuffer = manager.allocateShared(
|
||||
this, bytes.length, standardDrop(manager), allocationType);
|
||||
constantBuffer.writeBytes(bytes).makeReadOnly();
|
||||
return () -> manager.allocateConstChild(constantBuffer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
closed = true;
|
||||
}
|
||||
|
||||
public long getActiveAllocations() {
|
||||
return this.manager.getActiveAllocations();
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferAllocator getAllocator() {
|
||||
return this;
|
||||
}
|
||||
|
||||
private static class TestMemoryManager implements MemoryManager {
|
||||
|
||||
private final MemoryManager instance;
|
||||
private final LongAdder activeAllocations = new LongAdder();
|
||||
|
||||
public TestMemoryManager(MemoryManager instance) {
|
||||
this.instance = instance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Buffer allocateShared(AllocatorControl allocatorControl,
|
||||
long size,
|
||||
Function<Drop<Buffer>, Drop<Buffer>> dropDecorator,
|
||||
AllocationType allocationType) {
|
||||
return instance.allocateShared(allocatorControl, size, this::createDrop, allocationType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Buffer allocateConstChild(Buffer readOnlyConstParent) {
|
||||
return instance.allocateConstChild(readOnlyConstParent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object unwrapRecoverableMemory(Buffer buf) {
|
||||
return instance.unwrapRecoverableMemory(buf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemory, Drop<Buffer> drop) {
|
||||
return instance.recoverMemory(allocatorControl, recoverableMemory, drop);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object sliceMemory(Object memory, int offset, int length) {
|
||||
return instance.sliceMemory(memory, offset, length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clearMemory(Object o) {
|
||||
instance.clearMemory(o);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String implementationName() {
|
||||
return instance.implementationName();
|
||||
}
|
||||
|
||||
private Drop<Buffer> createDrop(Drop<Buffer> drop) {
|
||||
activeAllocations.increment();
|
||||
return new Drop<>() {
|
||||
@Override
|
||||
public void drop(Buffer obj) {
|
||||
activeAllocations.decrement();
|
||||
drop.drop(obj);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Drop<Buffer> fork() {
|
||||
return createDrop(drop.fork());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(Buffer obj) {
|
||||
drop.attach(obj);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public long getActiveAllocations() {
|
||||
return activeAllocations.longValue();
|
||||
}
|
||||
}
|
||||
}
|
@ -1,56 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.isCIMode;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDb;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
|
||||
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import java.util.Arrays;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.test.StepVerifier;
|
||||
|
||||
public abstract class TestDictionary {
|
||||
|
||||
private TestAllocator allocator;
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
private static Stream<Arguments> provideArgumentsCreate() {
|
||||
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
if (!isCIMode()) {
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
}
|
||||
destroyAllocator(allocator);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsCreate")
|
||||
public void testCreate(UpdateMode updateMode) {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempDictionary(db, updateMode)
|
||||
.flatMap(LLDictionary::clear)
|
||||
.then()
|
||||
))
|
||||
.verifyComplete();
|
||||
}
|
||||
}
|
@ -1,157 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import io.netty5.buffer.Buffer;
|
||||
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
|
||||
import it.cavallium.dbengine.lucene.HugePqCodec;
|
||||
import it.cavallium.dbengine.lucene.HugePqPriorityQueue;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.function.Function;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class TestHugePq {
|
||||
|
||||
private LLTempHugePqEnv env;
|
||||
private HugePqPriorityQueue<Integer> queue;
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() throws IOException {
|
||||
this.env = new LLTempHugePqEnv();
|
||||
this.queue = new HugePqPriorityQueue<>(env, new HugePqCodec<Integer>() {
|
||||
@Override
|
||||
public Buffer serialize(Function<Integer, Buffer> allocator, Integer data) {
|
||||
var buf = allocator.apply(Integer.BYTES);
|
||||
HugePqCodec.setLexInt(buf, 0, false, data);
|
||||
return buf.writerOffset(Integer.BYTES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer deserialize(Buffer b) {
|
||||
return HugePqCodec.getLexInt(b, 0, false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoOp() {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptyTop() {
|
||||
Assertions.assertNull(queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddSingle() {
|
||||
queue.add(2);
|
||||
Assertions.assertEquals(2, queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddSame() {
|
||||
queue.add(2);
|
||||
queue.add(2);
|
||||
Assertions.assertEquals(2, queue.top());
|
||||
Assertions.assertEquals(2, queue.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMulti() {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
queue.add(i);
|
||||
}
|
||||
Assertions.assertEquals(0, queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddRandomMulti() {
|
||||
var list = new ArrayList<Integer>(1000);
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
var n = ThreadLocalRandom.current().nextInt(-20, 20);
|
||||
queue.add(n);
|
||||
list.add(n);
|
||||
}
|
||||
list.sort(Comparator.reverseOrder());
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
Assertions.assertEquals(list.remove(list.size() - 1), queue.pop());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMultiClear() {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
queue.add(i);
|
||||
}
|
||||
queue.clear();
|
||||
Assertions.assertNull(queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddRemove() {
|
||||
queue.add(0);
|
||||
queue.remove(0);
|
||||
Assertions.assertNull(queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddRemoveNonexistent() {
|
||||
queue.add(0);
|
||||
queue.remove(1);
|
||||
Assertions.assertEquals(0, queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMultiSameRemove() {
|
||||
queue.add(0);
|
||||
queue.add(0);
|
||||
queue.add(1);
|
||||
queue.remove(0);
|
||||
Assertions.assertEquals(2, queue.size());
|
||||
Assertions.assertEquals(0, queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMultiRemove() {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
queue.add(i);
|
||||
}
|
||||
queue.remove(0);
|
||||
Assertions.assertEquals(1, queue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSort() {
|
||||
var sortedNumbers = new ArrayList<Integer>();
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
sortedNumbers.add(i);
|
||||
}
|
||||
var shuffledNumbers = new ArrayList<>(sortedNumbers);
|
||||
Collections.shuffle(shuffledNumbers);
|
||||
for (Integer number : shuffledNumbers) {
|
||||
queue.add(number);
|
||||
}
|
||||
|
||||
var newSortedNumbers = new ArrayList<>();
|
||||
Integer popped;
|
||||
while ((popped = queue.pop()) != null) {
|
||||
newSortedNumbers.add(popped);
|
||||
}
|
||||
|
||||
Assertions.assertEquals(sortedNumbers, newSortedNumbers);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() throws IOException {
|
||||
queue.close();
|
||||
env.close();
|
||||
}
|
||||
}
|
@ -1,381 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import io.netty5.buffer.Buffer;
|
||||
import it.cavallium.dbengine.database.SafeCloseable;
|
||||
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
|
||||
import it.cavallium.dbengine.lucene.LLScoreDoc;
|
||||
import it.cavallium.dbengine.lucene.HugePqCodec;
|
||||
import it.cavallium.dbengine.lucene.HugePqPriorityQueue;
|
||||
import it.cavallium.dbengine.lucene.PriorityQueue;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.function.Function;
|
||||
import org.apache.lucene.search.HitQueue;
|
||||
import org.apache.lucene.search.ScoreDoc;
|
||||
import org.assertj.core.description.Description;
|
||||
import org.assertj.core.description.TextDescription;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public class TestHugePqHitQueue {
|
||||
|
||||
public static final int NUM_HITS = 1024;
|
||||
|
||||
private LLTempHugePqEnv env;
|
||||
private SafeCloseable hugePqQueue;
|
||||
|
||||
private TestingPriorityQueue testingPriorityQueue;
|
||||
|
||||
protected static boolean lessThan(ScoreDoc hitA, ScoreDoc hitB) {
|
||||
if (hitA.score == hitB.score) {
|
||||
return hitA.doc > hitB.doc;
|
||||
} else {
|
||||
return hitA.score < hitB.score;
|
||||
}
|
||||
}
|
||||
|
||||
private static int compareScoreDoc(ScoreDoc hitA, ScoreDoc hitB) {
|
||||
if (hitA.score == hitB.score) {
|
||||
if (hitA.doc == hitB.doc) {
|
||||
return Integer.compare(hitA.shardIndex, hitB.shardIndex);
|
||||
} else {
|
||||
return Integer.compare(hitB.doc, hitA.doc);
|
||||
}
|
||||
} else {
|
||||
return Float.compare(hitA.score, hitB.score);
|
||||
}
|
||||
}
|
||||
|
||||
private static void assertEqualsScoreDoc(Description description, ScoreDoc expected, ScoreDoc actual) {
|
||||
org.assertj.core.api.Assertions.assertThat(toLLScoreDoc(expected)).as(description).isEqualTo(toLLScoreDoc(actual));
|
||||
}
|
||||
|
||||
private static void assertEqualsScoreDoc(List<ScoreDoc> expected, List<ScoreDoc> actual) {
|
||||
var list1 = expected.iterator();
|
||||
var list2 = actual.iterator();
|
||||
Assertions.assertEquals(expected.size(), actual.size());
|
||||
while (list1.hasNext() && list2.hasNext()) {
|
||||
Assertions.assertFalse(lessThan(list1.next(), list2.next()));
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() throws IOException {
|
||||
this.env = new LLTempHugePqEnv();
|
||||
var hugePqQueue = new HugePqPriorityQueue<ScoreDoc>(env, new HugePqCodec<>() {
|
||||
|
||||
@Override
|
||||
public Buffer serialize(Function<Integer, Buffer> allocator, ScoreDoc data) {
|
||||
var buf = allocator.apply(Float.BYTES + Integer.BYTES + Integer.BYTES);
|
||||
buf.writerOffset(Float.BYTES + Integer.BYTES + Integer.BYTES);
|
||||
setScore(buf, data.score);
|
||||
setDoc(buf, data.doc);
|
||||
setShardIndex(buf, data.shardIndex);
|
||||
return buf;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScoreDoc deserialize(Buffer buf) {
|
||||
return new ScoreDoc(getDoc(buf), getScore(buf), getShardIndex(buf));
|
||||
}
|
||||
|
||||
private static float getScore(Buffer hit) {
|
||||
return HugePqCodec.getLexFloat(hit, 0, false);
|
||||
}
|
||||
|
||||
private static int getDoc(Buffer hit) {
|
||||
return HugePqCodec.getLexInt(hit, Float.BYTES, true);
|
||||
}
|
||||
|
||||
private static int getShardIndex(Buffer hit) {
|
||||
return HugePqCodec.getLexInt(hit, Float.BYTES + Integer.BYTES, false);
|
||||
}
|
||||
|
||||
private static void setScore(Buffer hit, float score) {
|
||||
HugePqCodec.setLexFloat(hit, 0, false, score);
|
||||
}
|
||||
|
||||
private static void setDoc(Buffer hit, int doc) {
|
||||
HugePqCodec.setLexInt(hit, Float.BYTES, true, doc);
|
||||
}
|
||||
|
||||
private static void setShardIndex(Buffer hit, int shardIndex) {
|
||||
HugePqCodec.setLexInt(hit, Float.BYTES + Integer.BYTES, false, shardIndex);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScoreDoc clone(ScoreDoc obj) {
|
||||
return new ScoreDoc(obj.doc, obj.score, obj.shardIndex);
|
||||
}
|
||||
});
|
||||
this.hugePqQueue = hugePqQueue;
|
||||
PriorityQueueAdaptor<ScoreDoc> hitQueue = new PriorityQueueAdaptor<>(new HitQueue(NUM_HITS, false));
|
||||
Assertions.assertEquals(0, hugePqQueue.size());
|
||||
Assertions.assertEquals(0, hitQueue.size());
|
||||
this.testingPriorityQueue = new TestingPriorityQueue(hitQueue, hugePqQueue);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoOp() {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptyTop() {
|
||||
Assertions.assertNull(testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddSingle() {
|
||||
var item = new ScoreDoc(0, 0, 0);
|
||||
testingPriorityQueue.add(item);
|
||||
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), item, testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMulti() {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
var item = new ScoreDoc(i, i >> 1, -1);
|
||||
testingPriorityQueue.addUnsafe(item);
|
||||
}
|
||||
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), new ScoreDoc(1, 0, -1), testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMultiRandom() {
|
||||
var list = new ArrayList<Integer>(1000);
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
var ri = ThreadLocalRandom.current().nextInt(0, 20);
|
||||
list.add(ri);
|
||||
var item = new ScoreDoc(ri, ri << 1, ri % 4);
|
||||
testingPriorityQueue.addUnsafe(item);
|
||||
}
|
||||
list.sort(Comparator.reverseOrder());
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
var top = list.remove(list.size() - 1);
|
||||
assertEqualsScoreDoc(new TextDescription("%d value of %s", i, testingPriorityQueue), new ScoreDoc(top, top << 1, top % 4), testingPriorityQueue.pop());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMultiClear() {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
var item = new ScoreDoc(i, i >> 1, -1);
|
||||
testingPriorityQueue.addUnsafe(item);
|
||||
}
|
||||
testingPriorityQueue.clear();
|
||||
Assertions.assertNull(testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddRemove() {
|
||||
var item = new ScoreDoc(0, 0, -1);
|
||||
testingPriorityQueue.add(item);
|
||||
testingPriorityQueue.remove(item);
|
||||
Assertions.assertNull(testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddRemoveNonexistent() {
|
||||
var item = new ScoreDoc(0, 0, 0);
|
||||
testingPriorityQueue.addUnsafe(item);
|
||||
testingPriorityQueue.remove(new ScoreDoc(2, 0, 0));
|
||||
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), item, testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMultiRemove1() {
|
||||
ScoreDoc toRemove = null;
|
||||
ScoreDoc top = null;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
var item = new ScoreDoc(i, i >> 1, -1);
|
||||
if (i == 1) {
|
||||
toRemove = item;
|
||||
} else if (i == 0) {
|
||||
top = item;
|
||||
}
|
||||
testingPriorityQueue.addUnsafe(item);
|
||||
}
|
||||
testingPriorityQueue.removeUnsafe(toRemove);
|
||||
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), top, testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddMultiRemove2() {
|
||||
ScoreDoc toRemove = null;
|
||||
ScoreDoc top = null;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
var item = new ScoreDoc(i, i >> 1, -1);
|
||||
if (i == 0) {
|
||||
toRemove = item;
|
||||
} else if (i == 1) {
|
||||
top = item;
|
||||
}
|
||||
testingPriorityQueue.addUnsafe(item);
|
||||
}
|
||||
testingPriorityQueue.removeUnsafe(new ScoreDoc(0, 0, -1));
|
||||
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), top, testingPriorityQueue.top());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSort() {
|
||||
var sortedNumbers = new ArrayList<ScoreDoc>();
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
sortedNumbers.add(new ScoreDoc(i, i >> 1, -1));
|
||||
}
|
||||
sortedNumbers.sort(TestHugePqHitQueue::compareScoreDoc);
|
||||
var shuffledNumbers = new ArrayList<>(sortedNumbers);
|
||||
Collections.shuffle(shuffledNumbers, new Random(1000));
|
||||
|
||||
org.assertj.core.api.Assertions.assertThat(testingPriorityQueue.size()).isEqualTo(0);
|
||||
|
||||
for (ScoreDoc scoreDoc : shuffledNumbers) {
|
||||
testingPriorityQueue.addUnsafe(scoreDoc);
|
||||
}
|
||||
|
||||
org.assertj.core.api.Assertions.assertThat(testingPriorityQueue.size()).isEqualTo(sortedNumbers.size());
|
||||
|
||||
var newSortedNumbers = new ArrayList<ScoreDoc>();
|
||||
ScoreDoc popped;
|
||||
while ((popped = testingPriorityQueue.popUnsafe()) != null) {
|
||||
newSortedNumbers.add(popped);
|
||||
}
|
||||
org.assertj.core.api.Assertions.assertThat(testingPriorityQueue.size()).isEqualTo(0);
|
||||
|
||||
assertEqualsScoreDoc(sortedNumbers, newSortedNumbers);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() throws IOException {
|
||||
hugePqQueue.close();
|
||||
env.close();
|
||||
}
|
||||
|
||||
private static class TestingPriorityQueue implements PriorityQueue<ScoreDoc> {
|
||||
|
||||
private final PriorityQueue<ScoreDoc> referenceQueue;
|
||||
private final PriorityQueue<ScoreDoc> myQueue;
|
||||
|
||||
public TestingPriorityQueue(PriorityQueue<ScoreDoc> referenceQueue, PriorityQueue<ScoreDoc> myQueue) {
|
||||
this.referenceQueue = referenceQueue;
|
||||
this.myQueue = myQueue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(ScoreDoc element) {
|
||||
referenceQueue.add(element);
|
||||
myQueue.add(element);
|
||||
ensureEquality();
|
||||
}
|
||||
|
||||
public void addUnsafe(ScoreDoc element) {
|
||||
referenceQueue.add(element);
|
||||
myQueue.add(element);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScoreDoc top() {
|
||||
var top1 = referenceQueue.top();
|
||||
var top2 = myQueue.top();
|
||||
assertEqualsScoreDoc(new TextDescription("top value of %s", myQueue), top1, top2);
|
||||
return top2;
|
||||
}
|
||||
|
||||
public ScoreDoc topUnsafe() {
|
||||
var top1 = referenceQueue.top();
|
||||
var top2 = myQueue.top();
|
||||
return top2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScoreDoc pop() {
|
||||
var top1 = referenceQueue.pop();
|
||||
var top2 = myQueue.pop();
|
||||
assertEqualsScoreDoc(new TextDescription("top value of %s", myQueue), top1, top2);
|
||||
return top2;
|
||||
}
|
||||
|
||||
public ScoreDoc popUnsafe() {
|
||||
var top1 = referenceQueue.pop();
|
||||
var top2 = myQueue.pop();
|
||||
return top2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void replaceTop(ScoreDoc oldTop, ScoreDoc newTop) {
|
||||
referenceQueue.replaceTop(oldTop, newTop);
|
||||
myQueue.replaceTop(oldTop, newTop);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size() {
|
||||
var size1 = referenceQueue.size();
|
||||
var size2 = myQueue.size();
|
||||
Assertions.assertEquals(size1, size2);
|
||||
return size2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
referenceQueue.clear();
|
||||
myQueue.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean remove(ScoreDoc element) {
|
||||
var removedRef = referenceQueue.remove(element);
|
||||
var removedMy = myQueue.remove(element);
|
||||
Assertions.assertEquals(removedRef, removedMy);
|
||||
return removedMy;
|
||||
}
|
||||
|
||||
public boolean removeUnsafe(ScoreDoc element) {
|
||||
var removed1 = referenceQueue.remove(element);
|
||||
var removed2 = myQueue.remove(element);
|
||||
return removed2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ScoreDoc> iterate() {
|
||||
//noinspection BlockingMethodInNonBlockingContext
|
||||
var it1 = referenceQueue.iterate().collectList().blockOptional().orElseThrow();
|
||||
//noinspection BlockingMethodInNonBlockingContext
|
||||
var it2 = myQueue.iterate().collectList().blockOptional().orElseThrow();
|
||||
assertEqualsScoreDoc(it1, it2);
|
||||
return Flux.fromIterable(it2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
referenceQueue.close();
|
||||
myQueue.close();
|
||||
}
|
||||
|
||||
private void ensureEquality() {
|
||||
Assertions.assertEquals(referenceQueue.size(), myQueue.size());
|
||||
var referenceQueueElements = Lists.newArrayList(referenceQueue
|
||||
.iterate()
|
||||
.map(TestHugePqHitQueue::toLLScoreDoc)
|
||||
.toIterable());
|
||||
var testQueueElements = Lists.newArrayList(myQueue
|
||||
.iterate()
|
||||
.map(TestHugePqHitQueue::toLLScoreDoc)
|
||||
.toIterable());
|
||||
Assertions.assertEquals(referenceQueueElements, testQueueElements);
|
||||
}
|
||||
}
|
||||
|
||||
public static LLScoreDoc toLLScoreDoc(ScoreDoc scoreDoc) {
|
||||
if (scoreDoc == null) return null;
|
||||
return new LLScoreDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
|
||||
}
|
||||
}
|
@ -1,283 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.SyncUtils.run;
|
||||
import static it.cavallium.dbengine.SyncUtils.runVoid;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import io.netty5.buffer.Buffer;
|
||||
import io.netty5.util.Resource;
|
||||
import io.netty5.util.Send;
|
||||
import it.cavallium.dbengine.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.UpdateReturnMode;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public abstract class TestLLDictionary {
|
||||
|
||||
private final Logger log = LogManager.getLogger(this.getClass());
|
||||
private static final Mono<LLRange> RANGE_ALL = Mono.fromCallable(LLRange::all);
|
||||
private TestAllocator allocator;
|
||||
private TempDb tempDb;
|
||||
private LLKeyValueDatabase db;
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(allocator).block(), "TempDB");
|
||||
db = tempDb.db();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
getTempDbGenerator().closeTempDb(tempDb).block();
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
destroyAllocator(allocator);
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideArguments() {
|
||||
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
|
||||
}
|
||||
|
||||
public static Stream<Arguments> providePutArguments() {
|
||||
var updateModes = Arrays.stream(UpdateMode.values());
|
||||
return updateModes.flatMap(updateMode -> {
|
||||
var resultTypes = Arrays.stream(LLDictionaryResultType.values());
|
||||
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
|
||||
});
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideUpdateArguments() {
|
||||
var updateModes = Arrays.stream(UpdateMode.values());
|
||||
return updateModes.flatMap(updateMode -> {
|
||||
var resultTypes = Arrays.stream(UpdateReturnMode.values());
|
||||
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
|
||||
});
|
||||
}
|
||||
|
||||
private LLDictionary getDict(UpdateMode updateMode) {
|
||||
var dict = DbTestUtils.tempDictionary(db, updateMode).blockOptional().orElseThrow();
|
||||
var key1 = Mono.fromCallable(() -> fromString("test-key-1"));
|
||||
var key2 = Mono.fromCallable(() -> fromString("test-key-2"));
|
||||
var key3 = Mono.fromCallable(() -> fromString("test-key-3"));
|
||||
var key4 = Mono.fromCallable(() -> fromString("test-key-4"));
|
||||
var value = Mono.fromCallable(() -> fromString("test-value"));
|
||||
dict.put(key1, value, LLDictionaryResultType.VOID).block();
|
||||
dict.put(key2, value, LLDictionaryResultType.VOID).block();
|
||||
dict.put(key3, value, LLDictionaryResultType.VOID).block();
|
||||
dict.put(key4, value, LLDictionaryResultType.VOID).block();
|
||||
return dict;
|
||||
}
|
||||
|
||||
private Buffer fromString(String s) {
|
||||
var sb = s.getBytes(StandardCharsets.UTF_8);
|
||||
try (var b = db.getAllocator().allocate(sb.length + 3 + 13)) {
|
||||
assert b.writerOffset() == 0;
|
||||
assert b.readerOffset() == 0;
|
||||
b.writerOffset(3).writeBytes(sb);
|
||||
b.readerOffset(3);
|
||||
assert b.readableBytes() == sb.length;
|
||||
|
||||
var part1 = b.split();
|
||||
|
||||
return LLUtils.compositeBuffer(db.getAllocator(), part1.send(), b.send());
|
||||
}
|
||||
}
|
||||
|
||||
private String toString(Buffer bb) {
|
||||
try (bb) {
|
||||
byte[] data = new byte[bb.readableBytes()];
|
||||
bb.copyInto(bb.readerOffset(), data, 0, data.length);
|
||||
return new String(data, StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoOp() {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoOpAllocation() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
var a = allocator.allocator().allocate(i * 512);
|
||||
a.send().receive().close();
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetDict(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
Assertions.assertNotNull(dict);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetColumnName(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
Assertions.assertEquals("hash_map_testmap", dict.getColumnName());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetAllocator(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var alloc = dict.getAllocator();
|
||||
Assertions.assertEquals(alloc, alloc);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGet(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyEx = Mono.fromCallable(() -> fromString("test-key-1"));
|
||||
var keyNonEx = Mono.fromCallable(() -> fromString("test-nonexistent"));
|
||||
Assertions.assertEquals("test-value", run(dict.get(null, keyEx).map(this::toString)));
|
||||
Assertions.assertEquals("test-value", run(dict.get(null, keyEx).map(this::toString)));
|
||||
Assertions.assertEquals("test-value", run(dict.get(null, keyEx).map(this::toString)));
|
||||
Assertions.assertEquals((String) null, run(dict.get(null, keyNonEx).map(this::toString)));
|
||||
Assertions.assertEquals((String) null, run(dict.get(null, keyNonEx).map(this::toString)));
|
||||
Assertions.assertEquals((String) null, run(dict.get(null, keyNonEx).map(this::toString)));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testPutExisting(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyEx = Mono.fromCallable(() -> fromString("test-key-1"));
|
||||
var value = Mono.fromCallable(() -> fromString("test-value"));
|
||||
|
||||
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
|
||||
runVoid(dict.put(keyEx, value, resultType).then().doOnDiscard(Resource.class, Resource::close));
|
||||
|
||||
var afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
Assertions.assertEquals(0, afterSize - beforeSize);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testPutNew(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyNonEx = Mono.fromCallable(() -> fromString("test-nonexistent"));
|
||||
var value = Mono.fromCallable(() -> fromString("test-value"));
|
||||
|
||||
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
|
||||
runVoid(dict.put(keyNonEx, value, resultType).then().doOnDiscard(Resource.class, Resource::close));
|
||||
|
||||
var afterSize = run(dict.sizeRange(null, Mono.fromCallable(LLRange::all), false));
|
||||
Assertions.assertEquals(1, afterSize - beforeSize);
|
||||
|
||||
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).collectList()).contains("test-nonexistent"));
|
||||
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).collectList()).contains("test-nonexistent"));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetUpdateMode(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
assertEquals(updateMode, dict.getUpdateMode());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideUpdateArguments")
|
||||
public void testUpdateExisting(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyEx = Mono.fromCallable(() -> fromString("test-key-1"));
|
||||
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
long afterSize;
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(keyEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
|
||||
);
|
||||
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
assertEquals(0, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(keyEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
|
||||
);
|
||||
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
assertEquals(0, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(keyEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
|
||||
);
|
||||
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
assertEquals(0, afterSize - beforeSize);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideUpdateArguments")
|
||||
public void testUpdateNew(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
|
||||
int expected = updateMode == UpdateMode.DISALLOW ? 0 : 1;
|
||||
var dict = getDict(updateMode);
|
||||
var keyNonEx = Mono.fromCallable(() -> fromString("test-nonexistent"));
|
||||
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
long afterSize;
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
|
||||
);
|
||||
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
assertEquals(expected, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
|
||||
);
|
||||
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
assertEquals(expected, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
|
||||
);
|
||||
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
|
||||
assertEquals(expected, afterSize - beforeSize);
|
||||
|
||||
if (updateMode != UpdateMode.DISALLOW) {
|
||||
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).collectList()).contains(
|
||||
"test-nonexistent"));
|
||||
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).collectList()).contains(
|
||||
"test-nonexistent"));
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testUpdateAndGetDelta(UpdateMode updateMode) {
|
||||
log.warn("Test not implemented");
|
||||
//todo: implement
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testClear(UpdateMode updateMode) {
|
||||
log.warn("Test not implemented");
|
||||
//todo: implement
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
log.warn("Test not implemented");
|
||||
//todo: implement
|
||||
}
|
||||
}
|
@ -1,183 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDb;
|
||||
|
||||
import it.cavallium.data.generator.nativedata.StringSerializer;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.LLSingleton;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseInt;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseLong;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseSingleton;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.test.StepVerifier;
|
||||
|
||||
public abstract class TestSingletons {
|
||||
|
||||
private TestAllocator allocator;
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
private static Stream<Arguments> provideNumberWithRepeats() {
|
||||
return Stream.of(
|
||||
Arguments.of(Integer.MIN_VALUE, 2),
|
||||
Arguments.of(-11, 2),
|
||||
Arguments.of(0, 3),
|
||||
Arguments.of(102, 5)
|
||||
);
|
||||
}
|
||||
|
||||
private static Stream<Arguments> provideLongNumberWithRepeats() {
|
||||
return Stream.of(
|
||||
Arguments.of(Long.MIN_VALUE, 2),
|
||||
Arguments.of(-11L, 2),
|
||||
Arguments.of(0L, 3),
|
||||
Arguments.of(102L, 5)
|
||||
);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
destroyAllocator(allocator);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateInteger() {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", 0)
|
||||
.flatMap(dbInt -> dbInt.get(null))
|
||||
.then()
|
||||
))
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateIntegerNoop() {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", 0)
|
||||
.then()
|
||||
))
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateLong() {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempLong(db, "test", 0)
|
||||
.flatMap(dbLong -> dbLong.get(null))
|
||||
.then()
|
||||
))
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateSingleton() {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempSingleton(db, "testsingleton")
|
||||
.flatMap(dbSingleton -> dbSingleton.get(null))
|
||||
))
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints = {Integer.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Integer.MAX_VALUE})
|
||||
public void testDefaultValueInteger(int i) {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", i)
|
||||
.flatMap(dbInt -> dbInt.get(null))
|
||||
))
|
||||
.expectNext(i)
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(longs = {Long.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Long.MAX_VALUE})
|
||||
public void testDefaultValueLong(long i) {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempLong(db, "test", i)
|
||||
.flatMap(dbLong -> dbLong.get(null))
|
||||
))
|
||||
.expectNext(i)
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideNumberWithRepeats")
|
||||
public void testSetInteger(Integer i, Integer repeats) {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", 0)
|
||||
.flatMap(dbInt -> Mono
|
||||
.defer(() -> dbInt.set((int) System.currentTimeMillis()))
|
||||
.repeat(repeats)
|
||||
.then(dbInt.set(i))
|
||||
.then(dbInt.get(null)))
|
||||
))
|
||||
.expectNext(i)
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideLongNumberWithRepeats")
|
||||
public void testSetLong(Long i, Integer repeats) {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempLong(db, "test", 0)
|
||||
.flatMap(dbLong -> Mono
|
||||
.defer(() -> dbLong.set(System.currentTimeMillis()))
|
||||
.repeat(repeats)
|
||||
.then(dbLong.set(i))
|
||||
.then(dbLong.get(null)))
|
||||
))
|
||||
.expectNext(i)
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideLongNumberWithRepeats")
|
||||
public void testSetSingleton(Long i, Integer repeats) {
|
||||
StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempSingleton(db, "test")
|
||||
.flatMap(dbSingleton -> Mono
|
||||
.defer(() -> dbSingleton.set(Long.toString(System.currentTimeMillis())))
|
||||
.repeat(repeats)
|
||||
.then(dbSingleton.set(Long.toString(i)))
|
||||
.then(dbSingleton.get(null)))
|
||||
))
|
||||
.expectNext(Long.toString(i))
|
||||
.verifyComplete();
|
||||
}
|
||||
|
||||
public static Mono<DatabaseInt> tempInt(LLKeyValueDatabase database, String name, int defaultValue) {
|
||||
return database
|
||||
.getInteger("ints", name, defaultValue);
|
||||
}
|
||||
|
||||
public static Mono<DatabaseLong> tempLong(LLKeyValueDatabase database, String name, long defaultValue) {
|
||||
return database
|
||||
.getLong("longs", name, defaultValue);
|
||||
}
|
||||
|
||||
public static Mono<DatabaseSingleton<String>> tempSingleton(LLKeyValueDatabase database, String name) {
|
||||
return database
|
||||
.getSingleton("longs", name)
|
||||
.map(singleton -> new DatabaseSingleton<>(singleton, Serializer.UTF8_SERIALIZER));
|
||||
}
|
||||
}
|
@ -1,289 +0,0 @@
|
||||
package it.cavallium.dbengine.database.remote;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import io.netty.handler.ssl.ClientAuth;
|
||||
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
|
||||
import io.netty.handler.ssl.util.SelfSignedCertificate;
|
||||
import io.netty.incubator.codec.quic.InsecureQuicTokenHandler;
|
||||
import io.netty.incubator.codec.quic.QuicConnectionIdGenerator;
|
||||
import io.netty.incubator.codec.quic.QuicSslContext;
|
||||
import io.netty.incubator.codec.quic.QuicSslContextBuilder;
|
||||
import it.cavallium.dbengine.database.remote.RPCCodecs.RPCEventCodec;
|
||||
import it.cavallium.dbengine.rpc.current.data.Empty;
|
||||
import it.cavallium.dbengine.rpc.current.data.RPCEvent;
|
||||
import it.cavallium.dbengine.rpc.current.data.SingletonGet;
|
||||
import it.cavallium.dbengine.rpc.current.data.nullables.NullableLLSnapshot;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.logging.Level;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.publisher.SignalType;
|
||||
import reactor.netty.Connection;
|
||||
import reactor.netty.incubator.quic.QuicClient;
|
||||
import reactor.netty.incubator.quic.QuicConnection;
|
||||
|
||||
class QuicUtilsTest {
|
||||
|
||||
private static final int NORMAL = 0;
|
||||
private static final int WAIT_TIME = 1;
|
||||
private static final int FAIL_IMMEDIATELY = 2;
|
||||
private static final int WAIT_TIME_THEN_FAIL = 3;
|
||||
|
||||
|
||||
private Connection serverConn;
|
||||
private QuicConnection clientConn;
|
||||
private InetSocketAddress clientAddress;
|
||||
private InetSocketAddress serverAddress;
|
||||
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws CertificateException {
|
||||
var selfSignedCert = new SelfSignedCertificate();
|
||||
this.clientAddress = new InetSocketAddress("localhost", 8081);
|
||||
this.serverAddress = new InetSocketAddress("localhost", 8080);
|
||||
QuicSslContext sslContext = QuicSslContextBuilder
|
||||
.forServer(selfSignedCert.key(), null, selfSignedCert.cert())
|
||||
.applicationProtocols("db/0.9")
|
||||
.clientAuth(ClientAuth.NONE)
|
||||
.build();
|
||||
var qs = reactor.netty.incubator.quic.QuicServer
|
||||
.create()
|
||||
.tokenHandler(InsecureQuicTokenHandler.INSTANCE)
|
||||
.bindAddress(() -> serverAddress)
|
||||
.secure(sslContext)
|
||||
.idleTimeout(Duration.ofSeconds(30))
|
||||
.connectionIdAddressGenerator(QuicConnectionIdGenerator.randomGenerator())
|
||||
.initialSettings(spec -> spec
|
||||
.maxData(10000000)
|
||||
.maxStreamDataBidirectionalLocal(1000000)
|
||||
.maxStreamDataBidirectionalRemote(1000000)
|
||||
.maxStreamsBidirectional(100)
|
||||
.maxStreamsUnidirectional(100)
|
||||
)
|
||||
.handleStream((in, out) -> in
|
||||
.withConnection(conn -> conn.addHandler(new RPCEventCodec()))
|
||||
.receiveObject()
|
||||
.cast(RPCEvent.class)
|
||||
.log("recv", Level.FINEST)
|
||||
.flatMapSequential(req -> (switch ((int) ((SingletonGet) req).singletonId()) {
|
||||
case NORMAL -> Mono.<RPCEvent>just(Empty.of());
|
||||
case FAIL_IMMEDIATELY -> Mono.<RPCEvent>error(new Throwable("Expected error"));
|
||||
case WAIT_TIME -> Mono.delay(Duration.ofSeconds(3)).<RPCEvent>thenReturn(Empty.of());
|
||||
case WAIT_TIME_THEN_FAIL -> Mono
|
||||
.delay(Duration.ofSeconds(3))
|
||||
.then(Mono.<RPCEvent>error(new Throwable("Expected error")));
|
||||
default -> Mono.<RPCEvent>error(new UnsupportedOperationException("Unsupported request id " + req));
|
||||
}).log("Server", Level.SEVERE, SignalType.ON_ERROR).onErrorResume(QuicUtils::catchRPCErrors))
|
||||
.concatMap(message -> Mono.defer(() -> out
|
||||
.withConnection(conn -> conn.addHandler(new RPCEventCodec()))
|
||||
.sendObject(message)
|
||||
.then())
|
||||
.log("send", Level.FINEST)
|
||||
)
|
||||
);
|
||||
this.serverConn = qs.bindNow();
|
||||
|
||||
var clientSslContext = QuicSslContextBuilder
|
||||
.forClient()
|
||||
.trustManager(InsecureTrustManagerFactory.INSTANCE)
|
||||
.applicationProtocols("db/0.9")
|
||||
.build();
|
||||
this.clientConn = QuicClient.create()
|
||||
.bindAddress(() -> new InetSocketAddress(0))
|
||||
.remoteAddress(() -> serverAddress)
|
||||
.secure(clientSslContext)
|
||||
.idleTimeout(Duration.ofSeconds(30))
|
||||
.initialSettings(spec -> spec
|
||||
.maxData(10000000)
|
||||
.maxStreamDataBidirectionalLocal(1000000)
|
||||
)
|
||||
.connectNow();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (clientConn != null) {
|
||||
clientConn.disposeNow();
|
||||
}
|
||||
if (serverConn != null) {
|
||||
serverConn.disposeNow();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendSimpleRequest() {
|
||||
RPCEvent response = QuicUtils.<RPCEvent, RPCEvent>sendSimpleRequest(clientConn,
|
||||
RPCEventCodec::new,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty())
|
||||
).blockOptional().orElseThrow();
|
||||
assertEquals(Empty.of(), response);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendSimpleRequestFlux() {
|
||||
List<RPCEvent> results = QuicUtils.<RPCEvent, RPCEvent>sendSimpleRequestFlux(clientConn,
|
||||
RPCEventCodec::new,
|
||||
RPCEventCodec::new,
|
||||
Flux.just(
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty())
|
||||
)
|
||||
).collectList().blockOptional().orElseThrow();
|
||||
assertEquals(5, results.size());
|
||||
assertEquals(List.of(Empty.of(), Empty.of(), Empty.of(), Empty.of(), Empty.of()), results);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendUpdateFluxNormal() {
|
||||
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
serverData -> Mono.fromCallable(() -> {
|
||||
assertEquals(Empty.of(), serverData);
|
||||
return new SingletonGet(NORMAL, NullableLLSnapshot.empty());
|
||||
})
|
||||
).blockOptional().orElseThrow();
|
||||
assertEquals(Empty.of(), results);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendUpdateFluxSlowClient() {
|
||||
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
serverData -> Mono.<RPCEvent>fromCallable(() -> {
|
||||
assertEquals(Empty.of(), serverData);
|
||||
return new SingletonGet(NORMAL, NullableLLSnapshot.empty());
|
||||
}).delayElement(Duration.ofSeconds(2))
|
||||
).blockOptional().orElseThrow();
|
||||
assertEquals(Empty.of(), results);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendUpdateFluxSlowServer() {
|
||||
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty()),
|
||||
serverData -> Mono.fromCallable(() -> {
|
||||
assertEquals(Empty.of(), serverData);
|
||||
return new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty());
|
||||
})
|
||||
).blockOptional().orElseThrow();
|
||||
assertEquals(Empty.of(), results);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendUpdateFluxSlowClientAndServer() {
|
||||
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty()),
|
||||
serverData -> Mono.<RPCEvent>fromCallable(() -> {
|
||||
assertEquals(Empty.of(), serverData);
|
||||
return new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty());
|
||||
}).delayElement(Duration.ofSeconds(2))
|
||||
).blockOptional().orElseThrow();
|
||||
assertEquals(Empty.of(), results);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendUpdateClientFail() {
|
||||
class ExpectedException extends Throwable {}
|
||||
|
||||
assertThrows(ExpectedException.class, () -> {
|
||||
try {
|
||||
RPCEvent results = QuicUtils
|
||||
.<RPCEvent>sendUpdate(clientConn,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
serverData -> Mono.error(new ExpectedException())
|
||||
)
|
||||
.blockOptional()
|
||||
.orElseThrow();
|
||||
} catch (Throwable e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendUpdateServerFail1() {
|
||||
assertThrows(RPCException.class,
|
||||
() -> QuicUtils
|
||||
.<RPCEvent>sendUpdate(clientConn,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(FAIL_IMMEDIATELY, NullableLLSnapshot.empty()),
|
||||
serverData -> Mono.fromCallable(() -> {
|
||||
fail("Called update");
|
||||
return new SingletonGet(NORMAL, NullableLLSnapshot.empty());
|
||||
})
|
||||
)
|
||||
.blockOptional()
|
||||
.orElseThrow()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendUpdateServerFail2() {
|
||||
assertThrows(RPCException.class,
|
||||
() -> QuicUtils
|
||||
.<RPCEvent>sendUpdate(clientConn,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
|
||||
serverData -> Mono.fromCallable(() -> {
|
||||
assertEquals(Empty.of(), serverData);
|
||||
return new SingletonGet(FAIL_IMMEDIATELY, NullableLLSnapshot.empty());
|
||||
})
|
||||
)
|
||||
.blockOptional()
|
||||
.orElseThrow()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendSimpleRequestConcurrently() {
|
||||
// Send the request a second time
|
||||
var requestMono = QuicUtils.<RPCEvent, RPCEvent>sendSimpleRequest(clientConn,
|
||||
RPCEventCodec::new,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(NORMAL, NullableLLSnapshot.empty())
|
||||
);
|
||||
var results = Flux
|
||||
.merge(requestMono, requestMono, requestMono, requestMono, requestMono)
|
||||
.collectList()
|
||||
.blockOptional()
|
||||
.orElseThrow();
|
||||
assertEquals(5, results.size());
|
||||
assertEquals(List.of(Empty.of(), Empty.of(), Empty.of(), Empty.of(), Empty.of()), results);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendFailedRequest() {
|
||||
assertThrows(RPCException.class,
|
||||
() -> QuicUtils
|
||||
.<RPCEvent, RPCEvent>sendSimpleRequest(clientConn,
|
||||
RPCEventCodec::new,
|
||||
RPCEventCodec::new,
|
||||
new SingletonGet(FAIL_IMMEDIATELY, NullableLLSnapshot.empty())
|
||||
)
|
||||
.blockOptional()
|
||||
.orElseThrow()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void createStream() {
|
||||
}
|
||||
}
|
@ -1,174 +0,0 @@
|
||||
package it.cavallium.dbengine.lucene.hugepq.search;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
|
||||
import it.cavallium.dbengine.lucene.LLFieldDoc;
|
||||
import it.cavallium.dbengine.lucene.LLScoreDoc;
|
||||
import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.searcher.ShardIndexSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.SharedShardStatistics;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import org.apache.lucene.document.Field.Store;
|
||||
import org.apache.lucene.document.LongPoint;
|
||||
import org.apache.lucene.document.SortedNumericDocValuesField;
|
||||
import org.apache.lucene.document.SortedNumericDocValuesField;
|
||||
import org.apache.lucene.document.SortedNumericDocValuesField;
|
||||
import org.apache.lucene.document.SortedNumericDocValuesField;
|
||||
import org.apache.lucene.document.TextField;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.FieldDoc;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.search.SortField.Type;
|
||||
import org.apache.lucene.search.SortedNumericSortField;
|
||||
import org.apache.lucene.search.TopFieldCollector;
|
||||
import org.apache.lucene.search.TopScoreDocCollector;
|
||||
import org.apache.lucene.search.TotalHits.Relation;
|
||||
import org.apache.lucene.store.ByteBuffersDirectory;
|
||||
import org.apache.lucene.util.QueryBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class HugePqFullFieldDocCollectorTest {
|
||||
Sort sort = new Sort(new SortedNumericSortField("number_sort", Type.LONG));
|
||||
Query luceneQuery = LongPoint.newRangeQuery("number", -100, 100);
|
||||
|
||||
@Test
|
||||
public void testSingleShard() throws IOException {
|
||||
try (var dir = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
|
||||
var analyzer = new WordAnalyzer(true, true);
|
||||
var writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
|
||||
writer.updateDocument(new Term("id", "00"), List.of(new SortedNumericDocValuesField("number_sort", 1), new LongPoint("number", 1)));
|
||||
writer.updateDocument(new Term("id", "01"), List.of(new SortedNumericDocValuesField("number_sort", 44), new LongPoint("number", 44)));
|
||||
writer.updateDocument(new Term("id", "02"), List.of(new SortedNumericDocValuesField("number_sort", 203), new LongPoint("number", 203)));
|
||||
writer.updateDocument(new Term("id", "03"), List.of(new SortedNumericDocValuesField("number_sort", 209), new LongPoint("number", 209)));
|
||||
writer.updateDocument(new Term("id", "04"), List.of(new SortedNumericDocValuesField("number_sort", -33), new LongPoint("number", -33)));
|
||||
writer.updateDocument(new Term("id", "05"), List.of(new SortedNumericDocValuesField("number_sort", 0), new LongPoint("number", 0)));
|
||||
writer.updateDocument(new Term("id", "06"), List.of(new SortedNumericDocValuesField("number_sort", 933), new LongPoint("number", 933)));
|
||||
writer.updateDocument(new Term("id", "07"), List.of(new SortedNumericDocValuesField("number_sort", 6), new LongPoint("number", 6)));
|
||||
writer.updateDocument(new Term("id", "08"), List.of(new SortedNumericDocValuesField("number_sort", -11), new LongPoint("number", -11)));
|
||||
writer.updateDocument(new Term("id", "09"), List.of(new SortedNumericDocValuesField("number_sort", 9996), new LongPoint("number", 9996)));
|
||||
writer.updateDocument(new Term("id", "10"), List.of(new SortedNumericDocValuesField("number_sort", 9), new LongPoint("number", 9)));
|
||||
writer.updateDocument(new Term("id", "11"), List.of(new SortedNumericDocValuesField("number_sort", 66), new LongPoint("number", 66)));
|
||||
writer.updateDocument(new Term("id", "12"), List.of(new SortedNumericDocValuesField("number_sort", 88), new LongPoint("number", 88)));
|
||||
writer.updateDocument(new Term("id", "13"), List.of(new SortedNumericDocValuesField("number_sort", 222), new LongPoint("number", 222)));
|
||||
writer.updateDocument(new Term("id", "14"), List.of(new SortedNumericDocValuesField("number_sort", -2), new LongPoint("number", -2)));
|
||||
writer.updateDocument(new Term("id", "15"), List.of(new SortedNumericDocValuesField("number_sort", 7), new LongPoint("number", 7)));
|
||||
writer.updateDocument(new Term("id", "16"), List.of(new SortedNumericDocValuesField("number_sort", 1010912093), new LongPoint("number", 1010912093)));
|
||||
writer.updateDocument(new Term("id", "17"), List.of(new SortedNumericDocValuesField("number_sort", -3894789), new LongPoint("number", -3894789)));
|
||||
writer.updateDocument(new Term("id", "18"), List.of(new SortedNumericDocValuesField("number_sort", 122), new LongPoint("number", 122)));
|
||||
writer.updateDocument(new Term("id", "19"), List.of(new SortedNumericDocValuesField("number_sort", 2), new LongPoint("number", 2)));
|
||||
writer.flush();
|
||||
writer.commit();
|
||||
try (var reader = DirectoryReader.open(writer, true, true)) {
|
||||
var searcher = new IndexSearcher(reader);
|
||||
var expectedResults = searcher.search(luceneQuery, 20, sort, false);
|
||||
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
|
||||
var expectedDocs = Arrays
|
||||
.stream(expectedResults.scoreDocs)
|
||||
.map(sd -> (FieldDoc) sd)
|
||||
.map(fieldDoc -> new LLFieldDoc(fieldDoc.doc, fieldDoc.score, fieldDoc.shardIndex, Arrays.asList(fieldDoc.fields)))
|
||||
.toList();
|
||||
try (var collector = HugePqFullFieldDocCollector.create(env, sort, 20, Integer.MAX_VALUE)) {
|
||||
searcher.search(luceneQuery, collector);
|
||||
var docs = collector.fullDocs().iterate().collectList().blockOptional().orElseThrow();
|
||||
System.out.println("Expected docs:");
|
||||
for (var expectedDoc : expectedDocs) {
|
||||
System.out.println(expectedDoc);
|
||||
}
|
||||
System.out.println("");
|
||||
System.out.println("Obtained docs:");
|
||||
for (var doc : docs) {
|
||||
System.out.println(doc);
|
||||
}
|
||||
assertEquals(expectedDocs,
|
||||
docs.stream().map(elem -> new LLFieldDoc(elem.doc(), elem.score(), -1, elem.fields())).toList()
|
||||
);
|
||||
assertEquals(expectedTotalHits, new TotalHitsCount(collector.getTotalHits(), true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiShard() throws IOException {
|
||||
try (var dir1 = new ByteBuffersDirectory(); var dir2 = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
|
||||
var analyzer = new WordAnalyzer(true, true);
|
||||
var writer1 = new IndexWriter(dir1, new IndexWriterConfig(analyzer));
|
||||
var writer2 = new IndexWriter(dir2, new IndexWriterConfig(analyzer));
|
||||
writer1.updateDocument(new Term("id", "00"), List.of(new SortedNumericDocValuesField("number_sort", 1), new LongPoint("number", 1)));
|
||||
writer1.updateDocument(new Term("id", "01"), List.of(new SortedNumericDocValuesField("number_sort", 44), new LongPoint("number", 44)));
|
||||
writer1.updateDocument(new Term("id", "02"), List.of(new SortedNumericDocValuesField("number_sort", 203), new LongPoint("number", 203)));
|
||||
writer1.updateDocument(new Term("id", "03"), List.of(new SortedNumericDocValuesField("number_sort", 209), new LongPoint("number", 209)));
|
||||
writer1.updateDocument(new Term("id", "04"), List.of(new SortedNumericDocValuesField("number_sort", -33), new LongPoint("number", -33)));
|
||||
writer1.updateDocument(new Term("id", "05"), List.of(new SortedNumericDocValuesField("number_sort", 0), new LongPoint("number", 0)));
|
||||
writer1.updateDocument(new Term("id", "06"), List.of(new SortedNumericDocValuesField("number_sort", 933), new LongPoint("number", 933)));
|
||||
writer1.updateDocument(new Term("id", "07"), List.of(new SortedNumericDocValuesField("number_sort", 6), new LongPoint("number", 6)));
|
||||
writer1.updateDocument(new Term("id", "08"), List.of(new SortedNumericDocValuesField("number_sort", -11), new LongPoint("number", -11)));
|
||||
writer1.updateDocument(new Term("id", "09"), List.of(new SortedNumericDocValuesField("number_sort", 9996), new LongPoint("number", 9996)));
|
||||
writer2.updateDocument(new Term("id", "10"), List.of(new SortedNumericDocValuesField("number_sort", 9), new LongPoint("number", 9)));
|
||||
writer2.updateDocument(new Term("id", "11"), List.of(new SortedNumericDocValuesField("number_sort", 66), new LongPoint("number", 66)));
|
||||
writer2.updateDocument(new Term("id", "12"), List.of(new SortedNumericDocValuesField("number_sort", 88), new LongPoint("number", 88)));
|
||||
writer2.updateDocument(new Term("id", "13"), List.of(new SortedNumericDocValuesField("number_sort", 222), new LongPoint("number", 222)));
|
||||
writer2.updateDocument(new Term("id", "14"), List.of(new SortedNumericDocValuesField("number_sort", -2), new LongPoint("number", -2)));
|
||||
writer2.updateDocument(new Term("id", "15"), List.of(new SortedNumericDocValuesField("number_sort", 7), new LongPoint("number", 7)));
|
||||
writer2.updateDocument(new Term("id", "16"), List.of(new SortedNumericDocValuesField("number_sort", 1010912093), new LongPoint("number", 1010912093)));
|
||||
writer2.updateDocument(new Term("id", "17"), List.of(new SortedNumericDocValuesField("number_sort", -3894789), new LongPoint("number", -3894789)));
|
||||
writer2.updateDocument(new Term("id", "18"), List.of(new SortedNumericDocValuesField("number_sort", 122), new LongPoint("number", 122)));
|
||||
writer2.updateDocument(new Term("id", "19"), List.of(new SortedNumericDocValuesField("number_sort", 2), new LongPoint("number", 2)));
|
||||
writer1.flush();
|
||||
writer2.flush();
|
||||
writer1.commit();
|
||||
writer2.commit();
|
||||
var sharedStats = new SharedShardStatistics();
|
||||
try (var reader1 = DirectoryReader.open(writer1, true, true);
|
||||
var reader2 = DirectoryReader.open(writer2, true, true)) {
|
||||
var searcher1 = new IndexSearcher(reader1);
|
||||
var searcher2 = new IndexSearcher(reader2);
|
||||
var shardSearcher1 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 0);
|
||||
var shardSearcher2 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 1);
|
||||
var standardSharedManager = TopFieldCollector.createSharedManager(sort, 20, null, Integer.MAX_VALUE);
|
||||
var standardCollector1 = standardSharedManager.newCollector();
|
||||
var standardCollector2 = standardSharedManager.newCollector();
|
||||
shardSearcher1.search(luceneQuery, standardCollector1);
|
||||
shardSearcher2.search(luceneQuery, standardCollector2);
|
||||
var expectedResults = standardSharedManager.reduce(List.of(standardCollector1, standardCollector2));
|
||||
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
|
||||
var expectedDocs = Arrays
|
||||
.stream(expectedResults.scoreDocs)
|
||||
.map(sd -> (FieldDoc) sd)
|
||||
.map(fieldDoc -> new LLFieldDoc(fieldDoc.doc, fieldDoc.score, fieldDoc.shardIndex, Arrays.asList(fieldDoc.fields)))
|
||||
.toList();
|
||||
var collectorManager = HugePqFullFieldDocCollector.createSharedManager(env, sort, 20, Integer.MAX_VALUE);
|
||||
var collector1 = collectorManager.newCollector();
|
||||
var collector2 = collectorManager.newCollector();
|
||||
shardSearcher1.search(luceneQuery, collector1);
|
||||
shardSearcher2.search(luceneQuery, collector2);
|
||||
try (var results = collectorManager.reduce(List.of(collector1, collector2))) {
|
||||
var docs = results.iterate().collectList().blockOptional().orElseThrow();
|
||||
System.out.println("Expected docs:");
|
||||
for (var expectedDoc : expectedDocs) {
|
||||
System.out.println(expectedDoc);
|
||||
}
|
||||
System.out.println("");
|
||||
System.out.println("Obtained docs:");
|
||||
for (var doc : docs) {
|
||||
System.out.println(doc);
|
||||
}
|
||||
assertEquals(expectedDocs,
|
||||
docs.stream().map(elem -> new LLFieldDoc(elem.doc(), elem.score(), -1, elem.fields())).toList()
|
||||
);
|
||||
assertEquals(expectedTotalHits, new TotalHitsCount(results.totalHits().value, results.totalHits().relation == Relation.EQUAL_TO));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,178 +0,0 @@
|
||||
package it.cavallium.dbengine.lucene.hugepq.search;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import it.cavallium.dbengine.client.query.QueryUtils;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.disk.IndexSearcherManager;
|
||||
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
|
||||
import it.cavallium.dbengine.lucene.LLScoreDoc;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
import it.cavallium.dbengine.lucene.analyzer.LegacyWordAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.searcher.ShardIndexSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.SharedShardStatistics;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import org.apache.lucene.document.Field.Store;
|
||||
import org.apache.lucene.document.TextField;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.CollectorManager;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.PhraseQuery;
|
||||
import org.apache.lucene.search.ScoreDoc;
|
||||
import org.apache.lucene.search.SearcherManager;
|
||||
import org.apache.lucene.search.SimpleCollector;
|
||||
import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.search.TopDocsCollector;
|
||||
import org.apache.lucene.search.TopScoreDocCollector;
|
||||
import org.apache.lucene.search.TotalHits.Relation;
|
||||
import org.apache.lucene.store.ByteBuffersDirectory;
|
||||
import org.apache.lucene.util.QueryBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class HugePqFullScoreDocCollectorTest {
|
||||
|
||||
@Test
|
||||
public void testSingleShard() throws IOException {
|
||||
try (var dir = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
|
||||
var analyzer = new WordAnalyzer(true, true);
|
||||
var writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
|
||||
writer.updateDocument(new Term("id", "00"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "01"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "02"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "03"), List.of(new TextField("text", "Marios Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "04"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "05"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "06"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "07"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "08"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "09"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "10"), List.of(new TextField("text", "ROSSI UA", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "11"), List.of(new TextField("text", "Mario Barman", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "12"), List.of(new TextField("text", "Mario batman", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "13"), List.of(new TextField("text", "Admin Rossi desk", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "14"), List.of(new TextField("text", "MRI Marios bot", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "15"), List.of(new TextField("text", "Mario Rossi [beta]", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "16"), List.of(new TextField("text", "Mario Music Bot", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "17"), List.of(new TextField("text", "Mario night mode", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "18"), List.of(new TextField("text", "Mario stats bot", Store.YES)));
|
||||
writer.updateDocument(new Term("id", "19"), List.of(new TextField("text", "Very very long text with Mario Giovanni and Rossi inside", Store.YES)));
|
||||
writer.flush();
|
||||
writer.commit();
|
||||
try (var reader = DirectoryReader.open(writer, true, true)) {
|
||||
var searcher = new IndexSearcher(reader);
|
||||
var qb = new QueryBuilder(analyzer);
|
||||
var luceneQuery = qb.createMinShouldMatchQuery("text", "Mario rossi", 0.3f);
|
||||
var expectedResults = searcher.search(luceneQuery, 20);
|
||||
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
|
||||
var expectedDocs = Arrays
|
||||
.stream(expectedResults.scoreDocs)
|
||||
.map(scoreDoc -> new LLScoreDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex))
|
||||
.toList();
|
||||
try (var collector = HugePqFullScoreDocCollector.create(env, 20)) {
|
||||
searcher.search(luceneQuery, collector);
|
||||
var docs = collector
|
||||
.fullDocs()
|
||||
.iterate()
|
||||
.collectList()
|
||||
.transform(LLUtils::handleDiscard)
|
||||
.blockOptional()
|
||||
.orElseThrow();
|
||||
System.out.println("Expected docs:");
|
||||
for (LLScoreDoc expectedDoc : expectedDocs) {
|
||||
System.out.println(expectedDoc);
|
||||
}
|
||||
System.out.println("");
|
||||
System.out.println("Obtained docs:");
|
||||
for (LLScoreDoc doc : docs) {
|
||||
System.out.println(doc);
|
||||
}
|
||||
assertEquals(expectedDocs, docs.stream().map(elem -> new LLScoreDoc(elem.doc(), elem.score(), -1)).toList());
|
||||
assertEquals(expectedTotalHits, new TotalHitsCount(collector.getTotalHits(), true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiShard() throws IOException {
|
||||
try (var dir1 = new ByteBuffersDirectory(); var dir2 = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
|
||||
var analyzer = new WordAnalyzer(true, true);
|
||||
var writer1 = new IndexWriter(dir1, new IndexWriterConfig(analyzer));
|
||||
var writer2 = new IndexWriter(dir2, new IndexWriterConfig(analyzer));
|
||||
writer1.updateDocument(new Term("id", "00"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "01"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "02"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "03"), List.of(new TextField("text", "Marios Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "04"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "05"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "06"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "07"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "08"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer1.updateDocument(new Term("id", "09"), List.of(new TextField("text", "Rossi", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "10"), List.of(new TextField("text", "ROSSI UA", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "11"), List.of(new TextField("text", "Mario Barman", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "12"), List.of(new TextField("text", "Mario batman", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "13"), List.of(new TextField("text", "Admin Rossi desk", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "14"), List.of(new TextField("text", "MRI Marios bot", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "15"), List.of(new TextField("text", "Mario Rossi [beta]", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "16"), List.of(new TextField("text", "Mario Music Bot", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "17"), List.of(new TextField("text", "Mario night mode", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "18"), List.of(new TextField("text", "Mario stats bot", Store.YES)));
|
||||
writer2.updateDocument(new Term("id", "19"), List.of(new TextField("text", "Very very long text with Mario Giovanni and Rossi inside", Store.YES)));
|
||||
writer1.flush();
|
||||
writer2.flush();
|
||||
writer1.commit();
|
||||
writer2.commit();
|
||||
var sharedStats = new SharedShardStatistics();
|
||||
try (var reader1 = DirectoryReader.open(writer1, true, true);
|
||||
var reader2 = DirectoryReader.open(writer2, true, true)) {
|
||||
var searcher1 = new IndexSearcher(reader1);
|
||||
var searcher2 = new IndexSearcher(reader2);
|
||||
var shardSearcher1 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 0);
|
||||
var shardSearcher2 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 1);
|
||||
var qb = new QueryBuilder(analyzer);
|
||||
var luceneQuery = qb.createMinShouldMatchQuery("text", "Mario rossi", 0.3f);
|
||||
var standardSharedManager = TopScoreDocCollector.createSharedManager(20, null, Integer.MAX_VALUE);
|
||||
var standardCollector1 = standardSharedManager.newCollector();
|
||||
var standardCollector2 = standardSharedManager.newCollector();
|
||||
shardSearcher1.search(luceneQuery, standardCollector1);
|
||||
shardSearcher2.search(luceneQuery, standardCollector2);
|
||||
var expectedResults = standardSharedManager.reduce(List.of(standardCollector1, standardCollector2));
|
||||
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
|
||||
var expectedDocs = Arrays
|
||||
.stream(expectedResults.scoreDocs)
|
||||
.map(scoreDoc -> new LLScoreDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex))
|
||||
.toList();
|
||||
var collectorManager = HugePqFullScoreDocCollector.createSharedManager(env, 20, Integer.MAX_VALUE);
|
||||
try (var collector1 = collectorManager.newCollector();
|
||||
var collector2 = collectorManager.newCollector()) {
|
||||
shardSearcher1.search(luceneQuery, collector1);
|
||||
shardSearcher2.search(luceneQuery, collector2);
|
||||
try (var results = collectorManager.reduce(List.of(collector1, collector2))) {
|
||||
var docs = results.iterate().collectList().blockOptional().orElseThrow();
|
||||
System.out.println("Expected docs:");
|
||||
for (LLScoreDoc expectedDoc : expectedDocs) {
|
||||
System.out.println(expectedDoc);
|
||||
}
|
||||
System.out.println("");
|
||||
System.out.println("Obtained docs:");
|
||||
for (LLScoreDoc doc : docs) {
|
||||
System.out.println(doc);
|
||||
}
|
||||
assertEquals(expectedDocs, docs.stream().map(elem -> new LLScoreDoc(elem.doc(), elem.score(), -1)).toList());
|
||||
assertEquals(expectedTotalHits, new TotalHitsCount(results.totalHits().value, results.totalHits().relation == Relation.EQUAL_TO));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,23 +1,17 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import io.netty5.buffer.Buffer;
|
||||
import io.netty5.buffer.MemoryManager;
|
||||
import io.netty5.buffer.internal.LeakDetection;
|
||||
import io.netty5.buffer.internal.LifecycleTracer;
|
||||
import io.netty5.buffer.pool.PoolArenaMetric;
|
||||
import io.netty5.buffer.pool.PooledBufferAllocator;
|
||||
import io.netty5.util.ResourceLeakDetector;
|
||||
import io.netty5.util.ResourceLeakDetector.Level;
|
||||
import io.netty5.util.internal.PlatformDependent;
|
||||
import io.netty.util.ResourceLeakDetector;
|
||||
import io.netty.util.ResourceLeakDetector.Level;
|
||||
import it.cavallium.dbengine.buffers.BufDataInput;
|
||||
import it.cavallium.dbengine.buffers.BufDataOutput;
|
||||
import it.cavallium.dbengine.client.LuceneIndex;
|
||||
import it.cavallium.dbengine.client.LuceneIndexImpl;
|
||||
import it.cavallium.dbengine.database.LLDatabaseConnection;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseMapDictionary;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep;
|
||||
@ -26,23 +20,20 @@ import it.cavallium.dbengine.database.collections.DatabaseStageEntry;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseStageMap;
|
||||
import it.cavallium.dbengine.database.collections.SubStageGetterHashMap;
|
||||
import it.cavallium.dbengine.database.collections.SubStageGetterMap;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationException;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.Function;
|
||||
import org.apache.lucene.util.IOSupplier;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.reactivestreams.Publisher;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
|
||||
public class DbTestUtils {
|
||||
|
||||
static {
|
||||
LLUtils.initHooks();
|
||||
}
|
||||
|
||||
public static final String BIG_STRING = generateBigString();
|
||||
public static final int MAX_IN_MEMORY_RESULT_ENTRIES = 8192;
|
||||
|
||||
@ -50,84 +41,61 @@ public class DbTestUtils {
|
||||
return "0123456789".repeat(1024);
|
||||
}
|
||||
|
||||
public record TestAllocator(TestAllocatorImpl allocator) {}
|
||||
|
||||
public static TestAllocator newAllocator() {
|
||||
return new TestAllocator(TestAllocatorImpl.create());
|
||||
}
|
||||
|
||||
public static void destroyAllocator(TestAllocator testAllocator) {
|
||||
testAllocator.allocator().close();
|
||||
}
|
||||
|
||||
@SuppressWarnings("SameParameterValue")
|
||||
private static long getActiveAllocations(TestAllocatorImpl allocator, boolean printStats) {
|
||||
long activeAllocations = allocator.getActiveAllocations();
|
||||
if (printStats) {
|
||||
System.out.println("activeAllocations=" + activeAllocations);
|
||||
}
|
||||
return activeAllocations;
|
||||
}
|
||||
|
||||
public static boolean isCIMode() {
|
||||
return System.getProperty("dbengine.ci", "false").equalsIgnoreCase("true");
|
||||
}
|
||||
|
||||
public static <U> Flux<U> tempDb(TemporaryDbGenerator temporaryDbGenerator,
|
||||
TestAllocator alloc,
|
||||
Function<LLKeyValueDatabase, Publisher<U>> action) {
|
||||
return Flux.usingWhen(
|
||||
temporaryDbGenerator.openTempDb(alloc),
|
||||
tempDb -> Flux
|
||||
.from(action.apply(tempDb.db()))
|
||||
.doOnDiscard(Object.class, o -> System.out.println("Discarded: " + o.getClass().getName() + ", " + o)),
|
||||
temporaryDbGenerator::closeTempDb
|
||||
);
|
||||
public static <U> U tempDb(TemporaryDbGenerator temporaryDbGenerator,
|
||||
Function<LLKeyValueDatabase, U> action) throws IOException {
|
||||
var tempDb = temporaryDbGenerator.openTempDb();
|
||||
try {
|
||||
return action.apply(tempDb.db());
|
||||
} finally {
|
||||
temporaryDbGenerator.closeTempDb(tempDb);
|
||||
}
|
||||
}
|
||||
|
||||
public record TempDb(TestAllocator allocator, LLDatabaseConnection connection, LLKeyValueDatabase db,
|
||||
public static void runVoid(boolean shouldFail, Runnable consumer) {
|
||||
if (shouldFail) {
|
||||
Assertions.assertThrows(Throwable.class, consumer::run);
|
||||
} else {
|
||||
Assertions.assertDoesNotThrow(consumer::run);
|
||||
}
|
||||
}
|
||||
|
||||
public static <X> X run(boolean shouldFail, IOSupplier<X> consumer) {
|
||||
AtomicReference<X> result = new AtomicReference<>(null);
|
||||
if (shouldFail) {
|
||||
Assertions.assertThrows(Throwable.class, consumer::get);
|
||||
} else {
|
||||
Assertions.assertDoesNotThrow(() -> result.set(consumer.get()));
|
||||
}
|
||||
return result.get();
|
||||
}
|
||||
|
||||
public record TempDb(LLDatabaseConnection connection, LLKeyValueDatabase db,
|
||||
LLLuceneIndex luceneSingle,
|
||||
LLLuceneIndex luceneMulti,
|
||||
SwappableLuceneSearcher swappableLuceneSearcher,
|
||||
Path path) {}
|
||||
|
||||
static boolean computeCanUseNettyDirect() {
|
||||
boolean canUse = true;
|
||||
if (!PlatformDependent.hasUnsafe()) {
|
||||
System.err.println("Warning! Unsafe is not available!"
|
||||
+ " Netty direct buffers will not be used in tests!");
|
||||
canUse = false;
|
||||
}
|
||||
return canUse;
|
||||
}
|
||||
|
||||
public static void ensureNoLeaks(TestAllocatorImpl allocator, boolean printStats, boolean useClassicException) {
|
||||
public static void ensureNoLeaks(boolean printStats, boolean useClassicException) {
|
||||
ResourceLeakDetector.setLevel(Level.PARANOID);
|
||||
System.gc();
|
||||
if (allocator != null) {
|
||||
var allocs = getActiveAllocations(allocator, printStats);
|
||||
if (useClassicException) {
|
||||
if (allocs != 0) {
|
||||
throw new IllegalStateException("Active allocations: " + allocs);
|
||||
}
|
||||
} else {
|
||||
assertEquals(0L, allocs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static Mono<? extends LLDictionary> tempDictionary(LLKeyValueDatabase database, UpdateMode updateMode) {
|
||||
public static LLDictionary tempDictionary(LLKeyValueDatabase database, UpdateMode updateMode) {
|
||||
return tempDictionary(database, "testmap", updateMode);
|
||||
}
|
||||
|
||||
public static Mono<? extends LLDictionary> tempDictionary(LLKeyValueDatabase database,
|
||||
public static LLDictionary tempDictionary(LLKeyValueDatabase database,
|
||||
String name,
|
||||
UpdateMode updateMode) {
|
||||
return database.getDictionary(name, updateMode);
|
||||
}
|
||||
|
||||
public static Mono<? extends LuceneIndex<String, String>> tempLuceneIndex(LLLuceneIndex index) {
|
||||
return Mono.fromCallable(() -> new LuceneIndexImpl<>(index, new StringIndicizer()));
|
||||
public static LuceneIndex<String, String> tempLuceneIndex(LLLuceneIndex index) {
|
||||
return new LuceneIndexImpl<>(index, new StringIndicizer());
|
||||
}
|
||||
|
||||
|
||||
@ -157,14 +125,13 @@ public class DbTestUtils {
|
||||
}
|
||||
|
||||
@Override
|
||||
public @NotNull Short deserialize(@NotNull Buffer serialized) {
|
||||
Objects.requireNonNull(serialized);
|
||||
return serialized.readShort();
|
||||
public @NotNull Short deserialize(@NotNull BufDataInput in) throws SerializationException {
|
||||
return in.readShort();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(@NotNull Short deserialized, Buffer output) {
|
||||
output.writeShort(deserialized);
|
||||
public void serialize(@NotNull Short deserialized, BufDataOutput out) throws SerializationException {
|
||||
out.writeShort(deserialized);
|
||||
}
|
||||
}
|
||||
);
|
||||
@ -195,7 +162,7 @@ public class DbTestUtils {
|
||||
new SubStageGetterHashMap<>(Serializer.UTF8_SERIALIZER,
|
||||
Serializer.UTF8_SERIALIZER,
|
||||
String::hashCode,
|
||||
SerializerFixedBinaryLength.intSerializer(dictionary.getAllocator())
|
||||
SerializerFixedBinaryLength.intSerializer()
|
||||
)
|
||||
);
|
||||
}
|
||||
@ -206,7 +173,7 @@ public class DbTestUtils {
|
||||
Serializer.UTF8_SERIALIZER,
|
||||
Serializer.UTF8_SERIALIZER,
|
||||
String::hashCode,
|
||||
SerializerFixedBinaryLength.intSerializer(dictionary.getAllocator())
|
||||
SerializerFixedBinaryLength.intSerializer()
|
||||
);
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import it.cavallium.dbengine.client.Sort;
|
||||
import it.cavallium.dbengine.client.query.BaseType;
|
@ -0,0 +1,113 @@
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import it.cavallium.data.generator.nativedata.Nullableboolean;
|
||||
import it.cavallium.data.generator.nativedata.Nullabledouble;
|
||||
import it.cavallium.data.generator.nativedata.Nullableint;
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
|
||||
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
||||
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
||||
import it.cavallium.dbengine.database.ColumnUtils;
|
||||
import it.cavallium.dbengine.database.LLDatabaseConnection;
|
||||
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
|
||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
|
||||
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||
|
||||
private static final AtomicInteger dbId = new AtomicInteger(0);
|
||||
|
||||
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofSeconds(5),
|
||||
false,
|
||||
new ByteBuffersDirectory(),
|
||||
Nullableboolean.empty(),
|
||||
Nullabledouble.empty(),
|
||||
Nullableint.empty(),
|
||||
Nullableboolean.empty(),
|
||||
Nullableboolean.empty(),
|
||||
MAX_IN_MEMORY_RESULT_ENTRIES,
|
||||
LuceneUtils.getDefaultMergePolicy()
|
||||
);
|
||||
|
||||
@Override
|
||||
public TempDb openTempDb() throws IOException {
|
||||
var wrkspcPath = Path.of("/tmp/.cache/tempdb-" + dbId.incrementAndGet() + "/");
|
||||
|
||||
if (Files.exists(wrkspcPath)) {
|
||||
Files.walk(wrkspcPath).sorted(Comparator.reverseOrder()).forEach(file -> {
|
||||
try {
|
||||
Files.delete(file);
|
||||
} catch (IOException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
Files.createDirectories(wrkspcPath);
|
||||
|
||||
LLDatabaseConnection conn = new LLLocalDatabaseConnection(
|
||||
new SimpleMeterRegistry(),
|
||||
wrkspcPath,
|
||||
true
|
||||
).connect();
|
||||
|
||||
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
|
||||
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
|
||||
return new TempDb(conn,
|
||||
conn.getDatabase("testdb",
|
||||
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
|
||||
DefaultDatabaseOptions.builder().build()
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex1",
|
||||
LuceneUtils.singleStructure(),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex16",
|
||||
LuceneUtils.shardsStructure(3),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
searcher,
|
||||
wrkspcPath
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeTempDb(TempDb tempDb) throws IOException {
|
||||
tempDb.db().close();
|
||||
tempDb.connection().disconnect();
|
||||
ensureNoLeaks(false, false);
|
||||
if (Files.exists(tempDb.path())) {
|
||||
Files.walk(tempDb.path()).sorted(Comparator.reverseOrder()).forEach(file -> {
|
||||
try {
|
||||
Files.delete(file);
|
||||
} catch (IOException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
@ -1,6 +1,8 @@
|
||||
package it.cavallium.dbengine.lucene.searcher;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import it.cavallium.dbengine.lucene.ExponentialPageLimits;
|
||||
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
|
||||
import it.cavallium.dbengine.lucene.searcher.LuceneGenerator;
|
||||
import it.unimi.dsi.fastutil.longs.LongList;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
@ -80,7 +82,7 @@ public class LuceneGeneratorTest {
|
||||
|
||||
var reactiveGenerator = LuceneGenerator.reactive(is, localQueryParams, -1);
|
||||
var results = fixResults(localQueryParams.isSorted(),
|
||||
localQueryParams.needsScores(), reactiveGenerator.collectList().block());
|
||||
localQueryParams.needsScores(), reactiveGenerator.toList());
|
||||
|
||||
Assertions.assertNotEquals(0, results.size());
|
||||
|
||||
@ -98,7 +100,7 @@ public class LuceneGeneratorTest {
|
||||
localQueryParams.needsScores(), List.of(is.search(query, limit).scoreDocs));
|
||||
|
||||
var reactiveGenerator = LuceneGenerator.reactive(is, localQueryParams, -1);
|
||||
var results = fixResults(localQueryParams.isSorted(), localQueryParams.needsScores(), reactiveGenerator.collectList().block());
|
||||
var results = fixResults(localQueryParams.isSorted(), localQueryParams.needsScores(), reactiveGenerator.toList());
|
||||
|
||||
Assertions.assertNotEquals(0, results.size());
|
||||
|
||||
@ -123,7 +125,7 @@ public class LuceneGeneratorTest {
|
||||
),
|
||||
-1
|
||||
);
|
||||
var results = reactiveGenerator.collectList().block();
|
||||
var results = reactiveGenerator.toList();
|
||||
|
||||
Assertions.assertNotNull(results);
|
||||
Assertions.assertEquals(0, results.size());
|
||||
@ -144,7 +146,7 @@ public class LuceneGeneratorTest {
|
||||
),
|
||||
-1
|
||||
);
|
||||
var results = reactiveGenerator.collectList().block();
|
||||
var results = reactiveGenerator.toList();
|
||||
|
||||
Assertions.assertNotNull(results);
|
||||
Assertions.assertEquals(limit, results.size());
|
||||
@ -161,7 +163,7 @@ public class LuceneGeneratorTest {
|
||||
|
||||
var reactiveGenerator = LuceneGenerator.reactive(is, localQueryParams, -1);
|
||||
var results = fixResults(localQueryParams.isSorted(),
|
||||
localQueryParams.needsScores(), reactiveGenerator.collectList().block());
|
||||
localQueryParams.needsScores(), reactiveGenerator.toList());
|
||||
|
||||
Assertions.assertEquals(4, results.size());
|
||||
Assertions.assertEquals(expectedResults, results);
|
@ -0,0 +1,75 @@
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import it.cavallium.data.generator.nativedata.Nullableboolean;
|
||||
import it.cavallium.data.generator.nativedata.Nullabledouble;
|
||||
import it.cavallium.data.generator.nativedata.Nullableint;
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
|
||||
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
||||
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
||||
import it.cavallium.dbengine.database.ColumnUtils;
|
||||
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
|
||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
|
||||
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
|
||||
|
||||
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofSeconds(5),
|
||||
false,
|
||||
new ByteBuffersDirectory(),
|
||||
Nullableboolean.empty(),
|
||||
Nullabledouble.empty(),
|
||||
Nullableint.empty(),
|
||||
Nullableboolean.empty(),
|
||||
Nullableboolean.empty(),
|
||||
MAX_IN_MEMORY_RESULT_ENTRIES,
|
||||
LuceneUtils.getDefaultMergePolicy()
|
||||
);
|
||||
|
||||
@Override
|
||||
public TempDb openTempDb() {
|
||||
var conn = new LLMemoryDatabaseConnection(new SimpleMeterRegistry());
|
||||
|
||||
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
|
||||
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
|
||||
return new TempDb(conn,
|
||||
conn.getDatabase("testdb",
|
||||
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
|
||||
DefaultDatabaseOptions.builder().build()
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex1",
|
||||
LuceneUtils.singleStructure(),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
conn.getLuceneIndex("testluceneindex16",
|
||||
LuceneUtils.shardsStructure(3),
|
||||
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
|
||||
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
|
||||
LUCENE_OPTS,
|
||||
luceneHacks
|
||||
),
|
||||
searcher,
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeTempDb(TempDb db) {
|
||||
db.db().close();
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import it.cavallium.dbengine.database.DiscardingCloseable;
|
||||
import it.cavallium.dbengine.lucene.PriorityQueue;
|
||||
@ -7,8 +7,8 @@ import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.lucene.search.HitQueue;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public class PriorityQueueAdaptor<T> extends SimpleResource implements PriorityQueue<T>, DiscardingCloseable {
|
||||
|
||||
@ -61,7 +61,7 @@ public class PriorityQueueAdaptor<T> extends SimpleResource implements PriorityQ
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<T> iterate() {
|
||||
public Stream<T> iterate() {
|
||||
List<T> items = new ArrayList<>(hitQueue.size());
|
||||
T item;
|
||||
while ((item = hitQueue.pop()) != null) {
|
||||
@ -70,7 +70,7 @@ public class PriorityQueueAdaptor<T> extends SimpleResource implements PriorityQ
|
||||
for (T t : items) {
|
||||
hitQueue.insertWithOverflow(t);
|
||||
}
|
||||
return Flux.fromIterable(items);
|
||||
return items.stream();
|
||||
}
|
||||
|
||||
@Override
|
@ -1,3 +1,3 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
record Scored(String key, float score) {}
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.google.common.primitives.Longs;
|
||||
@ -16,30 +16,27 @@ import org.apache.lucene.document.Field.Store;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class StringIndicizer extends Indicizer<String, String> {
|
||||
|
||||
@Override
|
||||
public @NotNull Mono<LLUpdateDocument> toIndexRequest(@NotNull String key, @NotNull String value) {
|
||||
return Mono.fromCallable(() -> {
|
||||
var fields = new LinkedList<LLItem>();
|
||||
fields.add(LLItem.newStringField("uid", key, Field.Store.YES));
|
||||
fields.add(LLItem.newTextField("text", value, Store.NO));
|
||||
@SuppressWarnings("UnstableApiUsage")
|
||||
var numInt = Ints.tryParse(value);
|
||||
if (numInt != null) {
|
||||
fields.add(LLItem.newIntPoint("intpoint", numInt));
|
||||
fields.add(LLItem.newNumericDocValuesField("intsort", numInt));
|
||||
}
|
||||
@SuppressWarnings("UnstableApiUsage")
|
||||
var numLong = Longs.tryParse(value);
|
||||
if (numLong != null) {
|
||||
fields.add(LLItem.newLongPoint("longpoint", numLong));
|
||||
fields.add(LLItem.newNumericDocValuesField("longsort", numLong));
|
||||
}
|
||||
return new LLUpdateDocument(fields);
|
||||
});
|
||||
public @NotNull LLUpdateDocument toIndexRequest(@NotNull String key, @NotNull String value) {
|
||||
var fields = new LinkedList<LLItem>();
|
||||
fields.add(LLItem.newStringField("uid", key, Field.Store.YES));
|
||||
fields.add(LLItem.newTextField("text", value, Store.NO));
|
||||
@SuppressWarnings("UnstableApiUsage")
|
||||
var numInt = Ints.tryParse(value);
|
||||
if (numInt != null) {
|
||||
fields.add(LLItem.newIntPoint("intpoint", numInt));
|
||||
fields.add(LLItem.newNumericDocValuesField("intsort", numInt));
|
||||
}
|
||||
@SuppressWarnings("UnstableApiUsage")
|
||||
var numLong = Longs.tryParse(value);
|
||||
if (numLong != null) {
|
||||
fields.add(LLItem.newLongPoint("longpoint", numLong));
|
||||
fields.add(LLItem.newNumericDocValuesField("longsort", numLong));
|
||||
}
|
||||
return new LLUpdateDocument(fields);
|
||||
}
|
||||
|
||||
@Override
|
@ -1,9 +1,8 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
import static java.util.Objects.requireNonNullElseGet;
|
||||
|
||||
import io.netty5.util.Send;
|
||||
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
|
||||
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
||||
import it.cavallium.dbengine.lucene.searcher.GlobalQueryRewrite;
|
||||
@ -15,7 +14,6 @@ import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Closeable {
|
||||
|
||||
@ -27,7 +25,7 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
|
||||
public LuceneSearchResult collect(LLIndexSearcher indexSearcher,
|
||||
LocalQueryParams queryParams,
|
||||
@Nullable String keyFieldName,
|
||||
GlobalQueryRewrite transformer) {
|
||||
@ -36,7 +34,7 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
|
||||
single = this.multi.get();
|
||||
}
|
||||
requireNonNull(single, "LuceneLocalSearcher not set");
|
||||
return single.collect(indexSearcherMono, queryParams, keyFieldName, transformer);
|
||||
return single.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -55,12 +53,12 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||
public LuceneSearchResult collectMulti(LLIndexSearchers indexSearchers,
|
||||
LocalQueryParams queryParams,
|
||||
String keyFieldName,
|
||||
GlobalQueryRewrite transformer) {
|
||||
var multi = requireNonNull(this.multi.get(), "LuceneMultiSearcher not set");
|
||||
return multi.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
return multi.collectMulti(indexSearchers, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
|
||||
public void setSingle(LocalSearcher single) {
|
@ -0,0 +1,11 @@
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
import java.io.IOException;
|
||||
|
||||
public interface TemporaryDbGenerator {
|
||||
|
||||
TempDb openTempDb() throws IOException;
|
||||
|
||||
void closeTempDb(TempDb db) throws IOException;
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import it.cavallium.dbengine.lucene.DirectNIOFSDirectory;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
@ -0,0 +1,46 @@
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.isCIMode;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.tempDb;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.tempDictionary;
|
||||
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
public abstract class TestDictionary {
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
private static Stream<Arguments> provideArgumentsCreate() {
|
||||
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
ensureNoLeaks(false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
if (!isCIMode()) {
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsCreate")
|
||||
public void testCreate(UpdateMode updateMode) throws IOException {
|
||||
tempDb(getTempDbGenerator(), db -> {
|
||||
tempDictionary(db, updateMode).clear();
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
@ -1,42 +1,25 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.*;
|
||||
import static it.cavallium.dbengine.SyncUtils.*;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.*;
|
||||
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.test.StepVerifier;
|
||||
import reactor.test.StepVerifier.Step;
|
||||
import reactor.util.function.Tuple2;
|
||||
import reactor.util.function.Tuples;
|
||||
|
||||
public abstract class TestDictionaryMap {
|
||||
|
||||
private static final Logger log = LogManager.getLogger(TestDictionaryMap.class);
|
||||
private TestAllocator allocator;
|
||||
private boolean checkLeaks = true;
|
||||
|
||||
private static boolean isTestBadKeysEnabled() {
|
||||
@ -45,6 +28,11 @@ public abstract class TestDictionaryMap {
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
record Tuple2<X, Y>(X getT1, Y getT2) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
|
||||
|
||||
private static Stream<Arguments> provideArgumentsPut() {
|
||||
var goodKeys = List.of("12345");
|
||||
List<String> badKeys;
|
||||
@ -54,7 +42,7 @@ public abstract class TestDictionaryMap {
|
||||
badKeys = List.of();
|
||||
}
|
||||
List<Tuple2<String, Boolean>> keys = Stream
|
||||
.concat(goodKeys.stream().map(s -> Tuples.of(s, false)), badKeys.stream().map(s -> Tuples.of(s, true)))
|
||||
.concat(goodKeys.stream().map(s -> new Tuple2<>(s, false)), badKeys.stream().map(s -> new Tuple2<>(s, true)))
|
||||
.toList();
|
||||
var values = isCIMode() ? List.of("val") : List.of("", "\0", BIG_STRING);
|
||||
|
||||
@ -67,18 +55,18 @@ public abstract class TestDictionaryMap {
|
||||
} else {
|
||||
strm = values.stream();
|
||||
}
|
||||
return strm.map(val -> Tuples.of(keyTuple.getT1(), val, keyTuple.getT2()));
|
||||
return strm.map(val -> new Tuple3<>(keyTuple.getT1(), val, keyTuple.getT2()));
|
||||
})
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple4<>(updateMode,
|
||||
entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
entryTuple.getT3()
|
||||
)))
|
||||
.flatMap(entryTuple -> Stream.of(Tuples.of(MapType.MAP, entryTuple.getT1(),
|
||||
.flatMap(entryTuple -> Stream.of(new Tuple5<>(MapType.MAP, entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
entryTuple.getT3(),
|
||||
entryTuple.getT4()
|
||||
), Tuples.of(MapType.HASH_MAP, entryTuple.getT1(),
|
||||
), new Tuple5<>(MapType.HASH_MAP, entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
entryTuple.getT3(),
|
||||
false
|
||||
@ -89,36 +77,33 @@ public abstract class TestDictionaryMap {
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
ensureNoLeaks(false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
if (!isCIMode() && checkLeaks) {
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
destroyAllocator(allocator);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsPut")
|
||||
public void testPut(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
|
||||
public void testPut(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail)
|
||||
throws IOException {
|
||||
var gen = getTempDbGenerator();
|
||||
var db = run(gen.openTempDb(allocator));
|
||||
var dict = run(tempDictionary(db.db(), updateMode));
|
||||
var db = gen.openTempDb();
|
||||
var dict = tempDictionary(db.db(), updateMode);
|
||||
var map = tempDatabaseMapDictionaryMap(dict, mapType, 5);
|
||||
|
||||
runVoid(shouldFail, map.putValue(key, value));
|
||||
runVoid(shouldFail, () -> map.putValue(key, value));
|
||||
|
||||
var resultingMapSize = run(map.leavesCount(null, false));
|
||||
var resultingMapSize = map.leavesCount(null, false);
|
||||
Assertions.assertEquals(shouldFail ? 0 : 1, resultingMapSize);
|
||||
|
||||
var resultingMap = run(map.get(null));
|
||||
var resultingMap = map.get(null);
|
||||
Assertions.assertEquals(shouldFail ? null : Object2ObjectSortedMaps.singleton(key, value), resultingMap);
|
||||
|
||||
map.close();
|
||||
|
||||
//if (shouldFail) this.checkLeaks = false;
|
||||
|
||||
gen.closeTempDb(db);
|
||||
@ -126,47 +111,43 @@ public abstract class TestDictionaryMap {
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsPut")
|
||||
public void testAtSetAtGet(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
|
||||
var stpVer = StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempDictionary(db, updateMode)
|
||||
.map(dict -> tempDatabaseMapDictionaryMap(dict, mapType, 5))
|
||||
.flatMap(map -> Mono
|
||||
.usingWhen(map.at(null, key), v -> v.set(value), LLUtils::finalizeResource)
|
||||
.then(Mono.usingWhen(map.at(null, key), v -> v.get(null), LLUtils::finalizeResource))
|
||||
.doFinally(s -> map.close())
|
||||
)
|
||||
));
|
||||
public void testAtSetAtGet(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail)
|
||||
throws IOException {
|
||||
var result = tempDb(getTempDbGenerator(), db -> {
|
||||
var map = tempDatabaseMapDictionaryMap(tempDictionary(db, updateMode), mapType, 5);
|
||||
return run(shouldFail, () -> {
|
||||
map.at(null, key).set(value);
|
||||
return map.at(null, key).get(null);
|
||||
});
|
||||
});
|
||||
if (shouldFail) {
|
||||
this.checkLeaks = false;
|
||||
stpVer.verifyError();
|
||||
} else {
|
||||
stpVer.expectNext(value).verifyComplete();
|
||||
Assertions.assertEquals(value, result);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsPut")
|
||||
public void testPutAndGetPrevious(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
|
||||
var stpVer = StepVerifier
|
||||
.create(tempDb(getTempDbGenerator(), allocator, db -> tempDictionary(db, updateMode)
|
||||
.map(dict -> tempDatabaseMapDictionaryMap(dict, mapType, 5))
|
||||
.flatMapMany(map -> Flux
|
||||
.concat(
|
||||
map.putValueAndGetPrevious(key, "error?"),
|
||||
map.putValueAndGetPrevious(key, value),
|
||||
map.putValueAndGetPrevious(key, value)
|
||||
)
|
||||
.doFinally(s -> map.close())
|
||||
)
|
||||
));
|
||||
public void testPutAndGetPrevious(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail)
|
||||
throws IOException {
|
||||
var result = tempDb(getTempDbGenerator(), db -> {
|
||||
var map = tempDatabaseMapDictionaryMap(tempDictionary(db, updateMode), mapType, 5);
|
||||
return run(shouldFail,
|
||||
() -> Arrays.asList(map.putValueAndGetPrevious(key, "error?"),
|
||||
map.putValueAndGetPrevious(key, value),
|
||||
map.putValueAndGetPrevious(key, value)
|
||||
)
|
||||
);
|
||||
});
|
||||
if (shouldFail) {
|
||||
this.checkLeaks = false;
|
||||
stpVer.verifyError();
|
||||
} else {
|
||||
stpVer.expectNext("error?").expectNext(value).verifyComplete();
|
||||
Assertions.assertArrayEquals(new String[] {null, "error?", value}, result.toArray(String[]::new));
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsPut")
|
||||
public void testPutValueRemoveAndGetPrevious(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
|
||||
@ -370,7 +351,7 @@ public abstract class TestDictionaryMap {
|
||||
badKeys = List.of();
|
||||
}
|
||||
List<Tuple2<List<String>, Boolean>> keys = Stream
|
||||
.concat(goodKeys.stream().map(s -> Tuples.of(s, false)), badKeys.stream().map(s -> Tuples.of(s, true)))
|
||||
.concat(goodKeys.stream().map(s -> new Tuple2<>(s, false)), badKeys.stream().map(s -> new Tuple2<>(s, true)))
|
||||
.toList();
|
||||
var values = isCIMode() ? List.of("val") : List.of("", "\0", BIG_STRING);
|
||||
|
||||
@ -381,14 +362,14 @@ public abstract class TestDictionaryMap {
|
||||
.collectMap(Tuple2::getT1, Tuple2::getT2, Object2ObjectLinkedOpenHashMap::new)
|
||||
.block()
|
||||
))
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple2<>(updateMode,
|
||||
entryTuple.getT1(),
|
||||
entryTuple.getT2()
|
||||
)))
|
||||
.flatMap(entryTuple -> Stream.of(Tuples.of(MapType.MAP, entryTuple.getT1(),
|
||||
.flatMap(entryTuple -> Stream.of(new Tuple2<>(MapType.MAP, entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
entryTuple.getT3()
|
||||
), Tuples.of(MapType.HASH_MAP, entryTuple.getT1(),
|
||||
), new Tuple2<>(MapType.HASH_MAP, entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
false
|
||||
)))
|
||||
@ -761,4 +742,5 @@ public abstract class TestDictionaryMap {
|
||||
|
||||
Assertions.assertEquals(true, result.get(2));
|
||||
}
|
||||
*/
|
||||
}
|
@ -1,29 +1,22 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.BIG_STRING;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.isCIMode;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMap;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDb;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
|
||||
import static it.cavallium.dbengine.SyncUtils.*;
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.BIG_STRING;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.isCIMode;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.run;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.tempDatabaseMapDictionaryDeepMap;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.tempDictionary;
|
||||
|
||||
import io.netty5.buffer.internal.ResourceSupport;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import com.google.common.collect.Streams;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.utils.SimpleResource;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
@ -36,20 +29,11 @@ import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.test.StepVerifier;
|
||||
import reactor.test.StepVerifier.Step;
|
||||
import reactor.util.function.Tuple2;
|
||||
import reactor.util.function.Tuple3;
|
||||
import reactor.util.function.Tuple4;
|
||||
import reactor.util.function.Tuples;
|
||||
|
||||
@TestMethodOrder(MethodOrderer.MethodName.class)
|
||||
public abstract class TestDictionaryMapDeep {
|
||||
|
||||
private final Logger log = LogManager.getLogger(this.getClass());
|
||||
private TestAllocator allocator;
|
||||
private boolean checkLeaks = true;
|
||||
|
||||
private static boolean isTestBadKeysEnabled() {
|
||||
@ -58,6 +42,11 @@ public abstract class TestDictionaryMapDeep {
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
record Tuple2<X, Y>(X getT1, Y getT2) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
|
||||
|
||||
private static Stream<Arguments> provideArgumentsSet() {
|
||||
var goodKeys = Set.of("12345");
|
||||
Set<String> badKeys;
|
||||
@ -67,8 +56,8 @@ public abstract class TestDictionaryMapDeep {
|
||||
badKeys = Set.of();
|
||||
}
|
||||
Set<Tuple2<String, Boolean>> keys = Stream.concat(
|
||||
goodKeys.stream().map(s -> Tuples.of(s, false)),
|
||||
badKeys.stream().map(s -> Tuples.of(s, true))
|
||||
goodKeys.stream().map(s -> new Tuple2<>(s, false)),
|
||||
badKeys.stream().map(s -> new Tuple2<>(s, true))
|
||||
).collect(Collectors.toSet());
|
||||
var values = Set.of(
|
||||
new Object2ObjectLinkedOpenHashMap<>(Map.of("123456", "a", "234567", "")),
|
||||
@ -84,9 +73,9 @@ public abstract class TestDictionaryMapDeep {
|
||||
} else {
|
||||
strm = values.stream();
|
||||
}
|
||||
return strm.map(val -> Tuples.of(keyTuple.getT1(), val, keyTuple.getT2()));
|
||||
return strm.map(val -> new Tuple3<>(keyTuple.getT1(), val, keyTuple.getT2()));
|
||||
})
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple4<>(updateMode,
|
||||
entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
entryTuple.getT3()
|
||||
@ -112,43 +101,38 @@ public abstract class TestDictionaryMapDeep {
|
||||
|
||||
var values = isCIMode() ? List.of("val") : List.of("a", "", "\0", "\0\0", "z", "azzszgzczqz", BIG_STRING);
|
||||
|
||||
Flux<Tuple4<String, String, String, Boolean>> failOnKeys1 = Flux
|
||||
.fromIterable(badKeys1)
|
||||
.map(badKey1 -> Tuples.of(
|
||||
Stream<Tuple4<String, String, String, Boolean>> failOnKeys1 = badKeys1.stream()
|
||||
.map(badKey1 -> new Tuple4<>(
|
||||
badKey1,
|
||||
goodKeys2.stream().findFirst().orElseThrow(),
|
||||
values.stream().findFirst().orElseThrow(),
|
||||
true
|
||||
));
|
||||
Flux<Tuple4<String, String, String, Boolean>> failOnKeys2 = Flux
|
||||
.fromIterable(badKeys2)
|
||||
.map(badKey2 -> Tuples.of(
|
||||
Stream<Tuple4<String, String, String, Boolean>> failOnKeys2 = badKeys2.stream()
|
||||
.map(badKey2 -> new Tuple4<>(
|
||||
goodKeys1.stream().findFirst().orElseThrow(),
|
||||
badKey2,
|
||||
values.stream().findFirst().orElseThrow(),
|
||||
true
|
||||
));
|
||||
|
||||
Flux<Tuple4<String, String, String, Boolean>> goodKeys1And2 = Flux
|
||||
.fromIterable(values)
|
||||
.map(value -> Tuples.of(
|
||||
Stream<Tuple4<String, String, String, Boolean>> goodKeys1And2 = values.stream()
|
||||
.map(value -> new Tuple4<>(
|
||||
goodKeys1.stream().findFirst().orElseThrow(),
|
||||
goodKeys2.stream().findFirst().orElseThrow(),
|
||||
value,
|
||||
false
|
||||
));
|
||||
|
||||
Flux<Tuple4<String, String, String, Boolean>> keys1And2 = Flux
|
||||
.concat(
|
||||
Stream<Tuple4<String, String, String, Boolean>> keys1And2 = Streams.concat(
|
||||
goodKeys1And2,
|
||||
failOnKeys1,
|
||||
failOnKeys2
|
||||
);
|
||||
|
||||
return keys1And2
|
||||
.concatMap(entryTuple -> Flux
|
||||
.fromArray(UpdateMode.values())
|
||||
.map(updateMode -> Tuples.of(updateMode,
|
||||
.flatMap(entryTuple -> Stream.of(UpdateMode.values())
|
||||
.map(updateMode -> new Tuple5<>(updateMode,
|
||||
entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
entryTuple.getT3(),
|
||||
@ -161,22 +145,19 @@ public abstract class TestDictionaryMapDeep {
|
||||
fullTuple.getT4(),
|
||||
fullTuple.getT5()
|
||||
))
|
||||
.toStream()
|
||||
.sequential();
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
ensureNoLeaks(false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
if (!isCIMode() && checkLeaks) {
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
destroyAllocator(allocator);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ -184,25 +165,21 @@ public abstract class TestDictionaryMapDeep {
|
||||
public void testPutValue(UpdateMode updateMode,
|
||||
String key,
|
||||
Object2ObjectSortedMap<String, String> value,
|
||||
boolean shouldFail) {
|
||||
boolean shouldFail) throws IOException {
|
||||
var gen = getTempDbGenerator();
|
||||
var db = run(gen.openTempDb(allocator));
|
||||
var dict = run(tempDictionary(db.db(), updateMode));
|
||||
var db = gen.openTempDb();
|
||||
var dict = tempDictionary(db.db(), updateMode);
|
||||
var map = tempDatabaseMapDictionaryDeepMap(dict, 5, 6);
|
||||
|
||||
log.debug("Put \"{}\" = \"{}\"", key, value);
|
||||
runVoid(shouldFail, map.putValue(key, value));
|
||||
runVoid(shouldFail, () -> map.putValue(key, value));
|
||||
|
||||
var resultingMapSize = run(map.leavesCount(null, false));
|
||||
var resultingMapSize = map.leavesCount(null, false);
|
||||
Assertions.assertEquals(shouldFail ? 0 : value.size(), resultingMapSize);
|
||||
|
||||
var resultingMap = run(map.get(null));
|
||||
var resultingMap = map.get(null);
|
||||
Assertions.assertEquals(shouldFail ? null : Map.of(key, value), resultingMap);
|
||||
|
||||
map.close();
|
||||
|
||||
//if (shouldFail) this.checkLeaks = false;
|
||||
|
||||
gen.closeTempDb(db);
|
||||
}
|
||||
|
||||
@ -211,27 +188,23 @@ public abstract class TestDictionaryMapDeep {
|
||||
public void testGetValue(UpdateMode updateMode,
|
||||
String key,
|
||||
Object2ObjectSortedMap<String, String> value,
|
||||
boolean shouldFail) {
|
||||
boolean shouldFail) throws IOException {
|
||||
var gen = getTempDbGenerator();
|
||||
var db = run(gen.openTempDb(allocator));
|
||||
var dict = run(tempDictionary(db.db(), updateMode));
|
||||
var db = gen.openTempDb();
|
||||
var dict = tempDictionary(db.db(), updateMode);
|
||||
var map = tempDatabaseMapDictionaryDeepMap(dict, 5, 6);
|
||||
|
||||
log.debug("Put \"{}\" = \"{}\"", key, value);
|
||||
runVoid(shouldFail, map.putValue(key, value));
|
||||
runVoid(shouldFail, () -> map.putValue(key, value));
|
||||
|
||||
log.debug("Get \"{}\"", key);
|
||||
var returnedValue = run(shouldFail, map.getValue(null, key));
|
||||
var returnedValue = run(shouldFail, () -> map.getValue(null, key));
|
||||
|
||||
Assertions.assertEquals(shouldFail ? null : value, returnedValue);
|
||||
|
||||
map.close();
|
||||
|
||||
//if (shouldFail) this.checkLeaks = false;
|
||||
|
||||
gen.closeTempDb(db);
|
||||
}
|
||||
|
||||
/*
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsSet")
|
||||
public void testSetValueGetAllValues(UpdateMode updateMode, String key, Object2ObjectSortedMap<String, String> value,
|
||||
@ -284,7 +257,7 @@ public abstract class TestDictionaryMapDeep {
|
||||
.getAllStages(null, false)
|
||||
.flatMap(v -> v.getValue()
|
||||
.getAllValues(null, false)
|
||||
.map(result -> Tuples.of(v.getKey(), result.getKey(), result.getValue()))
|
||||
.map(result -> new Tuple2<>(v.getKey(), result.getKey(), result.getValue()))
|
||||
.doFinally(s -> v.getValue().close())
|
||||
)
|
||||
),
|
||||
@ -295,9 +268,9 @@ public abstract class TestDictionaryMapDeep {
|
||||
this.checkLeaks = false;
|
||||
stpVer.verifyError();
|
||||
} else {
|
||||
value.forEach((k, v) -> remainingEntries.add(Tuples.of(key, k, v)));
|
||||
remainingEntries.add(Tuples.of("capra", "normal", "123"));
|
||||
remainingEntries.add(Tuples.of("capra", "ormaln", "456"));
|
||||
value.forEach((k, v) -> remainingEntries.add(new Tuple2<>(key, k, v)));
|
||||
remainingEntries.add(new Tuple2<>("capra", "normal", "123"));
|
||||
remainingEntries.add(new Tuple2<>("capra", "ormaln", "456"));
|
||||
for (Tuple3<String, String, String> ignored : remainingEntries) {
|
||||
stpVer = stpVer.expectNextMatches(remainingEntries::remove);
|
||||
}
|
||||
@ -771,7 +744,7 @@ public abstract class TestDictionaryMapDeep {
|
||||
badKeys = List.of();
|
||||
}
|
||||
List<Tuple2<List<String>, Boolean>> keys = Stream
|
||||
.concat(goodKeys.stream().map(s -> Tuples.of(s, false)), badKeys.stream().map(s -> Tuples.of(s, true)))
|
||||
.concat(goodKeys.stream().map(s -> new Tuple2<>(s, false)), badKeys.stream().map(s -> new Tuple2<>(s, true)))
|
||||
.toList();
|
||||
var values = isCIMode() ? List.of(new Object2ObjectLinkedOpenHashMap<>(Map.of("123456", "val"))) : List.of(
|
||||
new Object2ObjectLinkedOpenHashMap<>(Map.of("123456", "a", "234567", "")),
|
||||
@ -785,7 +758,7 @@ public abstract class TestDictionaryMapDeep {
|
||||
.collectMap(Tuple2::getT1, Tuple2::getT2, Object2ObjectLinkedOpenHashMap::new)
|
||||
.block()
|
||||
))
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
|
||||
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple2<>(updateMode,
|
||||
entryTuple.getT1(),
|
||||
entryTuple.getT2()
|
||||
)))
|
||||
@ -1142,4 +1115,6 @@ public abstract class TestDictionaryMapDeep {
|
||||
stpVer.expectNext(true, entries.isEmpty(), true).verifyComplete();
|
||||
}
|
||||
}
|
||||
|
||||
*/
|
||||
}
|
@ -1,32 +1,19 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.BIG_STRING;
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.isCIMode;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMapHashMap;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDb;
|
||||
import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.BIG_STRING;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.isCIMode;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.tempDictionary;
|
||||
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import com.google.common.collect.Streams;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import java.util.List;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.test.StepVerifier;
|
||||
import reactor.util.function.Tuple4;
|
||||
import reactor.util.function.Tuples;
|
||||
|
||||
public abstract class TestDictionaryMapDeepHashMap {
|
||||
|
||||
private TestAllocator allocator;
|
||||
private boolean checkLeaks = true;
|
||||
|
||||
private static boolean isTestBadKeysEnabled() {
|
||||
@ -35,6 +22,11 @@ public abstract class TestDictionaryMapDeepHashMap {
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
record Tuple2<X, Y>(X getT1, Y getT2) {}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
|
||||
|
||||
private static Stream<Arguments> provideArgumentsPut() {
|
||||
var goodKeys1 = isCIMode() ? List.of("12345") : List.of("12345", "zebra");
|
||||
List<String> badKeys1;
|
||||
@ -47,34 +39,31 @@ public abstract class TestDictionaryMapDeepHashMap {
|
||||
|
||||
var values = isCIMode() ? List.of("val") : List.of("a", "", "\0", "\0\0", "z", "azzszgzczqz", BIG_STRING);
|
||||
|
||||
Flux<Tuple4<String, String, String, Boolean>> failOnKeys1 = Flux
|
||||
.fromIterable(badKeys1)
|
||||
.map(badKey1 -> Tuples.of(
|
||||
Stream<Tuple4<String, String, String, Boolean>> failOnKeys1 = badKeys1.stream()
|
||||
.map(badKey1 -> new Tuple4<>(
|
||||
badKey1,
|
||||
goodKeys2.stream().findAny().orElseThrow(),
|
||||
values.stream().findAny().orElseThrow(),
|
||||
true
|
||||
));
|
||||
|
||||
Flux<Tuple4<String, String, String, Boolean>> goodKeys1And2 = Flux
|
||||
.fromIterable(values)
|
||||
.map(value -> Tuples.of(
|
||||
Stream<Tuple4<String, String, String, Boolean>> goodKeys1And2 = values.stream()
|
||||
.map(value -> new Tuple4<>(
|
||||
goodKeys1.stream().findAny().orElseThrow(),
|
||||
goodKeys2.stream().findAny().orElseThrow(),
|
||||
value,
|
||||
false
|
||||
));
|
||||
|
||||
Flux<Tuple4<String, String, String, Boolean>> keys1And2 = Flux
|
||||
Stream<Tuple4<String, String, String, Boolean>> keys1And2 = Streams
|
||||
.concat(
|
||||
goodKeys1And2,
|
||||
failOnKeys1
|
||||
);
|
||||
|
||||
return keys1And2
|
||||
.flatMap(entryTuple -> Flux
|
||||
.fromArray(UpdateMode.values())
|
||||
.map(updateMode -> Tuples.of(updateMode,
|
||||
.flatMap(entryTuple -> Stream.of(UpdateMode.values())
|
||||
.map(updateMode -> new Tuple5<>(updateMode,
|
||||
entryTuple.getT1(),
|
||||
entryTuple.getT2(),
|
||||
entryTuple.getT3(),
|
||||
@ -86,24 +75,22 @@ public abstract class TestDictionaryMapDeepHashMap {
|
||||
fullTuple.getT3(),
|
||||
fullTuple.getT4(),
|
||||
fullTuple.getT1() != UpdateMode.ALLOW || fullTuple.getT5()
|
||||
))
|
||||
.toStream();
|
||||
));
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
ensureNoLeaks(false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
if (!isCIMode() && checkLeaks) {
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
destroyAllocator(allocator);
|
||||
}
|
||||
|
||||
/*
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArgumentsPut")
|
||||
public void testAtPutValueGetAllValues(UpdateMode updateMode, String key1, String key2, String value, boolean shouldFail) {
|
||||
@ -129,4 +116,6 @@ public abstract class TestDictionaryMapDeepHashMap {
|
||||
}
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
}
|
@ -1,8 +1,7 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static java.util.Map.entry;
|
||||
|
||||
import io.netty5.buffer.BufferAllocator;
|
||||
import it.cavallium.dbengine.database.disk.KeyMayExistGetter;
|
||||
import it.unimi.dsi.fastutil.bytes.ByteList;
|
||||
import java.nio.ByteBuffer;
|
||||
@ -41,7 +40,7 @@ public class TestGetter {
|
||||
return new String(bytes.toByteArray(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public KeyMayExistGetter getter = new KeyMayExistGetter(BufferAllocator.offHeapUnpooled(), true) {
|
||||
public KeyMayExistGetter getter = new KeyMayExistGetter() {
|
||||
@Override
|
||||
protected KeyMayExist keyMayExist(ReadOptions readOptions, ByteBuffer key, ByteBuffer value) {
|
||||
return null;
|
247
src/test/java/it/cavallium/dbengine/tests/TestLLDictionary.java
Normal file
247
src/test/java/it/cavallium/dbengine/tests/TestLLDictionary.java
Normal file
@ -0,0 +1,247 @@
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.UpdateReturnMode;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
public abstract class TestLLDictionary {
|
||||
|
||||
private final Logger log = LogManager.getLogger(this.getClass());
|
||||
private static final LLRange RANGE_ALL = LLRange.all();
|
||||
private TempDb tempDb;
|
||||
private LLKeyValueDatabase db;
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() throws IOException {
|
||||
ensureNoLeaks(false, false);
|
||||
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(), "TempDB");
|
||||
db = tempDb.db();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() throws IOException {
|
||||
getTempDbGenerator().closeTempDb(tempDb);
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideArguments() {
|
||||
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
|
||||
}
|
||||
|
||||
public static Stream<Arguments> providePutArguments() {
|
||||
var updateModes = Arrays.stream(UpdateMode.values());
|
||||
return updateModes.flatMap(updateMode -> {
|
||||
var resultTypes = Arrays.stream(LLDictionaryResultType.values());
|
||||
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
|
||||
});
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideUpdateArguments() {
|
||||
var updateModes = Arrays.stream(UpdateMode.values());
|
||||
return updateModes.flatMap(updateMode -> {
|
||||
var resultTypes = Arrays.stream(UpdateReturnMode.values());
|
||||
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
|
||||
});
|
||||
}
|
||||
|
||||
private LLDictionary getDict(UpdateMode updateMode) {
|
||||
var dict = DbTestUtils.tempDictionary(db, updateMode);
|
||||
var key1 = fromString("test-key-1");
|
||||
var key2 = fromString("test-key-2");
|
||||
var key3 = fromString("test-key-3");
|
||||
var key4 = fromString("test-key-4");
|
||||
var value = fromString("test-value");
|
||||
dict.put(key1, value, LLDictionaryResultType.VOID);
|
||||
dict.put(key2, value, LLDictionaryResultType.VOID);
|
||||
dict.put(key3, value, LLDictionaryResultType.VOID);
|
||||
dict.put(key4, value, LLDictionaryResultType.VOID);
|
||||
return dict;
|
||||
}
|
||||
|
||||
private Buf fromString(String s) {
|
||||
var sb = s.getBytes(StandardCharsets.UTF_8);
|
||||
Buf b = Buf.create(sb.length + 3 + 13);
|
||||
b.addElements(0, sb);
|
||||
assert b.size() == sb.length;
|
||||
return b;
|
||||
}
|
||||
|
||||
private String toString(Buf bb) {
|
||||
return bb != null ? bb.toString(StandardCharsets.UTF_8) : null;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoOp() {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoOpAllocation() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
var a = Buf.create(i * 512);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetDict(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
Assertions.assertNotNull(dict);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetColumnName(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
Assertions.assertEquals("hash_map_testmap", dict.getColumnName());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGet(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyEx = fromString("test-key-1");
|
||||
var keyNonEx = fromString("test-nonexistent");
|
||||
Assertions.assertEquals("test-value", toString(dict.get(null, keyEx)));
|
||||
Assertions.assertEquals("test-value", toString(dict.get(null, keyEx)));
|
||||
Assertions.assertEquals("test-value", toString(dict.get(null, keyEx)));
|
||||
Assertions.assertEquals((String) null, toString(dict.get(null, keyNonEx)));
|
||||
Assertions.assertEquals((String) null, toString(dict.get(null, keyNonEx)));
|
||||
Assertions.assertEquals((String) null, toString(dict.get(null, keyNonEx)));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testPutExisting(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyEx = fromString("test-key-1");
|
||||
var value = fromString("test-value");
|
||||
|
||||
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
|
||||
dict.put(keyEx, value, resultType);
|
||||
|
||||
var afterSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
Assertions.assertEquals(0, afterSize - beforeSize);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testPutNew(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyNonEx = fromString("test-nonexistent");
|
||||
var value = fromString("test-value");
|
||||
|
||||
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
|
||||
dict.put(keyNonEx, value, resultType);
|
||||
|
||||
var afterSize = dict.sizeRange(null, LLRange.all(), false);
|
||||
Assertions.assertEquals(1, afterSize - beforeSize);
|
||||
|
||||
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).toList().contains("test-nonexistent"));
|
||||
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).toList().contains("test-nonexistent"));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetUpdateMode(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
assertEquals(updateMode, dict.getUpdateMode());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideUpdateArguments")
|
||||
public void testUpdateExisting(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var keyEx = fromString("test-key-1");
|
||||
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
long afterSize;
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyEx, old -> fromString("test-value"), updateReturnMode));
|
||||
afterSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
assertEquals(0, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyEx, old -> fromString("test-value"), updateReturnMode));
|
||||
afterSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
assertEquals(0, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyEx, old -> fromString("test-value"), updateReturnMode));
|
||||
afterSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
assertEquals(0, afterSize - beforeSize);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideUpdateArguments")
|
||||
public void testUpdateNew(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
|
||||
int expected = updateMode == UpdateMode.DISALLOW ? 0 : 1;
|
||||
var dict = getDict(updateMode);
|
||||
var keyNonEx = fromString("test-nonexistent");
|
||||
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
long afterSize;
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode));
|
||||
afterSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
assertEquals(expected, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode));
|
||||
afterSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
assertEquals(expected, afterSize - beforeSize);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode));
|
||||
afterSize = dict.sizeRange(null, RANGE_ALL, false);
|
||||
assertEquals(expected, afterSize - beforeSize);
|
||||
|
||||
if (updateMode != UpdateMode.DISALLOW) {
|
||||
Assertions.assertTrue(dict
|
||||
.getRangeKeys(null, RANGE_ALL, false, false)
|
||||
.map(this::toString)
|
||||
.toList()
|
||||
.contains("test-nonexistent"));
|
||||
Assertions.assertTrue(dict
|
||||
.getRangeKeys(null, RANGE_ALL, true, false)
|
||||
.map(this::toString)
|
||||
.toList()
|
||||
.contains("test-nonexistent"));
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testUpdateAndGetDelta(UpdateMode updateMode) {
|
||||
log.warn("Test not implemented");
|
||||
//todo: implement
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testClear(UpdateMode updateMode) {
|
||||
log.warn("Test not implemented");
|
||||
//todo: implement
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
log.warn("Test not implemented");
|
||||
//todo: implement
|
||||
}
|
||||
}
|
@ -1,21 +1,18 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.SyncUtils.*;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import io.netty5.buffer.Buffer;
|
||||
import io.netty5.util.Send;
|
||||
import it.cavallium.dbengine.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.UpdateReturnMode;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
@ -27,29 +24,25 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public abstract class TestLLDictionaryLeaks {
|
||||
|
||||
private TestAllocator allocator;
|
||||
private TempDb tempDb;
|
||||
private LLKeyValueDatabase db;
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(allocator).block(), "TempDB");
|
||||
public void beforeEach() throws IOException {
|
||||
ensureNoLeaks(false, false);
|
||||
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(), "TempDB");
|
||||
db = tempDb.db();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
getTempDbGenerator().closeTempDb(tempDb).block();
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
destroyAllocator(allocator);
|
||||
public void afterEach() throws IOException {
|
||||
getTempDbGenerator().closeTempDb(tempDb);
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideArguments() {
|
||||
@ -73,30 +66,25 @@ public abstract class TestLLDictionaryLeaks {
|
||||
}
|
||||
|
||||
private LLDictionary getDict(UpdateMode updateMode) {
|
||||
var dict = DbTestUtils.tempDictionary(db, updateMode).blockOptional().orElseThrow();
|
||||
var key1 = Mono.fromCallable(() -> fromString("test-key-1"));
|
||||
var key2 = Mono.fromCallable(() -> fromString("test-key-2"));
|
||||
var key3 = Mono.fromCallable(() -> fromString("test-key-3"));
|
||||
var key4 = Mono.fromCallable(() -> fromString("test-key-4"));
|
||||
var value = Mono.fromCallable(() -> fromString("test-value"));
|
||||
dict.put(key1, value, LLDictionaryResultType.VOID).block();
|
||||
dict.put(key2, value, LLDictionaryResultType.VOID).block();
|
||||
dict.put(key3, value, LLDictionaryResultType.VOID).block();
|
||||
dict.put(key4, value, LLDictionaryResultType.VOID).block();
|
||||
var dict = DbTestUtils.tempDictionary(db, updateMode);
|
||||
var key1 = fromString("test-key-1");
|
||||
var key2 = fromString("test-key-2");
|
||||
var key3 = fromString("test-key-3");
|
||||
var key4 = fromString("test-key-4");
|
||||
var value = fromString("test-value");
|
||||
dict.put(key1, value, LLDictionaryResultType.VOID);
|
||||
dict.put(key2, value, LLDictionaryResultType.VOID);
|
||||
dict.put(key3, value, LLDictionaryResultType.VOID);
|
||||
dict.put(key4, value, LLDictionaryResultType.VOID);
|
||||
return dict;
|
||||
}
|
||||
|
||||
private Buffer fromString(String s) {
|
||||
private Buf fromString(String s) {
|
||||
var sb = s.getBytes(StandardCharsets.UTF_8);
|
||||
var b = db.getAllocator().allocate(sb.length);
|
||||
try {
|
||||
b.writeBytes(sb);
|
||||
assert b.readableBytes() == sb.length;
|
||||
return b;
|
||||
} catch (Throwable ex) {
|
||||
b.close();
|
||||
throw ex;
|
||||
}
|
||||
var b = Buf.create(sb.length);
|
||||
b.addElements(0, sb);
|
||||
assert b.size() == sb.length;
|
||||
return b;
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -106,8 +94,7 @@ public abstract class TestLLDictionaryLeaks {
|
||||
@Test
|
||||
public void testNoOpAllocation() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
var a = allocator.allocator().allocate(i * 512);
|
||||
a.send().receive().close();
|
||||
var a = Buf.create(i * 512);
|
||||
}
|
||||
}
|
||||
|
||||
@ -124,30 +111,23 @@ public abstract class TestLLDictionaryLeaks {
|
||||
dict.getColumnName();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGetAllocator(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
dict.getAllocator();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testGet(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var key = Mono.fromCallable(() -> fromString("test"));
|
||||
runVoid(dict.get(null, key).then());
|
||||
runVoid(dict.get(null, key).then());
|
||||
runVoid(dict.get(null, key).then());
|
||||
var key = fromString("test");
|
||||
dict.get(null, key);
|
||||
dict.get(null, key);
|
||||
dict.get(null, key);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testPut(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
var dict = getDict(updateMode);
|
||||
var key = Mono.fromCallable(() -> fromString("test-key"));
|
||||
var value = Mono.fromCallable(() -> fromString("test-value"));
|
||||
runVoid(dict.put(key, value, resultType).then().doOnDiscard(Buffer.class, Buffer::close));
|
||||
var key = fromString("test-key");
|
||||
var value = fromString("test-value");
|
||||
dict.put(key, value, resultType);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ -161,19 +141,13 @@ public abstract class TestLLDictionaryLeaks {
|
||||
@MethodSource("provideUpdateArguments")
|
||||
public void testUpdate(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var key = Mono.fromCallable(() -> fromString("test-key"));
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(key, this::pass, updateReturnMode).then()
|
||||
);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(key, this::pass, updateReturnMode).then()
|
||||
);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.update(key, this::pass, updateReturnMode).then()
|
||||
);
|
||||
var key = fromString("test-key");
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(key, this::pass, updateReturnMode));
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(key, this::pass, updateReturnMode));
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(key, this::pass, updateReturnMode));
|
||||
}
|
||||
|
||||
private Buffer pass(@Nullable Buffer old) {
|
||||
private Buf pass(@Nullable Buf old) {
|
||||
if (old == null) return null;
|
||||
return old.copy();
|
||||
}
|
||||
@ -182,30 +156,24 @@ public abstract class TestLLDictionaryLeaks {
|
||||
@MethodSource("provideArguments")
|
||||
public void testUpdateAndGetDelta(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
var key = Mono.fromCallable(() -> fromString("test-key"));
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.updateAndGetDelta(key, this::pass).then()
|
||||
);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.updateAndGetDelta(key, this::pass).then()
|
||||
);
|
||||
runVoid(updateMode == UpdateMode.DISALLOW,
|
||||
dict.updateAndGetDelta(key, this::pass).then()
|
||||
);
|
||||
var key = fromString("test-key");
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.updateAndGetDelta(key, this::pass));
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.updateAndGetDelta(key, this::pass));
|
||||
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.updateAndGetDelta(key, this::pass));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideArguments")
|
||||
public void testClear(UpdateMode updateMode) {
|
||||
var dict = getDict(updateMode);
|
||||
runVoid(dict.clear());
|
||||
dict.clear();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("providePutArguments")
|
||||
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
|
||||
var dict = getDict(updateMode);
|
||||
var key = Mono.fromCallable(() -> fromString("test-key"));
|
||||
runVoid(dict.remove(key, resultType).then().doOnDiscard(Buffer.class, Buffer::close));
|
||||
var key = fromString("test-key");
|
||||
dict.remove(key, resultType);
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestLocalDictionary extends TestDictionary {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestLocalDictionaryMap extends TestDictionaryMap {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestLocalDictionaryMapDeep extends TestDictionaryMapDeep {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestLocalDictionaryMapDeepHashMap extends TestDictionaryMapDeepHashMap {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestLocalLLDictionary extends TestLLDictionary {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestLocalLLDictionaryLeaks extends TestLLDictionaryLeaks {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestLocalSingletons extends TestSingletons {
|
||||
|
@ -1,29 +1,26 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.SyncUtils.*;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import it.cavallium.dbengine.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.client.LuceneIndex;
|
||||
import it.cavallium.dbengine.client.Sort;
|
||||
import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery;
|
||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||
import it.cavallium.dbengine.database.LLScoreMode;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
|
||||
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.CountMultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@ -37,15 +34,10 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.util.function.Tuples;
|
||||
|
||||
public class TestLuceneIndex {
|
||||
|
||||
private final Logger log = LogManager.getLogger(this.getClass());
|
||||
private static LLTempHugePqEnv ENV;
|
||||
|
||||
private TestAllocator allocator;
|
||||
private TempDb tempDb;
|
||||
private LLLuceneIndex luceneSingle;
|
||||
private LLLuceneIndex luceneMulti;
|
||||
@ -56,14 +48,12 @@ public class TestLuceneIndex {
|
||||
|
||||
@BeforeAll
|
||||
public static void beforeAll() throws IOException {
|
||||
ENV = new LLTempHugePqEnv();
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
this.allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(allocator).block(), "TempDB");
|
||||
public void beforeEach() throws IOException {
|
||||
ensureNoLeaks(false, false);
|
||||
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(), "TempDB");
|
||||
luceneSingle = tempDb.luceneSingle();
|
||||
luceneMulti = tempDb.luceneMulti();
|
||||
}
|
||||
@ -72,13 +62,13 @@ public class TestLuceneIndex {
|
||||
return Stream.of(false, true).map(Arguments::of);
|
||||
}
|
||||
|
||||
private static final Flux<Boolean> multi = Flux.just(false, true);
|
||||
private static final Flux<LLScoreMode> scoreModes = Flux.just(LLScoreMode.NO_SCORES,
|
||||
private static final List<Boolean> multi = List.of(false, true);
|
||||
private static final List<LLScoreMode> scoreModes = List.of(LLScoreMode.NO_SCORES,
|
||||
LLScoreMode.TOP_SCORES,
|
||||
LLScoreMode.COMPLETE_NO_SCORES,
|
||||
LLScoreMode.COMPLETE
|
||||
);
|
||||
private static final Flux<Sort> multiSort = Flux.just(Sort.score(),
|
||||
private static final List<Sort> multiSort = List.of(Sort.score(),
|
||||
Sort.random(),
|
||||
Sort.no(),
|
||||
Sort.doc(),
|
||||
@ -86,62 +76,78 @@ public class TestLuceneIndex {
|
||||
Sort.numeric("longsort", true)
|
||||
);
|
||||
|
||||
record Tuple2<X, Y>(X getT1, Y getT2) {
|
||||
|
||||
public Object[] toArray() {
|
||||
return new Object[] {getT1, getT2};
|
||||
}
|
||||
}
|
||||
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {
|
||||
|
||||
public Object[] toArray() {
|
||||
return new Object[] {getT1, getT2, getT3};
|
||||
}
|
||||
}
|
||||
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {
|
||||
|
||||
public Object[] toArray() {
|
||||
return new Object[] {getT1, getT2, getT3, getT4};
|
||||
}
|
||||
}
|
||||
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {
|
||||
|
||||
public Object[] toArray() {
|
||||
return new Object[] {getT1, getT2, getT3, getT4, getT5};
|
||||
}
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideQueryArgumentsScoreMode() {
|
||||
return multi
|
||||
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()))
|
||||
.toStream();
|
||||
return multi.stream()
|
||||
.flatMap(shard -> scoreModes.stream().map(scoreMode -> new Tuple2<>(shard, scoreMode)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()));
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideQueryArgumentsSort() {
|
||||
return multi
|
||||
.concatMap(shard -> multiSort.map(multiSort -> Tuples.of(shard, multiSort)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()))
|
||||
.toStream();
|
||||
return multi.stream()
|
||||
.flatMap(shard -> multiSort.stream().map(multiSort -> new Tuple2<>(shard, multiSort)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()));
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideQueryArgumentsScoreModeAndSort() {
|
||||
return multi
|
||||
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
|
||||
.concatMap(tuple -> multiSort.map(multiSort -> Tuples.of(tuple.getT1(), tuple.getT2(), multiSort)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()))
|
||||
.toStream();
|
||||
return multi.stream()
|
||||
.flatMap(shard -> scoreModes.stream().map(scoreMode -> new Tuple2<>(shard, scoreMode)))
|
||||
.flatMap(tuple -> multiSort.stream().map(multiSort -> new Tuple3<>(tuple.getT1(), tuple.getT2(), multiSort)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
getTempDbGenerator().closeTempDb(tempDb).block();
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
destroyAllocator(allocator);
|
||||
public void afterEach() throws IOException {
|
||||
getTempDbGenerator().closeTempDb(tempDb);
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
public static void afterAll() throws IOException {
|
||||
ENV.close();
|
||||
}
|
||||
|
||||
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
|
||||
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
|
||||
index.updateDocument("test-key-1", "0123456789").block();
|
||||
index.updateDocument("test-key-2", "test 0123456789 test word").block();
|
||||
index.updateDocument("test-key-3", "0123456789 test example string").block();
|
||||
index.updateDocument("test-key-4", "hello world the quick brown fox jumps over the lazy dog").block();
|
||||
index.updateDocument("test-key-5", "hello the quick brown fox jumps over the lazy dog").block();
|
||||
index.updateDocument("test-key-6", "hello the quick brown fox jumps over the world dog").block();
|
||||
index.updateDocument("test-key-7", "the quick brown fox jumps over the world dog").block();
|
||||
index.updateDocument("test-key-8", "the quick brown fox jumps over the lazy dog").block();
|
||||
index.updateDocument("test-key-9", "Example1").block();
|
||||
index.updateDocument("test-key-10", "Example2").block();
|
||||
index.updateDocument("test-key-11", "Example3").block();
|
||||
index.updateDocument("test-key-12", "-234").block();
|
||||
index.updateDocument("test-key-13", "2111").block();
|
||||
index.updateDocument("test-key-14", "2999").block();
|
||||
index.updateDocument("test-key-15", "3902").block();
|
||||
Flux
|
||||
.range(1, 1000)
|
||||
.concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i))
|
||||
.transform(LLUtils::handleDiscard)
|
||||
.blockLast();
|
||||
LuceneIndex<String, String> index = DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti);
|
||||
index.updateDocument("test-key-1", "0123456789");
|
||||
index.updateDocument("test-key-2", "test 0123456789 test word");
|
||||
index.updateDocument("test-key-3", "0123456789 test example string");
|
||||
index.updateDocument("test-key-4", "hello world the quick brown fox jumps over the lazy dog");
|
||||
index.updateDocument("test-key-5", "hello the quick brown fox jumps over the lazy dog");
|
||||
index.updateDocument("test-key-6", "hello the quick brown fox jumps over the world dog");
|
||||
index.updateDocument("test-key-7", "the quick brown fox jumps over the world dog");
|
||||
index.updateDocument("test-key-8", "the quick brown fox jumps over the lazy dog");
|
||||
index.updateDocument("test-key-9", "Example1");
|
||||
index.updateDocument("test-key-10", "Example2");
|
||||
index.updateDocument("test-key-11", "Example3");
|
||||
index.updateDocument("test-key-12", "-234");
|
||||
index.updateDocument("test-key-13", "2111");
|
||||
index.updateDocument("test-key-14", "2999");
|
||||
index.updateDocument("test-key-15", "3902");
|
||||
IntStream.rangeClosed(1, 1000).forEach(i -> index.updateDocument("test-key-" + (15 + i), "" + i));
|
||||
tempDb.swappableLuceneSearcher().setSingle(new CountMultiSearcher());
|
||||
tempDb.swappableLuceneSearcher().setMulti(new CountMultiSearcher());
|
||||
assertCount(index, 1000 + 15);
|
||||
@ -155,8 +161,8 @@ public class TestLuceneIndex {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
}
|
||||
return index;
|
||||
}
|
||||
@ -166,8 +172,8 @@ public class TestLuceneIndex {
|
||||
}
|
||||
|
||||
private long getCount(LuceneIndex<String, String> luceneIndex) {
|
||||
luceneIndex.refresh(true).block();
|
||||
var totalHitsCount = run(luceneIndex.count(null, new MatchAllDocsQuery()));
|
||||
luceneIndex.refresh(true);
|
||||
var totalHitsCount = luceneIndex.count(null, new MatchAllDocsQuery());
|
||||
Assertions.assertTrue(totalHitsCount.exact(), "Can't get count because the total hits count is not exact");
|
||||
return totalHitsCount.value();
|
||||
}
|
||||
@ -179,8 +185,7 @@ public class TestLuceneIndex {
|
||||
@Test
|
||||
public void testNoOpAllocation() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
var a = allocator.allocator().allocate(i * 512);
|
||||
a.send().receive().close();
|
||||
var a = Buf.create(i * 512);
|
||||
}
|
||||
}
|
||||
|
||||
@ -195,7 +200,7 @@ public class TestLuceneIndex {
|
||||
@MethodSource("provideArguments")
|
||||
public void testDeleteAll(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
runVoid(luceneIndex.deleteAll());
|
||||
luceneIndex.deleteAll();
|
||||
assertCount(luceneIndex, 0);
|
||||
}
|
||||
|
||||
@ -204,7 +209,7 @@ public class TestLuceneIndex {
|
||||
public void testDelete(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
var prevCount = getCount(luceneIndex);
|
||||
runVoid(luceneIndex.deleteDocument("test-key-1"));
|
||||
luceneIndex.deleteDocument("test-key-1");
|
||||
assertCount(luceneIndex, prevCount - 1);
|
||||
}
|
||||
|
||||
@ -213,7 +218,7 @@ public class TestLuceneIndex {
|
||||
public void testUpdateSameDoc(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
var prevCount = getCount(luceneIndex);
|
||||
runVoid(luceneIndex.updateDocument("test-key-1", "new-value"));
|
||||
luceneIndex.updateDocument("test-key-1", "new-value");
|
||||
assertCount(luceneIndex, prevCount );
|
||||
}
|
||||
|
||||
@ -222,7 +227,7 @@ public class TestLuceneIndex {
|
||||
public void testUpdateNewDoc(boolean shards) {
|
||||
var luceneIndex = getLuceneIndex(shards, null);
|
||||
var prevCount = getCount(luceneIndex);
|
||||
runVoid(luceneIndex.updateDocument("test-key-new", "new-value"));
|
||||
luceneIndex.updateDocument("test-key-new", "new-value");
|
||||
assertCount(luceneIndex, prevCount + 1);
|
||||
}
|
||||
|
@ -1,21 +1,17 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
|
||||
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
|
||||
import static it.cavallium.dbengine.SyncUtils.*;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import io.netty.buffer.PooledByteBufAllocator;
|
||||
import it.cavallium.dbengine.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
|
||||
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
|
||||
import it.cavallium.dbengine.tests.TestLuceneIndex.Tuple2;
|
||||
import it.cavallium.dbengine.client.HitKey;
|
||||
import it.cavallium.dbengine.client.Hits;
|
||||
import it.cavallium.dbengine.client.LuceneIndex;
|
||||
import it.cavallium.dbengine.client.Sort;
|
||||
import it.cavallium.dbengine.client.LazyHitKey;
|
||||
import it.cavallium.dbengine.client.query.ClientQueryParams;
|
||||
import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder;
|
||||
import it.cavallium.dbengine.client.query.current.data.BooleanQuery;
|
||||
@ -29,7 +25,6 @@ import it.cavallium.dbengine.client.query.current.data.Term;
|
||||
import it.cavallium.dbengine.client.query.current.data.TermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
|
||||
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.CountMultiSearcher;
|
||||
@ -38,18 +33,17 @@ import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.StandardSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.ScoredPagedMultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.PagedLocalSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.SortedScoredFullMultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.SortedByScoreFullMultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.UnsortedStreamingMultiSearcher;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.commons.lang3.function.FailableConsumer;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
@ -63,18 +57,12 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.FluxSink.OverflowStrategy;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
import reactor.util.function.Tuples;
|
||||
|
||||
public class TestLuceneSearches {
|
||||
|
||||
private static final Logger log = LogManager.getLogger(TestLuceneSearches.class);
|
||||
private static LLTempHugePqEnv ENV;
|
||||
private static final MemoryTemporaryDbGenerator TEMP_DB_GENERATOR = new MemoryTemporaryDbGenerator();
|
||||
|
||||
private static TestAllocator allocator;
|
||||
private static TempDb tempDb;
|
||||
private static LLLuceneIndex luceneSingle;
|
||||
private static LLLuceneIndex luceneMulti;
|
||||
@ -102,31 +90,25 @@ public class TestLuceneSearches {
|
||||
modifiableElements.put("test-key-13", "2111");
|
||||
modifiableElements.put("test-key-14", "2999");
|
||||
modifiableElements.put("test-key-15", "3902");
|
||||
runVoid(Flux.range(1, 1000).doOnNext(i -> modifiableElements.put("test-key-" + (15 + i), "" + i)).then());
|
||||
IntStream.rangeClosed(1, 1000).forEach(i -> modifiableElements.put("test-key-" + (15 + i), "" + i));
|
||||
ELEMENTS = Collections.unmodifiableMap(modifiableElements);
|
||||
}
|
||||
|
||||
@BeforeAll
|
||||
public static void beforeAll() throws IOException {
|
||||
allocator = newAllocator();
|
||||
ensureNoLeaks(allocator.allocator(), false, false);
|
||||
tempDb = Objects.requireNonNull(TEMP_DB_GENERATOR.openTempDb(allocator).block(), "TempDB");
|
||||
ensureNoLeaks(false, false);
|
||||
tempDb = Objects.requireNonNull(TEMP_DB_GENERATOR.openTempDb(), "TempDB");
|
||||
luceneSingle = tempDb.luceneSingle();
|
||||
luceneMulti = tempDb.luceneMulti();
|
||||
ENV = new LLTempHugePqEnv();
|
||||
|
||||
setUpIndex(true);
|
||||
setUpIndex(false);
|
||||
}
|
||||
|
||||
private static void setUpIndex(boolean shards) {
|
||||
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
|
||||
LuceneIndex<String, String> index = DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti);
|
||||
|
||||
Flux
|
||||
.fromIterable(ELEMENTS.entrySet())
|
||||
.concatMap(entry -> index.updateDocument(entry.getKey(), entry.getValue()))
|
||||
.subscribeOn(Schedulers.boundedElastic())
|
||||
.blockLast();
|
||||
ELEMENTS.forEach(index::updateDocument);
|
||||
tempDb.swappableLuceneSearcher().setSingle(new CountMultiSearcher());
|
||||
tempDb.swappableLuceneSearcher().setMulti(new CountMultiSearcher());
|
||||
assertCount(index, 1000 + 15);
|
||||
@ -141,8 +123,8 @@ public class TestLuceneSearches {
|
||||
return Stream.of(false, true).map(Arguments::of);
|
||||
}
|
||||
|
||||
private static final Flux<Boolean> multi = Flux.just(false, true);
|
||||
private static final Flux<Sort> multiSort = Flux.just(
|
||||
private static final List<Boolean> multi = List.of(false, true);
|
||||
private static final List<Sort> multiSort = List.of(
|
||||
Sort.score(),
|
||||
//todo: fix random sort field
|
||||
//Sort.randomSortField(),
|
||||
@ -154,50 +136,43 @@ public class TestLuceneSearches {
|
||||
Sort.numeric("intsort", true)
|
||||
);
|
||||
|
||||
private static Flux<LocalSearcher> getSearchers(ExpectedQueryType info) {
|
||||
return Flux.push(sink -> {
|
||||
if (info.shard()) {
|
||||
if (info.onlyCount()) {
|
||||
sink.next(new CountMultiSearcher());
|
||||
} else {
|
||||
sink.next(new ScoredPagedMultiSearcher());
|
||||
if (info.sorted() && !info.sortedByScore()) {
|
||||
sink.next(new SortedScoredFullMultiSearcher(ENV));
|
||||
} else {
|
||||
sink.next(new SortedByScoreFullMultiSearcher(ENV));
|
||||
}
|
||||
if (!info.sorted()) {
|
||||
sink.next(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher()));
|
||||
sink.next(new UnsortedStreamingMultiSearcher());
|
||||
}
|
||||
}
|
||||
sink.next(new AdaptiveMultiSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
private static List<LocalSearcher> getSearchers(ExpectedQueryType info) {
|
||||
var sink = new ArrayList<LocalSearcher>();
|
||||
if (info.shard()) {
|
||||
if (info.onlyCount()) {
|
||||
sink.add(new CountMultiSearcher());
|
||||
} else {
|
||||
if (info.onlyCount()) {
|
||||
sink.next(new CountMultiSearcher());
|
||||
} else {
|
||||
sink.next(new PagedLocalSearcher());
|
||||
sink.add(new ScoredPagedMultiSearcher());
|
||||
if (!info.sorted()) {
|
||||
sink.add(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher()));
|
||||
sink.add(new UnsortedStreamingMultiSearcher());
|
||||
}
|
||||
sink.next(new AdaptiveLocalSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
}
|
||||
sink.complete();
|
||||
}, OverflowStrategy.BUFFER);
|
||||
sink.add(new AdaptiveMultiSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
} else {
|
||||
if (info.onlyCount()) {
|
||||
sink.add(new CountMultiSearcher());
|
||||
} else {
|
||||
sink.add(new PagedLocalSearcher());
|
||||
}
|
||||
sink.add(new AdaptiveLocalSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
}
|
||||
return sink;
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideQueryArgumentsScoreMode() {
|
||||
return multi.map(tuple -> Arguments.of(multi)).toStream();
|
||||
return multi.stream().map(tuple -> Arguments.of(multi));
|
||||
}
|
||||
|
||||
public static Stream<Arguments> provideQueryArgumentsScoreModeAndSort() {
|
||||
return multi
|
||||
.concatMap(multi -> multiSort.map(multiSort -> Tuples.of(multi, multiSort)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()))
|
||||
.toStream();
|
||||
return multi.stream()
|
||||
.flatMap(multi -> multiSort.stream().map(multiSort -> new Tuple2<>(multi, multiSort)))
|
||||
.map(tuple -> Arguments.of(tuple.toArray()));
|
||||
}
|
||||
|
||||
private static void runSearchers(ExpectedQueryType expectedQueryType, FailableConsumer<LocalSearcher, Throwable> consumer)
|
||||
throws Throwable {
|
||||
var searchers = run(getSearchers(expectedQueryType).collectList());
|
||||
var searchers = getSearchers(expectedQueryType);
|
||||
for (LocalSearcher searcher : searchers) {
|
||||
log.info("Using searcher \"{}\"", searcher.getName());
|
||||
consumer.accept(searcher);
|
||||
@ -214,10 +189,8 @@ public class TestLuceneSearches {
|
||||
|
||||
@AfterAll
|
||||
public static void afterAll() throws IOException {
|
||||
TEMP_DB_GENERATOR.closeTempDb(tempDb).block();
|
||||
ENV.close();
|
||||
ensureNoLeaks(allocator.allocator(), true, false);
|
||||
destroyAllocator(allocator);
|
||||
TEMP_DB_GENERATOR.closeTempDb(tempDb);
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
|
||||
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
|
||||
@ -231,8 +204,8 @@ public class TestLuceneSearches {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
|
||||
}
|
||||
return shards ? multiIndex : localIndex;
|
||||
}
|
||||
@ -242,8 +215,8 @@ public class TestLuceneSearches {
|
||||
}
|
||||
|
||||
private static long getCount(LuceneIndex<String, String> luceneIndex) {
|
||||
luceneIndex.refresh(true).block();
|
||||
var totalHitsCount = run(luceneIndex.count(null, new MatchAllDocsQuery()));
|
||||
luceneIndex.refresh(true);
|
||||
var totalHitsCount = luceneIndex.count(null, new MatchAllDocsQuery());
|
||||
Assertions.assertTrue(totalHitsCount.exact(), "Can't get count because the total hits count is not exact");
|
||||
return totalHitsCount.value();
|
||||
}
|
||||
@ -266,7 +239,7 @@ public class TestLuceneSearches {
|
||||
runSearchers(expectedQueryType, searcher -> {
|
||||
var luceneIndex = getLuceneIndex(expectedQueryType.shard(), searcher);
|
||||
var query = queryParamsBuilder.build();
|
||||
try (var results = run(luceneIndex.search(query))) {
|
||||
try (var results = luceneIndex.search(query)) {
|
||||
var hits = results.totalHitsCount();
|
||||
var keys = getResults(results);
|
||||
if (hits.exact()) {
|
||||
@ -278,7 +251,7 @@ public class TestLuceneSearches {
|
||||
var standardSearcher = new StandardSearcher();
|
||||
luceneIndex = getLuceneIndex(expectedQueryType.shard(), standardSearcher);
|
||||
var officialQuery = queryParamsBuilder.limit(ELEMENTS.size() * 2L).build();
|
||||
try (var officialResults = run(luceneIndex.search(officialQuery))) {
|
||||
try (var officialResults = luceneIndex.search(officialQuery)) {
|
||||
var officialHits = officialResults.totalHitsCount();
|
||||
var officialKeys = getResults(officialResults);
|
||||
if (officialHits.exact()) {
|
||||
@ -303,7 +276,7 @@ public class TestLuceneSearches {
|
||||
@MethodSource("provideQueryArgumentsScoreModeAndSort")
|
||||
public void testSearchNoDocs(boolean shards, Sort multiSort) throws Throwable {
|
||||
var queryBuilder = ClientQueryParams
|
||||
.<LazyHitKey<String>>builder()
|
||||
.<HitKey<String>>builder()
|
||||
.query(new MatchNoDocsQuery())
|
||||
.snapshot(null)
|
||||
.computePreciseHitsCount(true)
|
||||
@ -317,7 +290,7 @@ public class TestLuceneSearches {
|
||||
@MethodSource("provideQueryArgumentsScoreModeAndSort")
|
||||
public void testSearchAllDocs(boolean shards, Sort multiSort) throws Throwable {
|
||||
var queryBuilder = ClientQueryParams
|
||||
.<LazyHitKey<String>>builder()
|
||||
.<HitKey<String>>builder()
|
||||
.query(new MatchAllDocsQuery())
|
||||
.snapshot(null)
|
||||
.computePreciseHitsCount(true)
|
||||
@ -370,10 +343,10 @@ public class TestLuceneSearches {
|
||||
}
|
||||
|
||||
private List<Scored> getResults(Hits<HitKey<String>> results) {
|
||||
return run(results
|
||||
return results
|
||||
.results()
|
||||
.map(key -> new Scored(key.key(), key.score()))
|
||||
.collectList());
|
||||
.toList();
|
||||
}
|
||||
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestMemoryDictionary extends TestDictionary {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestMemoryDictionaryMap extends TestDictionaryMap {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestMemoryLLDictionary extends TestLLDictionary {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestMemoryLLDictionaryLeaks extends TestLLDictionaryLeaks {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
public class TestMemorySingletons extends TestSingletons {
|
||||
|
@ -1,4 +1,4 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import it.cavallium.dbengine.database.disk.RocksLog4jLogger;
|
||||
import java.io.IOException;
|
||||
@ -23,7 +23,6 @@ import org.rocksdb.PersistentCache;
|
||||
import org.rocksdb.ReadOptions;
|
||||
import org.rocksdb.RocksDB;
|
||||
import org.rocksdb.RocksDBException;
|
||||
import org.rocksdb.TableFormatConfig;
|
||||
import org.rocksdb.WriteOptions;
|
||||
import org.rocksdb.util.SizeUnit;
|
||||
|
@ -1,8 +1,8 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import io.netty5.buffer.Buffer;
|
||||
import io.netty5.buffer.BufferAllocator;
|
||||
import it.cavallium.dbengine.buffers.Buf;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep;
|
||||
import java.util.Arrays;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
@ -11,16 +11,12 @@ import org.junit.jupiter.api.Test;
|
||||
|
||||
public class TestRanges {
|
||||
|
||||
private static BufferAllocator alloc;
|
||||
|
||||
@BeforeAll
|
||||
public static void beforeAll() {
|
||||
alloc = BufferAllocator.offHeapPooled();
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
public static void afterAll() {
|
||||
alloc = BufferAllocator.offHeapPooled();
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -43,17 +39,15 @@ public class TestRanges {
|
||||
|
||||
public void testNextRangeKey(byte[] prefixKey) {
|
||||
byte[] firstRangeKey;
|
||||
Buffer firstRangeKeyBuf = alloc.allocate(prefixKey.length).writeBytes(prefixKey);
|
||||
try (firstRangeKeyBuf) {
|
||||
DatabaseMapDictionaryDeep.firstRangeKey(firstRangeKeyBuf, prefixKey.length, 7, 3);
|
||||
firstRangeKey = LLUtils.toArray(firstRangeKeyBuf);
|
||||
}
|
||||
Buf firstRangeKeyBuf = Buf.create(prefixKey.length);
|
||||
firstRangeKeyBuf.addElements(0, prefixKey);
|
||||
firstRangeKeyBuf = DatabaseMapDictionaryDeep.firstRangeKey(firstRangeKeyBuf, prefixKey.length, Buf.createZeroes(7 + 3));
|
||||
firstRangeKey = firstRangeKeyBuf.asArray();
|
||||
byte[] nextRangeKey;
|
||||
Buffer nextRangeKeyBuf = alloc.allocate(prefixKey.length).writeBytes(prefixKey);
|
||||
try (nextRangeKeyBuf) {
|
||||
DatabaseMapDictionaryDeep.nextRangeKey(nextRangeKeyBuf, prefixKey.length, 7, 3);
|
||||
nextRangeKey = LLUtils.toArray(nextRangeKeyBuf);
|
||||
}
|
||||
Buf nextRangeKeyBuf = Buf.create(prefixKey.length);
|
||||
nextRangeKeyBuf.addElements(0, prefixKey);
|
||||
nextRangeKeyBuf = DatabaseMapDictionaryDeep.nextRangeKey(nextRangeKeyBuf, prefixKey.length, Buf.createZeroes(7 + 3));
|
||||
nextRangeKey = nextRangeKeyBuf.asArray();
|
||||
|
||||
if (Arrays.equals(prefixKey, new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF})) {
|
||||
org.assertj.core.api.Assertions
|
136
src/test/java/it/cavallium/dbengine/tests/TestSingletons.java
Normal file
136
src/test/java/it/cavallium/dbengine/tests/TestSingletons.java
Normal file
@ -0,0 +1,136 @@
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
|
||||
import static it.cavallium.dbengine.tests.DbTestUtils.tempDb;
|
||||
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseInt;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseLong;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseSingleton;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import java.io.IOException;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
|
||||
public abstract class TestSingletons {
|
||||
|
||||
protected abstract TemporaryDbGenerator getTempDbGenerator();
|
||||
|
||||
private static Stream<Arguments> provideNumberWithRepeats() {
|
||||
return Stream.of(
|
||||
Arguments.of(Integer.MIN_VALUE, 2),
|
||||
Arguments.of(-11, 2),
|
||||
Arguments.of(0, 3),
|
||||
Arguments.of(102, 5)
|
||||
);
|
||||
}
|
||||
|
||||
private static Stream<Arguments> provideLongNumberWithRepeats() {
|
||||
return Stream.of(
|
||||
Arguments.of(Long.MIN_VALUE, 2),
|
||||
Arguments.of(-11L, 2),
|
||||
Arguments.of(0L, 3),
|
||||
Arguments.of(102L, 5)
|
||||
);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEach() {
|
||||
ensureNoLeaks(false, false);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void afterEach() {
|
||||
ensureNoLeaks(true, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateInteger() throws IOException {
|
||||
tempDb(getTempDbGenerator(), db -> tempInt(db, "test", 0).get(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateIntegerNoop() throws IOException {
|
||||
tempDb(getTempDbGenerator(), db -> tempInt(db, "test", 0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateLong() throws IOException {
|
||||
tempDb(getTempDbGenerator(), db -> tempLong(db, "test", 0).get(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateSingleton() throws IOException {
|
||||
tempDb(getTempDbGenerator(), db -> tempSingleton(db, "testsingleton").get(null));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints = {Integer.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Integer.MAX_VALUE})
|
||||
public void testDefaultValueInteger(int i) throws IOException {
|
||||
Assertions.assertEquals((Integer) i, tempDb(getTempDbGenerator(), db -> tempInt(db, "test", i).get(null)));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(longs = {Long.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Long.MAX_VALUE})
|
||||
public void testDefaultValueLong(long i) throws IOException {
|
||||
Assertions.assertEquals((Long) i, tempDb(getTempDbGenerator(), db -> tempLong(db, "test", i).get(null)));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideNumberWithRepeats")
|
||||
public void testSetInteger(Integer i, Integer repeats) throws IOException {
|
||||
Assertions.assertEquals(i, tempDb(getTempDbGenerator(), db -> {
|
||||
var dbInt = tempInt(db, "test", 0);
|
||||
for (int integer = 0; integer < repeats; integer++) {
|
||||
dbInt.set((int) System.currentTimeMillis());
|
||||
}
|
||||
dbInt.set(i);
|
||||
return dbInt.get(null);
|
||||
}));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideLongNumberWithRepeats")
|
||||
public void testSetLong(Long i, Integer repeats) throws IOException {
|
||||
Assertions.assertEquals(i, tempDb(getTempDbGenerator(), db -> {
|
||||
var dbLong = tempLong(db, "test", 0);
|
||||
for (int integer = 0; integer < repeats; integer++) {
|
||||
dbLong.set(System.currentTimeMillis());
|
||||
}
|
||||
dbLong.set(i);
|
||||
return dbLong.get(null);
|
||||
}));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provideLongNumberWithRepeats")
|
||||
public void testSetSingleton(Long i, Integer repeats) throws IOException {
|
||||
Assertions.assertEquals(Long.toString(i), tempDb(getTempDbGenerator(), db -> {
|
||||
var dbSingleton = tempSingleton(db, "test");
|
||||
for (int integer = 0; integer < repeats; integer++) {
|
||||
dbSingleton.set(Long.toString(System.currentTimeMillis()));
|
||||
}
|
||||
dbSingleton.set(Long.toString(i));
|
||||
return dbSingleton.get(null);
|
||||
}));
|
||||
}
|
||||
|
||||
public static DatabaseInt tempInt(LLKeyValueDatabase database, String name, int defaultValue) {
|
||||
return database.getInteger("ints", name, defaultValue);
|
||||
}
|
||||
|
||||
public static DatabaseLong tempLong(LLKeyValueDatabase database, String name, long defaultValue) {
|
||||
return database.getLong("longs", name, defaultValue);
|
||||
}
|
||||
|
||||
public static DatabaseSingleton<String> tempSingleton(LLKeyValueDatabase database, String name) {
|
||||
return new DatabaseSingleton<>(database.getSingleton("longs", name), Serializer.UTF8_SERIALIZER);
|
||||
}
|
||||
}
|
@ -1,14 +1,10 @@
|
||||
package it.cavallium.dbengine;
|
||||
package it.cavallium.dbengine.tests;
|
||||
|
||||
import static it.cavallium.dbengine.client.UninterruptibleScheduler.uninterruptibleScheduler;
|
||||
import static it.cavallium.dbengine.database.LLUtils.singleOrClose;
|
||||
import static it.cavallium.dbengine.lucene.searcher.GlobalQueryRewrite.NO_REWRITE;
|
||||
|
||||
import io.netty5.util.Send;
|
||||
import com.google.common.collect.Streams;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import it.cavallium.dbengine.database.LLKeyScore;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
|
||||
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
||||
import it.cavallium.dbengine.lucene.LuceneCloseable;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
@ -17,17 +13,13 @@ import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
|
||||
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult;
|
||||
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.ShardIndexSearcher;
|
||||
import it.cavallium.dbengine.utils.SimpleResource;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
||||
|
||||
@ -40,12 +32,12 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||
public LuceneSearchResult collectMulti(LLIndexSearchers indexSearchers,
|
||||
LocalQueryParams queryParams,
|
||||
String keyFieldName,
|
||||
GlobalQueryRewrite transformer) {
|
||||
if (transformer != NO_REWRITE) {
|
||||
return LuceneUtils.rewriteMulti(this, indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
return LuceneUtils.rewriteMulti(this, indexSearchers, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
if (queryParams.isSorted() && queryParams.limitLong() > 0) {
|
||||
throw new UnsupportedOperationException(
|
||||
@ -56,33 +48,28 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
||||
"Scored queries are not supported" + " by SimpleUnsortedUnscoredLuceneMultiSearcher");
|
||||
}
|
||||
|
||||
return singleOrClose(indexSearchersMono, indexSearchers -> {
|
||||
var localQueryParams = getLocalQueryParams(queryParams);
|
||||
return Flux
|
||||
.fromIterable(indexSearchers.llShards())
|
||||
.flatMap(searcher -> localSearcher.collect(Mono.just(searcher), localQueryParams, keyFieldName, transformer))
|
||||
.collectList()
|
||||
.map(results -> {
|
||||
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
|
||||
List<Flux<LLKeyScore>> resultsFluxes = new ArrayList<>(results.size());
|
||||
boolean exactTotalHitsCount = true;
|
||||
long totalHitsCountValue = 0;
|
||||
for (LuceneSearchResult result : results) {
|
||||
resultsToDrop.add(result);
|
||||
resultsFluxes.add(result.results());
|
||||
exactTotalHitsCount &= result.totalHitsCount().exact();
|
||||
totalHitsCountValue += result.totalHitsCount().value();
|
||||
}
|
||||
var localQueryParams = getLocalQueryParams(queryParams);
|
||||
var results = indexSearchers.llShards().stream()
|
||||
.map(searcher -> localSearcher.collect(searcher, localQueryParams, keyFieldName, transformer))
|
||||
.toList();
|
||||
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
|
||||
List<Stream<LLKeyScore>> resultsFluxes = new ArrayList<>(results.size());
|
||||
boolean exactTotalHitsCount = true;
|
||||
long totalHitsCountValue = 0;
|
||||
for (LuceneSearchResult result : results) {
|
||||
resultsToDrop.add(result);
|
||||
resultsFluxes.add(result.results());
|
||||
exactTotalHitsCount &= result.totalHitsCount().exact();
|
||||
totalHitsCountValue += result.totalHitsCount().value();
|
||||
}
|
||||
|
||||
var totalHitsCount = new TotalHitsCount(totalHitsCountValue, exactTotalHitsCount);
|
||||
Flux<LLKeyScore> mergedFluxes = Flux
|
||||
.merge(resultsFluxes)
|
||||
.skip(queryParams.offsetLong())
|
||||
.take(queryParams.limitLong(), true);
|
||||
var totalHitsCount = new TotalHitsCount(totalHitsCountValue, exactTotalHitsCount);
|
||||
//noinspection unchecked
|
||||
Stream<LLKeyScore> mergedFluxes = (Stream<LLKeyScore>) (Stream) Streams.concat(resultsFluxes.toArray(Stream<?>[]::new))
|
||||
.skip(queryParams.offsetLong())
|
||||
.limit(queryParams.limitLong());
|
||||
|
||||
return new MyLuceneSearchResult(totalHitsCount, mergedFluxes, resultsToDrop, indexSearchers);
|
||||
});
|
||||
});
|
||||
return new MyLuceneSearchResult(totalHitsCount, mergedFluxes, resultsToDrop, indexSearchers);
|
||||
}
|
||||
|
||||
private LocalQueryParams getLocalQueryParams(LocalQueryParams queryParams) {
|
||||
@ -107,7 +94,7 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
||||
private final LLIndexSearchers indexSearchers;
|
||||
|
||||
public MyLuceneSearchResult(TotalHitsCount totalHitsCount,
|
||||
Flux<LLKeyScore> mergedFluxes,
|
||||
Stream<LLKeyScore> mergedFluxes,
|
||||
List<LuceneSearchResult> resultsToDrop,
|
||||
LLIndexSearchers indexSearchers) {
|
||||
super(totalHitsCount, mergedFluxes);
|
19
src/test/java/module-info.java
Normal file
19
src/test/java/module-info.java
Normal file
@ -0,0 +1,19 @@
|
||||
module dbengine.tests {
|
||||
requires org.junit.jupiter.api;
|
||||
requires dbengine;
|
||||
requires data.generator.runtime;
|
||||
requires org.assertj.core;
|
||||
requires org.apache.lucene.core;
|
||||
requires it.unimi.dsi.fastutil;
|
||||
requires org.apache.lucene.queryparser;
|
||||
requires io.netty.common;
|
||||
requires org.jetbrains.annotations;
|
||||
requires micrometer.core;
|
||||
requires org.junit.jupiter.params;
|
||||
requires com.google.common;
|
||||
requires org.apache.logging.log4j;
|
||||
requires io.netty.buffer;
|
||||
requires org.apache.commons.lang3;
|
||||
requires rocksdbjni;
|
||||
opens it.cavallium.dbengine.tests;
|
||||
}
|
Loading…
Reference in New Issue
Block a user