Fix sigsegv

This commit is contained in:
Andrea Cavalli 2023-02-22 22:31:36 +01:00
parent cd15f8d23d
commit 59f9f01268
71 changed files with 1198 additions and 2834 deletions

View File

@ -489,6 +489,9 @@ public class LLUtils {
}
public static Buf unmodifiableBytes(Buf previous) {
if (previous == null) {
return null;
}
previous.freeze();
return previous;
}

View File

@ -176,7 +176,7 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
buf1.skipNBytes(keyPrefixLength);
suffixAndExtKeyConsistency(buf1.available());
key = deserializeSuffix(serializedValue);
key = deserializeSuffix(buf1);
U value = valueSerializer.deserialize(serializedValue);
deserializedEntry = Map.entry(key, value);
return deserializedEntry;
@ -295,6 +295,9 @@ public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U,
var keyMono = serializeKeySuffixToKey(keySuffix);
var valueMono = serializeValue(value);
var valueBuf = dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE);
if (valueBuf == null) {
return null;
}
return deserializeValue(keySuffix, BufDataInput.create(valueBuf));
}

View File

@ -18,6 +18,7 @@ import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletionException;
@ -28,6 +29,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;
// todo: implement optimized methods (which?)
public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implements DatabaseStageMap<T, U, US> {
@ -74,18 +76,25 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
}
}
static Buf firstRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
modifiablePrefixKey.addAll(prefixKey);
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
@VisibleForTesting
public static Buf firstRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
return createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
}
@VisibleForTesting
public static Buf nextRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
Buf modifiablePrefixKey = createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
incrementPrefix(modifiablePrefixKey, prefixLength);
return modifiablePrefixKey;
}
static Buf nextRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
private static Buf createFullKeyWithEmptySuffixAndExt(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
modifiablePrefixKey.addAll(prefixKey);
if (prefixKey != null) {
modifiablePrefixKey.addAll(prefixKey);
}
assert prefixKey != null || prefixLength == 0 : "Prefix length is " + prefixLength + " but the prefix key is null";
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
incrementPrefix(modifiablePrefixKey, prefixLength);
return modifiablePrefixKey;
}
@ -98,7 +107,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
var suffixLengthAndExtLength = suffixAndExtZeroes.size();
assert result.size() == prefixLength;
assert suffixLengthAndExtLength > 0 : "Suffix length + ext length is < 0: " + suffixLengthAndExtLength;
result.size(prefixLength + suffixLengthAndExtLength);
result.size(prefixLength);
modifiablePrefixKey.addAll(suffixAndExtZeroes);
assert modifiablePrefixKey.size() == prefixLength + suffixAndExtZeroes.size() : "Result buffer size is wrong";
}
@ -172,6 +181,10 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
@SuppressWarnings("unused")
protected boolean suffixKeyLengthConsistency(int keySuffixLength) {
assert
this.keySuffixLength == keySuffixLength :
"Key suffix length is " + keySuffixLength + ", but it should be " + this.keySuffixLength + " bytes long";
//noinspection ConstantValue
return this.keySuffixLength == keySuffixLength;
}
@ -313,8 +326,8 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
}
protected void serializeSuffixTo(T keySuffix, BufDataOutput output) throws SerializationException {
assert suffixKeyLengthConsistency(output.size());
var beforeWriterOffset = output.size();
assert beforeWriterOffset == keyPrefixLength;
keySuffixSerializer.serialize(keySuffix, output);
var afterWriterOffset = output.size();
assert suffixKeyLengthConsistency(afterWriterOffset - beforeWriterOffset)

View File

@ -88,10 +88,9 @@ public final class DatabaseMapSingle<U> implements DatabaseStageEntry<U> {
}
@Override
public U update(SerializationFunction<@Nullable U, @Nullable U> updater,
UpdateReturnMode updateReturnMode) {
public U update(SerializationFunction<@Nullable U, @Nullable U> updater, UpdateReturnMode updateReturnMode) {
Buf resultBytes = dictionary.update(key, this.createUpdater(updater), updateReturnMode);
return deserializeValue(resultBytes);
return resultBytes != null ? deserializeValue(resultBytes) : null;
}
@Override

View File

@ -39,6 +39,9 @@ public class DatabaseSingleton<U> implements DatabaseStageEntry<U> {
}
private U deserializeValue(Buf value) {
if (value == null) {
return null;
}
try {
return serializer.deserialize(BufDataInput.create(value));
} catch (IndexOutOfBoundsException ex) {

View File

@ -203,10 +203,11 @@ public interface DatabaseStageMap<T, U, US extends DatabaseStage<U>> extends Dat
@Override
default Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
Object2ObjectSortedMap<T, U> map = this
.getAllValues(snapshot, true)
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
return map.isEmpty() ? null : map;
try (var stream = this.getAllValues(snapshot, true)) {
Object2ObjectSortedMap<T, U> map = stream
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new));
return map.isEmpty() ? null : map;
}
}
@Override

View File

@ -5,6 +5,7 @@ import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
import static it.cavallium.dbengine.database.LLUtils.toStringSafe;
import static it.cavallium.dbengine.database.disk.UpdateAtomicResultMode.DELTA;
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
import static java.util.Objects.requireNonNull;
import static it.cavallium.dbengine.utils.StreamUtils.batches;
@ -699,7 +700,7 @@ public class LLLocalDictionary implements LLDictionary {
ro.close();
throw new DBException("Failed to open rocksdb iterator", ex);
}
return Stream.generate(() -> {
return streamWhileNonNull(() -> {
if (!rocksIterator.isValid()) return null;
Buf rawKey = null;
try {
@ -709,7 +710,7 @@ public class LLLocalDictionary implements LLDictionary {
return new BadBlock(databaseName, ColumnUtils.special(columnName), rawKey, ex);
}
return null;
}).takeWhile(x -> rocksIterator.isValid()).filter(Objects::nonNull).onClose(() -> {
}).takeWhile(x -> rocksIterator.isValid()).onClose(() -> {
rocksIterator.close();
ro.close();
});
@ -1030,14 +1031,16 @@ public class LLLocalDictionary implements LLDictionary {
if (USE_CURRENT_FASTSIZE_FOR_OLD_SNAPSHOTS || rocksdbSnapshot.snapshot() == null) {
try {
if (USE_NUM_ENTRIES_PRECISE_COUNTER) {
return exactSizeAll(null);
return getRocksDBNumEntries();
}
return db.getLongProperty("rocksdb.estimate-num-keys");
} catch (RocksDBException e) {
logger.error(MARKER_ROCKSDB, "Failed to get RocksDB estimated keys count property", e);
return 0;
}
} else if (USE_NUM_ENTRIES_PRECISE_COUNTER || PARALLEL_EXACT_SIZE) {
} else if (USE_NUM_ENTRIES_PRECISE_COUNTER && snapshot == null) {
return getRocksDBNumEntries();
} else if (PARALLEL_EXACT_SIZE) {
return exactSizeAll(snapshot);
} else {
rocksdbSnapshot.setFillCache(false);
@ -1057,17 +1060,18 @@ public class LLLocalDictionary implements LLDictionary {
}
}
private long getRocksDBNumEntries() {
try {
return db.getNumEntries();
} catch (RocksDBException ex) {
throw new IllegalStateException("Failed to read exact size", ex);
}
}
private long exactSizeAll(@Nullable LLSnapshot snapshot) {
if (LLUtils.isInNonBlockingThread()) {
throw new UnsupportedOperationException("Called exactSizeAll in a nonblocking thread");
}
if (snapshot == null && USE_NUM_ENTRIES_PRECISE_COUNTER) {
try {
return db.getNumEntries();
} catch (RocksDBException ex) {
throw new IllegalStateException("Failed to read exact size", ex);
}
}
try (var readOpts = LLUtils.generateCustomReadOptions(generateReadOptionsOrNull(snapshot), false, false, false)) {
if (LLUtils.MANUAL_READAHEAD) {
readOpts.setReadaheadSize(128 * 1024); // 128KiB

View File

@ -9,6 +9,7 @@ import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.rocksdb.RocksIteratorObj;
import it.cavallium.dbengine.utils.DBException;
import it.cavallium.dbengine.utils.StreamUtils;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
import java.util.List;
@ -64,7 +65,7 @@ public abstract class LLLocalGroupedReactiveRocksIterator<T> {
throw new DBException("Failed to iterate the range", e);
}
return Stream.<List<T>>generate(() -> {
return StreamUtils.<List<T>>streamWhileNonNull(() -> {
try {
ObjectArrayList<T> values = new ObjectArrayList<>();
Buf firstGroupKey = null;
@ -111,7 +112,7 @@ public abstract class LLLocalGroupedReactiveRocksIterator<T> {
}
throw new CompletionException(new DBException("Range failed", ex));
}
}).takeWhile(Objects::nonNull).onClose(() -> {
}).onClose(() -> {
rocksIterator.close();
readOptions.close();
});

View File

@ -3,12 +3,17 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static it.cavallium.dbengine.database.LLUtils.generateCustomReadOptions;
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
import com.google.common.collect.Iterators;
import com.google.common.collect.Streams;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.utils.DBException;
import it.cavallium.dbengine.utils.StreamUtils;
import java.io.IOException;
import java.util.Objects;
import java.util.concurrent.CompletionException;
import java.util.function.Supplier;
import java.util.stream.Stream;
@ -52,7 +57,7 @@ public class LLLocalKeyPrefixReactiveRocksIterator {
}
var rocksIterator = db.newRocksIterator(readOptions, range, false);
return Stream.generate(() -> {
return streamWhileNonNull(() -> {
try {
Buf firstGroupKey = null;
while (rocksIterator.isValid()) {

View File

@ -1,6 +1,7 @@
package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.generateCustomReadOptions;
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLEntry;
@ -39,7 +40,7 @@ public final class LLLocalMigrationReactiveRocksIterator {
} catch (RocksDBException e) {
throw new DBException("Failed to open iterator", e);
}
return Stream.generate(() -> {
return streamWhileNonNull(() -> {
try {
if (rocksIterator.isValid()) {
var key = rocksIterator.keyBuf().copy();
@ -52,7 +53,7 @@ public final class LLLocalMigrationReactiveRocksIterator {
} catch (RocksDBException ex) {
throw new CompletionException(new DBException("Failed to iterate", ex));
}
}).takeWhile(Objects::nonNull).onClose(() -> {
}).onClose(() -> {
rocksIterator.close();
readOptions.close();
});

View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.database.LLUtils.MARKER_ROCKSDB;
import static it.cavallium.dbengine.database.LLUtils.generateCustomReadOptions;
import static it.cavallium.dbengine.database.LLUtils.isBoundedRange;
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLRange;
@ -59,7 +60,7 @@ public abstract class LLLocalReactiveRocksIterator<T> {
throw new DBException("Failed to iterate the range", e);
}
return Stream.generate(() -> {
return streamWhileNonNull(() -> {
try {
if (rocksIterator.isValid()) {
// Note that the underlying array is subject to changes!
@ -100,7 +101,7 @@ public abstract class LLLocalReactiveRocksIterator<T> {
}
throw new CompletionException(ex);
}
}).takeWhile(Objects::nonNull).onClose(() -> {
}).onClose(() -> {
rocksIterator.close();
readOptions.close();
});

View File

@ -59,7 +59,7 @@ public class LLMemoryKeyValueDatabase implements LLKeyValueDatabase {
));
var singleton = new LLMemorySingleton(dict, columnNameString, singletonName);
Buf returnValue = singleton.get(null);
if (returnValue == null) {
if (returnValue == null && defaultValue != null) {
singleton.set(Buf.wrap(defaultValue));
}
return singleton;

View File

@ -16,7 +16,7 @@ public class CheckIndexInput extends IndexInput {
}
private static void checkThread() {
assert LuceneUtils.isLuceneThread();
warnLuceneThread();
}
@Override

View File

@ -15,7 +15,7 @@ public class CheckIndexOutput extends IndexOutput {
}
private static void checkThread() {
assert LuceneUtils.isLuceneThread();
LuceneUtils.warnLuceneThread();
}
@Override

View File

@ -49,7 +49,7 @@ public class CheckOutputDirectory extends Directory {
@Override
public IndexOutput createOutput(String name, IOContext context) {
LuceneUtils.checkLuceneThread();
LuceneUtils.warnLuceneThread();
try {
return new CheckIndexOutput(directory.createOutput(name, context));
} catch (IOException e) {
@ -59,7 +59,7 @@ public class CheckOutputDirectory extends Directory {
@Override
public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) {
LuceneUtils.checkLuceneThread();
LuceneUtils.warnLuceneThread();
try {
return new CheckIndexOutput(directory.createTempOutput(prefix, suffix, context));
} catch (IOException e) {
@ -69,7 +69,7 @@ public class CheckOutputDirectory extends Directory {
@Override
public void sync(Collection<String> names) {
LuceneUtils.checkLuceneThread();
LuceneUtils.warnLuceneThread();
try {
directory.sync(names);
} catch (IOException e) {
@ -79,7 +79,7 @@ public class CheckOutputDirectory extends Directory {
@Override
public void syncMetaData() {
LuceneUtils.checkLuceneThread();
LuceneUtils.warnLuceneThread();
try {
directory.syncMetaData();
} catch (IOException e) {
@ -89,7 +89,7 @@ public class CheckOutputDirectory extends Directory {
@Override
public void rename(String source, String dest) {
LuceneUtils.checkLuceneThread();
LuceneUtils.warnLuceneThread();
try {
directory.rename(source, dest);
} catch (IOException e) {
@ -99,7 +99,7 @@ public class CheckOutputDirectory extends Directory {
@Override
public IndexInput openInput(String name, IOContext context) {
LuceneUtils.checkLuceneThread();
LuceneUtils.warnLuceneThread();
try {
return new CheckIndexInput(directory.openInput(name, context));
} catch (IOException e) {
@ -109,7 +109,7 @@ public class CheckOutputDirectory extends Directory {
@Override
public Lock obtainLock(String name) {
LuceneUtils.checkLuceneThread();
LuceneUtils.warnLuceneThread();
try {
return directory.obtainLock(name);
} catch (IOException e) {

View File

@ -1,5 +1,7 @@
package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
import java.io.IOException;
import it.cavallium.dbengine.utils.DBException;
import java.util.Iterator;
@ -51,7 +53,7 @@ public class LuceneGenerator implements Supplier<ScoreDoc> {
throw new IllegalArgumentException("Sorting is not allowed");
}
var lg = new LuceneGenerator(shard, localQueryParams, shardIndex);
return Stream.generate(lg).takeWhile(Objects::nonNull);
return streamWhileNonNull(lg);
}
@Override

View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLKeyScore;
@ -130,14 +131,14 @@ public class PagedLocalSearcher implements LocalSearcher {
LocalQueryParams queryParams, String keyFieldName, CurrentPageInfo secondPageInfo) {
AtomicReference<CurrentPageInfo> pageInfo = new AtomicReference<>(secondPageInfo);
Object lock = new Object();
Stream<ScoreDoc> topFieldDocFlux = Stream.generate(() -> {
Stream<ScoreDoc> topFieldDocFlux = streamWhileNonNull(() -> {
synchronized (lock) {
var currentPageInfo = pageInfo.getPlain();
var result = searchPageSync(queryParams, indexSearchers, true, 0, currentPageInfo);
pageInfo.setPlain(result.nextPageToIterate());
return result.pageData();
}
}).takeWhile(Objects::nonNull).flatMap(pd -> Stream.of(pd.topDocs().scoreDocs));
}).flatMap(pd -> Stream.of(pd.topDocs().scoreDocs));
return LuceneUtils.convertHits(topFieldDocFlux, indexSearchers, keyFieldName);
}

View File

@ -1,6 +1,7 @@
package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
import com.google.common.collect.Streams;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
@ -133,7 +134,7 @@ public class ScoredPagedMultiSearcher implements MultiSearcher {
private Stream<LLKeyScore> searchOtherPages(List<IndexSearcher> indexSearchers,
LocalQueryParams queryParams, String keyFieldName, CurrentPageInfo secondPageInfo) {
AtomicReference<CurrentPageInfo> currentPageInfoRef = new AtomicReference<>(secondPageInfo);
Stream<ScoreDoc> topFieldDocStream = Stream.generate(() -> {
Stream<ScoreDoc> topFieldDocStream = streamWhileNonNull(() -> {
var currentPageInfo = currentPageInfoRef.getPlain();
if (currentPageInfo == null) return null;
LOG.trace("Current page info: {}", currentPageInfo);
@ -145,7 +146,7 @@ public class ScoredPagedMultiSearcher implements MultiSearcher {
} else {
return Arrays.asList(result.topDocs().scoreDocs);
}
}).takeWhile(Objects::nonNull).flatMap(Collection::stream);
}).flatMap(Collection::stream);
return LuceneUtils.convertHits(topFieldDocStream, indexSearchers, keyFieldName);
}

View File

@ -8,6 +8,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Spliterator;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@ -48,6 +49,31 @@ public class StreamUtils {
: StreamSupport.stream(new BatchSpliterator<>(stream.spliterator(), batchSize), stream.isParallel());
}
@SuppressWarnings("UnstableApiUsage")
public static <X> Stream<X> streamWhileNonNull(Supplier<X> supplier) {
var it = new Iterator<X>() {
private boolean nextSet = false;
private X next;
@Override
public boolean hasNext() {
if (!nextSet) {
next = supplier.get();
nextSet = true;
}
return next != null;
}
@Override
public X next() {
nextSet = false;
return next;
}
};
return Streams.stream(it);
}
private record BatchSpliterator<E>(Spliterator<E> base, int batchSize) implements Spliterator<List<E>> {
@Override

View File

@ -19,6 +19,7 @@ module dbengine {
exports it.cavallium.dbengine.utils;
exports it.cavallium.dbengine.database.disk.rocksdb;
exports it.cavallium.dbengine.buffers;
exports it.cavallium.dbengine.lucene.hugepq.search;
requires org.jetbrains.annotations;
requires com.google.common;
requires micrometer.core;

View File

@ -1,131 +0,0 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.client.DefaultDatabaseOptions.DEFAULT_DATABASE_OPTIONS;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import it.cavallium.data.generator.nativedata.Nullableboolean;
import it.cavallium.data.generator.nativedata.Nullabledouble;
import it.cavallium.data.generator.nativedata.Nullableint;
import it.cavallium.data.generator.nativedata.Nullablelong;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.database.ColumnUtils;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptionsBuilder;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletionException;
import java.util.concurrent.atomic.AtomicInteger;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
private static final AtomicInteger dbId = new AtomicInteger(0);
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
Duration.ofSeconds(5),
Duration.ofSeconds(5),
false,
new ByteBuffersDirectory(),
Nullableboolean.empty(),
Nullabledouble.empty(),
Nullableint.empty(),
Nullableboolean.empty(),
Nullableboolean.empty(),
true,
MAX_IN_MEMORY_RESULT_ENTRIES,
LuceneUtils.getDefaultMergePolicy()
);
@Override
public Mono<TempDb> openTempDb(TestAllocator allocator) {
boolean canUseNettyDirect = DbTestUtils.computeCanUseNettyDirect();
return Mono.defer(() -> {
var wrkspcPath = Path.of("/tmp/.cache/tempdb-" + dbId.incrementAndGet() + "/");
return Mono
.<LLKeyValueDatabase>fromCallable(() -> {
if (Files.exists(wrkspcPath)) {
Files.walk(wrkspcPath).sorted(Comparator.reverseOrder()).forEach(file -> {
try {
Files.delete(file);
} catch (IOException ex) {
throw new CompletionException(ex);
}
});
}
Files.createDirectories(wrkspcPath);
return null;
})
.subscribeOn(Schedulers.boundedElastic())
.then(new LLLocalDatabaseConnection(allocator.allocator(),
new SimpleMeterRegistry(),
wrkspcPath,
true,
null
).connect())
.flatMap(conn -> {
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
return Mono.zip(
conn.getDatabase("testdb",
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build()
),
conn.getLuceneIndex("testluceneindex1",
LuceneUtils.singleStructure(),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
conn.getLuceneIndex("testluceneindex16",
LuceneUtils.shardsStructure(3),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
Mono.just(searcher)
)
.map(tuple -> new TempDb(allocator, conn, tuple.getT1(), tuple.getT2(), tuple.getT3(), tuple.getT4(), wrkspcPath));
});
});
}
@Override
public Mono<Void> closeTempDb(TempDb tempDb) {
return tempDb.db().close().then(tempDb.connection().disconnect()).then(Mono.fromCallable(() -> {
ensureNoLeaks(tempDb.allocator().allocator(), false, false);
if (Files.exists(tempDb.path())) {
Files.walk(tempDb.path()).sorted(Comparator.reverseOrder()).forEach(file -> {
try {
Files.delete(file);
} catch (IOException ex) {
throw new CompletionException(ex);
}
});
}
return null;
}).subscribeOn(Schedulers.boundedElastic())).then();
}
}

View File

@ -1,87 +0,0 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import it.cavallium.data.generator.nativedata.Nullableboolean;
import it.cavallium.data.generator.nativedata.Nullabledouble;
import it.cavallium.data.generator.nativedata.Nullableint;
import it.cavallium.data.generator.nativedata.Nullablelong;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.database.ColumnUtils;
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
import it.cavallium.dbengine.rpc.current.data.LuceneIndexStructure;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter;
import it.unimi.dsi.fastutil.ints.IntList;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import reactor.core.publisher.Mono;
public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
Duration.ofSeconds(5),
Duration.ofSeconds(5),
false,
new ByteBuffersDirectory(),
Nullableboolean.empty(),
Nullabledouble.empty(),
Nullableint.empty(),
Nullableboolean.empty(),
Nullableboolean.empty(),
false,
MAX_IN_MEMORY_RESULT_ENTRIES,
LuceneUtils.getDefaultMergePolicy()
);
@Override
public Mono<TempDb> openTempDb(TestAllocator allocator) {
boolean canUseNettyDirect = DbTestUtils.computeCanUseNettyDirect();
return Mono
.fromCallable(() -> new LLMemoryDatabaseConnection(allocator.allocator(), new SimpleMeterRegistry()))
.flatMap(conn -> {
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
return Mono
.zip(
conn.getDatabase("testdb",
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
DefaultDatabaseOptions.builder().allowNettyDirect(canUseNettyDirect).build()
),
conn.getLuceneIndex("testluceneindex1",
LuceneUtils.singleStructure(),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
conn.getLuceneIndex("testluceneindex16",
LuceneUtils.shardsStructure(3),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
Mono.just(searcher)
)
.map(tuple -> new TempDb(allocator, conn, tuple.getT1(), tuple.getT2(), tuple.getT3(), tuple.getT4(), null));
});
}
@Override
public Mono<Void> closeTempDb(TempDb db) {
return db.db().close();
}
}

View File

@ -1,40 +0,0 @@
package it.cavallium.dbengine;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class SyncUtils {
public static void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
public static void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
public static void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
}

View File

@ -1,13 +0,0 @@
package it.cavallium.dbengine;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import reactor.core.publisher.Mono;
public interface TemporaryDbGenerator {
Mono<TempDb> openTempDb(TestAllocator allocator);
Mono<Void> closeTempDb(TempDb db);
}

View File

@ -1,62 +0,0 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import io.netty5.buffer.Buffer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
public class TestAllocator {
private DbTestUtils.TestAllocator allocator;
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
}
@AfterEach
public void afterEach() {
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
}
@Test
public void testNoOp() {
}
@Test
public void testShouldPass() {
Buffer allocated = allocator.allocator().allocate(5000);
allocated.close();
ensureNoLeaks(allocator.allocator(), true, false);
}
@Test
public void testShouldFail() {
Buffer allocated = null;
try {
boolean failed;
try {
allocated = allocator.allocator().allocate(5000);
ensureNoLeaks(allocator.allocator(), true, true);
failed = false;
} catch (Exception ex) {
failed = true;
}
if (!failed) {
Assertions.fail("A leak was not detected!");
}
} finally {
if (allocated != null) {
allocated.close();
}
}
}
}

View File

@ -1,149 +0,0 @@
package it.cavallium.dbengine;
import static io.netty5.buffer.internal.InternalBufferUtils.allocatorClosedException;
import static io.netty5.buffer.internal.InternalBufferUtils.assertValidBufferSize;
import static io.netty5.buffer.internal.InternalBufferUtils.standardDrop;
import io.netty5.buffer.AllocationType;
import io.netty5.buffer.AllocatorControl;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import io.netty5.buffer.Drop;
import io.netty5.buffer.MemoryManager;
import io.netty5.buffer.StandardAllocationTypes;
import io.netty5.buffer.pool.PooledBufferAllocator;
import java.util.concurrent.atomic.LongAdder;
import java.util.function.Function;
import java.util.function.Supplier;
public class TestAllocatorImpl implements BufferAllocator, AllocatorControl {
private final TestMemoryManager manager;
private final AllocationType allocationType = StandardAllocationTypes.ON_HEAP;
private volatile boolean closed;
private TestAllocatorImpl(TestMemoryManager testMemoryManager) {
this.manager = testMemoryManager;
}
public static TestAllocatorImpl create() {
return new TestAllocatorImpl(new TestMemoryManager(MemoryManager.instance()));
}
@Override
public boolean isPooling() {
return false;
}
@Override
public AllocationType getAllocationType() {
return allocationType;
}
@Override
public Buffer allocate(int size) {
if (closed) {
throw allocatorClosedException();
}
assertValidBufferSize(size);
return manager.allocateShared(this, size, standardDrop(manager), allocationType);
}
@Override
public Supplier<Buffer> constBufferSupplier(byte[] bytes) {
if (closed) {
throw allocatorClosedException();
}
Buffer constantBuffer = manager.allocateShared(
this, bytes.length, standardDrop(manager), allocationType);
constantBuffer.writeBytes(bytes).makeReadOnly();
return () -> manager.allocateConstChild(constantBuffer);
}
@Override
public void close() {
closed = true;
}
public long getActiveAllocations() {
return this.manager.getActiveAllocations();
}
@Override
public BufferAllocator getAllocator() {
return this;
}
private static class TestMemoryManager implements MemoryManager {
private final MemoryManager instance;
private final LongAdder activeAllocations = new LongAdder();
public TestMemoryManager(MemoryManager instance) {
this.instance = instance;
}
@Override
public Buffer allocateShared(AllocatorControl allocatorControl,
long size,
Function<Drop<Buffer>, Drop<Buffer>> dropDecorator,
AllocationType allocationType) {
return instance.allocateShared(allocatorControl, size, this::createDrop, allocationType);
}
@Override
public Buffer allocateConstChild(Buffer readOnlyConstParent) {
return instance.allocateConstChild(readOnlyConstParent);
}
@Override
public Object unwrapRecoverableMemory(Buffer buf) {
return instance.unwrapRecoverableMemory(buf);
}
@Override
public Buffer recoverMemory(AllocatorControl allocatorControl, Object recoverableMemory, Drop<Buffer> drop) {
return instance.recoverMemory(allocatorControl, recoverableMemory, drop);
}
@Override
public Object sliceMemory(Object memory, int offset, int length) {
return instance.sliceMemory(memory, offset, length);
}
@Override
public void clearMemory(Object o) {
instance.clearMemory(o);
}
@Override
public String implementationName() {
return instance.implementationName();
}
private Drop<Buffer> createDrop(Drop<Buffer> drop) {
activeAllocations.increment();
return new Drop<>() {
@Override
public void drop(Buffer obj) {
activeAllocations.decrement();
drop.drop(obj);
}
@Override
public Drop<Buffer> fork() {
return createDrop(drop.fork());
}
@Override
public void attach(Buffer obj) {
drop.attach(obj);
}
};
}
public long getActiveAllocations() {
return activeAllocations.longValue();
}
}
}

View File

@ -1,56 +0,0 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.DbTestUtils.tempDb;
import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.UpdateMode;
import java.util.Arrays;
import java.util.stream.Stream;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.test.StepVerifier;
public abstract class TestDictionary {
private TestAllocator allocator;
protected abstract TemporaryDbGenerator getTempDbGenerator();
private static Stream<Arguments> provideArgumentsCreate() {
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
}
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
}
@AfterEach
public void afterEach() {
if (!isCIMode()) {
ensureNoLeaks(allocator.allocator(), true, false);
}
destroyAllocator(allocator);
}
@ParameterizedTest
@MethodSource("provideArgumentsCreate")
public void testCreate(UpdateMode updateMode) {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempDictionary(db, updateMode)
.flatMap(LLDictionary::clear)
.then()
))
.verifyComplete();
}
}

View File

@ -1,157 +0,0 @@
package it.cavallium.dbengine;
import static org.junit.jupiter.api.Assertions.assertEquals;
import io.netty5.buffer.Buffer;
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
import it.cavallium.dbengine.lucene.HugePqCodec;
import it.cavallium.dbengine.lucene.HugePqPriorityQueue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Function;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
public class TestHugePq {
private LLTempHugePqEnv env;
private HugePqPriorityQueue<Integer> queue;
@BeforeEach
public void beforeEach() throws IOException {
this.env = new LLTempHugePqEnv();
this.queue = new HugePqPriorityQueue<>(env, new HugePqCodec<Integer>() {
@Override
public Buffer serialize(Function<Integer, Buffer> allocator, Integer data) {
var buf = allocator.apply(Integer.BYTES);
HugePqCodec.setLexInt(buf, 0, false, data);
return buf.writerOffset(Integer.BYTES);
}
@Override
public Integer deserialize(Buffer b) {
return HugePqCodec.getLexInt(b, 0, false);
}
});
}
@Test
public void testNoOp() {
}
@Test
public void testEmptyTop() {
Assertions.assertNull(queue.top());
}
@Test
public void testAddSingle() {
queue.add(2);
Assertions.assertEquals(2, queue.top());
}
@Test
public void testAddSame() {
queue.add(2);
queue.add(2);
Assertions.assertEquals(2, queue.top());
Assertions.assertEquals(2, queue.size());
}
@Test
public void testAddMulti() {
for (int i = 0; i < 1000; i++) {
queue.add(i);
}
Assertions.assertEquals(0, queue.top());
}
@Test
public void testAddRandomMulti() {
var list = new ArrayList<Integer>(1000);
for (int i = 0; i < 1000; i++) {
var n = ThreadLocalRandom.current().nextInt(-20, 20);
queue.add(n);
list.add(n);
}
list.sort(Comparator.reverseOrder());
for (int i = 0; i < 1000; i++) {
Assertions.assertEquals(list.remove(list.size() - 1), queue.pop());
}
}
@Test
public void testAddMultiClear() {
for (int i = 0; i < 1000; i++) {
queue.add(i);
}
queue.clear();
Assertions.assertNull(queue.top());
}
@Test
public void testAddRemove() {
queue.add(0);
queue.remove(0);
Assertions.assertNull(queue.top());
}
@Test
public void testAddRemoveNonexistent() {
queue.add(0);
queue.remove(1);
Assertions.assertEquals(0, queue.top());
}
@Test
public void testAddMultiSameRemove() {
queue.add(0);
queue.add(0);
queue.add(1);
queue.remove(0);
Assertions.assertEquals(2, queue.size());
Assertions.assertEquals(0, queue.top());
}
@Test
public void testAddMultiRemove() {
for (int i = 0; i < 1000; i++) {
queue.add(i);
}
queue.remove(0);
Assertions.assertEquals(1, queue.top());
}
@Test
public void testSort() {
var sortedNumbers = new ArrayList<Integer>();
for (int i = 0; i < 1000; i++) {
sortedNumbers.add(i);
}
var shuffledNumbers = new ArrayList<>(sortedNumbers);
Collections.shuffle(shuffledNumbers);
for (Integer number : shuffledNumbers) {
queue.add(number);
}
var newSortedNumbers = new ArrayList<>();
Integer popped;
while ((popped = queue.pop()) != null) {
newSortedNumbers.add(popped);
}
Assertions.assertEquals(sortedNumbers, newSortedNumbers);
}
@AfterEach
public void afterEach() throws IOException {
queue.close();
env.close();
}
}

View File

@ -1,381 +0,0 @@
package it.cavallium.dbengine;
import com.google.common.collect.Lists;
import io.netty5.buffer.Buffer;
import it.cavallium.dbengine.database.SafeCloseable;
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
import it.cavallium.dbengine.lucene.LLScoreDoc;
import it.cavallium.dbengine.lucene.HugePqCodec;
import it.cavallium.dbengine.lucene.HugePqPriorityQueue;
import it.cavallium.dbengine.lucene.PriorityQueue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Function;
import org.apache.lucene.search.HitQueue;
import org.apache.lucene.search.ScoreDoc;
import org.assertj.core.description.Description;
import org.assertj.core.description.TextDescription;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import reactor.core.publisher.Flux;
public class TestHugePqHitQueue {
public static final int NUM_HITS = 1024;
private LLTempHugePqEnv env;
private SafeCloseable hugePqQueue;
private TestingPriorityQueue testingPriorityQueue;
protected static boolean lessThan(ScoreDoc hitA, ScoreDoc hitB) {
if (hitA.score == hitB.score) {
return hitA.doc > hitB.doc;
} else {
return hitA.score < hitB.score;
}
}
private static int compareScoreDoc(ScoreDoc hitA, ScoreDoc hitB) {
if (hitA.score == hitB.score) {
if (hitA.doc == hitB.doc) {
return Integer.compare(hitA.shardIndex, hitB.shardIndex);
} else {
return Integer.compare(hitB.doc, hitA.doc);
}
} else {
return Float.compare(hitA.score, hitB.score);
}
}
private static void assertEqualsScoreDoc(Description description, ScoreDoc expected, ScoreDoc actual) {
org.assertj.core.api.Assertions.assertThat(toLLScoreDoc(expected)).as(description).isEqualTo(toLLScoreDoc(actual));
}
private static void assertEqualsScoreDoc(List<ScoreDoc> expected, List<ScoreDoc> actual) {
var list1 = expected.iterator();
var list2 = actual.iterator();
Assertions.assertEquals(expected.size(), actual.size());
while (list1.hasNext() && list2.hasNext()) {
Assertions.assertFalse(lessThan(list1.next(), list2.next()));
}
}
@BeforeEach
public void beforeEach() throws IOException {
this.env = new LLTempHugePqEnv();
var hugePqQueue = new HugePqPriorityQueue<ScoreDoc>(env, new HugePqCodec<>() {
@Override
public Buffer serialize(Function<Integer, Buffer> allocator, ScoreDoc data) {
var buf = allocator.apply(Float.BYTES + Integer.BYTES + Integer.BYTES);
buf.writerOffset(Float.BYTES + Integer.BYTES + Integer.BYTES);
setScore(buf, data.score);
setDoc(buf, data.doc);
setShardIndex(buf, data.shardIndex);
return buf;
}
@Override
public ScoreDoc deserialize(Buffer buf) {
return new ScoreDoc(getDoc(buf), getScore(buf), getShardIndex(buf));
}
private static float getScore(Buffer hit) {
return HugePqCodec.getLexFloat(hit, 0, false);
}
private static int getDoc(Buffer hit) {
return HugePqCodec.getLexInt(hit, Float.BYTES, true);
}
private static int getShardIndex(Buffer hit) {
return HugePqCodec.getLexInt(hit, Float.BYTES + Integer.BYTES, false);
}
private static void setScore(Buffer hit, float score) {
HugePqCodec.setLexFloat(hit, 0, false, score);
}
private static void setDoc(Buffer hit, int doc) {
HugePqCodec.setLexInt(hit, Float.BYTES, true, doc);
}
private static void setShardIndex(Buffer hit, int shardIndex) {
HugePqCodec.setLexInt(hit, Float.BYTES + Integer.BYTES, false, shardIndex);
}
@Override
public ScoreDoc clone(ScoreDoc obj) {
return new ScoreDoc(obj.doc, obj.score, obj.shardIndex);
}
});
this.hugePqQueue = hugePqQueue;
PriorityQueueAdaptor<ScoreDoc> hitQueue = new PriorityQueueAdaptor<>(new HitQueue(NUM_HITS, false));
Assertions.assertEquals(0, hugePqQueue.size());
Assertions.assertEquals(0, hitQueue.size());
this.testingPriorityQueue = new TestingPriorityQueue(hitQueue, hugePqQueue);
}
@Test
public void testNoOp() {
}
@Test
public void testEmptyTop() {
Assertions.assertNull(testingPriorityQueue.top());
}
@Test
public void testAddSingle() {
var item = new ScoreDoc(0, 0, 0);
testingPriorityQueue.add(item);
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), item, testingPriorityQueue.top());
}
@Test
public void testAddMulti() {
for (int i = 0; i < 1000; i++) {
var item = new ScoreDoc(i, i >> 1, -1);
testingPriorityQueue.addUnsafe(item);
}
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), new ScoreDoc(1, 0, -1), testingPriorityQueue.top());
}
@Test
public void testAddMultiRandom() {
var list = new ArrayList<Integer>(1000);
for (int i = 0; i < 1000; i++) {
var ri = ThreadLocalRandom.current().nextInt(0, 20);
list.add(ri);
var item = new ScoreDoc(ri, ri << 1, ri % 4);
testingPriorityQueue.addUnsafe(item);
}
list.sort(Comparator.reverseOrder());
for (int i = 0; i < 1000; i++) {
var top = list.remove(list.size() - 1);
assertEqualsScoreDoc(new TextDescription("%d value of %s", i, testingPriorityQueue), new ScoreDoc(top, top << 1, top % 4), testingPriorityQueue.pop());
}
}
@Test
public void testAddMultiClear() {
for (int i = 0; i < 1000; i++) {
var item = new ScoreDoc(i, i >> 1, -1);
testingPriorityQueue.addUnsafe(item);
}
testingPriorityQueue.clear();
Assertions.assertNull(testingPriorityQueue.top());
}
@Test
public void testAddRemove() {
var item = new ScoreDoc(0, 0, -1);
testingPriorityQueue.add(item);
testingPriorityQueue.remove(item);
Assertions.assertNull(testingPriorityQueue.top());
}
@Test
public void testAddRemoveNonexistent() {
var item = new ScoreDoc(0, 0, 0);
testingPriorityQueue.addUnsafe(item);
testingPriorityQueue.remove(new ScoreDoc(2, 0, 0));
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), item, testingPriorityQueue.top());
}
@Test
public void testAddMultiRemove1() {
ScoreDoc toRemove = null;
ScoreDoc top = null;
for (int i = 0; i < 1000; i++) {
var item = new ScoreDoc(i, i >> 1, -1);
if (i == 1) {
toRemove = item;
} else if (i == 0) {
top = item;
}
testingPriorityQueue.addUnsafe(item);
}
testingPriorityQueue.removeUnsafe(toRemove);
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), top, testingPriorityQueue.top());
}
@Test
public void testAddMultiRemove2() {
ScoreDoc toRemove = null;
ScoreDoc top = null;
for (int i = 0; i < 1000; i++) {
var item = new ScoreDoc(i, i >> 1, -1);
if (i == 0) {
toRemove = item;
} else if (i == 1) {
top = item;
}
testingPriorityQueue.addUnsafe(item);
}
testingPriorityQueue.removeUnsafe(new ScoreDoc(0, 0, -1));
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), top, testingPriorityQueue.top());
}
@Test
public void testSort() {
var sortedNumbers = new ArrayList<ScoreDoc>();
for (int i = 0; i < 1000; i++) {
sortedNumbers.add(new ScoreDoc(i, i >> 1, -1));
}
sortedNumbers.sort(TestHugePqHitQueue::compareScoreDoc);
var shuffledNumbers = new ArrayList<>(sortedNumbers);
Collections.shuffle(shuffledNumbers, new Random(1000));
org.assertj.core.api.Assertions.assertThat(testingPriorityQueue.size()).isEqualTo(0);
for (ScoreDoc scoreDoc : shuffledNumbers) {
testingPriorityQueue.addUnsafe(scoreDoc);
}
org.assertj.core.api.Assertions.assertThat(testingPriorityQueue.size()).isEqualTo(sortedNumbers.size());
var newSortedNumbers = new ArrayList<ScoreDoc>();
ScoreDoc popped;
while ((popped = testingPriorityQueue.popUnsafe()) != null) {
newSortedNumbers.add(popped);
}
org.assertj.core.api.Assertions.assertThat(testingPriorityQueue.size()).isEqualTo(0);
assertEqualsScoreDoc(sortedNumbers, newSortedNumbers);
}
@AfterEach
public void afterEach() throws IOException {
hugePqQueue.close();
env.close();
}
private static class TestingPriorityQueue implements PriorityQueue<ScoreDoc> {
private final PriorityQueue<ScoreDoc> referenceQueue;
private final PriorityQueue<ScoreDoc> myQueue;
public TestingPriorityQueue(PriorityQueue<ScoreDoc> referenceQueue, PriorityQueue<ScoreDoc> myQueue) {
this.referenceQueue = referenceQueue;
this.myQueue = myQueue;
}
@Override
public void add(ScoreDoc element) {
referenceQueue.add(element);
myQueue.add(element);
ensureEquality();
}
public void addUnsafe(ScoreDoc element) {
referenceQueue.add(element);
myQueue.add(element);
}
@Override
public ScoreDoc top() {
var top1 = referenceQueue.top();
var top2 = myQueue.top();
assertEqualsScoreDoc(new TextDescription("top value of %s", myQueue), top1, top2);
return top2;
}
public ScoreDoc topUnsafe() {
var top1 = referenceQueue.top();
var top2 = myQueue.top();
return top2;
}
@Override
public ScoreDoc pop() {
var top1 = referenceQueue.pop();
var top2 = myQueue.pop();
assertEqualsScoreDoc(new TextDescription("top value of %s", myQueue), top1, top2);
return top2;
}
public ScoreDoc popUnsafe() {
var top1 = referenceQueue.pop();
var top2 = myQueue.pop();
return top2;
}
@Override
public void replaceTop(ScoreDoc oldTop, ScoreDoc newTop) {
referenceQueue.replaceTop(oldTop, newTop);
myQueue.replaceTop(oldTop, newTop);
}
@Override
public long size() {
var size1 = referenceQueue.size();
var size2 = myQueue.size();
Assertions.assertEquals(size1, size2);
return size2;
}
@Override
public void clear() {
referenceQueue.clear();
myQueue.clear();
}
@Override
public boolean remove(ScoreDoc element) {
var removedRef = referenceQueue.remove(element);
var removedMy = myQueue.remove(element);
Assertions.assertEquals(removedRef, removedMy);
return removedMy;
}
public boolean removeUnsafe(ScoreDoc element) {
var removed1 = referenceQueue.remove(element);
var removed2 = myQueue.remove(element);
return removed2;
}
@Override
public Flux<ScoreDoc> iterate() {
//noinspection BlockingMethodInNonBlockingContext
var it1 = referenceQueue.iterate().collectList().blockOptional().orElseThrow();
//noinspection BlockingMethodInNonBlockingContext
var it2 = myQueue.iterate().collectList().blockOptional().orElseThrow();
assertEqualsScoreDoc(it1, it2);
return Flux.fromIterable(it2);
}
@Override
public void close() {
referenceQueue.close();
myQueue.close();
}
private void ensureEquality() {
Assertions.assertEquals(referenceQueue.size(), myQueue.size());
var referenceQueueElements = Lists.newArrayList(referenceQueue
.iterate()
.map(TestHugePqHitQueue::toLLScoreDoc)
.toIterable());
var testQueueElements = Lists.newArrayList(myQueue
.iterate()
.map(TestHugePqHitQueue::toLLScoreDoc)
.toIterable());
Assertions.assertEquals(referenceQueueElements, testQueueElements);
}
}
public static LLScoreDoc toLLScoreDoc(ScoreDoc scoreDoc) {
if (scoreDoc == null) return null;
return new LLScoreDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
}
}

View File

@ -1,283 +0,0 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.run;
import static it.cavallium.dbengine.SyncUtils.runVoid;
import static org.junit.jupiter.api.Assertions.assertEquals;
import io.netty5.buffer.Buffer;
import io.netty5.util.Resource;
import io.netty5.util.Send;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLDictionaryResultType;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Objects;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.core.publisher.Mono;
public abstract class TestLLDictionary {
private final Logger log = LogManager.getLogger(this.getClass());
private static final Mono<LLRange> RANGE_ALL = Mono.fromCallable(LLRange::all);
private TestAllocator allocator;
private TempDb tempDb;
private LLKeyValueDatabase db;
protected abstract TemporaryDbGenerator getTempDbGenerator();
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(allocator).block(), "TempDB");
db = tempDb.db();
}
@AfterEach
public void afterEach() {
getTempDbGenerator().closeTempDb(tempDb).block();
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
}
public static Stream<Arguments> provideArguments() {
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
}
public static Stream<Arguments> providePutArguments() {
var updateModes = Arrays.stream(UpdateMode.values());
return updateModes.flatMap(updateMode -> {
var resultTypes = Arrays.stream(LLDictionaryResultType.values());
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
});
}
public static Stream<Arguments> provideUpdateArguments() {
var updateModes = Arrays.stream(UpdateMode.values());
return updateModes.flatMap(updateMode -> {
var resultTypes = Arrays.stream(UpdateReturnMode.values());
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
});
}
private LLDictionary getDict(UpdateMode updateMode) {
var dict = DbTestUtils.tempDictionary(db, updateMode).blockOptional().orElseThrow();
var key1 = Mono.fromCallable(() -> fromString("test-key-1"));
var key2 = Mono.fromCallable(() -> fromString("test-key-2"));
var key3 = Mono.fromCallable(() -> fromString("test-key-3"));
var key4 = Mono.fromCallable(() -> fromString("test-key-4"));
var value = Mono.fromCallable(() -> fromString("test-value"));
dict.put(key1, value, LLDictionaryResultType.VOID).block();
dict.put(key2, value, LLDictionaryResultType.VOID).block();
dict.put(key3, value, LLDictionaryResultType.VOID).block();
dict.put(key4, value, LLDictionaryResultType.VOID).block();
return dict;
}
private Buffer fromString(String s) {
var sb = s.getBytes(StandardCharsets.UTF_8);
try (var b = db.getAllocator().allocate(sb.length + 3 + 13)) {
assert b.writerOffset() == 0;
assert b.readerOffset() == 0;
b.writerOffset(3).writeBytes(sb);
b.readerOffset(3);
assert b.readableBytes() == sb.length;
var part1 = b.split();
return LLUtils.compositeBuffer(db.getAllocator(), part1.send(), b.send());
}
}
private String toString(Buffer bb) {
try (bb) {
byte[] data = new byte[bb.readableBytes()];
bb.copyInto(bb.readerOffset(), data, 0, data.length);
return new String(data, StandardCharsets.UTF_8);
}
}
@Test
public void testNoOp() {
}
@Test
public void testNoOpAllocation() {
for (int i = 0; i < 10; i++) {
var a = allocator.allocator().allocate(i * 512);
a.send().receive().close();
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetDict(UpdateMode updateMode) {
var dict = getDict(updateMode);
Assertions.assertNotNull(dict);
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetColumnName(UpdateMode updateMode) {
var dict = getDict(updateMode);
Assertions.assertEquals("hash_map_testmap", dict.getColumnName());
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetAllocator(UpdateMode updateMode) {
var dict = getDict(updateMode);
var alloc = dict.getAllocator();
Assertions.assertEquals(alloc, alloc);
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGet(UpdateMode updateMode) {
var dict = getDict(updateMode);
var keyEx = Mono.fromCallable(() -> fromString("test-key-1"));
var keyNonEx = Mono.fromCallable(() -> fromString("test-nonexistent"));
Assertions.assertEquals("test-value", run(dict.get(null, keyEx).map(this::toString)));
Assertions.assertEquals("test-value", run(dict.get(null, keyEx).map(this::toString)));
Assertions.assertEquals("test-value", run(dict.get(null, keyEx).map(this::toString)));
Assertions.assertEquals((String) null, run(dict.get(null, keyNonEx).map(this::toString)));
Assertions.assertEquals((String) null, run(dict.get(null, keyNonEx).map(this::toString)));
Assertions.assertEquals((String) null, run(dict.get(null, keyNonEx).map(this::toString)));
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testPutExisting(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode);
var keyEx = Mono.fromCallable(() -> fromString("test-key-1"));
var value = Mono.fromCallable(() -> fromString("test-value"));
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
runVoid(dict.put(keyEx, value, resultType).then().doOnDiscard(Resource.class, Resource::close));
var afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
Assertions.assertEquals(0, afterSize - beforeSize);
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testPutNew(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode);
var keyNonEx = Mono.fromCallable(() -> fromString("test-nonexistent"));
var value = Mono.fromCallable(() -> fromString("test-value"));
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
runVoid(dict.put(keyNonEx, value, resultType).then().doOnDiscard(Resource.class, Resource::close));
var afterSize = run(dict.sizeRange(null, Mono.fromCallable(LLRange::all), false));
Assertions.assertEquals(1, afterSize - beforeSize);
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).collectList()).contains("test-nonexistent"));
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).collectList()).contains("test-nonexistent"));
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetUpdateMode(UpdateMode updateMode) {
var dict = getDict(updateMode);
assertEquals(updateMode, dict.getUpdateMode());
}
@ParameterizedTest
@MethodSource("provideUpdateArguments")
public void testUpdateExisting(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
var dict = getDict(updateMode);
var keyEx = Mono.fromCallable(() -> fromString("test-key-1"));
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
long afterSize;
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(keyEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
);
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
assertEquals(0, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(keyEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
);
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
assertEquals(0, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(keyEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
);
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
assertEquals(0, afterSize - beforeSize);
}
@ParameterizedTest
@MethodSource("provideUpdateArguments")
public void testUpdateNew(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
int expected = updateMode == UpdateMode.DISALLOW ? 0 : 1;
var dict = getDict(updateMode);
var keyNonEx = Mono.fromCallable(() -> fromString("test-nonexistent"));
var beforeSize = run(dict.sizeRange(null, RANGE_ALL, false));
long afterSize;
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
);
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
assertEquals(expected, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
);
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
assertEquals(expected, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode).doOnNext(Resource::close).then()
);
afterSize = run(dict.sizeRange(null, RANGE_ALL, false));
assertEquals(expected, afterSize - beforeSize);
if (updateMode != UpdateMode.DISALLOW) {
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).collectList()).contains(
"test-nonexistent"));
Assertions.assertTrue(run(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).collectList()).contains(
"test-nonexistent"));
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testUpdateAndGetDelta(UpdateMode updateMode) {
log.warn("Test not implemented");
//todo: implement
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testClear(UpdateMode updateMode) {
log.warn("Test not implemented");
//todo: implement
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
log.warn("Test not implemented");
//todo: implement
}
}

View File

@ -1,183 +0,0 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.DbTestUtils.tempDb;
import it.cavallium.data.generator.nativedata.StringSerializer;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLSingleton;
import it.cavallium.dbengine.database.collections.DatabaseInt;
import it.cavallium.dbengine.database.collections.DatabaseLong;
import it.cavallium.dbengine.database.collections.DatabaseSingleton;
import it.cavallium.dbengine.database.serialization.Serializer;
import java.util.stream.Stream;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
public abstract class TestSingletons {
private TestAllocator allocator;
protected abstract TemporaryDbGenerator getTempDbGenerator();
private static Stream<Arguments> provideNumberWithRepeats() {
return Stream.of(
Arguments.of(Integer.MIN_VALUE, 2),
Arguments.of(-11, 2),
Arguments.of(0, 3),
Arguments.of(102, 5)
);
}
private static Stream<Arguments> provideLongNumberWithRepeats() {
return Stream.of(
Arguments.of(Long.MIN_VALUE, 2),
Arguments.of(-11L, 2),
Arguments.of(0L, 3),
Arguments.of(102L, 5)
);
}
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
}
@AfterEach
public void afterEach() {
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
}
@Test
public void testCreateInteger() {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", 0)
.flatMap(dbInt -> dbInt.get(null))
.then()
))
.verifyComplete();
}
@Test
public void testCreateIntegerNoop() {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", 0)
.then()
))
.verifyComplete();
}
@Test
public void testCreateLong() {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempLong(db, "test", 0)
.flatMap(dbLong -> dbLong.get(null))
.then()
))
.verifyComplete();
}
@Test
public void testCreateSingleton() {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempSingleton(db, "testsingleton")
.flatMap(dbSingleton -> dbSingleton.get(null))
))
.verifyComplete();
}
@ParameterizedTest
@ValueSource(ints = {Integer.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Integer.MAX_VALUE})
public void testDefaultValueInteger(int i) {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", i)
.flatMap(dbInt -> dbInt.get(null))
))
.expectNext(i)
.verifyComplete();
}
@ParameterizedTest
@ValueSource(longs = {Long.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Long.MAX_VALUE})
public void testDefaultValueLong(long i) {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempLong(db, "test", i)
.flatMap(dbLong -> dbLong.get(null))
))
.expectNext(i)
.verifyComplete();
}
@ParameterizedTest
@MethodSource("provideNumberWithRepeats")
public void testSetInteger(Integer i, Integer repeats) {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempInt(db, "test", 0)
.flatMap(dbInt -> Mono
.defer(() -> dbInt.set((int) System.currentTimeMillis()))
.repeat(repeats)
.then(dbInt.set(i))
.then(dbInt.get(null)))
))
.expectNext(i)
.verifyComplete();
}
@ParameterizedTest
@MethodSource("provideLongNumberWithRepeats")
public void testSetLong(Long i, Integer repeats) {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempLong(db, "test", 0)
.flatMap(dbLong -> Mono
.defer(() -> dbLong.set(System.currentTimeMillis()))
.repeat(repeats)
.then(dbLong.set(i))
.then(dbLong.get(null)))
))
.expectNext(i)
.verifyComplete();
}
@ParameterizedTest
@MethodSource("provideLongNumberWithRepeats")
public void testSetSingleton(Long i, Integer repeats) {
StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempSingleton(db, "test")
.flatMap(dbSingleton -> Mono
.defer(() -> dbSingleton.set(Long.toString(System.currentTimeMillis())))
.repeat(repeats)
.then(dbSingleton.set(Long.toString(i)))
.then(dbSingleton.get(null)))
))
.expectNext(Long.toString(i))
.verifyComplete();
}
public static Mono<DatabaseInt> tempInt(LLKeyValueDatabase database, String name, int defaultValue) {
return database
.getInteger("ints", name, defaultValue);
}
public static Mono<DatabaseLong> tempLong(LLKeyValueDatabase database, String name, long defaultValue) {
return database
.getLong("longs", name, defaultValue);
}
public static Mono<DatabaseSingleton<String>> tempSingleton(LLKeyValueDatabase database, String name) {
return database
.getSingleton("longs", name)
.map(singleton -> new DatabaseSingleton<>(singleton, Serializer.UTF8_SERIALIZER));
}
}

View File

@ -1,289 +0,0 @@
package it.cavallium.dbengine.database.remote;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.fail;
import io.netty.handler.ssl.ClientAuth;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import io.netty.incubator.codec.quic.InsecureQuicTokenHandler;
import io.netty.incubator.codec.quic.QuicConnectionIdGenerator;
import io.netty.incubator.codec.quic.QuicSslContext;
import io.netty.incubator.codec.quic.QuicSslContextBuilder;
import it.cavallium.dbengine.database.remote.RPCCodecs.RPCEventCodec;
import it.cavallium.dbengine.rpc.current.data.Empty;
import it.cavallium.dbengine.rpc.current.data.RPCEvent;
import it.cavallium.dbengine.rpc.current.data.SingletonGet;
import it.cavallium.dbengine.rpc.current.data.nullables.NullableLLSnapshot;
import java.net.InetSocketAddress;
import java.security.cert.CertificateException;
import java.time.Duration;
import java.util.List;
import java.util.logging.Level;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
import reactor.netty.Connection;
import reactor.netty.incubator.quic.QuicClient;
import reactor.netty.incubator.quic.QuicConnection;
class QuicUtilsTest {
private static final int NORMAL = 0;
private static final int WAIT_TIME = 1;
private static final int FAIL_IMMEDIATELY = 2;
private static final int WAIT_TIME_THEN_FAIL = 3;
private Connection serverConn;
private QuicConnection clientConn;
private InetSocketAddress clientAddress;
private InetSocketAddress serverAddress;
@BeforeEach
void setUp() throws CertificateException {
var selfSignedCert = new SelfSignedCertificate();
this.clientAddress = new InetSocketAddress("localhost", 8081);
this.serverAddress = new InetSocketAddress("localhost", 8080);
QuicSslContext sslContext = QuicSslContextBuilder
.forServer(selfSignedCert.key(), null, selfSignedCert.cert())
.applicationProtocols("db/0.9")
.clientAuth(ClientAuth.NONE)
.build();
var qs = reactor.netty.incubator.quic.QuicServer
.create()
.tokenHandler(InsecureQuicTokenHandler.INSTANCE)
.bindAddress(() -> serverAddress)
.secure(sslContext)
.idleTimeout(Duration.ofSeconds(30))
.connectionIdAddressGenerator(QuicConnectionIdGenerator.randomGenerator())
.initialSettings(spec -> spec
.maxData(10000000)
.maxStreamDataBidirectionalLocal(1000000)
.maxStreamDataBidirectionalRemote(1000000)
.maxStreamsBidirectional(100)
.maxStreamsUnidirectional(100)
)
.handleStream((in, out) -> in
.withConnection(conn -> conn.addHandler(new RPCEventCodec()))
.receiveObject()
.cast(RPCEvent.class)
.log("recv", Level.FINEST)
.flatMapSequential(req -> (switch ((int) ((SingletonGet) req).singletonId()) {
case NORMAL -> Mono.<RPCEvent>just(Empty.of());
case FAIL_IMMEDIATELY -> Mono.<RPCEvent>error(new Throwable("Expected error"));
case WAIT_TIME -> Mono.delay(Duration.ofSeconds(3)).<RPCEvent>thenReturn(Empty.of());
case WAIT_TIME_THEN_FAIL -> Mono
.delay(Duration.ofSeconds(3))
.then(Mono.<RPCEvent>error(new Throwable("Expected error")));
default -> Mono.<RPCEvent>error(new UnsupportedOperationException("Unsupported request id " + req));
}).log("Server", Level.SEVERE, SignalType.ON_ERROR).onErrorResume(QuicUtils::catchRPCErrors))
.concatMap(message -> Mono.defer(() -> out
.withConnection(conn -> conn.addHandler(new RPCEventCodec()))
.sendObject(message)
.then())
.log("send", Level.FINEST)
)
);
this.serverConn = qs.bindNow();
var clientSslContext = QuicSslContextBuilder
.forClient()
.trustManager(InsecureTrustManagerFactory.INSTANCE)
.applicationProtocols("db/0.9")
.build();
this.clientConn = QuicClient.create()
.bindAddress(() -> new InetSocketAddress(0))
.remoteAddress(() -> serverAddress)
.secure(clientSslContext)
.idleTimeout(Duration.ofSeconds(30))
.initialSettings(spec -> spec
.maxData(10000000)
.maxStreamDataBidirectionalLocal(1000000)
)
.connectNow();
}
@AfterEach
void tearDown() {
if (clientConn != null) {
clientConn.disposeNow();
}
if (serverConn != null) {
serverConn.disposeNow();
}
}
@Test
void sendSimpleRequest() {
RPCEvent response = QuicUtils.<RPCEvent, RPCEvent>sendSimpleRequest(clientConn,
RPCEventCodec::new,
RPCEventCodec::new,
new SingletonGet(NORMAL, NullableLLSnapshot.empty())
).blockOptional().orElseThrow();
assertEquals(Empty.of(), response);
}
@Test
void sendSimpleRequestFlux() {
List<RPCEvent> results = QuicUtils.<RPCEvent, RPCEvent>sendSimpleRequestFlux(clientConn,
RPCEventCodec::new,
RPCEventCodec::new,
Flux.just(
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
new SingletonGet(NORMAL, NullableLLSnapshot.empty())
)
).collectList().blockOptional().orElseThrow();
assertEquals(5, results.size());
assertEquals(List.of(Empty.of(), Empty.of(), Empty.of(), Empty.of(), Empty.of()), results);
}
@Test
void sendUpdateFluxNormal() {
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
RPCEventCodec::new,
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
serverData -> Mono.fromCallable(() -> {
assertEquals(Empty.of(), serverData);
return new SingletonGet(NORMAL, NullableLLSnapshot.empty());
})
).blockOptional().orElseThrow();
assertEquals(Empty.of(), results);
}
@Test
void sendUpdateFluxSlowClient() {
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
RPCEventCodec::new,
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
serverData -> Mono.<RPCEvent>fromCallable(() -> {
assertEquals(Empty.of(), serverData);
return new SingletonGet(NORMAL, NullableLLSnapshot.empty());
}).delayElement(Duration.ofSeconds(2))
).blockOptional().orElseThrow();
assertEquals(Empty.of(), results);
}
@Test
void sendUpdateFluxSlowServer() {
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
RPCEventCodec::new,
new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty()),
serverData -> Mono.fromCallable(() -> {
assertEquals(Empty.of(), serverData);
return new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty());
})
).blockOptional().orElseThrow();
assertEquals(Empty.of(), results);
}
@Test
void sendUpdateFluxSlowClientAndServer() {
RPCEvent results = QuicUtils.<RPCEvent>sendUpdate(clientConn,
RPCEventCodec::new,
new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty()),
serverData -> Mono.<RPCEvent>fromCallable(() -> {
assertEquals(Empty.of(), serverData);
return new SingletonGet(WAIT_TIME, NullableLLSnapshot.empty());
}).delayElement(Duration.ofSeconds(2))
).blockOptional().orElseThrow();
assertEquals(Empty.of(), results);
}
@Test
void sendUpdateClientFail() {
class ExpectedException extends Throwable {}
assertThrows(ExpectedException.class, () -> {
try {
RPCEvent results = QuicUtils
.<RPCEvent>sendUpdate(clientConn,
RPCEventCodec::new,
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
serverData -> Mono.error(new ExpectedException())
)
.blockOptional()
.orElseThrow();
} catch (Throwable e) {
throw e.getCause();
}
});
}
@Test
void sendUpdateServerFail1() {
assertThrows(RPCException.class,
() -> QuicUtils
.<RPCEvent>sendUpdate(clientConn,
RPCEventCodec::new,
new SingletonGet(FAIL_IMMEDIATELY, NullableLLSnapshot.empty()),
serverData -> Mono.fromCallable(() -> {
fail("Called update");
return new SingletonGet(NORMAL, NullableLLSnapshot.empty());
})
)
.blockOptional()
.orElseThrow()
);
}
@Test
void sendUpdateServerFail2() {
assertThrows(RPCException.class,
() -> QuicUtils
.<RPCEvent>sendUpdate(clientConn,
RPCEventCodec::new,
new SingletonGet(NORMAL, NullableLLSnapshot.empty()),
serverData -> Mono.fromCallable(() -> {
assertEquals(Empty.of(), serverData);
return new SingletonGet(FAIL_IMMEDIATELY, NullableLLSnapshot.empty());
})
)
.blockOptional()
.orElseThrow()
);
}
@Test
void sendSimpleRequestConcurrently() {
// Send the request a second time
var requestMono = QuicUtils.<RPCEvent, RPCEvent>sendSimpleRequest(clientConn,
RPCEventCodec::new,
RPCEventCodec::new,
new SingletonGet(NORMAL, NullableLLSnapshot.empty())
);
var results = Flux
.merge(requestMono, requestMono, requestMono, requestMono, requestMono)
.collectList()
.blockOptional()
.orElseThrow();
assertEquals(5, results.size());
assertEquals(List.of(Empty.of(), Empty.of(), Empty.of(), Empty.of(), Empty.of()), results);
}
@Test
void sendFailedRequest() {
assertThrows(RPCException.class,
() -> QuicUtils
.<RPCEvent, RPCEvent>sendSimpleRequest(clientConn,
RPCEventCodec::new,
RPCEventCodec::new,
new SingletonGet(FAIL_IMMEDIATELY, NullableLLSnapshot.empty())
)
.blockOptional()
.orElseThrow()
);
}
@Test
void createStream() {
}
}

View File

@ -1,174 +0,0 @@
package it.cavallium.dbengine.lucene.hugepq.search;
import static org.junit.jupiter.api.Assertions.assertEquals;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
import it.cavallium.dbengine.lucene.LLFieldDoc;
import it.cavallium.dbengine.lucene.LLScoreDoc;
import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer;
import it.cavallium.dbengine.lucene.searcher.ShardIndexSearcher;
import it.cavallium.dbengine.lucene.searcher.SharedShardStatistics;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHits.Relation;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.util.QueryBuilder;
import org.junit.jupiter.api.Test;
public class HugePqFullFieldDocCollectorTest {
Sort sort = new Sort(new SortedNumericSortField("number_sort", Type.LONG));
Query luceneQuery = LongPoint.newRangeQuery("number", -100, 100);
@Test
public void testSingleShard() throws IOException {
try (var dir = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
var analyzer = new WordAnalyzer(true, true);
var writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
writer.updateDocument(new Term("id", "00"), List.of(new SortedNumericDocValuesField("number_sort", 1), new LongPoint("number", 1)));
writer.updateDocument(new Term("id", "01"), List.of(new SortedNumericDocValuesField("number_sort", 44), new LongPoint("number", 44)));
writer.updateDocument(new Term("id", "02"), List.of(new SortedNumericDocValuesField("number_sort", 203), new LongPoint("number", 203)));
writer.updateDocument(new Term("id", "03"), List.of(new SortedNumericDocValuesField("number_sort", 209), new LongPoint("number", 209)));
writer.updateDocument(new Term("id", "04"), List.of(new SortedNumericDocValuesField("number_sort", -33), new LongPoint("number", -33)));
writer.updateDocument(new Term("id", "05"), List.of(new SortedNumericDocValuesField("number_sort", 0), new LongPoint("number", 0)));
writer.updateDocument(new Term("id", "06"), List.of(new SortedNumericDocValuesField("number_sort", 933), new LongPoint("number", 933)));
writer.updateDocument(new Term("id", "07"), List.of(new SortedNumericDocValuesField("number_sort", 6), new LongPoint("number", 6)));
writer.updateDocument(new Term("id", "08"), List.of(new SortedNumericDocValuesField("number_sort", -11), new LongPoint("number", -11)));
writer.updateDocument(new Term("id", "09"), List.of(new SortedNumericDocValuesField("number_sort", 9996), new LongPoint("number", 9996)));
writer.updateDocument(new Term("id", "10"), List.of(new SortedNumericDocValuesField("number_sort", 9), new LongPoint("number", 9)));
writer.updateDocument(new Term("id", "11"), List.of(new SortedNumericDocValuesField("number_sort", 66), new LongPoint("number", 66)));
writer.updateDocument(new Term("id", "12"), List.of(new SortedNumericDocValuesField("number_sort", 88), new LongPoint("number", 88)));
writer.updateDocument(new Term("id", "13"), List.of(new SortedNumericDocValuesField("number_sort", 222), new LongPoint("number", 222)));
writer.updateDocument(new Term("id", "14"), List.of(new SortedNumericDocValuesField("number_sort", -2), new LongPoint("number", -2)));
writer.updateDocument(new Term("id", "15"), List.of(new SortedNumericDocValuesField("number_sort", 7), new LongPoint("number", 7)));
writer.updateDocument(new Term("id", "16"), List.of(new SortedNumericDocValuesField("number_sort", 1010912093), new LongPoint("number", 1010912093)));
writer.updateDocument(new Term("id", "17"), List.of(new SortedNumericDocValuesField("number_sort", -3894789), new LongPoint("number", -3894789)));
writer.updateDocument(new Term("id", "18"), List.of(new SortedNumericDocValuesField("number_sort", 122), new LongPoint("number", 122)));
writer.updateDocument(new Term("id", "19"), List.of(new SortedNumericDocValuesField("number_sort", 2), new LongPoint("number", 2)));
writer.flush();
writer.commit();
try (var reader = DirectoryReader.open(writer, true, true)) {
var searcher = new IndexSearcher(reader);
var expectedResults = searcher.search(luceneQuery, 20, sort, false);
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
var expectedDocs = Arrays
.stream(expectedResults.scoreDocs)
.map(sd -> (FieldDoc) sd)
.map(fieldDoc -> new LLFieldDoc(fieldDoc.doc, fieldDoc.score, fieldDoc.shardIndex, Arrays.asList(fieldDoc.fields)))
.toList();
try (var collector = HugePqFullFieldDocCollector.create(env, sort, 20, Integer.MAX_VALUE)) {
searcher.search(luceneQuery, collector);
var docs = collector.fullDocs().iterate().collectList().blockOptional().orElseThrow();
System.out.println("Expected docs:");
for (var expectedDoc : expectedDocs) {
System.out.println(expectedDoc);
}
System.out.println("");
System.out.println("Obtained docs:");
for (var doc : docs) {
System.out.println(doc);
}
assertEquals(expectedDocs,
docs.stream().map(elem -> new LLFieldDoc(elem.doc(), elem.score(), -1, elem.fields())).toList()
);
assertEquals(expectedTotalHits, new TotalHitsCount(collector.getTotalHits(), true));
}
}
}
}
@Test
public void testMultiShard() throws IOException {
try (var dir1 = new ByteBuffersDirectory(); var dir2 = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
var analyzer = new WordAnalyzer(true, true);
var writer1 = new IndexWriter(dir1, new IndexWriterConfig(analyzer));
var writer2 = new IndexWriter(dir2, new IndexWriterConfig(analyzer));
writer1.updateDocument(new Term("id", "00"), List.of(new SortedNumericDocValuesField("number_sort", 1), new LongPoint("number", 1)));
writer1.updateDocument(new Term("id", "01"), List.of(new SortedNumericDocValuesField("number_sort", 44), new LongPoint("number", 44)));
writer1.updateDocument(new Term("id", "02"), List.of(new SortedNumericDocValuesField("number_sort", 203), new LongPoint("number", 203)));
writer1.updateDocument(new Term("id", "03"), List.of(new SortedNumericDocValuesField("number_sort", 209), new LongPoint("number", 209)));
writer1.updateDocument(new Term("id", "04"), List.of(new SortedNumericDocValuesField("number_sort", -33), new LongPoint("number", -33)));
writer1.updateDocument(new Term("id", "05"), List.of(new SortedNumericDocValuesField("number_sort", 0), new LongPoint("number", 0)));
writer1.updateDocument(new Term("id", "06"), List.of(new SortedNumericDocValuesField("number_sort", 933), new LongPoint("number", 933)));
writer1.updateDocument(new Term("id", "07"), List.of(new SortedNumericDocValuesField("number_sort", 6), new LongPoint("number", 6)));
writer1.updateDocument(new Term("id", "08"), List.of(new SortedNumericDocValuesField("number_sort", -11), new LongPoint("number", -11)));
writer1.updateDocument(new Term("id", "09"), List.of(new SortedNumericDocValuesField("number_sort", 9996), new LongPoint("number", 9996)));
writer2.updateDocument(new Term("id", "10"), List.of(new SortedNumericDocValuesField("number_sort", 9), new LongPoint("number", 9)));
writer2.updateDocument(new Term("id", "11"), List.of(new SortedNumericDocValuesField("number_sort", 66), new LongPoint("number", 66)));
writer2.updateDocument(new Term("id", "12"), List.of(new SortedNumericDocValuesField("number_sort", 88), new LongPoint("number", 88)));
writer2.updateDocument(new Term("id", "13"), List.of(new SortedNumericDocValuesField("number_sort", 222), new LongPoint("number", 222)));
writer2.updateDocument(new Term("id", "14"), List.of(new SortedNumericDocValuesField("number_sort", -2), new LongPoint("number", -2)));
writer2.updateDocument(new Term("id", "15"), List.of(new SortedNumericDocValuesField("number_sort", 7), new LongPoint("number", 7)));
writer2.updateDocument(new Term("id", "16"), List.of(new SortedNumericDocValuesField("number_sort", 1010912093), new LongPoint("number", 1010912093)));
writer2.updateDocument(new Term("id", "17"), List.of(new SortedNumericDocValuesField("number_sort", -3894789), new LongPoint("number", -3894789)));
writer2.updateDocument(new Term("id", "18"), List.of(new SortedNumericDocValuesField("number_sort", 122), new LongPoint("number", 122)));
writer2.updateDocument(new Term("id", "19"), List.of(new SortedNumericDocValuesField("number_sort", 2), new LongPoint("number", 2)));
writer1.flush();
writer2.flush();
writer1.commit();
writer2.commit();
var sharedStats = new SharedShardStatistics();
try (var reader1 = DirectoryReader.open(writer1, true, true);
var reader2 = DirectoryReader.open(writer2, true, true)) {
var searcher1 = new IndexSearcher(reader1);
var searcher2 = new IndexSearcher(reader2);
var shardSearcher1 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 0);
var shardSearcher2 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 1);
var standardSharedManager = TopFieldCollector.createSharedManager(sort, 20, null, Integer.MAX_VALUE);
var standardCollector1 = standardSharedManager.newCollector();
var standardCollector2 = standardSharedManager.newCollector();
shardSearcher1.search(luceneQuery, standardCollector1);
shardSearcher2.search(luceneQuery, standardCollector2);
var expectedResults = standardSharedManager.reduce(List.of(standardCollector1, standardCollector2));
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
var expectedDocs = Arrays
.stream(expectedResults.scoreDocs)
.map(sd -> (FieldDoc) sd)
.map(fieldDoc -> new LLFieldDoc(fieldDoc.doc, fieldDoc.score, fieldDoc.shardIndex, Arrays.asList(fieldDoc.fields)))
.toList();
var collectorManager = HugePqFullFieldDocCollector.createSharedManager(env, sort, 20, Integer.MAX_VALUE);
var collector1 = collectorManager.newCollector();
var collector2 = collectorManager.newCollector();
shardSearcher1.search(luceneQuery, collector1);
shardSearcher2.search(luceneQuery, collector2);
try (var results = collectorManager.reduce(List.of(collector1, collector2))) {
var docs = results.iterate().collectList().blockOptional().orElseThrow();
System.out.println("Expected docs:");
for (var expectedDoc : expectedDocs) {
System.out.println(expectedDoc);
}
System.out.println("");
System.out.println("Obtained docs:");
for (var doc : docs) {
System.out.println(doc);
}
assertEquals(expectedDocs,
docs.stream().map(elem -> new LLFieldDoc(elem.doc(), elem.score(), -1, elem.fields())).toList()
);
assertEquals(expectedTotalHits, new TotalHitsCount(results.totalHits().value, results.totalHits().relation == Relation.EQUAL_TO));
}
}
}
}
}

View File

@ -1,178 +0,0 @@
package it.cavallium.dbengine.lucene.hugepq.search;
import static org.junit.jupiter.api.Assertions.*;
import it.cavallium.dbengine.client.query.QueryUtils;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.IndexSearcherManager;
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
import it.cavallium.dbengine.lucene.LLScoreDoc;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.LegacyWordAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer;
import it.cavallium.dbengine.lucene.searcher.ShardIndexSearcher;
import it.cavallium.dbengine.lucene.searcher.SharedShardStatistics;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.CollectorManager;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHits.Relation;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.util.QueryBuilder;
import org.junit.jupiter.api.Test;
public class HugePqFullScoreDocCollectorTest {
@Test
public void testSingleShard() throws IOException {
try (var dir = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
var analyzer = new WordAnalyzer(true, true);
var writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
writer.updateDocument(new Term("id", "00"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
writer.updateDocument(new Term("id", "01"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
writer.updateDocument(new Term("id", "02"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
writer.updateDocument(new Term("id", "03"), List.of(new TextField("text", "Marios Rossi", Store.YES)));
writer.updateDocument(new Term("id", "04"), List.of(new TextField("text", "Rossi", Store.YES)));
writer.updateDocument(new Term("id", "05"), List.of(new TextField("text", "Rossi", Store.YES)));
writer.updateDocument(new Term("id", "06"), List.of(new TextField("text", "Rossi", Store.YES)));
writer.updateDocument(new Term("id", "07"), List.of(new TextField("text", "Rossi", Store.YES)));
writer.updateDocument(new Term("id", "08"), List.of(new TextField("text", "Rossi", Store.YES)));
writer.updateDocument(new Term("id", "09"), List.of(new TextField("text", "Rossi", Store.YES)));
writer.updateDocument(new Term("id", "10"), List.of(new TextField("text", "ROSSI UA", Store.YES)));
writer.updateDocument(new Term("id", "11"), List.of(new TextField("text", "Mario Barman", Store.YES)));
writer.updateDocument(new Term("id", "12"), List.of(new TextField("text", "Mario batman", Store.YES)));
writer.updateDocument(new Term("id", "13"), List.of(new TextField("text", "Admin Rossi desk", Store.YES)));
writer.updateDocument(new Term("id", "14"), List.of(new TextField("text", "MRI Marios bot", Store.YES)));
writer.updateDocument(new Term("id", "15"), List.of(new TextField("text", "Mario Rossi [beta]", Store.YES)));
writer.updateDocument(new Term("id", "16"), List.of(new TextField("text", "Mario Music Bot", Store.YES)));
writer.updateDocument(new Term("id", "17"), List.of(new TextField("text", "Mario night mode", Store.YES)));
writer.updateDocument(new Term("id", "18"), List.of(new TextField("text", "Mario stats bot", Store.YES)));
writer.updateDocument(new Term("id", "19"), List.of(new TextField("text", "Very very long text with Mario Giovanni and Rossi inside", Store.YES)));
writer.flush();
writer.commit();
try (var reader = DirectoryReader.open(writer, true, true)) {
var searcher = new IndexSearcher(reader);
var qb = new QueryBuilder(analyzer);
var luceneQuery = qb.createMinShouldMatchQuery("text", "Mario rossi", 0.3f);
var expectedResults = searcher.search(luceneQuery, 20);
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
var expectedDocs = Arrays
.stream(expectedResults.scoreDocs)
.map(scoreDoc -> new LLScoreDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex))
.toList();
try (var collector = HugePqFullScoreDocCollector.create(env, 20)) {
searcher.search(luceneQuery, collector);
var docs = collector
.fullDocs()
.iterate()
.collectList()
.transform(LLUtils::handleDiscard)
.blockOptional()
.orElseThrow();
System.out.println("Expected docs:");
for (LLScoreDoc expectedDoc : expectedDocs) {
System.out.println(expectedDoc);
}
System.out.println("");
System.out.println("Obtained docs:");
for (LLScoreDoc doc : docs) {
System.out.println(doc);
}
assertEquals(expectedDocs, docs.stream().map(elem -> new LLScoreDoc(elem.doc(), elem.score(), -1)).toList());
assertEquals(expectedTotalHits, new TotalHitsCount(collector.getTotalHits(), true));
}
}
}
}
@Test
public void testMultiShard() throws IOException {
try (var dir1 = new ByteBuffersDirectory(); var dir2 = new ByteBuffersDirectory(); var env = new LLTempHugePqEnv()) {
var analyzer = new WordAnalyzer(true, true);
var writer1 = new IndexWriter(dir1, new IndexWriterConfig(analyzer));
var writer2 = new IndexWriter(dir2, new IndexWriterConfig(analyzer));
writer1.updateDocument(new Term("id", "00"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "01"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "02"), List.of(new TextField("text", "Mario Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "03"), List.of(new TextField("text", "Marios Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "04"), List.of(new TextField("text", "Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "05"), List.of(new TextField("text", "Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "06"), List.of(new TextField("text", "Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "07"), List.of(new TextField("text", "Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "08"), List.of(new TextField("text", "Rossi", Store.YES)));
writer1.updateDocument(new Term("id", "09"), List.of(new TextField("text", "Rossi", Store.YES)));
writer2.updateDocument(new Term("id", "10"), List.of(new TextField("text", "ROSSI UA", Store.YES)));
writer2.updateDocument(new Term("id", "11"), List.of(new TextField("text", "Mario Barman", Store.YES)));
writer2.updateDocument(new Term("id", "12"), List.of(new TextField("text", "Mario batman", Store.YES)));
writer2.updateDocument(new Term("id", "13"), List.of(new TextField("text", "Admin Rossi desk", Store.YES)));
writer2.updateDocument(new Term("id", "14"), List.of(new TextField("text", "MRI Marios bot", Store.YES)));
writer2.updateDocument(new Term("id", "15"), List.of(new TextField("text", "Mario Rossi [beta]", Store.YES)));
writer2.updateDocument(new Term("id", "16"), List.of(new TextField("text", "Mario Music Bot", Store.YES)));
writer2.updateDocument(new Term("id", "17"), List.of(new TextField("text", "Mario night mode", Store.YES)));
writer2.updateDocument(new Term("id", "18"), List.of(new TextField("text", "Mario stats bot", Store.YES)));
writer2.updateDocument(new Term("id", "19"), List.of(new TextField("text", "Very very long text with Mario Giovanni and Rossi inside", Store.YES)));
writer1.flush();
writer2.flush();
writer1.commit();
writer2.commit();
var sharedStats = new SharedShardStatistics();
try (var reader1 = DirectoryReader.open(writer1, true, true);
var reader2 = DirectoryReader.open(writer2, true, true)) {
var searcher1 = new IndexSearcher(reader1);
var searcher2 = new IndexSearcher(reader2);
var shardSearcher1 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 0);
var shardSearcher2 = new ShardIndexSearcher(sharedStats, List.of(searcher1, searcher2), 1);
var qb = new QueryBuilder(analyzer);
var luceneQuery = qb.createMinShouldMatchQuery("text", "Mario rossi", 0.3f);
var standardSharedManager = TopScoreDocCollector.createSharedManager(20, null, Integer.MAX_VALUE);
var standardCollector1 = standardSharedManager.newCollector();
var standardCollector2 = standardSharedManager.newCollector();
shardSearcher1.search(luceneQuery, standardCollector1);
shardSearcher2.search(luceneQuery, standardCollector2);
var expectedResults = standardSharedManager.reduce(List.of(standardCollector1, standardCollector2));
var expectedTotalHits = new TotalHitsCount(expectedResults.totalHits.value, expectedResults.totalHits.relation == Relation.EQUAL_TO);
var expectedDocs = Arrays
.stream(expectedResults.scoreDocs)
.map(scoreDoc -> new LLScoreDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex))
.toList();
var collectorManager = HugePqFullScoreDocCollector.createSharedManager(env, 20, Integer.MAX_VALUE);
try (var collector1 = collectorManager.newCollector();
var collector2 = collectorManager.newCollector()) {
shardSearcher1.search(luceneQuery, collector1);
shardSearcher2.search(luceneQuery, collector2);
try (var results = collectorManager.reduce(List.of(collector1, collector2))) {
var docs = results.iterate().collectList().blockOptional().orElseThrow();
System.out.println("Expected docs:");
for (LLScoreDoc expectedDoc : expectedDocs) {
System.out.println(expectedDoc);
}
System.out.println("");
System.out.println("Obtained docs:");
for (LLScoreDoc doc : docs) {
System.out.println(doc);
}
assertEquals(expectedDocs, docs.stream().map(elem -> new LLScoreDoc(elem.doc(), elem.score(), -1)).toList());
assertEquals(expectedTotalHits, new TotalHitsCount(results.totalHits().value, results.totalHits().relation == Relation.EQUAL_TO));
}
}
}
}
}
}

View File

@ -1,23 +1,17 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static org.junit.jupiter.api.Assertions.assertEquals;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.MemoryManager;
import io.netty5.buffer.internal.LeakDetection;
import io.netty5.buffer.internal.LifecycleTracer;
import io.netty5.buffer.pool.PoolArenaMetric;
import io.netty5.buffer.pool.PooledBufferAllocator;
import io.netty5.util.ResourceLeakDetector;
import io.netty5.util.ResourceLeakDetector.Level;
import io.netty5.util.internal.PlatformDependent;
import io.netty.util.ResourceLeakDetector;
import io.netty.util.ResourceLeakDetector.Level;
import it.cavallium.dbengine.buffers.BufDataInput;
import it.cavallium.dbengine.buffers.BufDataOutput;
import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.LuceneIndexImpl;
import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionary;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep;
@ -26,23 +20,20 @@ import it.cavallium.dbengine.database.collections.DatabaseStageEntry;
import it.cavallium.dbengine.database.collections.DatabaseStageMap;
import it.cavallium.dbengine.database.collections.SubStageGetterHashMap;
import it.cavallium.dbengine.database.collections.SubStageGetterMap;
import it.cavallium.dbengine.database.serialization.SerializationException;
import it.cavallium.dbengine.database.serialization.Serializer;
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import org.apache.lucene.util.IOSupplier;
import org.jetbrains.annotations.NotNull;
import org.reactivestreams.Publisher;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import org.junit.jupiter.api.Assertions;
public class DbTestUtils {
static {
LLUtils.initHooks();
}
public static final String BIG_STRING = generateBigString();
public static final int MAX_IN_MEMORY_RESULT_ENTRIES = 8192;
@ -50,84 +41,61 @@ public class DbTestUtils {
return "0123456789".repeat(1024);
}
public record TestAllocator(TestAllocatorImpl allocator) {}
public static TestAllocator newAllocator() {
return new TestAllocator(TestAllocatorImpl.create());
}
public static void destroyAllocator(TestAllocator testAllocator) {
testAllocator.allocator().close();
}
@SuppressWarnings("SameParameterValue")
private static long getActiveAllocations(TestAllocatorImpl allocator, boolean printStats) {
long activeAllocations = allocator.getActiveAllocations();
if (printStats) {
System.out.println("activeAllocations=" + activeAllocations);
}
return activeAllocations;
}
public static boolean isCIMode() {
return System.getProperty("dbengine.ci", "false").equalsIgnoreCase("true");
}
public static <U> Flux<U> tempDb(TemporaryDbGenerator temporaryDbGenerator,
TestAllocator alloc,
Function<LLKeyValueDatabase, Publisher<U>> action) {
return Flux.usingWhen(
temporaryDbGenerator.openTempDb(alloc),
tempDb -> Flux
.from(action.apply(tempDb.db()))
.doOnDiscard(Object.class, o -> System.out.println("Discarded: " + o.getClass().getName() + ", " + o)),
temporaryDbGenerator::closeTempDb
);
public static <U> U tempDb(TemporaryDbGenerator temporaryDbGenerator,
Function<LLKeyValueDatabase, U> action) throws IOException {
var tempDb = temporaryDbGenerator.openTempDb();
try {
return action.apply(tempDb.db());
} finally {
temporaryDbGenerator.closeTempDb(tempDb);
}
}
public record TempDb(TestAllocator allocator, LLDatabaseConnection connection, LLKeyValueDatabase db,
public static void runVoid(boolean shouldFail, Runnable consumer) {
if (shouldFail) {
Assertions.assertThrows(Throwable.class, consumer::run);
} else {
Assertions.assertDoesNotThrow(consumer::run);
}
}
public static <X> X run(boolean shouldFail, IOSupplier<X> consumer) {
AtomicReference<X> result = new AtomicReference<>(null);
if (shouldFail) {
Assertions.assertThrows(Throwable.class, consumer::get);
} else {
Assertions.assertDoesNotThrow(() -> result.set(consumer.get()));
}
return result.get();
}
public record TempDb(LLDatabaseConnection connection, LLKeyValueDatabase db,
LLLuceneIndex luceneSingle,
LLLuceneIndex luceneMulti,
SwappableLuceneSearcher swappableLuceneSearcher,
Path path) {}
static boolean computeCanUseNettyDirect() {
boolean canUse = true;
if (!PlatformDependent.hasUnsafe()) {
System.err.println("Warning! Unsafe is not available!"
+ " Netty direct buffers will not be used in tests!");
canUse = false;
}
return canUse;
}
public static void ensureNoLeaks(TestAllocatorImpl allocator, boolean printStats, boolean useClassicException) {
public static void ensureNoLeaks(boolean printStats, boolean useClassicException) {
ResourceLeakDetector.setLevel(Level.PARANOID);
System.gc();
if (allocator != null) {
var allocs = getActiveAllocations(allocator, printStats);
if (useClassicException) {
if (allocs != 0) {
throw new IllegalStateException("Active allocations: " + allocs);
}
} else {
assertEquals(0L, allocs);
}
}
}
public static Mono<? extends LLDictionary> tempDictionary(LLKeyValueDatabase database, UpdateMode updateMode) {
public static LLDictionary tempDictionary(LLKeyValueDatabase database, UpdateMode updateMode) {
return tempDictionary(database, "testmap", updateMode);
}
public static Mono<? extends LLDictionary> tempDictionary(LLKeyValueDatabase database,
public static LLDictionary tempDictionary(LLKeyValueDatabase database,
String name,
UpdateMode updateMode) {
return database.getDictionary(name, updateMode);
}
public static Mono<? extends LuceneIndex<String, String>> tempLuceneIndex(LLLuceneIndex index) {
return Mono.fromCallable(() -> new LuceneIndexImpl<>(index, new StringIndicizer()));
public static LuceneIndex<String, String> tempLuceneIndex(LLLuceneIndex index) {
return new LuceneIndexImpl<>(index, new StringIndicizer());
}
@ -157,14 +125,13 @@ public class DbTestUtils {
}
@Override
public @NotNull Short deserialize(@NotNull Buffer serialized) {
Objects.requireNonNull(serialized);
return serialized.readShort();
public @NotNull Short deserialize(@NotNull BufDataInput in) throws SerializationException {
return in.readShort();
}
@Override
public void serialize(@NotNull Short deserialized, Buffer output) {
output.writeShort(deserialized);
public void serialize(@NotNull Short deserialized, BufDataOutput out) throws SerializationException {
out.writeShort(deserialized);
}
}
);
@ -195,7 +162,7 @@ public class DbTestUtils {
new SubStageGetterHashMap<>(Serializer.UTF8_SERIALIZER,
Serializer.UTF8_SERIALIZER,
String::hashCode,
SerializerFixedBinaryLength.intSerializer(dictionary.getAllocator())
SerializerFixedBinaryLength.intSerializer()
)
);
}
@ -206,7 +173,7 @@ public class DbTestUtils {
Serializer.UTF8_SERIALIZER,
Serializer.UTF8_SERIALIZER,
String::hashCode,
SerializerFixedBinaryLength.intSerializer(dictionary.getAllocator())
SerializerFixedBinaryLength.intSerializer()
);
}
}

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import it.cavallium.dbengine.client.Sort;
import it.cavallium.dbengine.client.query.BaseType;

View File

@ -0,0 +1,113 @@
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import it.cavallium.data.generator.nativedata.Nullableboolean;
import it.cavallium.data.generator.nativedata.Nullabledouble;
import it.cavallium.data.generator.nativedata.Nullableint;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.database.ColumnUtils;
import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletionException;
import java.util.concurrent.atomic.AtomicInteger;
public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
private static final AtomicInteger dbId = new AtomicInteger(0);
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
Duration.ofSeconds(5),
Duration.ofSeconds(5),
false,
new ByteBuffersDirectory(),
Nullableboolean.empty(),
Nullabledouble.empty(),
Nullableint.empty(),
Nullableboolean.empty(),
Nullableboolean.empty(),
MAX_IN_MEMORY_RESULT_ENTRIES,
LuceneUtils.getDefaultMergePolicy()
);
@Override
public TempDb openTempDb() throws IOException {
var wrkspcPath = Path.of("/tmp/.cache/tempdb-" + dbId.incrementAndGet() + "/");
if (Files.exists(wrkspcPath)) {
Files.walk(wrkspcPath).sorted(Comparator.reverseOrder()).forEach(file -> {
try {
Files.delete(file);
} catch (IOException ex) {
throw new CompletionException(ex);
}
});
}
Files.createDirectories(wrkspcPath);
LLDatabaseConnection conn = new LLLocalDatabaseConnection(
new SimpleMeterRegistry(),
wrkspcPath,
true
).connect();
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
return new TempDb(conn,
conn.getDatabase("testdb",
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
DefaultDatabaseOptions.builder().build()
),
conn.getLuceneIndex("testluceneindex1",
LuceneUtils.singleStructure(),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
conn.getLuceneIndex("testluceneindex16",
LuceneUtils.shardsStructure(3),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
searcher,
wrkspcPath
);
}
@Override
public void closeTempDb(TempDb tempDb) throws IOException {
tempDb.db().close();
tempDb.connection().disconnect();
ensureNoLeaks(false, false);
if (Files.exists(tempDb.path())) {
Files.walk(tempDb.path()).sorted(Comparator.reverseOrder()).forEach(file -> {
try {
Files.delete(file);
} catch (IOException ex) {
throw new CompletionException(ex);
}
});
}
}
}

View File

@ -1,6 +1,8 @@
package it.cavallium.dbengine.lucene.searcher;
package it.cavallium.dbengine.tests;
import it.cavallium.dbengine.lucene.ExponentialPageLimits;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LuceneGenerator;
import it.unimi.dsi.fastutil.longs.LongList;
import java.io.IOException;
import java.time.Duration;
@ -80,7 +82,7 @@ public class LuceneGeneratorTest {
var reactiveGenerator = LuceneGenerator.reactive(is, localQueryParams, -1);
var results = fixResults(localQueryParams.isSorted(),
localQueryParams.needsScores(), reactiveGenerator.collectList().block());
localQueryParams.needsScores(), reactiveGenerator.toList());
Assertions.assertNotEquals(0, results.size());
@ -98,7 +100,7 @@ public class LuceneGeneratorTest {
localQueryParams.needsScores(), List.of(is.search(query, limit).scoreDocs));
var reactiveGenerator = LuceneGenerator.reactive(is, localQueryParams, -1);
var results = fixResults(localQueryParams.isSorted(), localQueryParams.needsScores(), reactiveGenerator.collectList().block());
var results = fixResults(localQueryParams.isSorted(), localQueryParams.needsScores(), reactiveGenerator.toList());
Assertions.assertNotEquals(0, results.size());
@ -123,7 +125,7 @@ public class LuceneGeneratorTest {
),
-1
);
var results = reactiveGenerator.collectList().block();
var results = reactiveGenerator.toList();
Assertions.assertNotNull(results);
Assertions.assertEquals(0, results.size());
@ -144,7 +146,7 @@ public class LuceneGeneratorTest {
),
-1
);
var results = reactiveGenerator.collectList().block();
var results = reactiveGenerator.toList();
Assertions.assertNotNull(results);
Assertions.assertEquals(limit, results.size());
@ -161,7 +163,7 @@ public class LuceneGeneratorTest {
var reactiveGenerator = LuceneGenerator.reactive(is, localQueryParams, -1);
var results = fixResults(localQueryParams.isSorted(),
localQueryParams.needsScores(), reactiveGenerator.collectList().block());
localQueryParams.needsScores(), reactiveGenerator.toList());
Assertions.assertEquals(4, results.size());
Assertions.assertEquals(expectedResults, results);

View File

@ -0,0 +1,75 @@
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import it.cavallium.data.generator.nativedata.Nullableboolean;
import it.cavallium.data.generator.nativedata.Nullabledouble;
import it.cavallium.data.generator.nativedata.Nullableint;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
import it.cavallium.dbengine.client.DefaultDatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.database.ColumnUtils;
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.rpc.current.data.ByteBuffersDirectory;
import it.cavallium.dbengine.rpc.current.data.LuceneOptions;
import java.time.Duration;
import java.util.List;
import java.util.Map;
public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(),
Duration.ofSeconds(5),
Duration.ofSeconds(5),
false,
new ByteBuffersDirectory(),
Nullableboolean.empty(),
Nullabledouble.empty(),
Nullableint.empty(),
Nullableboolean.empty(),
Nullableboolean.empty(),
MAX_IN_MEMORY_RESULT_ENTRIES,
LuceneUtils.getDefaultMergePolicy()
);
@Override
public TempDb openTempDb() {
var conn = new LLMemoryDatabaseConnection(new SimpleMeterRegistry());
SwappableLuceneSearcher searcher = new SwappableLuceneSearcher();
var luceneHacks = new LuceneHacks(() -> searcher, () -> searcher);
return new TempDb(conn,
conn.getDatabase("testdb",
List.of(ColumnUtils.dictionary("testmap"), ColumnUtils.special("ints"), ColumnUtils.special("longs")),
DefaultDatabaseOptions.builder().build()
),
conn.getLuceneIndex("testluceneindex1",
LuceneUtils.singleStructure(),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
conn.getLuceneIndex("testluceneindex16",
LuceneUtils.shardsStructure(3),
IndicizerAnalyzers.of(TextFieldsAnalyzer.ICUCollationKey),
IndicizerSimilarities.of(TextFieldsSimilarity.Boolean),
LUCENE_OPTS,
luceneHacks
),
searcher,
null
);
}
@Override
public void closeTempDb(TempDb db) {
db.db().close();
}
}

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import it.cavallium.dbengine.database.DiscardingCloseable;
import it.cavallium.dbengine.lucene.PriorityQueue;
@ -7,8 +7,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import org.apache.lucene.search.HitQueue;
import reactor.core.publisher.Flux;
public class PriorityQueueAdaptor<T> extends SimpleResource implements PriorityQueue<T>, DiscardingCloseable {
@ -61,7 +61,7 @@ public class PriorityQueueAdaptor<T> extends SimpleResource implements PriorityQ
}
@Override
public Flux<T> iterate() {
public Stream<T> iterate() {
List<T> items = new ArrayList<>(hitQueue.size());
T item;
while ((item = hitQueue.pop()) != null) {
@ -70,7 +70,7 @@ public class PriorityQueueAdaptor<T> extends SimpleResource implements PriorityQ
for (T t : items) {
hitQueue.insertWithOverflow(t);
}
return Flux.fromIterable(items);
return items.stream();
}
@Override

View File

@ -1,3 +1,3 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
record Scored(String key, float score) {}

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
@ -16,30 +16,27 @@ import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.jetbrains.annotations.NotNull;
import reactor.core.publisher.Mono;
public class StringIndicizer extends Indicizer<String, String> {
@Override
public @NotNull Mono<LLUpdateDocument> toIndexRequest(@NotNull String key, @NotNull String value) {
return Mono.fromCallable(() -> {
var fields = new LinkedList<LLItem>();
fields.add(LLItem.newStringField("uid", key, Field.Store.YES));
fields.add(LLItem.newTextField("text", value, Store.NO));
@SuppressWarnings("UnstableApiUsage")
var numInt = Ints.tryParse(value);
if (numInt != null) {
fields.add(LLItem.newIntPoint("intpoint", numInt));
fields.add(LLItem.newNumericDocValuesField("intsort", numInt));
}
@SuppressWarnings("UnstableApiUsage")
var numLong = Longs.tryParse(value);
if (numLong != null) {
fields.add(LLItem.newLongPoint("longpoint", numLong));
fields.add(LLItem.newNumericDocValuesField("longsort", numLong));
}
return new LLUpdateDocument(fields);
});
public @NotNull LLUpdateDocument toIndexRequest(@NotNull String key, @NotNull String value) {
var fields = new LinkedList<LLItem>();
fields.add(LLItem.newStringField("uid", key, Field.Store.YES));
fields.add(LLItem.newTextField("text", value, Store.NO));
@SuppressWarnings("UnstableApiUsage")
var numInt = Ints.tryParse(value);
if (numInt != null) {
fields.add(LLItem.newIntPoint("intpoint", numInt));
fields.add(LLItem.newNumericDocValuesField("intsort", numInt));
}
@SuppressWarnings("UnstableApiUsage")
var numLong = Longs.tryParse(value);
if (numLong != null) {
fields.add(LLItem.newLongPoint("longpoint", numLong));
fields.add(LLItem.newNumericDocValuesField("longsort", numLong));
}
return new LLUpdateDocument(fields);
}
@Override

View File

@ -1,9 +1,8 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static java.util.Objects.requireNonNull;
import static java.util.Objects.requireNonNullElseGet;
import io.netty5.util.Send;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.searcher.GlobalQueryRewrite;
@ -15,7 +14,6 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicReference;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Closeable {
@ -27,7 +25,7 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
}
@Override
public Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
public LuceneSearchResult collect(LLIndexSearcher indexSearcher,
LocalQueryParams queryParams,
@Nullable String keyFieldName,
GlobalQueryRewrite transformer) {
@ -36,7 +34,7 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
single = this.multi.get();
}
requireNonNull(single, "LuceneLocalSearcher not set");
return single.collect(indexSearcherMono, queryParams, keyFieldName, transformer);
return single.collect(indexSearcher, queryParams, keyFieldName, transformer);
}
@Override
@ -55,12 +53,12 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
}
@Override
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
public LuceneSearchResult collectMulti(LLIndexSearchers indexSearchers,
LocalQueryParams queryParams,
String keyFieldName,
GlobalQueryRewrite transformer) {
var multi = requireNonNull(this.multi.get(), "LuceneMultiSearcher not set");
return multi.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
return multi.collectMulti(indexSearchers, queryParams, keyFieldName, transformer);
}
public void setSingle(LocalSearcher single) {

View File

@ -0,0 +1,11 @@
package it.cavallium.dbengine.tests;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
import java.io.IOException;
public interface TemporaryDbGenerator {
TempDb openTempDb() throws IOException;
void closeTempDb(TempDb db) throws IOException;
}

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import it.cavallium.dbengine.lucene.DirectNIOFSDirectory;
import it.cavallium.dbengine.lucene.LuceneUtils;

View File

@ -0,0 +1,46 @@
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.tests.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.tests.DbTestUtils.tempDb;
import static it.cavallium.dbengine.tests.DbTestUtils.tempDictionary;
import it.cavallium.dbengine.database.UpdateMode;
import java.io.IOException;
import java.util.Arrays;
import java.util.stream.Stream;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
public abstract class TestDictionary {
protected abstract TemporaryDbGenerator getTempDbGenerator();
private static Stream<Arguments> provideArgumentsCreate() {
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
}
@BeforeEach
public void beforeEach() {
ensureNoLeaks(false, false);
}
@AfterEach
public void afterEach() {
if (!isCIMode()) {
ensureNoLeaks(true, false);
}
}
@ParameterizedTest
@MethodSource("provideArgumentsCreate")
public void testCreate(UpdateMode updateMode) throws IOException {
tempDb(getTempDbGenerator(), db -> {
tempDictionary(db, updateMode).clear();
return null;
});
}
}

View File

@ -1,42 +1,25 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.DbTestUtils.*;
import static it.cavallium.dbengine.SyncUtils.*;
import static it.cavallium.dbengine.tests.DbTestUtils.*;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.database.UpdateMode;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import reactor.test.StepVerifier.Step;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;
public abstract class TestDictionaryMap {
private static final Logger log = LogManager.getLogger(TestDictionaryMap.class);
private TestAllocator allocator;
private boolean checkLeaks = true;
private static boolean isTestBadKeysEnabled() {
@ -45,6 +28,11 @@ public abstract class TestDictionaryMap {
protected abstract TemporaryDbGenerator getTempDbGenerator();
record Tuple2<X, Y>(X getT1, Y getT2) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
private static Stream<Arguments> provideArgumentsPut() {
var goodKeys = List.of("12345");
List<String> badKeys;
@ -54,7 +42,7 @@ public abstract class TestDictionaryMap {
badKeys = List.of();
}
List<Tuple2<String, Boolean>> keys = Stream
.concat(goodKeys.stream().map(s -> Tuples.of(s, false)), badKeys.stream().map(s -> Tuples.of(s, true)))
.concat(goodKeys.stream().map(s -> new Tuple2<>(s, false)), badKeys.stream().map(s -> new Tuple2<>(s, true)))
.toList();
var values = isCIMode() ? List.of("val") : List.of("", "\0", BIG_STRING);
@ -67,18 +55,18 @@ public abstract class TestDictionaryMap {
} else {
strm = values.stream();
}
return strm.map(val -> Tuples.of(keyTuple.getT1(), val, keyTuple.getT2()));
return strm.map(val -> new Tuple3<>(keyTuple.getT1(), val, keyTuple.getT2()));
})
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple4<>(updateMode,
entryTuple.getT1(),
entryTuple.getT2(),
entryTuple.getT3()
)))
.flatMap(entryTuple -> Stream.of(Tuples.of(MapType.MAP, entryTuple.getT1(),
.flatMap(entryTuple -> Stream.of(new Tuple5<>(MapType.MAP, entryTuple.getT1(),
entryTuple.getT2(),
entryTuple.getT3(),
entryTuple.getT4()
), Tuples.of(MapType.HASH_MAP, entryTuple.getT1(),
), new Tuple5<>(MapType.HASH_MAP, entryTuple.getT1(),
entryTuple.getT2(),
entryTuple.getT3(),
false
@ -89,36 +77,33 @@ public abstract class TestDictionaryMap {
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
ensureNoLeaks(false, false);
}
@AfterEach
public void afterEach() {
if (!isCIMode() && checkLeaks) {
ensureNoLeaks(allocator.allocator(), true, false);
ensureNoLeaks(true, false);
}
destroyAllocator(allocator);
}
@ParameterizedTest
@MethodSource("provideArgumentsPut")
public void testPut(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
public void testPut(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail)
throws IOException {
var gen = getTempDbGenerator();
var db = run(gen.openTempDb(allocator));
var dict = run(tempDictionary(db.db(), updateMode));
var db = gen.openTempDb();
var dict = tempDictionary(db.db(), updateMode);
var map = tempDatabaseMapDictionaryMap(dict, mapType, 5);
runVoid(shouldFail, map.putValue(key, value));
runVoid(shouldFail, () -> map.putValue(key, value));
var resultingMapSize = run(map.leavesCount(null, false));
var resultingMapSize = map.leavesCount(null, false);
Assertions.assertEquals(shouldFail ? 0 : 1, resultingMapSize);
var resultingMap = run(map.get(null));
var resultingMap = map.get(null);
Assertions.assertEquals(shouldFail ? null : Object2ObjectSortedMaps.singleton(key, value), resultingMap);
map.close();
//if (shouldFail) this.checkLeaks = false;
gen.closeTempDb(db);
@ -126,47 +111,43 @@ public abstract class TestDictionaryMap {
@ParameterizedTest
@MethodSource("provideArgumentsPut")
public void testAtSetAtGet(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
var stpVer = StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempDictionary(db, updateMode)
.map(dict -> tempDatabaseMapDictionaryMap(dict, mapType, 5))
.flatMap(map -> Mono
.usingWhen(map.at(null, key), v -> v.set(value), LLUtils::finalizeResource)
.then(Mono.usingWhen(map.at(null, key), v -> v.get(null), LLUtils::finalizeResource))
.doFinally(s -> map.close())
)
));
public void testAtSetAtGet(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail)
throws IOException {
var result = tempDb(getTempDbGenerator(), db -> {
var map = tempDatabaseMapDictionaryMap(tempDictionary(db, updateMode), mapType, 5);
return run(shouldFail, () -> {
map.at(null, key).set(value);
return map.at(null, key).get(null);
});
});
if (shouldFail) {
this.checkLeaks = false;
stpVer.verifyError();
} else {
stpVer.expectNext(value).verifyComplete();
Assertions.assertEquals(value, result);
}
}
@ParameterizedTest
@MethodSource("provideArgumentsPut")
public void testPutAndGetPrevious(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
var stpVer = StepVerifier
.create(tempDb(getTempDbGenerator(), allocator, db -> tempDictionary(db, updateMode)
.map(dict -> tempDatabaseMapDictionaryMap(dict, mapType, 5))
.flatMapMany(map -> Flux
.concat(
map.putValueAndGetPrevious(key, "error?"),
map.putValueAndGetPrevious(key, value),
map.putValueAndGetPrevious(key, value)
)
.doFinally(s -> map.close())
)
));
public void testPutAndGetPrevious(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail)
throws IOException {
var result = tempDb(getTempDbGenerator(), db -> {
var map = tempDatabaseMapDictionaryMap(tempDictionary(db, updateMode), mapType, 5);
return run(shouldFail,
() -> Arrays.asList(map.putValueAndGetPrevious(key, "error?"),
map.putValueAndGetPrevious(key, value),
map.putValueAndGetPrevious(key, value)
)
);
});
if (shouldFail) {
this.checkLeaks = false;
stpVer.verifyError();
} else {
stpVer.expectNext("error?").expectNext(value).verifyComplete();
Assertions.assertArrayEquals(new String[] {null, "error?", value}, result.toArray(String[]::new));
}
}
/*
@ParameterizedTest
@MethodSource("provideArgumentsPut")
public void testPutValueRemoveAndGetPrevious(MapType mapType, UpdateMode updateMode, String key, String value, boolean shouldFail) {
@ -370,7 +351,7 @@ public abstract class TestDictionaryMap {
badKeys = List.of();
}
List<Tuple2<List<String>, Boolean>> keys = Stream
.concat(goodKeys.stream().map(s -> Tuples.of(s, false)), badKeys.stream().map(s -> Tuples.of(s, true)))
.concat(goodKeys.stream().map(s -> new Tuple2<>(s, false)), badKeys.stream().map(s -> new Tuple2<>(s, true)))
.toList();
var values = isCIMode() ? List.of("val") : List.of("", "\0", BIG_STRING);
@ -381,14 +362,14 @@ public abstract class TestDictionaryMap {
.collectMap(Tuple2::getT1, Tuple2::getT2, Object2ObjectLinkedOpenHashMap::new)
.block()
))
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple2<>(updateMode,
entryTuple.getT1(),
entryTuple.getT2()
)))
.flatMap(entryTuple -> Stream.of(Tuples.of(MapType.MAP, entryTuple.getT1(),
.flatMap(entryTuple -> Stream.of(new Tuple2<>(MapType.MAP, entryTuple.getT1(),
entryTuple.getT2(),
entryTuple.getT3()
), Tuples.of(MapType.HASH_MAP, entryTuple.getT1(),
), new Tuple2<>(MapType.HASH_MAP, entryTuple.getT1(),
entryTuple.getT2(),
false
)))
@ -761,4 +742,5 @@ public abstract class TestDictionaryMap {
Assertions.assertEquals(true, result.get(2));
}
*/
}

View File

@ -1,29 +1,22 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.DbTestUtils.BIG_STRING;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMap;
import static it.cavallium.dbengine.DbTestUtils.tempDb;
import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.assertj.core.api.Assertions.*;
import static it.cavallium.dbengine.tests.DbTestUtils.BIG_STRING;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.tests.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.tests.DbTestUtils.run;
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
import static it.cavallium.dbengine.tests.DbTestUtils.tempDatabaseMapDictionaryDeepMap;
import static it.cavallium.dbengine.tests.DbTestUtils.tempDictionary;
import io.netty5.buffer.internal.ResourceSupport;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import com.google.common.collect.Streams;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.utils.SimpleResource;
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
@ -36,20 +29,11 @@ import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import reactor.test.StepVerifier.Step;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuple3;
import reactor.util.function.Tuple4;
import reactor.util.function.Tuples;
@TestMethodOrder(MethodOrderer.MethodName.class)
public abstract class TestDictionaryMapDeep {
private final Logger log = LogManager.getLogger(this.getClass());
private TestAllocator allocator;
private boolean checkLeaks = true;
private static boolean isTestBadKeysEnabled() {
@ -58,6 +42,11 @@ public abstract class TestDictionaryMapDeep {
protected abstract TemporaryDbGenerator getTempDbGenerator();
record Tuple2<X, Y>(X getT1, Y getT2) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
private static Stream<Arguments> provideArgumentsSet() {
var goodKeys = Set.of("12345");
Set<String> badKeys;
@ -67,8 +56,8 @@ public abstract class TestDictionaryMapDeep {
badKeys = Set.of();
}
Set<Tuple2<String, Boolean>> keys = Stream.concat(
goodKeys.stream().map(s -> Tuples.of(s, false)),
badKeys.stream().map(s -> Tuples.of(s, true))
goodKeys.stream().map(s -> new Tuple2<>(s, false)),
badKeys.stream().map(s -> new Tuple2<>(s, true))
).collect(Collectors.toSet());
var values = Set.of(
new Object2ObjectLinkedOpenHashMap<>(Map.of("123456", "a", "234567", "")),
@ -84,9 +73,9 @@ public abstract class TestDictionaryMapDeep {
} else {
strm = values.stream();
}
return strm.map(val -> Tuples.of(keyTuple.getT1(), val, keyTuple.getT2()));
return strm.map(val -> new Tuple3<>(keyTuple.getT1(), val, keyTuple.getT2()));
})
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple4<>(updateMode,
entryTuple.getT1(),
entryTuple.getT2(),
entryTuple.getT3()
@ -112,43 +101,38 @@ public abstract class TestDictionaryMapDeep {
var values = isCIMode() ? List.of("val") : List.of("a", "", "\0", "\0\0", "z", "azzszgzczqz", BIG_STRING);
Flux<Tuple4<String, String, String, Boolean>> failOnKeys1 = Flux
.fromIterable(badKeys1)
.map(badKey1 -> Tuples.of(
Stream<Tuple4<String, String, String, Boolean>> failOnKeys1 = badKeys1.stream()
.map(badKey1 -> new Tuple4<>(
badKey1,
goodKeys2.stream().findFirst().orElseThrow(),
values.stream().findFirst().orElseThrow(),
true
));
Flux<Tuple4<String, String, String, Boolean>> failOnKeys2 = Flux
.fromIterable(badKeys2)
.map(badKey2 -> Tuples.of(
Stream<Tuple4<String, String, String, Boolean>> failOnKeys2 = badKeys2.stream()
.map(badKey2 -> new Tuple4<>(
goodKeys1.stream().findFirst().orElseThrow(),
badKey2,
values.stream().findFirst().orElseThrow(),
true
));
Flux<Tuple4<String, String, String, Boolean>> goodKeys1And2 = Flux
.fromIterable(values)
.map(value -> Tuples.of(
Stream<Tuple4<String, String, String, Boolean>> goodKeys1And2 = values.stream()
.map(value -> new Tuple4<>(
goodKeys1.stream().findFirst().orElseThrow(),
goodKeys2.stream().findFirst().orElseThrow(),
value,
false
));
Flux<Tuple4<String, String, String, Boolean>> keys1And2 = Flux
.concat(
Stream<Tuple4<String, String, String, Boolean>> keys1And2 = Streams.concat(
goodKeys1And2,
failOnKeys1,
failOnKeys2
);
return keys1And2
.concatMap(entryTuple -> Flux
.fromArray(UpdateMode.values())
.map(updateMode -> Tuples.of(updateMode,
.flatMap(entryTuple -> Stream.of(UpdateMode.values())
.map(updateMode -> new Tuple5<>(updateMode,
entryTuple.getT1(),
entryTuple.getT2(),
entryTuple.getT3(),
@ -161,22 +145,19 @@ public abstract class TestDictionaryMapDeep {
fullTuple.getT4(),
fullTuple.getT5()
))
.toStream()
.sequential();
}
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
ensureNoLeaks(false, false);
}
@AfterEach
public void afterEach() {
if (!isCIMode() && checkLeaks) {
ensureNoLeaks(allocator.allocator(), true, false);
ensureNoLeaks(true, false);
}
destroyAllocator(allocator);
}
@ParameterizedTest
@ -184,25 +165,21 @@ public abstract class TestDictionaryMapDeep {
public void testPutValue(UpdateMode updateMode,
String key,
Object2ObjectSortedMap<String, String> value,
boolean shouldFail) {
boolean shouldFail) throws IOException {
var gen = getTempDbGenerator();
var db = run(gen.openTempDb(allocator));
var dict = run(tempDictionary(db.db(), updateMode));
var db = gen.openTempDb();
var dict = tempDictionary(db.db(), updateMode);
var map = tempDatabaseMapDictionaryDeepMap(dict, 5, 6);
log.debug("Put \"{}\" = \"{}\"", key, value);
runVoid(shouldFail, map.putValue(key, value));
runVoid(shouldFail, () -> map.putValue(key, value));
var resultingMapSize = run(map.leavesCount(null, false));
var resultingMapSize = map.leavesCount(null, false);
Assertions.assertEquals(shouldFail ? 0 : value.size(), resultingMapSize);
var resultingMap = run(map.get(null));
var resultingMap = map.get(null);
Assertions.assertEquals(shouldFail ? null : Map.of(key, value), resultingMap);
map.close();
//if (shouldFail) this.checkLeaks = false;
gen.closeTempDb(db);
}
@ -211,27 +188,23 @@ public abstract class TestDictionaryMapDeep {
public void testGetValue(UpdateMode updateMode,
String key,
Object2ObjectSortedMap<String, String> value,
boolean shouldFail) {
boolean shouldFail) throws IOException {
var gen = getTempDbGenerator();
var db = run(gen.openTempDb(allocator));
var dict = run(tempDictionary(db.db(), updateMode));
var db = gen.openTempDb();
var dict = tempDictionary(db.db(), updateMode);
var map = tempDatabaseMapDictionaryDeepMap(dict, 5, 6);
log.debug("Put \"{}\" = \"{}\"", key, value);
runVoid(shouldFail, map.putValue(key, value));
runVoid(shouldFail, () -> map.putValue(key, value));
log.debug("Get \"{}\"", key);
var returnedValue = run(shouldFail, map.getValue(null, key));
var returnedValue = run(shouldFail, () -> map.getValue(null, key));
Assertions.assertEquals(shouldFail ? null : value, returnedValue);
map.close();
//if (shouldFail) this.checkLeaks = false;
gen.closeTempDb(db);
}
/*
@ParameterizedTest
@MethodSource("provideArgumentsSet")
public void testSetValueGetAllValues(UpdateMode updateMode, String key, Object2ObjectSortedMap<String, String> value,
@ -284,7 +257,7 @@ public abstract class TestDictionaryMapDeep {
.getAllStages(null, false)
.flatMap(v -> v.getValue()
.getAllValues(null, false)
.map(result -> Tuples.of(v.getKey(), result.getKey(), result.getValue()))
.map(result -> new Tuple2<>(v.getKey(), result.getKey(), result.getValue()))
.doFinally(s -> v.getValue().close())
)
),
@ -295,9 +268,9 @@ public abstract class TestDictionaryMapDeep {
this.checkLeaks = false;
stpVer.verifyError();
} else {
value.forEach((k, v) -> remainingEntries.add(Tuples.of(key, k, v)));
remainingEntries.add(Tuples.of("capra", "normal", "123"));
remainingEntries.add(Tuples.of("capra", "ormaln", "456"));
value.forEach((k, v) -> remainingEntries.add(new Tuple2<>(key, k, v)));
remainingEntries.add(new Tuple2<>("capra", "normal", "123"));
remainingEntries.add(new Tuple2<>("capra", "ormaln", "456"));
for (Tuple3<String, String, String> ignored : remainingEntries) {
stpVer = stpVer.expectNextMatches(remainingEntries::remove);
}
@ -771,7 +744,7 @@ public abstract class TestDictionaryMapDeep {
badKeys = List.of();
}
List<Tuple2<List<String>, Boolean>> keys = Stream
.concat(goodKeys.stream().map(s -> Tuples.of(s, false)), badKeys.stream().map(s -> Tuples.of(s, true)))
.concat(goodKeys.stream().map(s -> new Tuple2<>(s, false)), badKeys.stream().map(s -> new Tuple2<>(s, true)))
.toList();
var values = isCIMode() ? List.of(new Object2ObjectLinkedOpenHashMap<>(Map.of("123456", "val"))) : List.of(
new Object2ObjectLinkedOpenHashMap<>(Map.of("123456", "a", "234567", "")),
@ -785,7 +758,7 @@ public abstract class TestDictionaryMapDeep {
.collectMap(Tuple2::getT1, Tuple2::getT2, Object2ObjectLinkedOpenHashMap::new)
.block()
))
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> Tuples.of(updateMode,
.flatMap(entryTuple -> Arrays.stream(UpdateMode.values()).map(updateMode -> new Tuple2<>(updateMode,
entryTuple.getT1(),
entryTuple.getT2()
)))
@ -1142,4 +1115,6 @@ public abstract class TestDictionaryMapDeep {
stpVer.expectNext(true, entries.isEmpty(), true).verifyComplete();
}
}
*/
}

View File

@ -1,32 +1,19 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.DbTestUtils.BIG_STRING;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMapHashMap;
import static it.cavallium.dbengine.DbTestUtils.tempDb;
import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
import static it.cavallium.dbengine.tests.DbTestUtils.BIG_STRING;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.tests.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.tests.DbTestUtils.tempDictionary;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import com.google.common.collect.Streams;
import it.cavallium.dbengine.database.UpdateMode;
import java.util.List;
import java.util.Map.Entry;
import java.util.stream.Stream;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.core.publisher.Flux;
import reactor.test.StepVerifier;
import reactor.util.function.Tuple4;
import reactor.util.function.Tuples;
public abstract class TestDictionaryMapDeepHashMap {
private TestAllocator allocator;
private boolean checkLeaks = true;
private static boolean isTestBadKeysEnabled() {
@ -35,6 +22,11 @@ public abstract class TestDictionaryMapDeepHashMap {
protected abstract TemporaryDbGenerator getTempDbGenerator();
record Tuple2<X, Y>(X getT1, Y getT2) {}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {}
private static Stream<Arguments> provideArgumentsPut() {
var goodKeys1 = isCIMode() ? List.of("12345") : List.of("12345", "zebra");
List<String> badKeys1;
@ -47,34 +39,31 @@ public abstract class TestDictionaryMapDeepHashMap {
var values = isCIMode() ? List.of("val") : List.of("a", "", "\0", "\0\0", "z", "azzszgzczqz", BIG_STRING);
Flux<Tuple4<String, String, String, Boolean>> failOnKeys1 = Flux
.fromIterable(badKeys1)
.map(badKey1 -> Tuples.of(
Stream<Tuple4<String, String, String, Boolean>> failOnKeys1 = badKeys1.stream()
.map(badKey1 -> new Tuple4<>(
badKey1,
goodKeys2.stream().findAny().orElseThrow(),
values.stream().findAny().orElseThrow(),
true
));
Flux<Tuple4<String, String, String, Boolean>> goodKeys1And2 = Flux
.fromIterable(values)
.map(value -> Tuples.of(
Stream<Tuple4<String, String, String, Boolean>> goodKeys1And2 = values.stream()
.map(value -> new Tuple4<>(
goodKeys1.stream().findAny().orElseThrow(),
goodKeys2.stream().findAny().orElseThrow(),
value,
false
));
Flux<Tuple4<String, String, String, Boolean>> keys1And2 = Flux
Stream<Tuple4<String, String, String, Boolean>> keys1And2 = Streams
.concat(
goodKeys1And2,
failOnKeys1
);
return keys1And2
.flatMap(entryTuple -> Flux
.fromArray(UpdateMode.values())
.map(updateMode -> Tuples.of(updateMode,
.flatMap(entryTuple -> Stream.of(UpdateMode.values())
.map(updateMode -> new Tuple5<>(updateMode,
entryTuple.getT1(),
entryTuple.getT2(),
entryTuple.getT3(),
@ -86,24 +75,22 @@ public abstract class TestDictionaryMapDeepHashMap {
fullTuple.getT3(),
fullTuple.getT4(),
fullTuple.getT1() != UpdateMode.ALLOW || fullTuple.getT5()
))
.toStream();
));
}
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
ensureNoLeaks(false, false);
}
@AfterEach
public void afterEach() {
if (!isCIMode() && checkLeaks) {
ensureNoLeaks(allocator.allocator(), true, false);
ensureNoLeaks(true, false);
}
destroyAllocator(allocator);
}
/*
@ParameterizedTest
@MethodSource("provideArgumentsPut")
public void testAtPutValueGetAllValues(UpdateMode updateMode, String key1, String key2, String value, boolean shouldFail) {
@ -129,4 +116,6 @@ public abstract class TestDictionaryMapDeepHashMap {
}
}
*/
}

View File

@ -1,8 +1,7 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static java.util.Map.entry;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.database.disk.KeyMayExistGetter;
import it.unimi.dsi.fastutil.bytes.ByteList;
import java.nio.ByteBuffer;
@ -41,7 +40,7 @@ public class TestGetter {
return new String(bytes.toByteArray(), StandardCharsets.UTF_8);
}
public KeyMayExistGetter getter = new KeyMayExistGetter(BufferAllocator.offHeapUnpooled(), true) {
public KeyMayExistGetter getter = new KeyMayExistGetter() {
@Override
protected KeyMayExist keyMayExist(ReadOptions readOptions, ByteBuffer key, ByteBuffer value) {
return null;

View File

@ -0,0 +1,247 @@
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
import static org.junit.jupiter.api.Assertions.assertEquals;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLDictionaryResultType;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLRange;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Objects;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
public abstract class TestLLDictionary {
private final Logger log = LogManager.getLogger(this.getClass());
private static final LLRange RANGE_ALL = LLRange.all();
private TempDb tempDb;
private LLKeyValueDatabase db;
protected abstract TemporaryDbGenerator getTempDbGenerator();
@BeforeEach
public void beforeEach() throws IOException {
ensureNoLeaks(false, false);
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(), "TempDB");
db = tempDb.db();
}
@AfterEach
public void afterEach() throws IOException {
getTempDbGenerator().closeTempDb(tempDb);
ensureNoLeaks(true, false);
}
public static Stream<Arguments> provideArguments() {
return Arrays.stream(UpdateMode.values()).map(Arguments::of);
}
public static Stream<Arguments> providePutArguments() {
var updateModes = Arrays.stream(UpdateMode.values());
return updateModes.flatMap(updateMode -> {
var resultTypes = Arrays.stream(LLDictionaryResultType.values());
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
});
}
public static Stream<Arguments> provideUpdateArguments() {
var updateModes = Arrays.stream(UpdateMode.values());
return updateModes.flatMap(updateMode -> {
var resultTypes = Arrays.stream(UpdateReturnMode.values());
return resultTypes.map(resultType -> Arguments.of(updateMode, resultType));
});
}
private LLDictionary getDict(UpdateMode updateMode) {
var dict = DbTestUtils.tempDictionary(db, updateMode);
var key1 = fromString("test-key-1");
var key2 = fromString("test-key-2");
var key3 = fromString("test-key-3");
var key4 = fromString("test-key-4");
var value = fromString("test-value");
dict.put(key1, value, LLDictionaryResultType.VOID);
dict.put(key2, value, LLDictionaryResultType.VOID);
dict.put(key3, value, LLDictionaryResultType.VOID);
dict.put(key4, value, LLDictionaryResultType.VOID);
return dict;
}
private Buf fromString(String s) {
var sb = s.getBytes(StandardCharsets.UTF_8);
Buf b = Buf.create(sb.length + 3 + 13);
b.addElements(0, sb);
assert b.size() == sb.length;
return b;
}
private String toString(Buf bb) {
return bb != null ? bb.toString(StandardCharsets.UTF_8) : null;
}
@Test
public void testNoOp() {
}
@Test
public void testNoOpAllocation() {
for (int i = 0; i < 10; i++) {
var a = Buf.create(i * 512);
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetDict(UpdateMode updateMode) {
var dict = getDict(updateMode);
Assertions.assertNotNull(dict);
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetColumnName(UpdateMode updateMode) {
var dict = getDict(updateMode);
Assertions.assertEquals("hash_map_testmap", dict.getColumnName());
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGet(UpdateMode updateMode) {
var dict = getDict(updateMode);
var keyEx = fromString("test-key-1");
var keyNonEx = fromString("test-nonexistent");
Assertions.assertEquals("test-value", toString(dict.get(null, keyEx)));
Assertions.assertEquals("test-value", toString(dict.get(null, keyEx)));
Assertions.assertEquals("test-value", toString(dict.get(null, keyEx)));
Assertions.assertEquals((String) null, toString(dict.get(null, keyNonEx)));
Assertions.assertEquals((String) null, toString(dict.get(null, keyNonEx)));
Assertions.assertEquals((String) null, toString(dict.get(null, keyNonEx)));
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testPutExisting(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode);
var keyEx = fromString("test-key-1");
var value = fromString("test-value");
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
dict.put(keyEx, value, resultType);
var afterSize = dict.sizeRange(null, RANGE_ALL, false);
Assertions.assertEquals(0, afterSize - beforeSize);
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testPutNew(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode);
var keyNonEx = fromString("test-nonexistent");
var value = fromString("test-value");
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
dict.put(keyNonEx, value, resultType);
var afterSize = dict.sizeRange(null, LLRange.all(), false);
Assertions.assertEquals(1, afterSize - beforeSize);
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, false, false).map(this::toString).toList().contains("test-nonexistent"));
Assertions.assertTrue(dict.getRangeKeys(null, RANGE_ALL, true, false).map(this::toString).toList().contains("test-nonexistent"));
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetUpdateMode(UpdateMode updateMode) {
var dict = getDict(updateMode);
assertEquals(updateMode, dict.getUpdateMode());
}
@ParameterizedTest
@MethodSource("provideUpdateArguments")
public void testUpdateExisting(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
var dict = getDict(updateMode);
var keyEx = fromString("test-key-1");
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
long afterSize;
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyEx, old -> fromString("test-value"), updateReturnMode));
afterSize = dict.sizeRange(null, RANGE_ALL, false);
assertEquals(0, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyEx, old -> fromString("test-value"), updateReturnMode));
afterSize = dict.sizeRange(null, RANGE_ALL, false);
assertEquals(0, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyEx, old -> fromString("test-value"), updateReturnMode));
afterSize = dict.sizeRange(null, RANGE_ALL, false);
assertEquals(0, afterSize - beforeSize);
}
@ParameterizedTest
@MethodSource("provideUpdateArguments")
public void testUpdateNew(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
int expected = updateMode == UpdateMode.DISALLOW ? 0 : 1;
var dict = getDict(updateMode);
var keyNonEx = fromString("test-nonexistent");
var beforeSize = dict.sizeRange(null, RANGE_ALL, false);
long afterSize;
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode));
afterSize = dict.sizeRange(null, RANGE_ALL, false);
assertEquals(expected, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode));
afterSize = dict.sizeRange(null, RANGE_ALL, false);
assertEquals(expected, afterSize - beforeSize);
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(keyNonEx, old -> fromString("test-value"), updateReturnMode));
afterSize = dict.sizeRange(null, RANGE_ALL, false);
assertEquals(expected, afterSize - beforeSize);
if (updateMode != UpdateMode.DISALLOW) {
Assertions.assertTrue(dict
.getRangeKeys(null, RANGE_ALL, false, false)
.map(this::toString)
.toList()
.contains("test-nonexistent"));
Assertions.assertTrue(dict
.getRangeKeys(null, RANGE_ALL, true, false)
.map(this::toString)
.toList()
.contains("test-nonexistent"));
}
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testUpdateAndGetDelta(UpdateMode updateMode) {
log.warn("Test not implemented");
//todo: implement
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testClear(UpdateMode updateMode) {
log.warn("Test not implemented");
//todo: implement
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
log.warn("Test not implemented");
//todo: implement
}
}

View File

@ -1,21 +1,18 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.tests.DbTestUtils.runVoid;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import io.netty5.buffer.Buffer;
import io.netty5.util.Send;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLDictionary;
import it.cavallium.dbengine.database.LLDictionaryResultType;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.UpdateMode;
import it.cavallium.dbengine.database.UpdateReturnMode;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Objects;
@ -27,29 +24,25 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.core.publisher.Mono;
public abstract class TestLLDictionaryLeaks {
private TestAllocator allocator;
private TempDb tempDb;
private LLKeyValueDatabase db;
protected abstract TemporaryDbGenerator getTempDbGenerator();
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(allocator).block(), "TempDB");
public void beforeEach() throws IOException {
ensureNoLeaks(false, false);
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(), "TempDB");
db = tempDb.db();
}
@AfterEach
public void afterEach() {
getTempDbGenerator().closeTempDb(tempDb).block();
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
public void afterEach() throws IOException {
getTempDbGenerator().closeTempDb(tempDb);
ensureNoLeaks(true, false);
}
public static Stream<Arguments> provideArguments() {
@ -73,30 +66,25 @@ public abstract class TestLLDictionaryLeaks {
}
private LLDictionary getDict(UpdateMode updateMode) {
var dict = DbTestUtils.tempDictionary(db, updateMode).blockOptional().orElseThrow();
var key1 = Mono.fromCallable(() -> fromString("test-key-1"));
var key2 = Mono.fromCallable(() -> fromString("test-key-2"));
var key3 = Mono.fromCallable(() -> fromString("test-key-3"));
var key4 = Mono.fromCallable(() -> fromString("test-key-4"));
var value = Mono.fromCallable(() -> fromString("test-value"));
dict.put(key1, value, LLDictionaryResultType.VOID).block();
dict.put(key2, value, LLDictionaryResultType.VOID).block();
dict.put(key3, value, LLDictionaryResultType.VOID).block();
dict.put(key4, value, LLDictionaryResultType.VOID).block();
var dict = DbTestUtils.tempDictionary(db, updateMode);
var key1 = fromString("test-key-1");
var key2 = fromString("test-key-2");
var key3 = fromString("test-key-3");
var key4 = fromString("test-key-4");
var value = fromString("test-value");
dict.put(key1, value, LLDictionaryResultType.VOID);
dict.put(key2, value, LLDictionaryResultType.VOID);
dict.put(key3, value, LLDictionaryResultType.VOID);
dict.put(key4, value, LLDictionaryResultType.VOID);
return dict;
}
private Buffer fromString(String s) {
private Buf fromString(String s) {
var sb = s.getBytes(StandardCharsets.UTF_8);
var b = db.getAllocator().allocate(sb.length);
try {
b.writeBytes(sb);
assert b.readableBytes() == sb.length;
return b;
} catch (Throwable ex) {
b.close();
throw ex;
}
var b = Buf.create(sb.length);
b.addElements(0, sb);
assert b.size() == sb.length;
return b;
}
@Test
@ -106,8 +94,7 @@ public abstract class TestLLDictionaryLeaks {
@Test
public void testNoOpAllocation() {
for (int i = 0; i < 10; i++) {
var a = allocator.allocator().allocate(i * 512);
a.send().receive().close();
var a = Buf.create(i * 512);
}
}
@ -124,30 +111,23 @@ public abstract class TestLLDictionaryLeaks {
dict.getColumnName();
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetAllocator(UpdateMode updateMode) {
var dict = getDict(updateMode);
dict.getAllocator();
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGet(UpdateMode updateMode) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test"));
runVoid(dict.get(null, key).then());
runVoid(dict.get(null, key).then());
runVoid(dict.get(null, key).then());
var key = fromString("test");
dict.get(null, key);
dict.get(null, key);
dict.get(null, key);
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testPut(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
var value = Mono.fromCallable(() -> fromString("test-value"));
runVoid(dict.put(key, value, resultType).then().doOnDiscard(Buffer.class, Buffer::close));
var key = fromString("test-key");
var value = fromString("test-value");
dict.put(key, value, resultType);
}
@ParameterizedTest
@ -161,19 +141,13 @@ public abstract class TestLLDictionaryLeaks {
@MethodSource("provideUpdateArguments")
public void testUpdate(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, this::pass, updateReturnMode).then()
);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, this::pass, updateReturnMode).then()
);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, this::pass, updateReturnMode).then()
);
var key = fromString("test-key");
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(key, this::pass, updateReturnMode));
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(key, this::pass, updateReturnMode));
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.update(key, this::pass, updateReturnMode));
}
private Buffer pass(@Nullable Buffer old) {
private Buf pass(@Nullable Buf old) {
if (old == null) return null;
return old.copy();
}
@ -182,30 +156,24 @@ public abstract class TestLLDictionaryLeaks {
@MethodSource("provideArguments")
public void testUpdateAndGetDelta(UpdateMode updateMode) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, this::pass).then()
);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, this::pass).then()
);
runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, this::pass).then()
);
var key = fromString("test-key");
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.updateAndGetDelta(key, this::pass));
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.updateAndGetDelta(key, this::pass));
runVoid(updateMode == UpdateMode.DISALLOW, () -> dict.updateAndGetDelta(key, this::pass));
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testClear(UpdateMode updateMode) {
var dict = getDict(updateMode);
runVoid(dict.clear());
dict.clear();
}
@ParameterizedTest
@MethodSource("providePutArguments")
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
runVoid(dict.remove(key, resultType).then().doOnDiscard(Buffer.class, Buffer::close));
var key = fromString("test-key");
dict.remove(key, resultType);
}
}

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestLocalDictionary extends TestDictionary {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestLocalDictionaryMap extends TestDictionaryMap {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestLocalDictionaryMapDeep extends TestDictionaryMapDeep {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestLocalDictionaryMapDeepHashMap extends TestDictionaryMapDeepHashMap {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestLocalLLDictionary extends TestLLDictionary {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestLocalLLDictionaryLeaks extends TestLLDictionaryLeaks {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestLocalSingletons extends TestSingletons {

View File

@ -1,29 +1,26 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.Sort;
import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.CountMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -37,15 +34,10 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.core.publisher.Flux;
import reactor.util.function.Tuples;
public class TestLuceneIndex {
private final Logger log = LogManager.getLogger(this.getClass());
private static LLTempHugePqEnv ENV;
private TestAllocator allocator;
private TempDb tempDb;
private LLLuceneIndex luceneSingle;
private LLLuceneIndex luceneMulti;
@ -56,14 +48,12 @@ public class TestLuceneIndex {
@BeforeAll
public static void beforeAll() throws IOException {
ENV = new LLTempHugePqEnv();
}
@BeforeEach
public void beforeEach() {
this.allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(allocator).block(), "TempDB");
public void beforeEach() throws IOException {
ensureNoLeaks(false, false);
tempDb = Objects.requireNonNull(getTempDbGenerator().openTempDb(), "TempDB");
luceneSingle = tempDb.luceneSingle();
luceneMulti = tempDb.luceneMulti();
}
@ -72,13 +62,13 @@ public class TestLuceneIndex {
return Stream.of(false, true).map(Arguments::of);
}
private static final Flux<Boolean> multi = Flux.just(false, true);
private static final Flux<LLScoreMode> scoreModes = Flux.just(LLScoreMode.NO_SCORES,
private static final List<Boolean> multi = List.of(false, true);
private static final List<LLScoreMode> scoreModes = List.of(LLScoreMode.NO_SCORES,
LLScoreMode.TOP_SCORES,
LLScoreMode.COMPLETE_NO_SCORES,
LLScoreMode.COMPLETE
);
private static final Flux<Sort> multiSort = Flux.just(Sort.score(),
private static final List<Sort> multiSort = List.of(Sort.score(),
Sort.random(),
Sort.no(),
Sort.doc(),
@ -86,62 +76,78 @@ public class TestLuceneIndex {
Sort.numeric("longsort", true)
);
record Tuple2<X, Y>(X getT1, Y getT2) {
public Object[] toArray() {
return new Object[] {getT1, getT2};
}
}
record Tuple3<X, Y, Z>(X getT1, Y getT2, Y getT3) {
public Object[] toArray() {
return new Object[] {getT1, getT2, getT3};
}
}
record Tuple4<X, Y, Z, W>(X getT1, Y getT2, Y getT3, W getT4) {
public Object[] toArray() {
return new Object[] {getT1, getT2, getT3, getT4};
}
}
record Tuple5<X, Y, Z, W, X1>(X getT1, Y getT2, Y getT3, W getT4, X1 getT5) {
public Object[] toArray() {
return new Object[] {getT1, getT2, getT3, getT4, getT5};
}
}
public static Stream<Arguments> provideQueryArgumentsScoreMode() {
return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
return multi.stream()
.flatMap(shard -> scoreModes.stream().map(scoreMode -> new Tuple2<>(shard, scoreMode)))
.map(tuple -> Arguments.of(tuple.toArray()));
}
public static Stream<Arguments> provideQueryArgumentsSort() {
return multi
.concatMap(shard -> multiSort.map(multiSort -> Tuples.of(shard, multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
return multi.stream()
.flatMap(shard -> multiSort.stream().map(multiSort -> new Tuple2<>(shard, multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()));
}
public static Stream<Arguments> provideQueryArgumentsScoreModeAndSort() {
return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
.concatMap(tuple -> multiSort.map(multiSort -> Tuples.of(tuple.getT1(), tuple.getT2(), multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
return multi.stream()
.flatMap(shard -> scoreModes.stream().map(scoreMode -> new Tuple2<>(shard, scoreMode)))
.flatMap(tuple -> multiSort.stream().map(multiSort -> new Tuple3<>(tuple.getT1(), tuple.getT2(), multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()));
}
@AfterEach
public void afterEach() {
getTempDbGenerator().closeTempDb(tempDb).block();
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
public void afterEach() throws IOException {
getTempDbGenerator().closeTempDb(tempDb);
ensureNoLeaks(true, false);
}
@AfterAll
public static void afterAll() throws IOException {
ENV.close();
}
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
index.updateDocument("test-key-1", "0123456789").block();
index.updateDocument("test-key-2", "test 0123456789 test word").block();
index.updateDocument("test-key-3", "0123456789 test example string").block();
index.updateDocument("test-key-4", "hello world the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-5", "hello the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-6", "hello the quick brown fox jumps over the world dog").block();
index.updateDocument("test-key-7", "the quick brown fox jumps over the world dog").block();
index.updateDocument("test-key-8", "the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-9", "Example1").block();
index.updateDocument("test-key-10", "Example2").block();
index.updateDocument("test-key-11", "Example3").block();
index.updateDocument("test-key-12", "-234").block();
index.updateDocument("test-key-13", "2111").block();
index.updateDocument("test-key-14", "2999").block();
index.updateDocument("test-key-15", "3902").block();
Flux
.range(1, 1000)
.concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i))
.transform(LLUtils::handleDiscard)
.blockLast();
LuceneIndex<String, String> index = DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti);
index.updateDocument("test-key-1", "0123456789");
index.updateDocument("test-key-2", "test 0123456789 test word");
index.updateDocument("test-key-3", "0123456789 test example string");
index.updateDocument("test-key-4", "hello world the quick brown fox jumps over the lazy dog");
index.updateDocument("test-key-5", "hello the quick brown fox jumps over the lazy dog");
index.updateDocument("test-key-6", "hello the quick brown fox jumps over the world dog");
index.updateDocument("test-key-7", "the quick brown fox jumps over the world dog");
index.updateDocument("test-key-8", "the quick brown fox jumps over the lazy dog");
index.updateDocument("test-key-9", "Example1");
index.updateDocument("test-key-10", "Example2");
index.updateDocument("test-key-11", "Example3");
index.updateDocument("test-key-12", "-234");
index.updateDocument("test-key-13", "2111");
index.updateDocument("test-key-14", "2999");
index.updateDocument("test-key-15", "3902");
IntStream.rangeClosed(1, 1000).forEach(i -> index.updateDocument("test-key-" + (15 + i), "" + i));
tempDb.swappableLuceneSearcher().setSingle(new CountMultiSearcher());
tempDb.swappableLuceneSearcher().setMulti(new CountMultiSearcher());
assertCount(index, 1000 + 15);
@ -155,8 +161,8 @@ public class TestLuceneIndex {
}
}
} else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
}
return index;
}
@ -166,8 +172,8 @@ public class TestLuceneIndex {
}
private long getCount(LuceneIndex<String, String> luceneIndex) {
luceneIndex.refresh(true).block();
var totalHitsCount = run(luceneIndex.count(null, new MatchAllDocsQuery()));
luceneIndex.refresh(true);
var totalHitsCount = luceneIndex.count(null, new MatchAllDocsQuery());
Assertions.assertTrue(totalHitsCount.exact(), "Can't get count because the total hits count is not exact");
return totalHitsCount.value();
}
@ -179,8 +185,7 @@ public class TestLuceneIndex {
@Test
public void testNoOpAllocation() {
for (int i = 0; i < 10; i++) {
var a = allocator.allocator().allocate(i * 512);
a.send().receive().close();
var a = Buf.create(i * 512);
}
}
@ -195,7 +200,7 @@ public class TestLuceneIndex {
@MethodSource("provideArguments")
public void testDeleteAll(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
runVoid(luceneIndex.deleteAll());
luceneIndex.deleteAll();
assertCount(luceneIndex, 0);
}
@ -204,7 +209,7 @@ public class TestLuceneIndex {
public void testDelete(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
var prevCount = getCount(luceneIndex);
runVoid(luceneIndex.deleteDocument("test-key-1"));
luceneIndex.deleteDocument("test-key-1");
assertCount(luceneIndex, prevCount - 1);
}
@ -213,7 +218,7 @@ public class TestLuceneIndex {
public void testUpdateSameDoc(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
var prevCount = getCount(luceneIndex);
runVoid(luceneIndex.updateDocument("test-key-1", "new-value"));
luceneIndex.updateDocument("test-key-1", "new-value");
assertCount(luceneIndex, prevCount );
}
@ -222,7 +227,7 @@ public class TestLuceneIndex {
public void testUpdateNewDoc(boolean shards) {
var luceneIndex = getLuceneIndex(shards, null);
var prevCount = getCount(luceneIndex);
runVoid(luceneIndex.updateDocument("test-key-new", "new-value"));
luceneIndex.updateDocument("test-key-new", "new-value");
assertCount(luceneIndex, prevCount + 1);
}

View File

@ -1,21 +1,17 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static it.cavallium.dbengine.tests.DbTestUtils.MAX_IN_MEMORY_RESULT_ENTRIES;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import io.netty.buffer.PooledByteBufAllocator;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.tests.DbTestUtils.TempDb;
import it.cavallium.dbengine.tests.TestLuceneIndex.Tuple2;
import it.cavallium.dbengine.client.HitKey;
import it.cavallium.dbengine.client.Hits;
import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.Sort;
import it.cavallium.dbengine.client.LazyHitKey;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder;
import it.cavallium.dbengine.client.query.current.data.BooleanQuery;
@ -29,7 +25,6 @@ import it.cavallium.dbengine.client.query.current.data.Term;
import it.cavallium.dbengine.client.query.current.data.TermQuery;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.disk.LLTempHugePqEnv;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.CountMultiSearcher;
@ -38,18 +33,17 @@ import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.StandardSearcher;
import it.cavallium.dbengine.lucene.searcher.ScoredPagedMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.PagedLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.SortedScoredFullMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.SortedByScoreFullMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedStreamingMultiSearcher;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.commons.lang3.function.FailableConsumer;
import org.apache.logging.log4j.LogManager;
@ -63,18 +57,12 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink.OverflowStrategy;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuples;
public class TestLuceneSearches {
private static final Logger log = LogManager.getLogger(TestLuceneSearches.class);
private static LLTempHugePqEnv ENV;
private static final MemoryTemporaryDbGenerator TEMP_DB_GENERATOR = new MemoryTemporaryDbGenerator();
private static TestAllocator allocator;
private static TempDb tempDb;
private static LLLuceneIndex luceneSingle;
private static LLLuceneIndex luceneMulti;
@ -102,31 +90,25 @@ public class TestLuceneSearches {
modifiableElements.put("test-key-13", "2111");
modifiableElements.put("test-key-14", "2999");
modifiableElements.put("test-key-15", "3902");
runVoid(Flux.range(1, 1000).doOnNext(i -> modifiableElements.put("test-key-" + (15 + i), "" + i)).then());
IntStream.rangeClosed(1, 1000).forEach(i -> modifiableElements.put("test-key-" + (15 + i), "" + i));
ELEMENTS = Collections.unmodifiableMap(modifiableElements);
}
@BeforeAll
public static void beforeAll() throws IOException {
allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
tempDb = Objects.requireNonNull(TEMP_DB_GENERATOR.openTempDb(allocator).block(), "TempDB");
ensureNoLeaks(false, false);
tempDb = Objects.requireNonNull(TEMP_DB_GENERATOR.openTempDb(), "TempDB");
luceneSingle = tempDb.luceneSingle();
luceneMulti = tempDb.luceneMulti();
ENV = new LLTempHugePqEnv();
setUpIndex(true);
setUpIndex(false);
}
private static void setUpIndex(boolean shards) {
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
LuceneIndex<String, String> index = DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti);
Flux
.fromIterable(ELEMENTS.entrySet())
.concatMap(entry -> index.updateDocument(entry.getKey(), entry.getValue()))
.subscribeOn(Schedulers.boundedElastic())
.blockLast();
ELEMENTS.forEach(index::updateDocument);
tempDb.swappableLuceneSearcher().setSingle(new CountMultiSearcher());
tempDb.swappableLuceneSearcher().setMulti(new CountMultiSearcher());
assertCount(index, 1000 + 15);
@ -141,8 +123,8 @@ public class TestLuceneSearches {
return Stream.of(false, true).map(Arguments::of);
}
private static final Flux<Boolean> multi = Flux.just(false, true);
private static final Flux<Sort> multiSort = Flux.just(
private static final List<Boolean> multi = List.of(false, true);
private static final List<Sort> multiSort = List.of(
Sort.score(),
//todo: fix random sort field
//Sort.randomSortField(),
@ -154,50 +136,43 @@ public class TestLuceneSearches {
Sort.numeric("intsort", true)
);
private static Flux<LocalSearcher> getSearchers(ExpectedQueryType info) {
return Flux.push(sink -> {
if (info.shard()) {
if (info.onlyCount()) {
sink.next(new CountMultiSearcher());
} else {
sink.next(new ScoredPagedMultiSearcher());
if (info.sorted() && !info.sortedByScore()) {
sink.next(new SortedScoredFullMultiSearcher(ENV));
} else {
sink.next(new SortedByScoreFullMultiSearcher(ENV));
}
if (!info.sorted()) {
sink.next(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher()));
sink.next(new UnsortedStreamingMultiSearcher());
}
}
sink.next(new AdaptiveMultiSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
private static List<LocalSearcher> getSearchers(ExpectedQueryType info) {
var sink = new ArrayList<LocalSearcher>();
if (info.shard()) {
if (info.onlyCount()) {
sink.add(new CountMultiSearcher());
} else {
if (info.onlyCount()) {
sink.next(new CountMultiSearcher());
} else {
sink.next(new PagedLocalSearcher());
sink.add(new ScoredPagedMultiSearcher());
if (!info.sorted()) {
sink.add(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher()));
sink.add(new UnsortedStreamingMultiSearcher());
}
sink.next(new AdaptiveLocalSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
}
sink.complete();
}, OverflowStrategy.BUFFER);
sink.add(new AdaptiveMultiSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
} else {
if (info.onlyCount()) {
sink.add(new CountMultiSearcher());
} else {
sink.add(new PagedLocalSearcher());
}
sink.add(new AdaptiveLocalSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
}
return sink;
}
public static Stream<Arguments> provideQueryArgumentsScoreMode() {
return multi.map(tuple -> Arguments.of(multi)).toStream();
return multi.stream().map(tuple -> Arguments.of(multi));
}
public static Stream<Arguments> provideQueryArgumentsScoreModeAndSort() {
return multi
.concatMap(multi -> multiSort.map(multiSort -> Tuples.of(multi, multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
return multi.stream()
.flatMap(multi -> multiSort.stream().map(multiSort -> new Tuple2<>(multi, multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()));
}
private static void runSearchers(ExpectedQueryType expectedQueryType, FailableConsumer<LocalSearcher, Throwable> consumer)
throws Throwable {
var searchers = run(getSearchers(expectedQueryType).collectList());
var searchers = getSearchers(expectedQueryType);
for (LocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
consumer.accept(searcher);
@ -214,10 +189,8 @@ public class TestLuceneSearches {
@AfterAll
public static void afterAll() throws IOException {
TEMP_DB_GENERATOR.closeTempDb(tempDb).block();
ENV.close();
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
TEMP_DB_GENERATOR.closeTempDb(tempDb);
ensureNoLeaks(true, false);
}
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
@ -231,8 +204,8 @@ public class TestLuceneSearches {
}
}
} else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true, MAX_IN_MEMORY_RESULT_ENTRIES));
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(MAX_IN_MEMORY_RESULT_ENTRIES));
}
return shards ? multiIndex : localIndex;
}
@ -242,8 +215,8 @@ public class TestLuceneSearches {
}
private static long getCount(LuceneIndex<String, String> luceneIndex) {
luceneIndex.refresh(true).block();
var totalHitsCount = run(luceneIndex.count(null, new MatchAllDocsQuery()));
luceneIndex.refresh(true);
var totalHitsCount = luceneIndex.count(null, new MatchAllDocsQuery());
Assertions.assertTrue(totalHitsCount.exact(), "Can't get count because the total hits count is not exact");
return totalHitsCount.value();
}
@ -266,7 +239,7 @@ public class TestLuceneSearches {
runSearchers(expectedQueryType, searcher -> {
var luceneIndex = getLuceneIndex(expectedQueryType.shard(), searcher);
var query = queryParamsBuilder.build();
try (var results = run(luceneIndex.search(query))) {
try (var results = luceneIndex.search(query)) {
var hits = results.totalHitsCount();
var keys = getResults(results);
if (hits.exact()) {
@ -278,7 +251,7 @@ public class TestLuceneSearches {
var standardSearcher = new StandardSearcher();
luceneIndex = getLuceneIndex(expectedQueryType.shard(), standardSearcher);
var officialQuery = queryParamsBuilder.limit(ELEMENTS.size() * 2L).build();
try (var officialResults = run(luceneIndex.search(officialQuery))) {
try (var officialResults = luceneIndex.search(officialQuery)) {
var officialHits = officialResults.totalHitsCount();
var officialKeys = getResults(officialResults);
if (officialHits.exact()) {
@ -303,7 +276,7 @@ public class TestLuceneSearches {
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchNoDocs(boolean shards, Sort multiSort) throws Throwable {
var queryBuilder = ClientQueryParams
.<LazyHitKey<String>>builder()
.<HitKey<String>>builder()
.query(new MatchNoDocsQuery())
.snapshot(null)
.computePreciseHitsCount(true)
@ -317,7 +290,7 @@ public class TestLuceneSearches {
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchAllDocs(boolean shards, Sort multiSort) throws Throwable {
var queryBuilder = ClientQueryParams
.<LazyHitKey<String>>builder()
.<HitKey<String>>builder()
.query(new MatchAllDocsQuery())
.snapshot(null)
.computePreciseHitsCount(true)
@ -370,10 +343,10 @@ public class TestLuceneSearches {
}
private List<Scored> getResults(Hits<HitKey<String>> results) {
return run(results
return results
.results()
.map(key -> new Scored(key.key(), key.score()))
.collectList());
.toList();
}
}

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestMemoryDictionary extends TestDictionary {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestMemoryDictionaryMap extends TestDictionaryMap {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestMemoryLLDictionary extends TestLLDictionary {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestMemoryLLDictionaryLeaks extends TestLLDictionaryLeaks {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
public class TestMemorySingletons extends TestSingletons {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import it.cavallium.dbengine.database.disk.RocksLog4jLogger;
import java.io.IOException;
@ -23,7 +23,6 @@ import org.rocksdb.PersistentCache;
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.TableFormatConfig;
import org.rocksdb.WriteOptions;
import org.rocksdb.util.SizeUnit;

View File

@ -1,8 +1,8 @@
package it.cavallium.dbengine.database.collections;
package it.cavallium.dbengine.tests;
import io.netty5.buffer.Buffer;
import io.netty5.buffer.BufferAllocator;
import it.cavallium.dbengine.buffers.Buf;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep;
import java.util.Arrays;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
@ -11,16 +11,12 @@ import org.junit.jupiter.api.Test;
public class TestRanges {
private static BufferAllocator alloc;
@BeforeAll
public static void beforeAll() {
alloc = BufferAllocator.offHeapPooled();
}
@AfterAll
public static void afterAll() {
alloc = BufferAllocator.offHeapPooled();
}
@Test
@ -43,17 +39,15 @@ public class TestRanges {
public void testNextRangeKey(byte[] prefixKey) {
byte[] firstRangeKey;
Buffer firstRangeKeyBuf = alloc.allocate(prefixKey.length).writeBytes(prefixKey);
try (firstRangeKeyBuf) {
DatabaseMapDictionaryDeep.firstRangeKey(firstRangeKeyBuf, prefixKey.length, 7, 3);
firstRangeKey = LLUtils.toArray(firstRangeKeyBuf);
}
Buf firstRangeKeyBuf = Buf.create(prefixKey.length);
firstRangeKeyBuf.addElements(0, prefixKey);
firstRangeKeyBuf = DatabaseMapDictionaryDeep.firstRangeKey(firstRangeKeyBuf, prefixKey.length, Buf.createZeroes(7 + 3));
firstRangeKey = firstRangeKeyBuf.asArray();
byte[] nextRangeKey;
Buffer nextRangeKeyBuf = alloc.allocate(prefixKey.length).writeBytes(prefixKey);
try (nextRangeKeyBuf) {
DatabaseMapDictionaryDeep.nextRangeKey(nextRangeKeyBuf, prefixKey.length, 7, 3);
nextRangeKey = LLUtils.toArray(nextRangeKeyBuf);
}
Buf nextRangeKeyBuf = Buf.create(prefixKey.length);
nextRangeKeyBuf.addElements(0, prefixKey);
nextRangeKeyBuf = DatabaseMapDictionaryDeep.nextRangeKey(nextRangeKeyBuf, prefixKey.length, Buf.createZeroes(7 + 3));
nextRangeKey = nextRangeKeyBuf.asArray();
if (Arrays.equals(prefixKey, new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF})) {
org.assertj.core.api.Assertions

View File

@ -0,0 +1,136 @@
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.tests.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.tests.DbTestUtils.tempDb;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.collections.DatabaseInt;
import it.cavallium.dbengine.database.collections.DatabaseLong;
import it.cavallium.dbengine.database.collections.DatabaseSingleton;
import it.cavallium.dbengine.database.serialization.Serializer;
import java.io.IOException;
import java.util.stream.Stream;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
public abstract class TestSingletons {
protected abstract TemporaryDbGenerator getTempDbGenerator();
private static Stream<Arguments> provideNumberWithRepeats() {
return Stream.of(
Arguments.of(Integer.MIN_VALUE, 2),
Arguments.of(-11, 2),
Arguments.of(0, 3),
Arguments.of(102, 5)
);
}
private static Stream<Arguments> provideLongNumberWithRepeats() {
return Stream.of(
Arguments.of(Long.MIN_VALUE, 2),
Arguments.of(-11L, 2),
Arguments.of(0L, 3),
Arguments.of(102L, 5)
);
}
@BeforeEach
public void beforeEach() {
ensureNoLeaks(false, false);
}
@AfterEach
public void afterEach() {
ensureNoLeaks(true, false);
}
@Test
public void testCreateInteger() throws IOException {
tempDb(getTempDbGenerator(), db -> tempInt(db, "test", 0).get(null));
}
@Test
public void testCreateIntegerNoop() throws IOException {
tempDb(getTempDbGenerator(), db -> tempInt(db, "test", 0));
}
@Test
public void testCreateLong() throws IOException {
tempDb(getTempDbGenerator(), db -> tempLong(db, "test", 0).get(null));
}
@Test
public void testCreateSingleton() throws IOException {
tempDb(getTempDbGenerator(), db -> tempSingleton(db, "testsingleton").get(null));
}
@ParameterizedTest
@ValueSource(ints = {Integer.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Integer.MAX_VALUE})
public void testDefaultValueInteger(int i) throws IOException {
Assertions.assertEquals((Integer) i, tempDb(getTempDbGenerator(), db -> tempInt(db, "test", i).get(null)));
}
@ParameterizedTest
@ValueSource(longs = {Long.MIN_VALUE, -192, -2, -1, 0, 1, 2, 1292, Long.MAX_VALUE})
public void testDefaultValueLong(long i) throws IOException {
Assertions.assertEquals((Long) i, tempDb(getTempDbGenerator(), db -> tempLong(db, "test", i).get(null)));
}
@ParameterizedTest
@MethodSource("provideNumberWithRepeats")
public void testSetInteger(Integer i, Integer repeats) throws IOException {
Assertions.assertEquals(i, tempDb(getTempDbGenerator(), db -> {
var dbInt = tempInt(db, "test", 0);
for (int integer = 0; integer < repeats; integer++) {
dbInt.set((int) System.currentTimeMillis());
}
dbInt.set(i);
return dbInt.get(null);
}));
}
@ParameterizedTest
@MethodSource("provideLongNumberWithRepeats")
public void testSetLong(Long i, Integer repeats) throws IOException {
Assertions.assertEquals(i, tempDb(getTempDbGenerator(), db -> {
var dbLong = tempLong(db, "test", 0);
for (int integer = 0; integer < repeats; integer++) {
dbLong.set(System.currentTimeMillis());
}
dbLong.set(i);
return dbLong.get(null);
}));
}
@ParameterizedTest
@MethodSource("provideLongNumberWithRepeats")
public void testSetSingleton(Long i, Integer repeats) throws IOException {
Assertions.assertEquals(Long.toString(i), tempDb(getTempDbGenerator(), db -> {
var dbSingleton = tempSingleton(db, "test");
for (int integer = 0; integer < repeats; integer++) {
dbSingleton.set(Long.toString(System.currentTimeMillis()));
}
dbSingleton.set(Long.toString(i));
return dbSingleton.get(null);
}));
}
public static DatabaseInt tempInt(LLKeyValueDatabase database, String name, int defaultValue) {
return database.getInteger("ints", name, defaultValue);
}
public static DatabaseLong tempLong(LLKeyValueDatabase database, String name, long defaultValue) {
return database.getLong("longs", name, defaultValue);
}
public static DatabaseSingleton<String> tempSingleton(LLKeyValueDatabase database, String name) {
return new DatabaseSingleton<>(database.getSingleton("longs", name), Serializer.UTF8_SERIALIZER);
}
}

View File

@ -1,14 +1,10 @@
package it.cavallium.dbengine;
package it.cavallium.dbengine.tests;
import static it.cavallium.dbengine.client.UninterruptibleScheduler.uninterruptibleScheduler;
import static it.cavallium.dbengine.database.LLUtils.singleOrClose;
import static it.cavallium.dbengine.lucene.searcher.GlobalQueryRewrite.NO_REWRITE;
import io.netty5.util.Send;
import com.google.common.collect.Streams;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.LuceneCloseable;
import it.cavallium.dbengine.lucene.LuceneUtils;
@ -17,17 +13,13 @@ import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.ShardIndexSearcher;
import it.cavallium.dbengine.utils.SimpleResource;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
@ -40,12 +32,12 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
}
@Override
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
public LuceneSearchResult collectMulti(LLIndexSearchers indexSearchers,
LocalQueryParams queryParams,
String keyFieldName,
GlobalQueryRewrite transformer) {
if (transformer != NO_REWRITE) {
return LuceneUtils.rewriteMulti(this, indexSearchersMono, queryParams, keyFieldName, transformer);
return LuceneUtils.rewriteMulti(this, indexSearchers, queryParams, keyFieldName, transformer);
}
if (queryParams.isSorted() && queryParams.limitLong() > 0) {
throw new UnsupportedOperationException(
@ -56,33 +48,28 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
"Scored queries are not supported" + " by SimpleUnsortedUnscoredLuceneMultiSearcher");
}
return singleOrClose(indexSearchersMono, indexSearchers -> {
var localQueryParams = getLocalQueryParams(queryParams);
return Flux
.fromIterable(indexSearchers.llShards())
.flatMap(searcher -> localSearcher.collect(Mono.just(searcher), localQueryParams, keyFieldName, transformer))
.collectList()
.map(results -> {
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
List<Flux<LLKeyScore>> resultsFluxes = new ArrayList<>(results.size());
boolean exactTotalHitsCount = true;
long totalHitsCountValue = 0;
for (LuceneSearchResult result : results) {
resultsToDrop.add(result);
resultsFluxes.add(result.results());
exactTotalHitsCount &= result.totalHitsCount().exact();
totalHitsCountValue += result.totalHitsCount().value();
}
var localQueryParams = getLocalQueryParams(queryParams);
var results = indexSearchers.llShards().stream()
.map(searcher -> localSearcher.collect(searcher, localQueryParams, keyFieldName, transformer))
.toList();
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
List<Stream<LLKeyScore>> resultsFluxes = new ArrayList<>(results.size());
boolean exactTotalHitsCount = true;
long totalHitsCountValue = 0;
for (LuceneSearchResult result : results) {
resultsToDrop.add(result);
resultsFluxes.add(result.results());
exactTotalHitsCount &= result.totalHitsCount().exact();
totalHitsCountValue += result.totalHitsCount().value();
}
var totalHitsCount = new TotalHitsCount(totalHitsCountValue, exactTotalHitsCount);
Flux<LLKeyScore> mergedFluxes = Flux
.merge(resultsFluxes)
.skip(queryParams.offsetLong())
.take(queryParams.limitLong(), true);
var totalHitsCount = new TotalHitsCount(totalHitsCountValue, exactTotalHitsCount);
//noinspection unchecked
Stream<LLKeyScore> mergedFluxes = (Stream<LLKeyScore>) (Stream) Streams.concat(resultsFluxes.toArray(Stream<?>[]::new))
.skip(queryParams.offsetLong())
.limit(queryParams.limitLong());
return new MyLuceneSearchResult(totalHitsCount, mergedFluxes, resultsToDrop, indexSearchers);
});
});
return new MyLuceneSearchResult(totalHitsCount, mergedFluxes, resultsToDrop, indexSearchers);
}
private LocalQueryParams getLocalQueryParams(LocalQueryParams queryParams) {
@ -107,7 +94,7 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
private final LLIndexSearchers indexSearchers;
public MyLuceneSearchResult(TotalHitsCount totalHitsCount,
Flux<LLKeyScore> mergedFluxes,
Stream<LLKeyScore> mergedFluxes,
List<LuceneSearchResult> resultsToDrop,
LLIndexSearchers indexSearchers) {
super(totalHitsCount, mergedFluxes);

View File

@ -0,0 +1,19 @@
module dbengine.tests {
requires org.junit.jupiter.api;
requires dbengine;
requires data.generator.runtime;
requires org.assertj.core;
requires org.apache.lucene.core;
requires it.unimi.dsi.fastutil;
requires org.apache.lucene.queryparser;
requires io.netty.common;
requires org.jetbrains.annotations;
requires micrometer.core;
requires org.junit.jupiter.params;
requires com.google.common;
requires org.apache.logging.log4j;
requires io.netty.buffer;
requires org.apache.commons.lang3;
requires rocksdbjni;
opens it.cavallium.dbengine.tests;
}