Rename classes and improve readonly tests performance

This commit is contained in:
Andrea Cavalli 2021-10-13 12:25:32 +02:00
parent 09f60a3a99
commit 584115c5cc
42 changed files with 463 additions and 384 deletions

View File

@ -224,7 +224,6 @@
<dependency> <dependency>
<groupId>io.projectreactor</groupId> <groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId> <artifactId>reactor-test</artifactId>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.novasearch</groupId> <groupId>org.novasearch</groupId>

View File

@ -6,22 +6,20 @@ import static java.util.Objects.requireNonNullElseGet;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLLocalSingleton;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher; import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult; import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
public class SwappableLuceneSearcher implements LuceneLocalSearcher, LuceneMultiSearcher, Closeable { public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Closeable {
private final AtomicReference<LuceneLocalSearcher> single = new AtomicReference<>(null); private final AtomicReference<LocalSearcher> single = new AtomicReference<>(null);
private final AtomicReference<LuceneMultiSearcher> multi = new AtomicReference<>(null); private final AtomicReference<MultiSearcher> multi = new AtomicReference<>(null);
public SwappableLuceneSearcher() { public SwappableLuceneSearcher() {
@ -61,11 +59,11 @@ public class SwappableLuceneSearcher implements LuceneLocalSearcher, LuceneMulti
return multi.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); return multi.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
} }
public void setSingle(LuceneLocalSearcher single) { public void setSingle(LocalSearcher single) {
this.single.set(single); this.single.set(single);
} }
public void setMulti(LuceneMultiSearcher multi) { public void setMulti(MultiSearcher multi) {
this.multi.set(multi); this.multi.set(multi);
} }

View File

@ -5,7 +5,7 @@ import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.client.LuceneOptions; import it.cavallium.dbengine.client.LuceneOptions;
import it.cavallium.dbengine.database.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneHacks;
import java.util.List; import java.util.List;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;

View File

@ -5,7 +5,6 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache; import com.google.common.cache.LoadingCache;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.lucene.searcher.ExecutorSearcherFactory;
import java.io.IOException; import java.io.IOException;
import java.time.Duration; import java.time.Duration;
import java.util.concurrent.Executor; import java.util.concurrent.Executor;

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.database.disk;
import java.util.concurrent.Executor; import java.util.concurrent.Executor;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;

View File

@ -8,8 +8,7 @@ import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.database.LLDatabaseConnection; import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher;
import it.cavallium.dbengine.netty.JMXNettyMonitoringManager; import it.cavallium.dbengine.netty.JMXNettyMonitoringManager;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;

View File

@ -16,13 +16,12 @@ import it.cavallium.dbengine.database.LLSearchResultShard;
import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLTerm; import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.AlwaysDirectIOFSDirectory; import it.cavallium.dbengine.lucene.AlwaysDirectIOFSDirectory;
import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher; import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher; import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
@ -50,7 +49,6 @@ import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.Constants; import org.apache.lucene.util.Constants;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.reactivestreams.Publisher;
import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory; import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
@ -62,7 +60,7 @@ import reactor.util.function.Tuple2;
public class LLLocalLuceneIndex implements LLLuceneIndex { public class LLLocalLuceneIndex implements LLLuceneIndex {
protected static final Logger logger = LoggerFactory.getLogger(LLLocalLuceneIndex.class); protected static final Logger logger = LoggerFactory.getLogger(LLLocalLuceneIndex.class);
private final LuceneLocalSearcher localSearcher; private final LocalSearcher localSearcher;
/** /**
* Global lucene index scheduler. * Global lucene index scheduler.
* There is only a single thread globally to not overwhelm the disk with * There is only a single thread globally to not overwhelm the disk with
@ -170,7 +168,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
if (luceneHacks != null && luceneHacks.customLocalSearcher() != null) { if (luceneHacks != null && luceneHacks.customLocalSearcher() != null) {
localSearcher = luceneHacks.customLocalSearcher().get(); localSearcher = luceneHacks.customLocalSearcher().get();
} else { } else {
localSearcher = new AdaptiveLuceneLocalSearcher(); localSearcher = new AdaptiveLocalSearcher();
} }
var indexWriterConfig = new IndexWriterConfig(luceneAnalyzer); var indexWriterConfig = new IndexWriterConfig(luceneAnalyzer);

View File

@ -10,16 +10,15 @@ import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLSearchResultShard; import it.cavallium.dbengine.database.LLSearchResultShard;
import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLTerm; import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher; import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -46,7 +45,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
private final PerFieldAnalyzerWrapper luceneAnalyzer; private final PerFieldAnalyzerWrapper luceneAnalyzer;
private final PerFieldSimilarityWrapper luceneSimilarity; private final PerFieldSimilarityWrapper luceneSimilarity;
private final LuceneMultiSearcher multiSearcher; private final MultiSearcher multiSearcher;
public LLLocalMultiLuceneIndex(Path lucene, public LLLocalMultiLuceneIndex(Path lucene,
String name, String name,
@ -83,7 +82,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
if (luceneHacks != null && luceneHacks.customMultiSearcher() != null) { if (luceneHacks != null && luceneHacks.customMultiSearcher() != null) {
multiSearcher = luceneHacks.customMultiSearcher().get(); multiSearcher = luceneHacks.customMultiSearcher().get();
} else { } else {
multiSearcher = new AdaptiveLuceneMultiSearcher(); multiSearcher = new AdaptiveMultiSearcher();
} }
} }

View File

@ -1,10 +0,0 @@
package it.cavallium.dbengine.database.lucene;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher;
import java.util.function.Supplier;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public record LuceneHacks(@Nullable Supplier<@NotNull LuceneLocalSearcher> customLocalSearcher,
@Nullable Supplier<@NotNull LuceneMultiSearcher> customMultiSearcher) {}

View File

@ -10,7 +10,7 @@ import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.LLKeyValueDatabase; import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.disk.LLLocalLuceneIndex; import it.cavallium.dbengine.database.disk.LLLocalLuceneIndex;
import it.cavallium.dbengine.database.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.netty.JMXNettyMonitoringManager; import it.cavallium.dbengine.netty.JMXNettyMonitoringManager;
import java.util.List; import java.util.List;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;

View File

@ -1,6 +1,4 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene;
import it.cavallium.dbengine.lucene.LuceneUtils;
/** /**
* <pre>y = 2 ^ (x + pageIndexOffset) + firstPageLimit</pre> * <pre>y = 2 ^ (x + pageIndexOffset) + firstPageLimit</pre>

View File

@ -1,6 +1,4 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene;
import it.cavallium.dbengine.lucene.LuceneUtils;
/** /**
* <pre>y = (x * factor) + firstPageLimit</pre> * <pre>y = (x * factor) + firstPageLimit</pre>

View File

@ -0,0 +1,10 @@
package it.cavallium.dbengine.lucene;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import java.util.function.Supplier;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public record LuceneHacks(@Nullable Supplier<@NotNull LocalSearcher> customLocalSearcher,
@Nullable Supplier<@NotNull MultiSearcher> customMultiSearcher) {}

View File

@ -8,12 +8,10 @@ import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.EnglishItalianStopFilter; import it.cavallium.dbengine.database.EnglishItalianStopFilter;
import it.cavallium.dbengine.database.LLKeyScore; import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionary; import it.cavallium.dbengine.database.collections.DatabaseMapDictionary;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep; import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep;
import it.cavallium.dbengine.database.collections.ValueGetter; import it.cavallium.dbengine.database.collections.ValueGetter;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.analyzer.NCharGramAnalyzer; import it.cavallium.dbengine.lucene.analyzer.NCharGramAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.NCharGramEdgeAnalyzer; import it.cavallium.dbengine.lucene.analyzer.NCharGramEdgeAnalyzer;
@ -22,9 +20,7 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer; import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer;
import it.cavallium.dbengine.lucene.mlt.BigCompositeReader; import it.cavallium.dbengine.lucene.mlt.BigCompositeReader;
import it.cavallium.dbengine.lucene.mlt.MultiMoreLikeThis; import it.cavallium.dbengine.lucene.mlt.MultiMoreLikeThis;
import it.cavallium.dbengine.lucene.searcher.ExponentialPageLimits;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.PageLimits;
import it.cavallium.dbengine.lucene.similarity.NGramSimilarity; import it.cavallium.dbengine.lucene.similarity.NGramSimilarity;
import java.io.EOFException; import java.io.EOFException;
import java.io.IOException; import java.io.IOException;
@ -50,10 +46,8 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanQuery.Builder; import org.apache.lucene.search.BooleanQuery.Builder;
import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery;
@ -79,7 +73,6 @@ import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory; import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
import reactor.util.concurrent.Queues; import reactor.util.concurrent.Queues;
import reactor.util.function.Tuple2; import reactor.util.function.Tuple2;

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene;
import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.LuceneUtils;

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene;
public class SinglePageLimits implements PageLimits { public class SinglePageLimits implements PageLimits {

View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene.collector;
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.TIE_BREAKER; import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.TIE_BREAKER;

View File

@ -5,16 +5,14 @@ import io.net5.buffer.api.internal.ResourceSupport;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLIndexSearchers.UnshardedIndexSearchers;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import org.apache.lucene.search.IndexSearcher;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
public class AdaptiveLuceneLocalSearcher implements LuceneLocalSearcher { public class AdaptiveLocalSearcher implements LocalSearcher {
private static final LuceneLocalSearcher localSearcher = new SimpleLuceneLocalSearcher(); private static final LocalSearcher localSearcher = new PagedLocalSearcher();
private static final LuceneLocalSearcher countSearcher = new CountLuceneLocalSearcher(); private static final LocalSearcher countSearcher = new CountLocalSearcher();
@Override @Override
public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcher, public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcher,

View File

@ -8,23 +8,23 @@ import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
public class AdaptiveLuceneMultiSearcher implements LuceneMultiSearcher, Closeable { public class AdaptiveMultiSearcher implements MultiSearcher, Closeable {
private static final LuceneMultiSearcher count private static final MultiSearcher count
= new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher()); = new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher());
private static final LuceneMultiSearcher scoredSimple = new ScoredSimpleLuceneMultiSearcher(); private static final MultiSearcher scoredSimple = new ScoredPagedMultiSearcher();
private static final LuceneMultiSearcher unsortedUnscoredPaged private static final MultiSearcher unsortedUnscoredPaged
= new SimpleUnsortedUnscoredLuceneMultiSearcher(new SimpleLuceneLocalSearcher()); = new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher());
private static final LuceneMultiSearcher unsortedUnscoredContinuous private static final MultiSearcher unsortedUnscoredContinuous
= new UnsortedUnscoredContinuousLuceneMultiSearcher(); = new UnsortedUnscoredStreamingMultiSearcher();
private final UnsortedScoredFullLuceneMultiSearcher scoredFull; private final UnsortedScoredFullMultiSearcher scoredFull;
public AdaptiveLuceneMultiSearcher() throws IOException { public AdaptiveMultiSearcher() throws IOException {
scoredFull = new UnsortedScoredFullLuceneMultiSearcher(); scoredFull = new UnsortedScoredFullMultiSearcher();
} }
@Override @Override
@ -76,6 +76,6 @@ public class AdaptiveLuceneMultiSearcher implements LuceneMultiSearcher, Closeab
@Override @Override
public String getName() { public String getName() {
return "adaptivemulti"; return "adaptive local";
} }
} }

View File

@ -1,7 +0,0 @@
package it.cavallium.dbengine.lucene.searcher;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLKeyScore;
import reactor.core.publisher.Flux;
record CalculatedResults(TotalHitsCount totalHitsCount, Flux<LLKeyScore> firstPageHitsFlux) {}

View File

@ -6,12 +6,11 @@ import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import org.apache.lucene.search.IndexSearcher;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
public class CountLuceneLocalSearcher implements LuceneLocalSearcher { public class CountLocalSearcher implements LocalSearcher {
@Override @Override
public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono, public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
@ -45,6 +44,6 @@ public class CountLuceneLocalSearcher implements LuceneLocalSearcher {
@Override @Override
public String getName() { public String getName() {
return "count"; return "count local";
} }
} }

View File

@ -6,8 +6,10 @@ import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
record CurrentPageInfo(@Nullable ScoreDoc last, long remainingLimit, int pageIndex) { public record CurrentPageInfo(@Nullable ScoreDoc last, long remainingLimit, int pageIndex) {
public static final Comparator<ScoreDoc> TIE_BREAKER = Comparator.comparingInt((d) -> d.shardIndex); public static final Comparator<ScoreDoc> TIE_BREAKER = Comparator
.<ScoreDoc>comparingInt((d) -> d.shardIndex)
.thenComparingInt(d -> -d.doc);
public static final CurrentPageInfo EMPTY_STATUS = new CurrentPageInfo(null, 0, 0); public static final CurrentPageInfo EMPTY_STATUS = new CurrentPageInfo(null, 0, 0);
} }

View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene.searcher;
import it.cavallium.dbengine.lucene.PageLimits;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Sort; import org.apache.lucene.search.Sort;

View File

@ -4,7 +4,7 @@ import io.net5.buffer.api.Send;
import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
public interface LuceneLocalSearcher { public interface LocalSearcher {
/** /**
* @param indexSearcherMono Lucene index searcher * @param indexSearcherMono Lucene index searcher

View File

@ -5,7 +5,7 @@ import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
public interface LuceneMultiSearcher extends LuceneLocalSearcher { public interface MultiSearcher extends LocalSearcher {
/** /**
* @param indexSearchersMono Lucene index searcher * @param indexSearchersMono Lucene index searcher

View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS; import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.FIRST_PAGE_LIMIT;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT; import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
@ -10,13 +9,11 @@ import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLIndexSearchers.UnshardedIndexSearchers;
import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects; import java.util.Objects;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
@ -29,7 +26,7 @@ import reactor.core.publisher.Mono;
import reactor.core.publisher.SynchronousSink; import reactor.core.publisher.SynchronousSink;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher { public class PagedLocalSearcher implements LocalSearcher {
@Override @Override
public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono, public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
@ -69,7 +66,7 @@ public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher {
@Override @Override
public String getName() { public String getName() {
return "simplelocal"; return "paged local";
} }
/** /**
@ -187,7 +184,7 @@ public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher {
} else if (s.pageIndex() == 0 || (s.last() != null && s.remainingLimit() > 0)) { } else if (s.pageIndex() == 0 || (s.last() != null && s.remainingLimit() > 0)) {
TopDocs pageTopDocs; TopDocs pageTopDocs;
try { try {
TopDocsCollector<ScoreDoc> collector = TopDocsSearcher.getTopDocsCollector(queryParams.sort(), TopDocsCollector<ScoreDoc> collector = TopDocsCollectorUtils.getTopDocsCollector(queryParams.sort(),
currentPageLimit, s.last(), LuceneUtils.totalHitsThreshold(), currentPageLimit, s.last(), LuceneUtils.totalHitsThreshold(),
allowPagination, queryParams.isScored()); allowPagination, queryParams.isScored());
indexSearchers.get(0).search(queryParams.query(), collector); indexSearchers.get(0).search(queryParams.query(), collector);

View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene.searcher;
import java.util.Comparator; import it.cavallium.dbengine.lucene.PageLimits;
import org.apache.lucene.search.ScoreDoc;
public record PaginationInfo(long totalLimit, long firstPageOffset, PageLimits pageLimits, boolean forceSinglePage) { public record PaginationInfo(long totalLimit, long firstPageOffset, PageLimits pageLimits, boolean forceSinglePage) {

View File

@ -1,6 +1,5 @@
package it.cavallium.dbengine.lucene.searcher; package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT; import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
import io.net5.buffer.api.Send; import io.net5.buffer.api.Send;
@ -8,19 +7,16 @@ import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.PageLimits;
import it.cavallium.dbengine.lucene.collector.ScoringShardsCollectorManager;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort; import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.TotalHits.Relation;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory; import org.warp.commonutils.log.LoggerFactory;
@ -28,11 +24,11 @@ import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
public class ScoredSimpleLuceneMultiSearcher implements LuceneMultiSearcher { public class ScoredPagedMultiSearcher implements MultiSearcher {
protected static final Logger logger = LoggerFactory.getLogger(ScoredSimpleLuceneMultiSearcher.class); protected static final Logger logger = LoggerFactory.getLogger(ScoredPagedMultiSearcher.class);
public ScoredSimpleLuceneMultiSearcher() { public ScoredPagedMultiSearcher() {
} }
@Override @Override
@ -214,6 +210,6 @@ public class ScoredSimpleLuceneMultiSearcher implements LuceneMultiSearcher {
@Override @Override
public String getName() { public String getName() {
return "scoredsimplemulti"; return "scored paged multi";
} }
} }

View File

@ -10,7 +10,7 @@ import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector; import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.search.TopScoreDocCollector;
class TopDocsSearcher { class TopDocsCollectorUtils {
@SuppressWarnings({"unchecked", "rawtypes"}) @SuppressWarnings({"unchecked", "rawtypes"})
public static TopDocsCollector<ScoreDoc> getTopDocsCollector(Sort luceneSort, public static TopDocsCollector<ScoreDoc> getTopDocsCollector(Sort luceneSort,

View File

@ -20,13 +20,13 @@ import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearcher, Closeable { public class UnsortedScoredFullMultiSearcher implements MultiSearcher, Closeable {
protected static final Logger logger = LoggerFactory.getLogger(UnsortedScoredFullLuceneMultiSearcher.class); protected static final Logger logger = LoggerFactory.getLogger(UnsortedScoredFullMultiSearcher.class);
private final LLTempLMDBEnv env; private final LLTempLMDBEnv env;
public UnsortedScoredFullLuceneMultiSearcher() throws IOException { public UnsortedScoredFullMultiSearcher() throws IOException {
this.env = new LLTempLMDBEnv(); this.env = new LLTempLMDBEnv();
} }
@ -46,7 +46,7 @@ public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearche
return queryParamsMono.flatMap(queryParams2 -> { return queryParamsMono.flatMap(queryParams2 -> {
Objects.requireNonNull(queryParams2.scoreMode(), "ScoreMode must not be null"); Objects.requireNonNull(queryParams2.scoreMode(), "ScoreMode must not be null");
if (queryParams2.sort() != null && queryParams2.sort() != Sort.RELEVANCE) { if (queryParams2.sort() != null && queryParams2.sort() != Sort.RELEVANCE) {
throw new IllegalArgumentException(UnsortedScoredFullLuceneMultiSearcher.this.getClass().getSimpleName() throw new IllegalArgumentException(UnsortedScoredFullMultiSearcher.this.getClass().getSimpleName()
+ " doesn't support sorted queries"); + " doesn't support sorted queries");
} }
@ -115,6 +115,6 @@ public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearche
@Override @Override
public String getName() { public String getName() {
return "scoredfullmulti"; return "unsorted scored full multi";
} }
} }

View File

@ -13,11 +13,11 @@ import java.util.List;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
public class SimpleUnsortedUnscoredLuceneMultiSearcher implements LuceneMultiSearcher { public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
private final LuceneLocalSearcher localSearcher; private final LocalSearcher localSearcher;
public SimpleUnsortedUnscoredLuceneMultiSearcher(LuceneLocalSearcher localSearcher) { public UnsortedUnscoredSimpleMultiSearcher(LocalSearcher localSearcher) {
this.localSearcher = localSearcher; this.localSearcher = localSearcher;
} }
@ -103,6 +103,6 @@ public class SimpleUnsortedUnscoredLuceneMultiSearcher implements LuceneMultiSea
@Override @Override
public String getName() { public String getName() {
return "simpleunsortedunscoredmulti"; return "unsorted unscored simple multi";
} }
} }

View File

@ -23,7 +23,7 @@ import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
import reactor.util.concurrent.Queues; import reactor.util.concurrent.Queues;
public class UnsortedUnscoredContinuousLuceneMultiSearcher implements LuceneMultiSearcher { public class UnsortedUnscoredStreamingMultiSearcher implements MultiSearcher {
private static final Scheduler UNSCORED_UNSORTED_EXECUTOR = Schedulers.newBoundedElastic( private static final Scheduler UNSCORED_UNSORTED_EXECUTOR = Schedulers.newBoundedElastic(
Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE, Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE,
@ -117,6 +117,6 @@ public class UnsortedUnscoredContinuousLuceneMultiSearcher implements LuceneMult
@Override @Override
public String getName() { public String getName() {
return "unsortedunscoredcontinuousmulti"; return "unsorted unscored streaming multi";
} }
} }

View File

@ -47,38 +47,6 @@ public class DbTestUtils {
return "0123456789".repeat(1024); return "0123456789".repeat(1024);
} }
public static void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
public static void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
public static void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
public static record TestAllocator(PooledBufferAllocator allocator) {} public static record TestAllocator(PooledBufferAllocator allocator) {}
public static TestAllocator newAllocator() { public static TestAllocator newAllocator() {

View File

@ -12,11 +12,9 @@ import it.cavallium.dbengine.client.NRTCachingOptions;
import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.LLKeyValueDatabase; import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection; import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
import it.cavallium.dbengine.database.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer; import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity; import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -36,7 +34,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
private static final Optional<NRTCachingOptions> NRT = Optional.empty(); private static final Optional<NRTCachingOptions> NRT = Optional.empty();
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5), private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
false, true, Optional.empty(), true, NRT, -1, true, true); false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
@Override @Override
public Mono<TempDb> openTempDb(TestAllocator allocator) { public Mono<TempDb> openTempDb(TestAllocator allocator) {

View File

@ -8,7 +8,7 @@ import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.client.LuceneOptions; import it.cavallium.dbengine.client.LuceneOptions;
import it.cavallium.dbengine.client.NRTCachingOptions; import it.cavallium.dbengine.client.NRTCachingOptions;
import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection; import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer; import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity; import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
@ -22,7 +22,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
private static final Optional<NRTCachingOptions> NRT = Optional.empty(); private static final Optional<NRTCachingOptions> NRT = Optional.empty();
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5), private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
false, true, Optional.empty(), true, NRT, -1, true, true); false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
@Override @Override
public Mono<TempDb> openTempDb(TestAllocator allocator) { public Mono<TempDb> openTempDb(TestAllocator allocator) {

View File

@ -0,0 +1,40 @@
package it.cavallium.dbengine;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class SyncUtils {
public static void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
public static void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
public static void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
}

View File

@ -1,6 +1,7 @@
package it.cavallium.dbengine; package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.*; import static it.cavallium.dbengine.DbTestUtils.*;
import static it.cavallium.dbengine.SyncUtils.*;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.DbTestUtils.TestAllocator; import it.cavallium.dbengine.DbTestUtils.TestAllocator;

View File

@ -5,12 +5,11 @@ import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.isCIMode; import static it.cavallium.dbengine.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.DbTestUtils.newAllocator; import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.run;
import static it.cavallium.dbengine.DbTestUtils.runVoid;
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMap; import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMap;
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryMap; import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryMap;
import static it.cavallium.dbengine.DbTestUtils.tempDb; import static it.cavallium.dbengine.DbTestUtils.tempDb;
import static it.cavallium.dbengine.DbTestUtils.tempDictionary; import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
import static it.cavallium.dbengine.SyncUtils.*;
import io.net5.buffer.api.internal.ResourceSupport; import io.net5.buffer.api.internal.ResourceSupport;
import it.cavallium.dbengine.DbTestUtils.TestAllocator; import it.cavallium.dbengine.DbTestUtils.TestAllocator;

View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator; import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import io.net5.buffer.api.Buffer; import io.net5.buffer.api.Buffer;
@ -115,38 +116,6 @@ public abstract class TestLLDictionary {
} }
} }
private void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
private void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
private void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
@Test @Test
public void testNoOp() { public void testNoOp() {
} }

View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator; import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
@ -131,9 +132,9 @@ public abstract class TestLLDictionaryLeaks {
public void testGet(UpdateMode updateMode) { public void testGet(UpdateMode updateMode) {
var dict = getDict(updateMode); var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test")); var key = Mono.fromCallable(() -> fromString("test"));
DbTestUtils.runVoid(dict.get(null, key).then().transform(LLUtils::handleDiscard)); runVoid(dict.get(null, key).then().transform(LLUtils::handleDiscard));
DbTestUtils.runVoid(dict.get(null, key, true).then().transform(LLUtils::handleDiscard)); runVoid(dict.get(null, key, true).then().transform(LLUtils::handleDiscard));
DbTestUtils.runVoid(dict.get(null, key, false).then().transform(LLUtils::handleDiscard)); runVoid(dict.get(null, key, false).then().transform(LLUtils::handleDiscard));
} }
@ParameterizedTest @ParameterizedTest
@ -142,14 +143,14 @@ public abstract class TestLLDictionaryLeaks {
var dict = getDict(updateMode); var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key")); var key = Mono.fromCallable(() -> fromString("test-key"));
var value = Mono.fromCallable(() -> fromString("test-value")); var value = Mono.fromCallable(() -> fromString("test-value"));
DbTestUtils.runVoid(dict.put(key, value, resultType).then().doOnDiscard(Send.class, Send::close)); runVoid(dict.put(key, value, resultType).then().doOnDiscard(Send.class, Send::close));
} }
@ParameterizedTest @ParameterizedTest
@MethodSource("provideArguments") @MethodSource("provideArguments")
public void testGetUpdateMode(UpdateMode updateMode) { public void testGetUpdateMode(UpdateMode updateMode) {
var dict = getDict(updateMode); var dict = getDict(updateMode);
assertEquals(updateMode, DbTestUtils.run(dict.getUpdateMode())); assertEquals(updateMode, run(dict.getUpdateMode()));
} }
@ParameterizedTest @ParameterizedTest
@ -157,13 +158,13 @@ public abstract class TestLLDictionaryLeaks {
public void testUpdate(UpdateMode updateMode, UpdateReturnMode updateReturnMode) { public void testUpdate(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
var dict = getDict(updateMode); var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key")); var key = Mono.fromCallable(() -> fromString("test-key"));
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, old -> old, updateReturnMode, true).then().transform(LLUtils::handleDiscard) dict.update(key, old -> old, updateReturnMode, true).then().transform(LLUtils::handleDiscard)
); );
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, old -> old, updateReturnMode, false).then().transform(LLUtils::handleDiscard) dict.update(key, old -> old, updateReturnMode, false).then().transform(LLUtils::handleDiscard)
); );
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, old -> old, updateReturnMode).then().transform(LLUtils::handleDiscard) dict.update(key, old -> old, updateReturnMode).then().transform(LLUtils::handleDiscard)
); );
} }
@ -173,13 +174,13 @@ public abstract class TestLLDictionaryLeaks {
public void testUpdateAndGetDelta(UpdateMode updateMode) { public void testUpdateAndGetDelta(UpdateMode updateMode) {
var dict = getDict(updateMode); var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key")); var key = Mono.fromCallable(() -> fromString("test-key"));
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, old -> old, true).then().transform(LLUtils::handleDiscard) dict.updateAndGetDelta(key, old -> old, true).then().transform(LLUtils::handleDiscard)
); );
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, old -> old, false).then().transform(LLUtils::handleDiscard) dict.updateAndGetDelta(key, old -> old, false).then().transform(LLUtils::handleDiscard)
); );
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, old -> old).then().transform(LLUtils::handleDiscard) dict.updateAndGetDelta(key, old -> old).then().transform(LLUtils::handleDiscard)
); );
} }
@ -188,7 +189,7 @@ public abstract class TestLLDictionaryLeaks {
@MethodSource("provideArguments") @MethodSource("provideArguments")
public void testClear(UpdateMode updateMode) { public void testClear(UpdateMode updateMode) {
var dict = getDict(updateMode); var dict = getDict(updateMode);
DbTestUtils.runVoid(dict.clear()); runVoid(dict.clear());
} }
@ParameterizedTest @ParameterizedTest
@ -196,6 +197,6 @@ public abstract class TestLLDictionaryLeaks {
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) { public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode); var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key")); var key = Mono.fromCallable(() -> fromString("test-key"));
DbTestUtils.runVoid(dict.remove(key, resultType).then().doOnDiscard(Send.class, Send::close)); runVoid(dict.remove(key, resultType).then().doOnDiscard(Send.class, Send::close));
} }
} }

View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator; import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.fail;
@ -11,29 +12,16 @@ import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.LuceneIndex; import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.MultiSort; import it.cavallium.dbengine.client.MultiSort;
import it.cavallium.dbengine.client.SearchResultKey; import it.cavallium.dbengine.client.SearchResultKey;
import it.cavallium.dbengine.client.SearchResultKeys;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder;
import it.cavallium.dbengine.client.query.QueryParser;
import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery; import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery;
import it.cavallium.dbengine.client.query.current.data.MatchNoDocsQuery;
import it.cavallium.dbengine.client.query.current.data.NoSort;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode; import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher; import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher; import it.cavallium.dbengine.lucene.searcher.CountLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.CountLuceneLocalSearcher; import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher; import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredSimpleMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedScoredFullLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.ScoredSimpleLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.SimpleLuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.SimpleUnsortedUnscoredLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredContinuousLuceneMultiSearcher;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
@ -47,9 +35,6 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory; import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux; import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink.OverflowStrategy;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuples; import reactor.util.function.Tuples;
public class TestLuceneIndex { public class TestLuceneIndex {
@ -92,38 +77,6 @@ public class TestLuceneIndex {
MultiSort.numericSort("longsort", true) MultiSort.numericSort("longsort", true)
); );
private static Flux<LuceneLocalSearcher> getSearchers(ExpectedQueryType info) {
return Flux.push(sink -> {
try {
if (info.shard()) {
sink.next(new AdaptiveLuceneMultiSearcher());
if (info.onlyCount()) {
sink.next(new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher()));
} else {
sink.next(new ScoredSimpleLuceneMultiSearcher());
if (!info.sorted()) {
sink.next(new UnsortedScoredFullLuceneMultiSearcher());
}
if (!info.scored() && !info.sorted()) {
sink.next(new SimpleUnsortedUnscoredLuceneMultiSearcher(new SimpleLuceneLocalSearcher()));
sink.next(new UnsortedUnscoredContinuousLuceneMultiSearcher());
}
}
} else {
sink.next(new AdaptiveLuceneLocalSearcher());
if (info.onlyCount()) {
sink.next(new CountLuceneLocalSearcher());
} else {
sink.next(new SimpleLuceneLocalSearcher());
}
}
sink.complete();
} catch (IOException e) {
sink.error(e);
}
}, OverflowStrategy.BUFFER);
}
public static Stream<Arguments> provideQueryArgumentsScoreMode() { public static Stream<Arguments> provideQueryArgumentsScoreMode() {
return multi return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode))) .concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
@ -153,7 +106,7 @@ public class TestLuceneIndex {
destroyAllocator(allocator); destroyAllocator(allocator);
} }
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LuceneLocalSearcher customSearcher) { private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti)); LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
index.updateDocument("test-key-1", "0123456789").block(); index.updateDocument("test-key-1", "0123456789").block();
index.updateDocument("test-key-2", "test 0123456789 test word").block(); index.updateDocument("test-key-2", "test 0123456789 test word").block();
@ -171,22 +124,22 @@ public class TestLuceneIndex {
index.updateDocument("test-key-14", "2999").block(); index.updateDocument("test-key-14", "2999").block();
index.updateDocument("test-key-15", "3902").block(); index.updateDocument("test-key-15", "3902").block();
Flux.range(1, 1000).concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i)).blockLast(); Flux.range(1, 1000).concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i)).blockLast();
tempDb.swappableLuceneSearcher().setSingle(new CountLuceneLocalSearcher()); tempDb.swappableLuceneSearcher().setSingle(new CountLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher())); tempDb.swappableLuceneSearcher().setMulti(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher()));
assertCount(index, 1000 + 15); assertCount(index, 1000 + 15);
try { try {
if (customSearcher != null) { if (customSearcher != null) {
tempDb.swappableLuceneSearcher().setSingle(customSearcher); tempDb.swappableLuceneSearcher().setSingle(customSearcher);
if (shards) { if (shards) {
if (customSearcher instanceof LuceneMultiSearcher multiSearcher) { if (customSearcher instanceof MultiSearcher multiSearcher) {
tempDb.swappableLuceneSearcher().setMulti(multiSearcher); tempDb.swappableLuceneSearcher().setMulti(multiSearcher);
} else { } else {
throw new IllegalArgumentException("Expected a LuceneMultiSearcher, got a LuceneLocalSearcher: " + customSearcher.getName()); throw new IllegalArgumentException("Expected a LuceneMultiSearcher, got a LuceneLocalSearcher: " + customSearcher.getName());
} }
} }
} else { } else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLuceneLocalSearcher()); tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveLuceneMultiSearcher()); tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher());
} }
} catch (IOException e) { } catch (IOException e) {
fail(e); fail(e);
@ -194,38 +147,6 @@ public class TestLuceneIndex {
return index; return index;
} }
private void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
private void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
private void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
private void assertCount(LuceneIndex<String, String> luceneIndex, long expected) { private void assertCount(LuceneIndex<String, String> luceneIndex, long expected) {
Assertions.assertEquals(expected, getCount(luceneIndex)); Assertions.assertEquals(expected, getCount(luceneIndex));
} }
@ -291,95 +212,4 @@ public class TestLuceneIndex {
assertCount(luceneIndex, prevCount + 1); assertCount(luceneIndex, prevCount + 1);
} }
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchNoDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LuceneLocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean supportsPreciseHitsCount(LuceneLocalSearcher searcher,
ClientQueryParams<SearchResultKey<String>> query) {
if (searcher instanceof UnsortedUnscoredContinuousLuceneMultiSearcher) {
return false;
}
var scored = isScored(query.scoreMode(), Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
var sorted = isSorted(Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
if (!sorted && !scored) {
if (searcher instanceof AdaptiveLuceneMultiSearcher || searcher instanceof AdaptiveLuceneLocalSearcher) {
return false;
}
}
return true;
}
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchAllDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LuceneLocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean isSorted(MultiSort<SearchResultKey<String>> multiSort) {
return !(multiSort.getQuerySort() instanceof NoSort);
}
private boolean isScored(LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var needsScores = LLUtils.toScoreMode(scoreMode).needsScores();
var sort =QueryParser.toSort(multiSort.getQuerySort());
if (sort != null) {
needsScores |= sort.needsScores();
}
return needsScores;
}
private List<Scored> getResults(SearchResultKeys<String> results) {
return run(results
.results()
.flatMapSequential(searchResultKey -> searchResultKey
.key()
.single()
.map(key -> new Scored(key, searchResultKey.score()))
)
.collectList());
}
} }

View File

@ -0,0 +1,307 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.MultiSort;
import it.cavallium.dbengine.client.SearchResultKey;
import it.cavallium.dbengine.client.SearchResultKeys;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder;
import it.cavallium.dbengine.client.query.QueryParser;
import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery;
import it.cavallium.dbengine.client.query.current.data.MatchNoDocsQuery;
import it.cavallium.dbengine.client.query.current.data.NoSort;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.CountLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.ScoredPagedMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.PagedLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredSimpleMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedScoredFullMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredStreamingMultiSearcher;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink.OverflowStrategy;
import reactor.util.function.Tuples;
public class TestLuceneSearches {
private static final Logger log = LoggerFactory.getLogger(TestLuceneSearches.class);
private static final MemoryTemporaryDbGenerator TEMP_DB_GENERATOR = new MemoryTemporaryDbGenerator();
private static TestAllocator allocator;
private static TempDb tempDb;
private static LLLuceneIndex luceneSingle;
private static LLLuceneIndex luceneMulti;
private static LuceneIndex<String, String> multiIndex;
private static LuceneIndex<String, String> localIndex;
@BeforeAll
public static void beforeAll() {
allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
tempDb = Objects.requireNonNull(TEMP_DB_GENERATOR.openTempDb(allocator).block(), "TempDB");
luceneSingle = tempDb.luceneSingle();
luceneMulti = tempDb.luceneMulti();
setUpIndex(true);
setUpIndex(false);
}
private static void setUpIndex(boolean shards) {
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
index.updateDocument("test-key-1", "0123456789").block();
index.updateDocument("test-key-2", "test 0123456789 test word").block();
index.updateDocument("test-key-3", "0123456789 test example string").block();
index.updateDocument("test-key-4", "hello world the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-5", "hello the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-6", "hello the quick brown fox jumps over the world dog").block();
index.updateDocument("test-key-7", "the quick brown fox jumps over the world dog").block();
index.updateDocument("test-key-8", "the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-9", "Example1").block();
index.updateDocument("test-key-10", "Example2").block();
index.updateDocument("test-key-11", "Example3").block();
index.updateDocument("test-key-12", "-234").block();
index.updateDocument("test-key-13", "2111").block();
index.updateDocument("test-key-14", "2999").block();
index.updateDocument("test-key-15", "3902").block();
Flux.range(1, 1000).concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i)).blockLast();
tempDb.swappableLuceneSearcher().setSingle(new CountLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher()));
assertCount(index, 1000 + 15);
if (shards) {
multiIndex = index;
} else {
localIndex = index;
}
}
public static Stream<Arguments> provideArguments() {
return Stream.of(false, true).map(Arguments::of);
}
private static final Flux<Boolean> multi = Flux.just(false, true);
private static final Flux<LLScoreMode> scoreModes = Flux.just(LLScoreMode.NO_SCORES,
LLScoreMode.TOP_SCORES,
LLScoreMode.COMPLETE_NO_SCORES,
LLScoreMode.COMPLETE
);
private static final Flux<MultiSort<SearchResultKey<String>>> multiSort = Flux.just(MultiSort.topScore(),
MultiSort.randomSortField(),
MultiSort.noSort(),
MultiSort.docSort(),
MultiSort.numericSort("longsort", false),
MultiSort.numericSort("longsort", true)
);
private static Flux<LocalSearcher> getSearchers(ExpectedQueryType info) {
return Flux.push(sink -> {
try {
if (info.shard()) {
sink.next(new AdaptiveMultiSearcher());
if (info.onlyCount()) {
sink.next(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher()));
} else {
sink.next(new ScoredPagedMultiSearcher());
if (!info.sorted()) {
sink.next(new UnsortedScoredFullMultiSearcher());
}
if (!info.scored() && !info.sorted()) {
sink.next(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher()));
sink.next(new UnsortedUnscoredStreamingMultiSearcher());
}
}
} else {
sink.next(new AdaptiveLocalSearcher());
if (info.onlyCount()) {
sink.next(new CountLocalSearcher());
} else {
sink.next(new PagedLocalSearcher());
}
}
sink.complete();
} catch (IOException e) {
sink.error(e);
}
}, OverflowStrategy.BUFFER);
}
public static Stream<Arguments> provideQueryArgumentsScoreMode() {
return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
}
public static Stream<Arguments> provideQueryArgumentsSort() {
return multi
.concatMap(shard -> multiSort.map(multiSort -> Tuples.of(shard, multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
}
public static Stream<Arguments> provideQueryArgumentsScoreModeAndSort() {
return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
.concatMap(tuple -> multiSort.map(multiSort -> Tuples.of(tuple.getT1(), tuple.getT2(), multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
}
@AfterAll
public static void afterAll() {
TEMP_DB_GENERATOR.closeTempDb(tempDb).block();
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
}
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
try {
if (customSearcher != null) {
tempDb.swappableLuceneSearcher().setSingle(customSearcher);
if (shards) {
if (customSearcher instanceof MultiSearcher multiSearcher) {
tempDb.swappableLuceneSearcher().setMulti(multiSearcher);
} else {
throw new IllegalArgumentException("Expected a LuceneMultiSearcher, got a LuceneLocalSearcher: " + customSearcher.getName());
}
}
} else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher());
}
} catch (IOException e) {
fail(e);
}
return shards ? multiIndex : localIndex;
}
private static void assertCount(LuceneIndex<String, String> luceneIndex, long expected) {
Assertions.assertEquals(expected, getCount(luceneIndex));
}
private static long getCount(LuceneIndex<String, String> luceneIndex) {
luceneIndex.refresh(true).block();
var totalHitsCount = run(luceneIndex.count(null, new MatchAllDocsQuery()));
Assertions.assertTrue(totalHitsCount.exact(), "Can't get count because the total hits count is not exact");
return totalHitsCount.value();
}
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchNoDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean supportsPreciseHitsCount(LocalSearcher searcher,
ClientQueryParams<SearchResultKey<String>> query) {
if (searcher instanceof UnsortedUnscoredStreamingMultiSearcher) {
return false;
}
var scored = isScored(query.scoreMode(), Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
var sorted = isSorted(Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
if (!sorted && !scored) {
if (searcher instanceof AdaptiveMultiSearcher || searcher instanceof AdaptiveLocalSearcher) {
return false;
}
}
return true;
}
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchAllDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean isSorted(MultiSort<SearchResultKey<String>> multiSort) {
return !(multiSort.getQuerySort() instanceof NoSort);
}
private boolean isScored(LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var needsScores = LLUtils.toScoreMode(scoreMode).needsScores();
var sort =QueryParser.toSort(multiSort.getQuerySort());
if (sort != null) {
needsScores |= sort.needsScores();
}
return needsScores;
}
private List<Scored> getResults(SearchResultKeys<String> results) {
return run(results
.results()
.flatMapSequential(searchResultKey -> searchResultKey
.key()
.single()
.map(key -> new Scored(key, searchResultKey.score()))
)
.collectList());
}
}