diff --git a/pom.xml b/pom.xml index 9ed8d6f..f9960a3 100644 --- a/pom.xml +++ b/pom.xml @@ -224,7 +224,6 @@ io.projectreactor reactor-test - test org.novasearch diff --git a/src/main/java/it/cavallium/dbengine/SwappableLuceneSearcher.java b/src/main/java/it/cavallium/dbengine/SwappableLuceneSearcher.java index bc14b08..203a187 100644 --- a/src/main/java/it/cavallium/dbengine/SwappableLuceneSearcher.java +++ b/src/main/java/it/cavallium/dbengine/SwappableLuceneSearcher.java @@ -6,22 +6,20 @@ import static java.util.Objects.requireNonNullElseGet; import io.net5.buffer.api.Send; import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearchers; -import it.cavallium.dbengine.database.disk.LLLocalSingleton; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; -import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.LocalSearcher; +import it.cavallium.dbengine.lucene.searcher.MultiSearcher; import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult; import java.io.Closeable; import java.io.IOException; -import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import reactor.core.publisher.Mono; -public class SwappableLuceneSearcher implements LuceneLocalSearcher, LuceneMultiSearcher, Closeable { +public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Closeable { - private final AtomicReference single = new AtomicReference<>(null); - private final AtomicReference multi = new AtomicReference<>(null); + private final AtomicReference single = new AtomicReference<>(null); + private final AtomicReference multi = new AtomicReference<>(null); public SwappableLuceneSearcher() { @@ -61,11 +59,11 @@ public class SwappableLuceneSearcher implements LuceneLocalSearcher, LuceneMulti return multi.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); } - public void setSingle(LuceneLocalSearcher single) { + public void setSingle(LocalSearcher single) { this.single.set(single); } - public void setMulti(LuceneMultiSearcher multi) { + public void setMulti(MultiSearcher multi) { this.multi.set(multi); } diff --git a/src/main/java/it/cavallium/dbengine/database/LLDatabaseConnection.java b/src/main/java/it/cavallium/dbengine/database/LLDatabaseConnection.java index ce293d4..c519730 100644 --- a/src/main/java/it/cavallium/dbengine/database/LLDatabaseConnection.java +++ b/src/main/java/it/cavallium/dbengine/database/LLDatabaseConnection.java @@ -5,7 +5,7 @@ import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.client.IndicizerAnalyzers; import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.client.LuceneOptions; -import it.cavallium.dbengine.database.lucene.LuceneHacks; +import it.cavallium.dbengine.lucene.LuceneHacks; import java.util.List; import org.jetbrains.annotations.Nullable; import reactor.core.publisher.Mono; diff --git a/src/main/java/it/cavallium/dbengine/database/disk/CachedIndexSearcherManager.java b/src/main/java/it/cavallium/dbengine/database/disk/CachedIndexSearcherManager.java index dbe0185..0fef9b1 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/CachedIndexSearcherManager.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/CachedIndexSearcherManager.java @@ -5,7 +5,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import io.net5.buffer.api.Send; import it.cavallium.dbengine.database.LLSnapshot; -import it.cavallium.dbengine.lucene.searcher.ExecutorSearcherFactory; import java.io.IOException; import java.time.Duration; import java.util.concurrent.Executor; diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/ExecutorSearcherFactory.java b/src/main/java/it/cavallium/dbengine/database/disk/ExecutorSearcherFactory.java similarity index 91% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/ExecutorSearcherFactory.java rename to src/main/java/it/cavallium/dbengine/database/disk/ExecutorSearcherFactory.java index 55b2d1f..28ab6d9 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/ExecutorSearcherFactory.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/ExecutorSearcherFactory.java @@ -1,4 +1,4 @@ -package it.cavallium.dbengine.lucene.searcher; +package it.cavallium.dbengine.database.disk; import java.util.concurrent.Executor; import org.apache.lucene.index.IndexReader; diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDatabaseConnection.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDatabaseConnection.java index a3c853c..f19d4ad 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDatabaseConnection.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalDatabaseConnection.java @@ -8,8 +8,7 @@ import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.client.DatabaseOptions; import it.cavallium.dbengine.database.LLDatabaseConnection; import it.cavallium.dbengine.database.LLLuceneIndex; -import it.cavallium.dbengine.database.lucene.LuceneHacks; -import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher; +import it.cavallium.dbengine.lucene.LuceneHacks; import it.cavallium.dbengine.netty.JMXNettyMonitoringManager; import java.nio.file.Files; import java.nio.file.Path; diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java index ba109c0..bdbf7c3 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java @@ -16,13 +16,12 @@ import it.cavallium.dbengine.database.LLSearchResultShard; import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLTerm; import it.cavallium.dbengine.database.LLUtils; -import it.cavallium.dbengine.database.lucene.LuceneHacks; +import it.cavallium.dbengine.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.AlwaysDirectIOFSDirectory; import it.cavallium.dbengine.lucene.LuceneUtils; -import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher; +import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; -import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.LocalSearcher; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer; import java.io.IOException; import java.nio.file.Path; @@ -50,7 +49,6 @@ import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.util.Constants; import org.jetbrains.annotations.Nullable; -import org.reactivestreams.Publisher; import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.LoggerFactory; import reactor.core.publisher.Flux; @@ -62,7 +60,7 @@ import reactor.util.function.Tuple2; public class LLLocalLuceneIndex implements LLLuceneIndex { protected static final Logger logger = LoggerFactory.getLogger(LLLocalLuceneIndex.class); - private final LuceneLocalSearcher localSearcher; + private final LocalSearcher localSearcher; /** * Global lucene index scheduler. * There is only a single thread globally to not overwhelm the disk with @@ -170,7 +168,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { if (luceneHacks != null && luceneHacks.customLocalSearcher() != null) { localSearcher = luceneHacks.customLocalSearcher().get(); } else { - localSearcher = new AdaptiveLuceneLocalSearcher(); + localSearcher = new AdaptiveLocalSearcher(); } var indexWriterConfig = new IndexWriterConfig(luceneAnalyzer); diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java index aa218fa..8465c9f 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java @@ -10,16 +10,15 @@ import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.LLSearchResultShard; import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLTerm; -import it.cavallium.dbengine.database.lucene.LuceneHacks; +import it.cavallium.dbengine.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.LuceneUtils; -import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; -import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.MultiSearcher; import java.io.Closeable; import java.io.IOException; import java.nio.file.Path; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -46,7 +45,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex { private final PerFieldAnalyzerWrapper luceneAnalyzer; private final PerFieldSimilarityWrapper luceneSimilarity; - private final LuceneMultiSearcher multiSearcher; + private final MultiSearcher multiSearcher; public LLLocalMultiLuceneIndex(Path lucene, String name, @@ -83,7 +82,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex { if (luceneHacks != null && luceneHacks.customMultiSearcher() != null) { multiSearcher = luceneHacks.customMultiSearcher().get(); } else { - multiSearcher = new AdaptiveLuceneMultiSearcher(); + multiSearcher = new AdaptiveMultiSearcher(); } } diff --git a/src/main/java/it/cavallium/dbengine/database/lucene/LuceneHacks.java b/src/main/java/it/cavallium/dbengine/database/lucene/LuceneHacks.java deleted file mode 100644 index 3552a73..0000000 --- a/src/main/java/it/cavallium/dbengine/database/lucene/LuceneHacks.java +++ /dev/null @@ -1,10 +0,0 @@ -package it.cavallium.dbengine.database.lucene; - -import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; -import java.util.function.Supplier; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -public record LuceneHacks(@Nullable Supplier<@NotNull LuceneLocalSearcher> customLocalSearcher, - @Nullable Supplier<@NotNull LuceneMultiSearcher> customMultiSearcher) {} diff --git a/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDatabaseConnection.java b/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDatabaseConnection.java index 84d4d34..f31f4e2 100644 --- a/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDatabaseConnection.java +++ b/src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDatabaseConnection.java @@ -10,7 +10,7 @@ import it.cavallium.dbengine.database.LLDatabaseConnection; import it.cavallium.dbengine.database.LLKeyValueDatabase; import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.disk.LLLocalLuceneIndex; -import it.cavallium.dbengine.database.lucene.LuceneHacks; +import it.cavallium.dbengine.lucene.LuceneHacks; import it.cavallium.dbengine.netty.JMXNettyMonitoringManager; import java.util.List; import org.jetbrains.annotations.Nullable; diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/ExponentialPageLimits.java b/src/main/java/it/cavallium/dbengine/lucene/ExponentialPageLimits.java similarity index 92% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/ExponentialPageLimits.java rename to src/main/java/it/cavallium/dbengine/lucene/ExponentialPageLimits.java index ba0d6ef..470e49c 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/ExponentialPageLimits.java +++ b/src/main/java/it/cavallium/dbengine/lucene/ExponentialPageLimits.java @@ -1,6 +1,4 @@ -package it.cavallium.dbengine.lucene.searcher; - -import it.cavallium.dbengine.lucene.LuceneUtils; +package it.cavallium.dbengine.lucene; /** * 
y = 2 ^ (x + pageIndexOffset) + firstPageLimit
diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/LinearPageLimits.java b/src/main/java/it/cavallium/dbengine/lucene/LinearPageLimits.java similarity index 90% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/LinearPageLimits.java rename to src/main/java/it/cavallium/dbengine/lucene/LinearPageLimits.java index 28a2e2d..6850d07 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/LinearPageLimits.java +++ b/src/main/java/it/cavallium/dbengine/lucene/LinearPageLimits.java @@ -1,6 +1,4 @@ -package it.cavallium.dbengine.lucene.searcher; - -import it.cavallium.dbengine.lucene.LuceneUtils; +package it.cavallium.dbengine.lucene; /** * 
y = (x * factor) + firstPageLimit
diff --git a/src/main/java/it/cavallium/dbengine/lucene/LuceneHacks.java b/src/main/java/it/cavallium/dbengine/lucene/LuceneHacks.java new file mode 100644 index 0000000..e8a10e7 --- /dev/null +++ b/src/main/java/it/cavallium/dbengine/lucene/LuceneHacks.java @@ -0,0 +1,10 @@ +package it.cavallium.dbengine.lucene; + +import it.cavallium.dbengine.lucene.searcher.LocalSearcher; +import it.cavallium.dbengine.lucene.searcher.MultiSearcher; +import java.util.function.Supplier; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public record LuceneHacks(@Nullable Supplier<@NotNull LocalSearcher> customLocalSearcher, + @Nullable Supplier<@NotNull MultiSearcher> customMultiSearcher) {} diff --git a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java index 50eff75..e1dc9f6 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java +++ b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java @@ -8,12 +8,10 @@ import it.cavallium.dbengine.client.query.current.data.QueryParams; import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.database.EnglishItalianStopFilter; import it.cavallium.dbengine.database.LLKeyScore; -import it.cavallium.dbengine.database.LLSnapshot; import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.collections.DatabaseMapDictionary; import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep; import it.cavallium.dbengine.database.collections.ValueGetter; -import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.lucene.analyzer.NCharGramAnalyzer; import it.cavallium.dbengine.lucene.analyzer.NCharGramEdgeAnalyzer; @@ -22,9 +20,7 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity; import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer; import it.cavallium.dbengine.lucene.mlt.BigCompositeReader; import it.cavallium.dbengine.lucene.mlt.MultiMoreLikeThis; -import it.cavallium.dbengine.lucene.searcher.ExponentialPageLimits; import it.cavallium.dbengine.lucene.searcher.LocalQueryParams; -import it.cavallium.dbengine.lucene.searcher.PageLimits; import it.cavallium.dbengine.lucene.similarity.NGramSimilarity; import java.io.EOFException; import java.io.IOException; @@ -50,10 +46,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BooleanQuery.Builder; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -79,7 +73,6 @@ import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.LoggerFactory; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; import reactor.util.concurrent.Queues; import reactor.util.function.Tuple2; diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/PageLimits.java b/src/main/java/it/cavallium/dbengine/lucene/PageLimits.java similarity index 80% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/PageLimits.java rename to src/main/java/it/cavallium/dbengine/lucene/PageLimits.java index 2051d86..790acf1 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/PageLimits.java +++ b/src/main/java/it/cavallium/dbengine/lucene/PageLimits.java @@ -1,4 +1,4 @@ -package it.cavallium.dbengine.lucene.searcher; +package it.cavallium.dbengine.lucene; import it.cavallium.dbengine.lucene.LuceneUtils; diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/SinglePageLimits.java b/src/main/java/it/cavallium/dbengine/lucene/SinglePageLimits.java similarity index 89% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/SinglePageLimits.java rename to src/main/java/it/cavallium/dbengine/lucene/SinglePageLimits.java index 7d019ee..6dbd5e1 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/SinglePageLimits.java +++ b/src/main/java/it/cavallium/dbengine/lucene/SinglePageLimits.java @@ -1,4 +1,4 @@ -package it.cavallium.dbengine.lucene.searcher; +package it.cavallium.dbengine.lucene; public class SinglePageLimits implements PageLimits { diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoringShardsCollectorManager.java b/src/main/java/it/cavallium/dbengine/lucene/collector/ScoringShardsCollectorManager.java similarity index 98% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/ScoringShardsCollectorManager.java rename to src/main/java/it/cavallium/dbengine/lucene/collector/ScoringShardsCollectorManager.java index 5805e4b..61306bd 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoringShardsCollectorManager.java +++ b/src/main/java/it/cavallium/dbengine/lucene/collector/ScoringShardsCollectorManager.java @@ -1,4 +1,4 @@ -package it.cavallium.dbengine.lucene.searcher; +package it.cavallium.dbengine.lucene.collector; import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.TIE_BREAKER; diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneLocalSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLocalSearcher.java similarity index 81% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneLocalSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLocalSearcher.java index db315d0..d06a2be 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneLocalSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLocalSearcher.java @@ -5,16 +5,14 @@ import io.net5.buffer.api.internal.ResourceSupport; import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearchers; -import it.cavallium.dbengine.database.disk.LLIndexSearchers.UnshardedIndexSearchers; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; -import org.apache.lucene.search.IndexSearcher; import reactor.core.publisher.Mono; -public class AdaptiveLuceneLocalSearcher implements LuceneLocalSearcher { +public class AdaptiveLocalSearcher implements LocalSearcher { - private static final LuceneLocalSearcher localSearcher = new SimpleLuceneLocalSearcher(); + private static final LocalSearcher localSearcher = new PagedLocalSearcher(); - private static final LuceneLocalSearcher countSearcher = new CountLuceneLocalSearcher(); + private static final LocalSearcher countSearcher = new CountLocalSearcher(); @Override public Mono> collect(Mono> indexSearcher, diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java similarity index 76% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneMultiSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java index 8476a95..0ca4853 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java @@ -8,23 +8,23 @@ import java.io.Closeable; import java.io.IOException; import reactor.core.publisher.Mono; -public class AdaptiveLuceneMultiSearcher implements LuceneMultiSearcher, Closeable { +public class AdaptiveMultiSearcher implements MultiSearcher, Closeable { - private static final LuceneMultiSearcher count - = new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher()); + private static final MultiSearcher count + = new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher()); - private static final LuceneMultiSearcher scoredSimple = new ScoredSimpleLuceneMultiSearcher(); + private static final MultiSearcher scoredSimple = new ScoredPagedMultiSearcher(); - private static final LuceneMultiSearcher unsortedUnscoredPaged - = new SimpleUnsortedUnscoredLuceneMultiSearcher(new SimpleLuceneLocalSearcher()); + private static final MultiSearcher unsortedUnscoredPaged + = new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher()); - private static final LuceneMultiSearcher unsortedUnscoredContinuous - = new UnsortedUnscoredContinuousLuceneMultiSearcher(); + private static final MultiSearcher unsortedUnscoredContinuous + = new UnsortedUnscoredStreamingMultiSearcher(); - private final UnsortedScoredFullLuceneMultiSearcher scoredFull; + private final UnsortedScoredFullMultiSearcher scoredFull; - public AdaptiveLuceneMultiSearcher() throws IOException { - scoredFull = new UnsortedScoredFullLuceneMultiSearcher(); + public AdaptiveMultiSearcher() throws IOException { + scoredFull = new UnsortedScoredFullMultiSearcher(); } @Override @@ -76,6 +76,6 @@ public class AdaptiveLuceneMultiSearcher implements LuceneMultiSearcher, Closeab @Override public String getName() { - return "adaptivemulti"; + return "adaptive local"; } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/CalculatedResults.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/CalculatedResults.java deleted file mode 100644 index f4cef12..0000000 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/CalculatedResults.java +++ /dev/null @@ -1,7 +0,0 @@ -package it.cavallium.dbengine.lucene.searcher; - -import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; -import it.cavallium.dbengine.database.LLKeyScore; -import reactor.core.publisher.Flux; - -record CalculatedResults(TotalHitsCount totalHitsCount, Flux firstPageHitsFlux) {} diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLocalSearcher.java similarity index 92% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/CountLocalSearcher.java index ddbd9ba..08ab49c 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLocalSearcher.java @@ -6,12 +6,11 @@ import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; -import org.apache.lucene.search.IndexSearcher; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -public class CountLuceneLocalSearcher implements LuceneLocalSearcher { +public class CountLocalSearcher implements LocalSearcher { @Override public Mono> collect(Mono> indexSearcherMono, @@ -45,6 +44,6 @@ public class CountLuceneLocalSearcher implements LuceneLocalSearcher { @Override public String getName() { - return "count"; + return "count local"; } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/CurrentPageInfo.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/CurrentPageInfo.java index 6437f65..61c318d 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/CurrentPageInfo.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/CurrentPageInfo.java @@ -6,8 +6,10 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.jetbrains.annotations.Nullable; -record CurrentPageInfo(@Nullable ScoreDoc last, long remainingLimit, int pageIndex) { +public record CurrentPageInfo(@Nullable ScoreDoc last, long remainingLimit, int pageIndex) { - public static final Comparator TIE_BREAKER = Comparator.comparingInt((d) -> d.shardIndex); + public static final Comparator TIE_BREAKER = Comparator + .comparingInt((d) -> d.shardIndex) + .thenComparingInt(d -> -d.doc); public static final CurrentPageInfo EMPTY_STATUS = new CurrentPageInfo(null, 0, 0); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java index 8967054..32b0293 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java @@ -1,5 +1,6 @@ package it.cavallium.dbengine.lucene.searcher; +import it.cavallium.dbengine.lucene.PageLimits; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Sort; diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneLocalSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalSearcher.java similarity index 94% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneLocalSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/LocalSearcher.java index 1a57498..119fbd2 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneLocalSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalSearcher.java @@ -4,7 +4,7 @@ import io.net5.buffer.api.Send; import it.cavallium.dbengine.database.disk.LLIndexSearcher; import reactor.core.publisher.Mono; -public interface LuceneLocalSearcher { +public interface LocalSearcher { /** * @param indexSearcherMono Lucene index searcher diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java similarity index 94% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneMultiSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java index fb81170..00b3fbe 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java @@ -5,7 +5,7 @@ import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearchers; import reactor.core.publisher.Mono; -public interface LuceneMultiSearcher extends LuceneLocalSearcher { +public interface MultiSearcher extends LocalSearcher { /** * @param indexSearchersMono Lucene index searcher diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java similarity index 95% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java index 3642134..0ddb1b5 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java @@ -1,7 +1,6 @@ package it.cavallium.dbengine.lucene.searcher; import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS; -import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.FIRST_PAGE_LIMIT; import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT; import io.net5.buffer.api.Send; @@ -10,13 +9,11 @@ import it.cavallium.dbengine.database.LLKeyScore; import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.disk.LLIndexSearcher; import it.cavallium.dbengine.database.disk.LLIndexSearchers; -import it.cavallium.dbengine.database.disk.LLIndexSearchers.UnshardedIndexSearchers; import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.NoSuchElementException; import java.util.Objects; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; @@ -29,7 +26,7 @@ import reactor.core.publisher.Mono; import reactor.core.publisher.SynchronousSink; import reactor.core.scheduler.Schedulers; -public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher { +public class PagedLocalSearcher implements LocalSearcher { @Override public Mono> collect(Mono> indexSearcherMono, @@ -69,7 +66,7 @@ public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher { @Override public String getName() { - return "simplelocal"; + return "paged local"; } /** @@ -187,7 +184,7 @@ public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher { } else if (s.pageIndex() == 0 || (s.last() != null && s.remainingLimit() > 0)) { TopDocs pageTopDocs; try { - TopDocsCollector collector = TopDocsSearcher.getTopDocsCollector(queryParams.sort(), + TopDocsCollector collector = TopDocsCollectorUtils.getTopDocsCollector(queryParams.sort(), currentPageLimit, s.last(), LuceneUtils.totalHitsThreshold(), allowPagination, queryParams.isScored()); indexSearchers.get(0).search(queryParams.query(), collector); diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/PaginationInfo.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/PaginationInfo.java index ad207c0..2d37dd8 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/PaginationInfo.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/PaginationInfo.java @@ -1,7 +1,6 @@ package it.cavallium.dbengine.lucene.searcher; -import java.util.Comparator; -import org.apache.lucene.search.ScoreDoc; +import it.cavallium.dbengine.lucene.PageLimits; public record PaginationInfo(long totalLimit, long firstPageOffset, PageLimits pageLimits, boolean forceSinglePage) { diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java similarity index 94% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneMultiSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java index 1e63219..9b3c74f 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java @@ -1,6 +1,5 @@ package it.cavallium.dbengine.lucene.searcher; -import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS; import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT; import io.net5.buffer.api.Send; @@ -8,19 +7,16 @@ import it.cavallium.dbengine.database.LLKeyScore; import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.database.disk.LLIndexSearchers; import it.cavallium.dbengine.lucene.LuceneUtils; +import it.cavallium.dbengine.lucene.PageLimits; +import it.cavallium.dbengine.lucene.collector.ScoringShardsCollectorManager; import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput; -import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.TotalHits.Relation; import org.jetbrains.annotations.Nullable; import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.LoggerFactory; @@ -28,11 +24,11 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -public class ScoredSimpleLuceneMultiSearcher implements LuceneMultiSearcher { +public class ScoredPagedMultiSearcher implements MultiSearcher { - protected static final Logger logger = LoggerFactory.getLogger(ScoredSimpleLuceneMultiSearcher.class); + protected static final Logger logger = LoggerFactory.getLogger(ScoredPagedMultiSearcher.class); - public ScoredSimpleLuceneMultiSearcher() { + public ScoredPagedMultiSearcher() { } @Override @@ -214,6 +210,6 @@ public class ScoredSimpleLuceneMultiSearcher implements LuceneMultiSearcher { @Override public String getName() { - return "scoredsimplemulti"; + return "scored paged multi"; } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsCollectorUtils.java similarity index 98% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsCollectorUtils.java index 60171d8..1216e3f 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsCollectorUtils.java @@ -10,7 +10,7 @@ import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.search.TopFieldCollector; import org.apache.lucene.search.TopScoreDocCollector; -class TopDocsSearcher { +class TopDocsCollectorUtils { @SuppressWarnings({"unchecked", "rawtypes"}) public static TopDocsCollector getTopDocsCollector(Sort luceneSort, diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullLuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java similarity index 91% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullLuceneMultiSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java index 8fb6770..462dccc 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullLuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java @@ -20,13 +20,13 @@ import org.warp.commonutils.log.LoggerFactory; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearcher, Closeable { +public class UnsortedScoredFullMultiSearcher implements MultiSearcher, Closeable { - protected static final Logger logger = LoggerFactory.getLogger(UnsortedScoredFullLuceneMultiSearcher.class); + protected static final Logger logger = LoggerFactory.getLogger(UnsortedScoredFullMultiSearcher.class); private final LLTempLMDBEnv env; - public UnsortedScoredFullLuceneMultiSearcher() throws IOException { + public UnsortedScoredFullMultiSearcher() throws IOException { this.env = new LLTempLMDBEnv(); } @@ -46,7 +46,7 @@ public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearche return queryParamsMono.flatMap(queryParams2 -> { Objects.requireNonNull(queryParams2.scoreMode(), "ScoreMode must not be null"); if (queryParams2.sort() != null && queryParams2.sort() != Sort.RELEVANCE) { - throw new IllegalArgumentException(UnsortedScoredFullLuceneMultiSearcher.this.getClass().getSimpleName() + throw new IllegalArgumentException(UnsortedScoredFullMultiSearcher.this.getClass().getSimpleName() + " doesn't support sorted queries"); } @@ -115,6 +115,6 @@ public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearche @Override public String getName() { - return "scoredfullmulti"; + return "unsorted scored full multi"; } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleUnsortedUnscoredLuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java similarity index 93% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleUnsortedUnscoredLuceneMultiSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java index e3e085d..9e37675 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleUnsortedUnscoredLuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java @@ -13,11 +13,11 @@ import java.util.List; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class SimpleUnsortedUnscoredLuceneMultiSearcher implements LuceneMultiSearcher { +public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher { - private final LuceneLocalSearcher localSearcher; + private final LocalSearcher localSearcher; - public SimpleUnsortedUnscoredLuceneMultiSearcher(LuceneLocalSearcher localSearcher) { + public UnsortedUnscoredSimpleMultiSearcher(LocalSearcher localSearcher) { this.localSearcher = localSearcher; } @@ -103,6 +103,6 @@ public class SimpleUnsortedUnscoredLuceneMultiSearcher implements LuceneMultiSea @Override public String getName() { - return "simpleunsortedunscoredmulti"; + return "unsorted unscored simple multi"; } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredContinuousLuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java similarity index 97% rename from src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredContinuousLuceneMultiSearcher.java rename to src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java index 3be653b..927766a 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredContinuousLuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java @@ -23,7 +23,7 @@ import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; import reactor.util.concurrent.Queues; -public class UnsortedUnscoredContinuousLuceneMultiSearcher implements LuceneMultiSearcher { +public class UnsortedUnscoredStreamingMultiSearcher implements MultiSearcher { private static final Scheduler UNSCORED_UNSORTED_EXECUTOR = Schedulers.newBoundedElastic( Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE, @@ -117,6 +117,6 @@ public class UnsortedUnscoredContinuousLuceneMultiSearcher implements LuceneMult @Override public String getName() { - return "unsortedunscoredcontinuousmulti"; + return "unsorted unscored streaming multi"; } } diff --git a/src/test/java/it/cavallium/dbengine/DbTestUtils.java b/src/test/java/it/cavallium/dbengine/DbTestUtils.java index dd7faad..fe30d7a 100644 --- a/src/test/java/it/cavallium/dbengine/DbTestUtils.java +++ b/src/test/java/it/cavallium/dbengine/DbTestUtils.java @@ -47,38 +47,6 @@ public class DbTestUtils { return "0123456789".repeat(1024); } - public static void run(Flux publisher) { - publisher.subscribeOn(Schedulers.immediate()).blockLast(); - } - - public static void runVoid(Mono publisher) { - publisher.then().subscribeOn(Schedulers.immediate()).block(); - } - - public static T run(Mono publisher) { - return publisher.subscribeOn(Schedulers.immediate()).block(); - } - - public static T run(boolean shouldFail, Mono publisher) { - return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> { - if (shouldFail) { - return mono.onErrorResume(ex -> Mono.empty()); - } else { - return mono; - } - }).block(); - } - - public static void runVoid(boolean shouldFail, Mono publisher) { - publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> { - if (shouldFail) { - return mono.onErrorResume(ex -> Mono.empty()); - } else { - return mono; - } - }).block(); - } - public static record TestAllocator(PooledBufferAllocator allocator) {} public static TestAllocator newAllocator() { diff --git a/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java b/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java index 3fe96aa..616296b 100644 --- a/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java +++ b/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java @@ -12,11 +12,9 @@ import it.cavallium.dbengine.client.NRTCachingOptions; import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.LLKeyValueDatabase; import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection; -import it.cavallium.dbengine.database.lucene.LuceneHacks; +import it.cavallium.dbengine.lucene.LuceneHacks; import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer; import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity; -import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -36,7 +34,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator { private static final Optional NRT = Optional.empty(); private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5), - false, true, Optional.empty(), true, NRT, -1, true, true); + false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false); @Override public Mono openTempDb(TestAllocator allocator) { diff --git a/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java b/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java index 86b81ab..a71ff6d 100644 --- a/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java +++ b/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java @@ -8,7 +8,7 @@ import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.client.LuceneOptions; import it.cavallium.dbengine.client.NRTCachingOptions; import it.cavallium.dbengine.database.Column; -import it.cavallium.dbengine.database.lucene.LuceneHacks; +import it.cavallium.dbengine.lucene.LuceneHacks; import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection; import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer; import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity; @@ -22,7 +22,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator { private static final Optional NRT = Optional.empty(); private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5), - false, true, Optional.empty(), true, NRT, -1, true, true); + false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false); @Override public Mono openTempDb(TestAllocator allocator) { diff --git a/src/test/java/it/cavallium/dbengine/SyncUtils.java b/src/test/java/it/cavallium/dbengine/SyncUtils.java new file mode 100644 index 0000000..f685d7d --- /dev/null +++ b/src/test/java/it/cavallium/dbengine/SyncUtils.java @@ -0,0 +1,40 @@ +package it.cavallium.dbengine; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +public class SyncUtils { + + public static void run(Flux publisher) { + publisher.subscribeOn(Schedulers.immediate()).blockLast(); + } + + public static void runVoid(Mono publisher) { + publisher.then().subscribeOn(Schedulers.immediate()).block(); + } + + public static T run(Mono publisher) { + return publisher.subscribeOn(Schedulers.immediate()).block(); + } + + public static T run(boolean shouldFail, Mono publisher) { + return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> { + if (shouldFail) { + return mono.onErrorResume(ex -> Mono.empty()); + } else { + return mono; + } + }).block(); + } + + public static void runVoid(boolean shouldFail, Mono publisher) { + publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> { + if (shouldFail) { + return mono.onErrorResume(ex -> Mono.empty()); + } else { + return mono; + } + }).block(); + } +} diff --git a/src/test/java/it/cavallium/dbengine/TestDictionaryMap.java b/src/test/java/it/cavallium/dbengine/TestDictionaryMap.java index cc1e3f6..ae4d638 100644 --- a/src/test/java/it/cavallium/dbengine/TestDictionaryMap.java +++ b/src/test/java/it/cavallium/dbengine/TestDictionaryMap.java @@ -1,6 +1,7 @@ package it.cavallium.dbengine; import static it.cavallium.dbengine.DbTestUtils.*; +import static it.cavallium.dbengine.SyncUtils.*; import it.cavallium.dbengine.database.LLUtils; import it.cavallium.dbengine.DbTestUtils.TestAllocator; diff --git a/src/test/java/it/cavallium/dbengine/TestDictionaryMapDeep.java b/src/test/java/it/cavallium/dbengine/TestDictionaryMapDeep.java index 8a29604..0083429 100644 --- a/src/test/java/it/cavallium/dbengine/TestDictionaryMapDeep.java +++ b/src/test/java/it/cavallium/dbengine/TestDictionaryMapDeep.java @@ -5,12 +5,11 @@ import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.isCIMode; import static it.cavallium.dbengine.DbTestUtils.newAllocator; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; -import static it.cavallium.dbengine.DbTestUtils.run; -import static it.cavallium.dbengine.DbTestUtils.runVoid; import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMap; import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryMap; import static it.cavallium.dbengine.DbTestUtils.tempDb; import static it.cavallium.dbengine.DbTestUtils.tempDictionary; +import static it.cavallium.dbengine.SyncUtils.*; import io.net5.buffer.api.internal.ResourceSupport; import it.cavallium.dbengine.DbTestUtils.TestAllocator; diff --git a/src/test/java/it/cavallium/dbengine/TestLLDictionary.java b/src/test/java/it/cavallium/dbengine/TestLLDictionary.java index 05fded5..1798b1c 100644 --- a/src/test/java/it/cavallium/dbengine/TestLLDictionary.java +++ b/src/test/java/it/cavallium/dbengine/TestLLDictionary.java @@ -3,6 +3,7 @@ package it.cavallium.dbengine; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.newAllocator; +import static it.cavallium.dbengine.SyncUtils.*; import static org.junit.jupiter.api.Assertions.assertEquals; import io.net5.buffer.api.Buffer; @@ -115,38 +116,6 @@ public abstract class TestLLDictionary { } } - private void run(Flux publisher) { - publisher.subscribeOn(Schedulers.immediate()).blockLast(); - } - - private void runVoid(Mono publisher) { - publisher.then().subscribeOn(Schedulers.immediate()).block(); - } - - private T run(Mono publisher) { - return publisher.subscribeOn(Schedulers.immediate()).block(); - } - - private T run(boolean shouldFail, Mono publisher) { - return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> { - if (shouldFail) { - return mono.onErrorResume(ex -> Mono.empty()); - } else { - return mono; - } - }).block(); - } - - private void runVoid(boolean shouldFail, Mono publisher) { - publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> { - if (shouldFail) { - return mono.onErrorResume(ex -> Mono.empty()); - } else { - return mono; - } - }).block(); - } - @Test public void testNoOp() { } diff --git a/src/test/java/it/cavallium/dbengine/TestLLDictionaryLeaks.java b/src/test/java/it/cavallium/dbengine/TestLLDictionaryLeaks.java index 026107d..7793086 100644 --- a/src/test/java/it/cavallium/dbengine/TestLLDictionaryLeaks.java +++ b/src/test/java/it/cavallium/dbengine/TestLLDictionaryLeaks.java @@ -3,6 +3,7 @@ package it.cavallium.dbengine; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.newAllocator; +import static it.cavallium.dbengine.SyncUtils.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -131,9 +132,9 @@ public abstract class TestLLDictionaryLeaks { public void testGet(UpdateMode updateMode) { var dict = getDict(updateMode); var key = Mono.fromCallable(() -> fromString("test")); - DbTestUtils.runVoid(dict.get(null, key).then().transform(LLUtils::handleDiscard)); - DbTestUtils.runVoid(dict.get(null, key, true).then().transform(LLUtils::handleDiscard)); - DbTestUtils.runVoid(dict.get(null, key, false).then().transform(LLUtils::handleDiscard)); + runVoid(dict.get(null, key).then().transform(LLUtils::handleDiscard)); + runVoid(dict.get(null, key, true).then().transform(LLUtils::handleDiscard)); + runVoid(dict.get(null, key, false).then().transform(LLUtils::handleDiscard)); } @ParameterizedTest @@ -142,14 +143,14 @@ public abstract class TestLLDictionaryLeaks { var dict = getDict(updateMode); var key = Mono.fromCallable(() -> fromString("test-key")); var value = Mono.fromCallable(() -> fromString("test-value")); - DbTestUtils.runVoid(dict.put(key, value, resultType).then().doOnDiscard(Send.class, Send::close)); + runVoid(dict.put(key, value, resultType).then().doOnDiscard(Send.class, Send::close)); } @ParameterizedTest @MethodSource("provideArguments") public void testGetUpdateMode(UpdateMode updateMode) { var dict = getDict(updateMode); - assertEquals(updateMode, DbTestUtils.run(dict.getUpdateMode())); + assertEquals(updateMode, run(dict.getUpdateMode())); } @ParameterizedTest @@ -157,13 +158,13 @@ public abstract class TestLLDictionaryLeaks { public void testUpdate(UpdateMode updateMode, UpdateReturnMode updateReturnMode) { var dict = getDict(updateMode); var key = Mono.fromCallable(() -> fromString("test-key")); - DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, + runVoid(updateMode == UpdateMode.DISALLOW, dict.update(key, old -> old, updateReturnMode, true).then().transform(LLUtils::handleDiscard) ); - DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, + runVoid(updateMode == UpdateMode.DISALLOW, dict.update(key, old -> old, updateReturnMode, false).then().transform(LLUtils::handleDiscard) ); - DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, + runVoid(updateMode == UpdateMode.DISALLOW, dict.update(key, old -> old, updateReturnMode).then().transform(LLUtils::handleDiscard) ); } @@ -173,13 +174,13 @@ public abstract class TestLLDictionaryLeaks { public void testUpdateAndGetDelta(UpdateMode updateMode) { var dict = getDict(updateMode); var key = Mono.fromCallable(() -> fromString("test-key")); - DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, + runVoid(updateMode == UpdateMode.DISALLOW, dict.updateAndGetDelta(key, old -> old, true).then().transform(LLUtils::handleDiscard) ); - DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, + runVoid(updateMode == UpdateMode.DISALLOW, dict.updateAndGetDelta(key, old -> old, false).then().transform(LLUtils::handleDiscard) ); - DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW, + runVoid(updateMode == UpdateMode.DISALLOW, dict.updateAndGetDelta(key, old -> old).then().transform(LLUtils::handleDiscard) ); } @@ -188,7 +189,7 @@ public abstract class TestLLDictionaryLeaks { @MethodSource("provideArguments") public void testClear(UpdateMode updateMode) { var dict = getDict(updateMode); - DbTestUtils.runVoid(dict.clear()); + runVoid(dict.clear()); } @ParameterizedTest @@ -196,6 +197,6 @@ public abstract class TestLLDictionaryLeaks { public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) { var dict = getDict(updateMode); var key = Mono.fromCallable(() -> fromString("test-key")); - DbTestUtils.runVoid(dict.remove(key, resultType).then().doOnDiscard(Send.class, Send::close)); + runVoid(dict.remove(key, resultType).then().doOnDiscard(Send.class, Send::close)); } } diff --git a/src/test/java/it/cavallium/dbengine/TestLuceneIndex.java b/src/test/java/it/cavallium/dbengine/TestLuceneIndex.java index c3f93ca..e9611f2 100644 --- a/src/test/java/it/cavallium/dbengine/TestLuceneIndex.java +++ b/src/test/java/it/cavallium/dbengine/TestLuceneIndex.java @@ -3,6 +3,7 @@ package it.cavallium.dbengine; import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; import static it.cavallium.dbengine.DbTestUtils.newAllocator; +import static it.cavallium.dbengine.SyncUtils.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -11,29 +12,16 @@ import it.cavallium.dbengine.DbTestUtils.TestAllocator; import it.cavallium.dbengine.client.LuceneIndex; import it.cavallium.dbengine.client.MultiSort; import it.cavallium.dbengine.client.SearchResultKey; -import it.cavallium.dbengine.client.SearchResultKeys; -import it.cavallium.dbengine.client.query.ClientQueryParams; -import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder; -import it.cavallium.dbengine.client.query.QueryParser; import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery; -import it.cavallium.dbengine.client.query.current.data.MatchNoDocsQuery; -import it.cavallium.dbengine.client.query.current.data.NoSort; -import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.LLScoreMode; -import it.cavallium.dbengine.database.LLUtils; -import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher; -import it.cavallium.dbengine.lucene.searcher.CountLuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher; -import it.cavallium.dbengine.lucene.searcher.UnsortedScoredFullLuceneMultiSearcher; -import it.cavallium.dbengine.lucene.searcher.ScoredSimpleLuceneMultiSearcher; -import it.cavallium.dbengine.lucene.searcher.SimpleLuceneLocalSearcher; -import it.cavallium.dbengine.lucene.searcher.SimpleUnsortedUnscoredLuceneMultiSearcher; -import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredContinuousLuceneMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher; +import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.CountLocalSearcher; +import it.cavallium.dbengine.lucene.searcher.LocalSearcher; +import it.cavallium.dbengine.lucene.searcher.MultiSearcher; +import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredSimpleMultiSearcher; import java.io.IOException; -import java.util.List; import java.util.Objects; import java.util.stream.Stream; import org.jetbrains.annotations.Nullable; @@ -47,9 +35,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.LoggerFactory; import reactor.core.publisher.Flux; -import reactor.core.publisher.FluxSink.OverflowStrategy; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; import reactor.util.function.Tuples; public class TestLuceneIndex { @@ -92,38 +77,6 @@ public class TestLuceneIndex { MultiSort.numericSort("longsort", true) ); - private static Flux getSearchers(ExpectedQueryType info) { - return Flux.push(sink -> { - try { - if (info.shard()) { - sink.next(new AdaptiveLuceneMultiSearcher()); - if (info.onlyCount()) { - sink.next(new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher())); - } else { - sink.next(new ScoredSimpleLuceneMultiSearcher()); - if (!info.sorted()) { - sink.next(new UnsortedScoredFullLuceneMultiSearcher()); - } - if (!info.scored() && !info.sorted()) { - sink.next(new SimpleUnsortedUnscoredLuceneMultiSearcher(new SimpleLuceneLocalSearcher())); - sink.next(new UnsortedUnscoredContinuousLuceneMultiSearcher()); - } - } - } else { - sink.next(new AdaptiveLuceneLocalSearcher()); - if (info.onlyCount()) { - sink.next(new CountLuceneLocalSearcher()); - } else { - sink.next(new SimpleLuceneLocalSearcher()); - } - } - sink.complete(); - } catch (IOException e) { - sink.error(e); - } - }, OverflowStrategy.BUFFER); - } - public static Stream provideQueryArgumentsScoreMode() { return multi .concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode))) @@ -153,7 +106,7 @@ public class TestLuceneIndex { destroyAllocator(allocator); } - private LuceneIndex getLuceneIndex(boolean shards, @Nullable LuceneLocalSearcher customSearcher) { + private LuceneIndex getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) { LuceneIndex index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti)); index.updateDocument("test-key-1", "0123456789").block(); index.updateDocument("test-key-2", "test 0123456789 test word").block(); @@ -171,22 +124,22 @@ public class TestLuceneIndex { index.updateDocument("test-key-14", "2999").block(); index.updateDocument("test-key-15", "3902").block(); Flux.range(1, 1000).concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i)).blockLast(); - tempDb.swappableLuceneSearcher().setSingle(new CountLuceneLocalSearcher()); - tempDb.swappableLuceneSearcher().setMulti(new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher())); + tempDb.swappableLuceneSearcher().setSingle(new CountLocalSearcher()); + tempDb.swappableLuceneSearcher().setMulti(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher())); assertCount(index, 1000 + 15); try { if (customSearcher != null) { tempDb.swappableLuceneSearcher().setSingle(customSearcher); if (shards) { - if (customSearcher instanceof LuceneMultiSearcher multiSearcher) { + if (customSearcher instanceof MultiSearcher multiSearcher) { tempDb.swappableLuceneSearcher().setMulti(multiSearcher); } else { throw new IllegalArgumentException("Expected a LuceneMultiSearcher, got a LuceneLocalSearcher: " + customSearcher.getName()); } } } else { - tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLuceneLocalSearcher()); - tempDb.swappableLuceneSearcher().setMulti(new AdaptiveLuceneMultiSearcher()); + tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher()); + tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher()); } } catch (IOException e) { fail(e); @@ -194,38 +147,6 @@ public class TestLuceneIndex { return index; } - private void run(Flux publisher) { - publisher.subscribeOn(Schedulers.immediate()).blockLast(); - } - - private void runVoid(Mono publisher) { - publisher.then().subscribeOn(Schedulers.immediate()).block(); - } - - private T run(Mono publisher) { - return publisher.subscribeOn(Schedulers.immediate()).block(); - } - - private T run(boolean shouldFail, Mono publisher) { - return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> { - if (shouldFail) { - return mono.onErrorResume(ex -> Mono.empty()); - } else { - return mono; - } - }).block(); - } - - private void runVoid(boolean shouldFail, Mono publisher) { - publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> { - if (shouldFail) { - return mono.onErrorResume(ex -> Mono.empty()); - } else { - return mono; - } - }).block(); - } - private void assertCount(LuceneIndex luceneIndex, long expected) { Assertions.assertEquals(expected, getCount(luceneIndex)); } @@ -291,95 +212,4 @@ public class TestLuceneIndex { assertCount(luceneIndex, prevCount + 1); } - @ParameterizedTest - @MethodSource("provideQueryArgumentsScoreModeAndSort") - public void testSearchNoDocs(boolean shards, LLScoreMode scoreMode, MultiSort> multiSort) { - var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList()); - for (LuceneLocalSearcher searcher : searchers) { - log.info("Using searcher \"{}\"", searcher.getName()); - - var luceneIndex = getLuceneIndex(shards, searcher); - ClientQueryParamsBuilder> queryBuilder = ClientQueryParams.builder(); - queryBuilder.query(new MatchNoDocsQuery()); - queryBuilder.snapshot(null); - queryBuilder.scoreMode(scoreMode); - queryBuilder.sort(multiSort); - var query = queryBuilder.build(); - try (var results = run(luceneIndex.search(query)).receive()) { - var hits = results.totalHitsCount(); - if (supportsPreciseHitsCount(searcher, query)) { - assertEquals(new TotalHitsCount(0, true), hits); - } - - var keys = getResults(results); - assertEquals(List.of(), keys); - } - } - } - - private boolean supportsPreciseHitsCount(LuceneLocalSearcher searcher, - ClientQueryParams> query) { - if (searcher instanceof UnsortedUnscoredContinuousLuceneMultiSearcher) { - return false; - } - var scored = isScored(query.scoreMode(), Objects.requireNonNullElse(query.sort(), MultiSort.noSort())); - var sorted = isSorted(Objects.requireNonNullElse(query.sort(), MultiSort.noSort())); - if (!sorted && !scored) { - if (searcher instanceof AdaptiveLuceneMultiSearcher || searcher instanceof AdaptiveLuceneLocalSearcher) { - return false; - } - } - return true; - } - - @ParameterizedTest - @MethodSource("provideQueryArgumentsScoreModeAndSort") - public void testSearchAllDocs(boolean shards, LLScoreMode scoreMode, MultiSort> multiSort) { - var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList()); - for (LuceneLocalSearcher searcher : searchers) { - log.info("Using searcher \"{}\"", searcher.getName()); - - var luceneIndex = getLuceneIndex(shards, searcher); - ClientQueryParamsBuilder> queryBuilder = ClientQueryParams.builder(); - queryBuilder.query(new MatchNoDocsQuery()); - queryBuilder.snapshot(null); - queryBuilder.scoreMode(scoreMode); - queryBuilder.sort(multiSort); - var query = queryBuilder.build(); - try (var results = run(luceneIndex.search(query)).receive()) { - var hits = results.totalHitsCount(); - if (supportsPreciseHitsCount(searcher, query)) { - assertEquals(new TotalHitsCount(0, true), hits); - } - - var keys = getResults(results); - assertEquals(List.of(), keys); - } - } - } - - private boolean isSorted(MultiSort> multiSort) { - return !(multiSort.getQuerySort() instanceof NoSort); - } - - private boolean isScored(LLScoreMode scoreMode, MultiSort> multiSort) { - var needsScores = LLUtils.toScoreMode(scoreMode).needsScores(); - var sort =QueryParser.toSort(multiSort.getQuerySort()); - if (sort != null) { - needsScores |= sort.needsScores(); - } - return needsScores; - } - - private List getResults(SearchResultKeys results) { - return run(results - .results() - .flatMapSequential(searchResultKey -> searchResultKey - .key() - .single() - .map(key -> new Scored(key, searchResultKey.score())) - ) - .collectList()); - } - } diff --git a/src/test/java/it/cavallium/dbengine/TestLuceneSearches.java b/src/test/java/it/cavallium/dbengine/TestLuceneSearches.java new file mode 100644 index 0000000..69d462a --- /dev/null +++ b/src/test/java/it/cavallium/dbengine/TestLuceneSearches.java @@ -0,0 +1,307 @@ +package it.cavallium.dbengine; + +import static it.cavallium.dbengine.DbTestUtils.destroyAllocator; +import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks; +import static it.cavallium.dbengine.DbTestUtils.newAllocator; +import static it.cavallium.dbengine.SyncUtils.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import it.cavallium.dbengine.DbTestUtils.TempDb; +import it.cavallium.dbengine.DbTestUtils.TestAllocator; +import it.cavallium.dbengine.client.LuceneIndex; +import it.cavallium.dbengine.client.MultiSort; +import it.cavallium.dbengine.client.SearchResultKey; +import it.cavallium.dbengine.client.SearchResultKeys; +import it.cavallium.dbengine.client.query.ClientQueryParams; +import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder; +import it.cavallium.dbengine.client.query.QueryParser; +import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery; +import it.cavallium.dbengine.client.query.current.data.MatchNoDocsQuery; +import it.cavallium.dbengine.client.query.current.data.NoSort; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; +import it.cavallium.dbengine.database.LLLuceneIndex; +import it.cavallium.dbengine.database.LLScoreMode; +import it.cavallium.dbengine.database.LLUtils; +import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher; +import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.CountLocalSearcher; +import it.cavallium.dbengine.lucene.searcher.LocalSearcher; +import it.cavallium.dbengine.lucene.searcher.MultiSearcher; +import it.cavallium.dbengine.lucene.searcher.ScoredPagedMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.PagedLocalSearcher; +import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredSimpleMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.UnsortedScoredFullMultiSearcher; +import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredStreamingMultiSearcher; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; +import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.warp.commonutils.log.Logger; +import org.warp.commonutils.log.LoggerFactory; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink.OverflowStrategy; +import reactor.util.function.Tuples; + +public class TestLuceneSearches { + + private static final Logger log = LoggerFactory.getLogger(TestLuceneSearches.class); + private static final MemoryTemporaryDbGenerator TEMP_DB_GENERATOR = new MemoryTemporaryDbGenerator(); + + private static TestAllocator allocator; + private static TempDb tempDb; + private static LLLuceneIndex luceneSingle; + private static LLLuceneIndex luceneMulti; + private static LuceneIndex multiIndex; + private static LuceneIndex localIndex; + + @BeforeAll + public static void beforeAll() { + allocator = newAllocator(); + ensureNoLeaks(allocator.allocator(), false, false); + tempDb = Objects.requireNonNull(TEMP_DB_GENERATOR.openTempDb(allocator).block(), "TempDB"); + luceneSingle = tempDb.luceneSingle(); + luceneMulti = tempDb.luceneMulti(); + + setUpIndex(true); + setUpIndex(false); + } + + private static void setUpIndex(boolean shards) { + LuceneIndex index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti)); + index.updateDocument("test-key-1", "0123456789").block(); + index.updateDocument("test-key-2", "test 0123456789 test word").block(); + index.updateDocument("test-key-3", "0123456789 test example string").block(); + index.updateDocument("test-key-4", "hello world the quick brown fox jumps over the lazy dog").block(); + index.updateDocument("test-key-5", "hello the quick brown fox jumps over the lazy dog").block(); + index.updateDocument("test-key-6", "hello the quick brown fox jumps over the world dog").block(); + index.updateDocument("test-key-7", "the quick brown fox jumps over the world dog").block(); + index.updateDocument("test-key-8", "the quick brown fox jumps over the lazy dog").block(); + index.updateDocument("test-key-9", "Example1").block(); + index.updateDocument("test-key-10", "Example2").block(); + index.updateDocument("test-key-11", "Example3").block(); + index.updateDocument("test-key-12", "-234").block(); + index.updateDocument("test-key-13", "2111").block(); + index.updateDocument("test-key-14", "2999").block(); + index.updateDocument("test-key-15", "3902").block(); + Flux.range(1, 1000).concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i)).blockLast(); + tempDb.swappableLuceneSearcher().setSingle(new CountLocalSearcher()); + tempDb.swappableLuceneSearcher().setMulti(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher())); + assertCount(index, 1000 + 15); + if (shards) { + multiIndex = index; + } else { + localIndex = index; + } + } + + public static Stream provideArguments() { + return Stream.of(false, true).map(Arguments::of); + } + + private static final Flux multi = Flux.just(false, true); + private static final Flux scoreModes = Flux.just(LLScoreMode.NO_SCORES, + LLScoreMode.TOP_SCORES, + LLScoreMode.COMPLETE_NO_SCORES, + LLScoreMode.COMPLETE + ); + private static final Flux>> multiSort = Flux.just(MultiSort.topScore(), + MultiSort.randomSortField(), + MultiSort.noSort(), + MultiSort.docSort(), + MultiSort.numericSort("longsort", false), + MultiSort.numericSort("longsort", true) + ); + + private static Flux getSearchers(ExpectedQueryType info) { + return Flux.push(sink -> { + try { + if (info.shard()) { + sink.next(new AdaptiveMultiSearcher()); + if (info.onlyCount()) { + sink.next(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher())); + } else { + sink.next(new ScoredPagedMultiSearcher()); + if (!info.sorted()) { + sink.next(new UnsortedScoredFullMultiSearcher()); + } + if (!info.scored() && !info.sorted()) { + sink.next(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher())); + sink.next(new UnsortedUnscoredStreamingMultiSearcher()); + } + } + } else { + sink.next(new AdaptiveLocalSearcher()); + if (info.onlyCount()) { + sink.next(new CountLocalSearcher()); + } else { + sink.next(new PagedLocalSearcher()); + } + } + sink.complete(); + } catch (IOException e) { + sink.error(e); + } + }, OverflowStrategy.BUFFER); + } + + public static Stream provideQueryArgumentsScoreMode() { + return multi + .concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode))) + .map(tuple -> Arguments.of(tuple.toArray())) + .toStream(); + } + + public static Stream provideQueryArgumentsSort() { + return multi + .concatMap(shard -> multiSort.map(multiSort -> Tuples.of(shard, multiSort))) + .map(tuple -> Arguments.of(tuple.toArray())) + .toStream(); + } + + public static Stream provideQueryArgumentsScoreModeAndSort() { + return multi + .concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode))) + .concatMap(tuple -> multiSort.map(multiSort -> Tuples.of(tuple.getT1(), tuple.getT2(), multiSort))) + .map(tuple -> Arguments.of(tuple.toArray())) + .toStream(); + } + + @AfterAll + public static void afterAll() { + TEMP_DB_GENERATOR.closeTempDb(tempDb).block(); + ensureNoLeaks(allocator.allocator(), true, false); + destroyAllocator(allocator); + } + + private LuceneIndex getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) { + try { + if (customSearcher != null) { + tempDb.swappableLuceneSearcher().setSingle(customSearcher); + if (shards) { + if (customSearcher instanceof MultiSearcher multiSearcher) { + tempDb.swappableLuceneSearcher().setMulti(multiSearcher); + } else { + throw new IllegalArgumentException("Expected a LuceneMultiSearcher, got a LuceneLocalSearcher: " + customSearcher.getName()); + } + } + } else { + tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher()); + tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher()); + } + } catch (IOException e) { + fail(e); + } + return shards ? multiIndex : localIndex; + } + + private static void assertCount(LuceneIndex luceneIndex, long expected) { + Assertions.assertEquals(expected, getCount(luceneIndex)); + } + + private static long getCount(LuceneIndex luceneIndex) { + luceneIndex.refresh(true).block(); + var totalHitsCount = run(luceneIndex.count(null, new MatchAllDocsQuery())); + Assertions.assertTrue(totalHitsCount.exact(), "Can't get count because the total hits count is not exact"); + return totalHitsCount.value(); + } + + @ParameterizedTest + @MethodSource("provideQueryArgumentsScoreModeAndSort") + public void testSearchNoDocs(boolean shards, LLScoreMode scoreMode, MultiSort> multiSort) { + var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList()); + for (LocalSearcher searcher : searchers) { + log.info("Using searcher \"{}\"", searcher.getName()); + + var luceneIndex = getLuceneIndex(shards, searcher); + ClientQueryParamsBuilder> queryBuilder = ClientQueryParams.builder(); + queryBuilder.query(new MatchNoDocsQuery()); + queryBuilder.snapshot(null); + queryBuilder.scoreMode(scoreMode); + queryBuilder.sort(multiSort); + var query = queryBuilder.build(); + try (var results = run(luceneIndex.search(query)).receive()) { + var hits = results.totalHitsCount(); + if (supportsPreciseHitsCount(searcher, query)) { + assertEquals(new TotalHitsCount(0, true), hits); + } + + var keys = getResults(results); + assertEquals(List.of(), keys); + } + } + } + + private boolean supportsPreciseHitsCount(LocalSearcher searcher, + ClientQueryParams> query) { + if (searcher instanceof UnsortedUnscoredStreamingMultiSearcher) { + return false; + } + var scored = isScored(query.scoreMode(), Objects.requireNonNullElse(query.sort(), MultiSort.noSort())); + var sorted = isSorted(Objects.requireNonNullElse(query.sort(), MultiSort.noSort())); + if (!sorted && !scored) { + if (searcher instanceof AdaptiveMultiSearcher || searcher instanceof AdaptiveLocalSearcher) { + return false; + } + } + return true; + } + + @ParameterizedTest + @MethodSource("provideQueryArgumentsScoreModeAndSort") + public void testSearchAllDocs(boolean shards, LLScoreMode scoreMode, MultiSort> multiSort) { + var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList()); + for (LocalSearcher searcher : searchers) { + log.info("Using searcher \"{}\"", searcher.getName()); + + var luceneIndex = getLuceneIndex(shards, searcher); + ClientQueryParamsBuilder> queryBuilder = ClientQueryParams.builder(); + queryBuilder.query(new MatchNoDocsQuery()); + queryBuilder.snapshot(null); + queryBuilder.scoreMode(scoreMode); + queryBuilder.sort(multiSort); + var query = queryBuilder.build(); + try (var results = run(luceneIndex.search(query)).receive()) { + var hits = results.totalHitsCount(); + if (supportsPreciseHitsCount(searcher, query)) { + assertEquals(new TotalHitsCount(0, true), hits); + } + + var keys = getResults(results); + assertEquals(List.of(), keys); + } + } + } + + private boolean isSorted(MultiSort> multiSort) { + return !(multiSort.getQuerySort() instanceof NoSort); + } + + private boolean isScored(LLScoreMode scoreMode, MultiSort> multiSort) { + var needsScores = LLUtils.toScoreMode(scoreMode).needsScores(); + var sort =QueryParser.toSort(multiSort.getQuerySort()); + if (sort != null) { + needsScores |= sort.needsScores(); + } + return needsScores; + } + + private List getResults(SearchResultKeys results) { + return run(results + .results() + .flatMapSequential(searchResultKey -> searchResultKey + .key() + .single() + .map(key -> new Scored(key, searchResultKey.score())) + ) + .collectList()); + } + +}