From d6b6a211a8ba49148f42ef51d1a36367fe63c60d Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Thu, 14 Oct 2021 00:49:21 +0200 Subject: [PATCH] Replace scoreMode with boolean "complete" --- src/main/data-generator/lucene-query.yaml | 6 +- .../cavallium/dbengine/client/MultiSort.java | 5 ++ .../client/query/ClientQueryParams.java | 20 ++--- .../dbengine/client/query/QueryParser.java | 19 ---- .../dbengine/database/LLLuceneIndex.java | 3 +- .../dbengine/lucene/LuceneUtils.java | 12 +-- .../searcher/AdaptiveMultiSearcher.java | 2 +- .../lucene/searcher/LocalQueryParams.java | 34 ++++++- .../lucene/searcher/PagedLocalSearcher.java | 11 ++- .../searcher/ScoredPagedMultiSearcher.java | 10 ++- .../UnsortedScoredFullMultiSearcher.java | 11 ++- .../UnsortedUnscoredSimpleMultiSearcher.java | 4 +- ...nsortedUnscoredStreamingMultiSearcher.java | 10 ++- .../it/cavallium/dbengine/DbTestUtils.java | 1 - .../cavallium/dbengine/ExpectedQueryType.java | 2 +- .../dbengine/LocalTemporaryDbGenerator.java | 2 +- .../dbengine/MemoryTemporaryDbGenerator.java | 2 +- .../dbengine/TestLuceneSearches.java | 88 ++++++------------- 18 files changed, 115 insertions(+), 127 deletions(-) diff --git a/src/main/data-generator/lucene-query.yaml b/src/main/data-generator/lucene-query.yaml index 16297eb..d4a68dd 100644 --- a/src/main/data-generator/lucene-query.yaml +++ b/src/main/data-generator/lucene-query.yaml @@ -179,7 +179,7 @@ versions: limit: long minCompetitiveScore: -float sort: Sort - scoreMode: ScoreMode + complete: boolean NoSort: data: { } NumericSort: @@ -192,10 +192,6 @@ versions: data: { } DocSort: data: { } - ScoreMode: - data: - onlyTopScores: boolean - computeScores: boolean TotalHitsCount: stringRepresenter: "it.cavallium.dbengine.lucene.LuceneUtils.toHumanReadableString" data: diff --git a/src/main/java/it/cavallium/dbengine/client/MultiSort.java b/src/main/java/it/cavallium/dbengine/client/MultiSort.java index 86d42ff..3cfa0f1 100644 --- a/src/main/java/it/cavallium/dbengine/client/MultiSort.java +++ b/src/main/java/it/cavallium/dbengine/client/MultiSort.java @@ -1,5 +1,6 @@ package it.cavallium.dbengine.client; +import it.cavallium.dbengine.client.query.BasicType; import it.cavallium.dbengine.client.query.current.data.DocSort; import it.cavallium.dbengine.client.query.current.data.NoSort; import it.cavallium.dbengine.client.query.current.data.NumericSort; @@ -24,6 +25,10 @@ public class MultiSort { this.querySort = querySort; } + public boolean isSorted() { + return querySort.getBasicType$() != BasicType.NoSort; + } + /** * Sort a lucene field and the results by a numeric sort field and an int value * @param fieldName Lucene SortedNumericSortField field name diff --git a/src/main/java/it/cavallium/dbengine/client/query/ClientQueryParams.java b/src/main/java/it/cavallium/dbengine/client/query/ClientQueryParams.java index 6db636e..9dee0a1 100644 --- a/src/main/java/it/cavallium/dbengine/client/query/ClientQueryParams.java +++ b/src/main/java/it/cavallium/dbengine/client/query/ClientQueryParams.java @@ -8,7 +8,6 @@ import it.cavallium.dbengine.client.query.current.data.NoSort; import it.cavallium.dbengine.client.query.current.data.Query; import it.cavallium.dbengine.client.query.current.data.QueryParams; import it.cavallium.dbengine.client.query.current.data.QueryParamsBuilder; -import it.cavallium.dbengine.client.query.current.data.ScoreMode; import it.cavallium.dbengine.database.LLScoreMode; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -20,7 +19,7 @@ public final record ClientQueryParams(@Nullable CompositeSnapshot snapshot, long limit, @Nullable Float minCompetitiveScore, @Nullable MultiSort sort, - @NotNull LLScoreMode scoreMode) { + boolean complete) { public static ClientQueryParamsBuilder builder() { return ClientQueryParamsBuilder @@ -30,29 +29,22 @@ public final record ClientQueryParams(@Nullable CompositeSnapshot snapshot, .limit(Long.MAX_VALUE) .minCompetitiveScore(null) .sort(null) - .scoreMode(LLScoreMode.COMPLETE); + .complete(true); } - public ScoreMode toScoreMode() { - return switch (this.scoreMode()) { - case COMPLETE -> ScoreMode.of(false, true); - case COMPLETE_NO_SCORES -> ScoreMode.of(false, false); - case TOP_SCORES -> ScoreMode.of(true, true); - case NO_SCORES -> ScoreMode.of(true, false); - //noinspection UnnecessaryDefault - default -> throw new IllegalArgumentException(); - }; + public boolean isSorted() { + return sort != null && sort.isSorted(); } public QueryParams toQueryParams() { return QueryParamsBuilder .builder() .query(query()) - .sort(sort() != null ? sort().getQuerySort() : NoSort.of()) + .sort(sort != null ? sort.getQuerySort() : new NoSort()) .minCompetitiveScore(Nullablefloat.ofNullable(minCompetitiveScore())) .offset(offset()) .limit(limit()) - .scoreMode(toScoreMode()) + .complete(complete()) .build(); } } diff --git a/src/main/java/it/cavallium/dbengine/client/query/QueryParser.java b/src/main/java/it/cavallium/dbengine/client/query/QueryParser.java index 7b4052b..6ff5247 100644 --- a/src/main/java/it/cavallium/dbengine/client/query/QueryParser.java +++ b/src/main/java/it/cavallium/dbengine/client/query/QueryParser.java @@ -132,10 +132,6 @@ public class QueryParser { return new Term(term.field(), term.value()); } - public static boolean isScoringEnabled(QueryParams queryParams) { - return queryParams.scoreMode().computeScores(); - } - public static Sort toSort(it.cavallium.dbengine.client.query.current.data.Sort sort) { switch (sort.getBasicType$()) { case NoSort: @@ -154,21 +150,6 @@ public class QueryParser { } } - @SuppressWarnings("ConstantConditions") - public static ScoreMode toScoreMode(it.cavallium.dbengine.client.query.current.data.ScoreMode scoreMode) { - if (scoreMode.computeScores() && scoreMode.onlyTopScores()) { - return ScoreMode.TOP_SCORES; - } else if (scoreMode.computeScores() && !scoreMode.onlyTopScores()) { - return ScoreMode.COMPLETE; - } else if (!scoreMode.computeScores() && scoreMode.onlyTopScores()) { - return ScoreMode.TOP_DOCS; - } else if (!scoreMode.computeScores() && !scoreMode.onlyTopScores()) { - return ScoreMode.COMPLETE_NO_SCORES; - } else { - throw new IllegalStateException("Unexpected value: " + scoreMode); - } - } - public static it.cavallium.dbengine.client.query.current.data.Term toQueryTerm(Term term) { return it.cavallium.dbengine.client.query.current.data.Term.of(term.field(), term.text()); } diff --git a/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java index 2d2b58f..a0b5eb0 100644 --- a/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java @@ -5,7 +5,6 @@ import it.cavallium.data.generator.nativedata.Nullablefloat; import it.cavallium.dbengine.client.query.current.data.NoSort; import it.cavallium.dbengine.client.query.current.data.Query; import it.cavallium.dbengine.client.query.current.data.QueryParams; -import it.cavallium.dbengine.client.query.current.data.ScoreMode; import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.lucene.LuceneUtils; import java.util.List; @@ -54,7 +53,7 @@ public interface LLLuceneIndex extends LLSnapshottable { Mono> search(@Nullable LLSnapshot snapshot, QueryParams queryParams, String keyFieldName); default Mono count(@Nullable LLSnapshot snapshot, Query query) { - QueryParams params = QueryParams.of(query, 0, 0, Nullablefloat.empty(), NoSort.of(), ScoreMode.of(false, false)); + QueryParams params = QueryParams.of(query, 0, 0, Nullablefloat.empty(), NoSort.of(), false); return Mono.from(this.search(snapshot, params, null) .map(llSearchResultShardToReceive -> { try (var llSearchResultShard = llSearchResultShardToReceive.receive()) { diff --git a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java index e1dc9f6..56e7229 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java +++ b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java @@ -347,7 +347,7 @@ public class LuceneUtils { DEFAULT_PAGE_LIMITS, queryParams.minCompetitiveScore().getNullable(), QueryParser.toSort(queryParams.sort()), - QueryParser.toScoreMode(queryParams.scoreMode()) + queryParams.complete() ); } @@ -453,8 +453,8 @@ public class LuceneUtils { return result; } - public static int totalHitsThreshold() { - return 1; + public static int totalHitsThreshold(boolean complete) { + return complete ? Integer.MAX_VALUE : 1; } public static TotalHitsCount convertTotalHitsCount(TotalHits totalHits) { @@ -503,7 +503,7 @@ public class LuceneUtils { DEFAULT_PAGE_LIMITS, localQueryParams.minCompetitiveScore(), localQueryParams.sort(), - localQueryParams.scoreMode() + localQueryParams.complete() ); } MultiMoreLikeThis mlt; @@ -521,7 +521,7 @@ public class LuceneUtils { mlt.setMinTermFreq(1); mlt.setMinDocFreq(3); mlt.setMaxDocFreqPct(20); - mlt.setBoost(localQueryParams.scoreMode().needsScores()); + mlt.setBoost(localQueryParams.needsScores()); mlt.setStopWords(EnglishItalianStopFilter.getStopWordsString()); if (similarity instanceof TFIDFSimilarity tfidfSimilarity) { mlt.setSimilarity(tfidfSimilarity); @@ -548,7 +548,7 @@ public class LuceneUtils { DEFAULT_PAGE_LIMITS, localQueryParams.minCompetitiveScore(), localQueryParams.sort(), - localQueryParams.scoreMode() + localQueryParams.complete() ); }).subscribeOn(Schedulers.boundedElastic())); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java index 0ca4853..75cc6a2 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java @@ -53,7 +53,7 @@ public class AdaptiveMultiSearcher implements MultiSearcher, Closeable { return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> { if (queryParams.limit() == 0) { return count.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); - } else if (queryParams.isSorted() || queryParams.isScored()) { + } else if (queryParams.isSorted() || queryParams.needsScores()) { if (queryParams.isSorted() || realLimit <= (long) queryParams.pageLimits().getPageLimit(0)) { return scoredSimple.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); } else { diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java index 32b0293..0f55c5a 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java @@ -1,6 +1,8 @@ package it.cavallium.dbengine.lucene.searcher; +import it.cavallium.dbengine.lucene.LuceneUtils; import it.cavallium.dbengine.lucene.PageLimits; +import java.util.Optional; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Sort; @@ -8,14 +10,38 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; public record LocalQueryParams(@NotNull Query query, int offset, int limit, @NotNull PageLimits pageLimits, - @Nullable Float minCompetitiveScore, @Nullable Sort sort, - @NotNull ScoreMode scoreMode) { + @Nullable Float minCompetitiveScore, @Nullable Sort sort, boolean complete) { public boolean isSorted() { return sort != null; } - public boolean isScored() { - return (sort != null && sort.needsScores()) || scoreMode.needsScores(); + public boolean needsScores() { + return sort != null && sort.needsScores(); + } + + public Optional needsScoresOptional() { + if (sort == null) return Optional.empty(); + return Optional.of(sort.needsScores()); + } + + public ScoreMode getScoreMode() { + if (complete) { + return needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + } else { + return needsScores() ? ScoreMode.TOP_DOCS_WITH_SCORES : ScoreMode.TOP_DOCS; + } + } + + public Optional getScoreModeOptional() { + if (complete) { + return needsScoresOptional().map(needsScores -> needsScores ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES); + } else { + return needsScoresOptional().map(needsScores -> needsScores ? ScoreMode.TOP_DOCS_WITH_SCORES : ScoreMode.TOP_DOCS); + } + } + + public int getTotalHitsThreshold() { + return LuceneUtils.totalHitsThreshold(this.complete); } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java index 0ddb1b5..468e70d 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java @@ -33,8 +33,6 @@ public class PagedLocalSearcher implements LocalSearcher { LocalQueryParams queryParams, String keyFieldName, LLSearchTransformer transformer) { - - Objects.requireNonNull(queryParams.scoreMode(), "ScoreMode must not be null"); PaginationInfo paginationInfo = getPaginationInfo(queryParams); var indexSearchersMono = indexSearcherMono.map(LLIndexSearchers::unsharded).map(ResourceSupport::send); @@ -185,8 +183,13 @@ public class PagedLocalSearcher implements LocalSearcher { TopDocs pageTopDocs; try { TopDocsCollector collector = TopDocsCollectorUtils.getTopDocsCollector(queryParams.sort(), - currentPageLimit, s.last(), LuceneUtils.totalHitsThreshold(), - allowPagination, queryParams.isScored()); + currentPageLimit, s.last(), queryParams.getTotalHitsThreshold(), + allowPagination, queryParams.needsScores()); + assert queryParams.complete() == collector.scoreMode().isExhaustive(); + queryParams.getScoreModeOptional().ifPresent(scoreMode -> { + assert scoreMode == collector.scoreMode(); + }); + indexSearchers.get(0).search(queryParams.query(), collector); if (resultsOffset > 0) { pageTopDocs = collector.topDocs(resultsOffset, currentPageLimit); diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java index 9b3c74f..e212ee0 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java @@ -16,6 +16,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Sort; import org.jetbrains.annotations.Nullable; import org.warp.commonutils.log.Logger; @@ -45,7 +46,6 @@ public class ScoredPagedMultiSearcher implements MultiSearcher { } return queryParamsMono.flatMap(queryParams2 -> { - Objects.requireNonNull(queryParams2.scoreMode(), "ScoreMode must not be null"); PaginationInfo paginationInfo = getPaginationInfo(queryParams2); return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> this @@ -175,7 +175,7 @@ public class ScoredPagedMultiSearcher implements MultiSearcher { @Nullable var sort = getSort(queryParams); var pageLimit = pageLimits.getPageLimit(s.pageIndex()); var after = (FieldDoc) s.last(); - var totalHitsThreshold = LuceneUtils.totalHitsThreshold(); + var totalHitsThreshold = queryParams.getTotalHitsThreshold(); return new ScoringShardsCollectorManager(sort, pageLimit, after, totalHitsThreshold, resultsOffset); } else { @@ -186,7 +186,13 @@ public class ScoredPagedMultiSearcher implements MultiSearcher { .fromIterable(indexSearchers) .flatMap(shard -> Mono.fromCallable(() -> { LLUtils.ensureBlocking(); + var collector = sharedManager.newCollector(); + assert queryParams.complete() == collector.scoreMode().isExhaustive(); + queryParams.getScoreModeOptional().ifPresent(scoreMode -> { + assert scoreMode == collector.scoreMode(); + }); + shard.search(queryParams.query(), collector); return collector; })) diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java index 462dccc..faf2604 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java @@ -44,8 +44,7 @@ public class UnsortedScoredFullMultiSearcher implements MultiSearcher, Closeable } return queryParamsMono.flatMap(queryParams2 -> { - Objects.requireNonNull(queryParams2.scoreMode(), "ScoreMode must not be null"); - if (queryParams2.sort() != null && queryParams2.sort() != Sort.RELEVANCE) { + if (queryParams2.isSorted()) { throw new IllegalArgumentException(UnsortedScoredFullMultiSearcher.this.getClass().getSimpleName() + " doesn't support sorted queries"); } @@ -70,14 +69,20 @@ public class UnsortedScoredFullMultiSearcher implements MultiSearcher, Closeable return Mono .fromCallable(() -> { LLUtils.ensureBlocking(); - var totalHitsThreshold = LuceneUtils.totalHitsThreshold(); + var totalHitsThreshold = queryParams.getTotalHitsThreshold(); return LMDBFullScoreDocCollector.createSharedManager(env, totalHitsThreshold); }) .flatMap(sharedManager -> Flux .fromIterable(indexSearchers) .flatMap(shard -> Mono.fromCallable(() -> { LLUtils.ensureBlocking(); + var collector = sharedManager.newCollector(); + assert queryParams.complete() == collector.scoreMode().isExhaustive(); + queryParams.getScoreModeOptional().ifPresent(scoreMode -> { + assert scoreMode == collector.scoreMode(); + }); + shard.search(queryParams.query(), collector); return collector; })) diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java index 9e37675..5d8326b 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java @@ -46,7 +46,7 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher { throw new UnsupportedOperationException("Sorted queries are not supported" + " by SimpleUnsortedUnscoredLuceneMultiSearcher"); } - if (queryParams2.isScored() && queryParams2.limit() > 0) { + if (queryParams2.needsScores() && queryParams2.limit() > 0) { throw new UnsupportedOperationException("Scored queries are not supported" + " by SimpleUnsortedUnscoredLuceneMultiSearcher"); } @@ -97,7 +97,7 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher { queryParams.pageLimits(), queryParams.minCompetitiveScore(), queryParams.sort(), - queryParams.scoreMode() + queryParams.complete() ); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java index 927766a..a86ff7a 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java @@ -54,7 +54,7 @@ public class UnsortedUnscoredStreamingMultiSearcher implements MultiSearcher { return Mono.error(new UnsupportedOperationException("Sorted queries are not supported" + " by UnsortedUnscoredContinuousLuceneMultiSearcher")); } - if (queryParams2.isScored() && queryParams2.limit() > 0) { + if (queryParams2.needsScores() && queryParams2.limit() > 0) { return Mono.error(new UnsupportedOperationException("Scored queries are not supported" + " by UnsortedUnscoredContinuousLuceneMultiSearcher")); } @@ -75,7 +75,13 @@ public class UnsortedUnscoredStreamingMultiSearcher implements MultiSearcher { UNSCORED_UNSORTED_EXECUTOR.schedule(() -> { try { var collector = cm.newCollector(); + assert queryParams.complete() == collector.scoreMode().isExhaustive(); + queryParams.getScoreModeOptional().ifPresent(scoreMode -> { + assert scoreMode == collector.scoreMode(); + }); + collector.setShardIndex(shardIndex); + shard.search(localQueryParams.query(), collector); } catch (Throwable e) { while (scoreDocsSink.tryEmitError(e) == EmitResult.FAIL_NON_SERIALIZED) { @@ -111,7 +117,7 @@ public class UnsortedUnscoredStreamingMultiSearcher implements MultiSearcher { queryParams.pageLimits(), queryParams.minCompetitiveScore(), queryParams.sort(), - queryParams.scoreMode() + queryParams.complete() ); } diff --git a/src/test/java/it/cavallium/dbengine/DbTestUtils.java b/src/test/java/it/cavallium/dbengine/DbTestUtils.java index fe30d7a..1446fe2 100644 --- a/src/test/java/it/cavallium/dbengine/DbTestUtils.java +++ b/src/test/java/it/cavallium/dbengine/DbTestUtils.java @@ -40,7 +40,6 @@ import reactor.core.scheduler.Schedulers; public class DbTestUtils { - public static final String BIG_STRING = generateBigString(); private static String generateBigString() { diff --git a/src/test/java/it/cavallium/dbengine/ExpectedQueryType.java b/src/test/java/it/cavallium/dbengine/ExpectedQueryType.java index a468571..12dbf19 100644 --- a/src/test/java/it/cavallium/dbengine/ExpectedQueryType.java +++ b/src/test/java/it/cavallium/dbengine/ExpectedQueryType.java @@ -1,3 +1,3 @@ package it.cavallium.dbengine; -record ExpectedQueryType(boolean shard, boolean sorted, boolean scored, boolean unlimited, boolean onlyCount) {} +record ExpectedQueryType(boolean shard, boolean sorted, boolean complete, boolean onlyCount) {} diff --git a/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java b/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java index 616296b..41260f2 100644 --- a/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java +++ b/src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java @@ -74,7 +74,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator { luceneHacks ), conn.getLuceneIndex("testluceneindex16", - 1, + 3, IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple), IndicizerSimilarities.of(TextFieldsSimilarity.Boolean), LUCENE_OPTS, diff --git a/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java b/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java index a71ff6d..bcccc41 100644 --- a/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java +++ b/src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java @@ -46,7 +46,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator { luceneHacks ), conn.getLuceneIndex("testluceneindex16", - 1, + 3, IndicizerAnalyzers.of(TextFieldsAnalyzer.WordSimple), IndicizerSimilarities.of(TextFieldsSimilarity.Boolean), LUCENE_OPTS, diff --git a/src/test/java/it/cavallium/dbengine/TestLuceneSearches.java b/src/test/java/it/cavallium/dbengine/TestLuceneSearches.java index 067db96..924428d 100644 --- a/src/test/java/it/cavallium/dbengine/TestLuceneSearches.java +++ b/src/test/java/it/cavallium/dbengine/TestLuceneSearches.java @@ -72,7 +72,7 @@ public class TestLuceneSearches { private static LuceneIndex multiIndex; private static LuceneIndex localIndex; - private static Map elements; + private static final Map ELEMENTS; static { var modifiableElements = new HashMap(); modifiableElements.put("test-key-1", "0123456789"); @@ -91,7 +91,7 @@ public class TestLuceneSearches { modifiableElements.put("test-key-14", "2999"); modifiableElements.put("test-key-15", "3902"); runVoid(Flux.range(1, 1000).doOnNext(i -> modifiableElements.put("test-key-" + (15 + i), "" + i)).then()); - elements = Collections.unmodifiableMap(modifiableElements); + ELEMENTS = Collections.unmodifiableMap(modifiableElements); } @BeforeAll @@ -110,7 +110,7 @@ public class TestLuceneSearches { LuceneIndex index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti)); Flux - .fromIterable(elements.entrySet()) + .fromIterable(ELEMENTS.entrySet()) .flatMap(entry -> index.updateDocument(entry.getKey(), entry.getValue())) .subscribeOn(Schedulers.boundedElastic()) .blockLast(); @@ -129,11 +129,6 @@ public class TestLuceneSearches { } private static final Flux multi = Flux.just(false, true); - private static final Flux scoreModes = Flux.just(LLScoreMode.NO_SCORES, - LLScoreMode.TOP_SCORES, - LLScoreMode.COMPLETE_NO_SCORES, - LLScoreMode.COMPLETE - ); private static final Flux>> multiSort = Flux.just(MultiSort.topScore(), //todo: fix random sort field //MultiSort.randomSortField(), @@ -147,7 +142,6 @@ public class TestLuceneSearches { return Flux.push(sink -> { try { if (info.shard()) { - sink.next(new AdaptiveMultiSearcher()); if (info.onlyCount()) { sink.next(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher())); } else { @@ -155,18 +149,19 @@ public class TestLuceneSearches { if (!info.sorted()) { sink.next(new UnsortedScoredFullMultiSearcher()); } - if (!info.scored() && !info.sorted()) { + if (!info.sorted()) { sink.next(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher())); sink.next(new UnsortedUnscoredStreamingMultiSearcher()); } } + sink.next(new AdaptiveMultiSearcher()); } else { - sink.next(new AdaptiveLocalSearcher()); if (info.onlyCount()) { sink.next(new CountLocalSearcher()); } else { sink.next(new PagedLocalSearcher()); } + sink.next(new AdaptiveLocalSearcher()); } sink.complete(); } catch (IOException e) { @@ -176,35 +171,26 @@ public class TestLuceneSearches { } public static Stream provideQueryArgumentsScoreMode() { - return multi - .concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode))) - .map(tuple -> Arguments.of(tuple.toArray())) - .toStream(); - } - - public static Stream provideQueryArgumentsSort() { - return multi - .concatMap(shard -> multiSort.map(multiSort -> Tuples.of(shard, multiSort))) - .map(tuple -> Arguments.of(tuple.toArray())) - .toStream(); + return multi.map(tuple -> Arguments.of(multi)).toStream(); } public static Stream provideQueryArgumentsScoreModeAndSort() { return multi - .concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode))) - .concatMap(tuple -> multiSort.map(multiSort -> Tuples.of(tuple.getT1(), tuple.getT2(), multiSort))) + .concatMap(multi -> multiSort.map(multiSort -> Tuples.of(multi, multiSort))) .map(tuple -> Arguments.of(tuple.toArray())) .toStream(); } private static void runSearchers(ExpectedQueryType expectedQueryType, FailableConsumer consumer) { - Assertions.assertDoesNotThrow(() -> { - var searchers = run(getSearchers(expectedQueryType).collectList()); - for (LocalSearcher searcher : searchers) { - log.info("Using searcher \"{}\"", searcher.getName()); + var searchers = run(getSearchers(expectedQueryType).collectList()); + for (LocalSearcher searcher : searchers) { + log.info("Using searcher \"{}\"", searcher.getName()); + try { consumer.accept(searcher); + } catch (Throwable e) { + Assertions.fail(e); } - }); + } } @AfterAll @@ -248,28 +234,25 @@ public class TestLuceneSearches { private boolean supportsPreciseHitsCount(LocalSearcher searcher, ClientQueryParams> query) { + var sorted = query.isSorted(); if (searcher instanceof UnsortedUnscoredStreamingMultiSearcher) { return false; + } else if (!sorted) { + return !(searcher instanceof AdaptiveMultiSearcher) && !(searcher instanceof AdaptiveLocalSearcher); + } else { + return true; } - var scored = isScored(query.scoreMode(), Objects.requireNonNullElse(query.sort(), MultiSort.noSort())); - var sorted = isSorted(Objects.requireNonNullElse(query.sort(), MultiSort.noSort())); - if (!sorted && !scored) { - if (searcher instanceof AdaptiveMultiSearcher || searcher instanceof AdaptiveLocalSearcher) { - return false; - } - } - return true; } @ParameterizedTest @MethodSource("provideQueryArgumentsScoreModeAndSort") - public void testSearchNoDocs(boolean shards, LLScoreMode scoreMode, MultiSort> multiSort) { - runSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false), searcher -> { + public void testSearchNoDocs(boolean shards, MultiSort> multiSort) { + runSearchers(new ExpectedQueryType(shards, multiSort.isSorted(), true, false), searcher -> { var luceneIndex = getLuceneIndex(shards, searcher); ClientQueryParamsBuilder> queryBuilder = ClientQueryParams.builder(); queryBuilder.query(new MatchNoDocsQuery()); queryBuilder.snapshot(null); - queryBuilder.scoreMode(scoreMode); + queryBuilder.complete(true); queryBuilder.sort(multiSort); var query = queryBuilder.build(); try (var results = run(luceneIndex.search(query)).receive()) { @@ -284,22 +267,22 @@ public class TestLuceneSearches { @ParameterizedTest @MethodSource("provideQueryArgumentsScoreModeAndSort") - public void testSearchAllDocs(boolean shards, LLScoreMode scoreMode, MultiSort> multiSort) { - var sorted = isSorted(multiSort); - runSearchers(new ExpectedQueryType(shards, sorted, isScored(scoreMode, multiSort), true, false), (LocalSearcher searcher) -> { + public void testSearchAllDocs(boolean shards, MultiSort> multiSort) { + var sorted = multiSort.isSorted(); + runSearchers(new ExpectedQueryType(shards, sorted, true, false), (LocalSearcher searcher) -> { var luceneIndex = getLuceneIndex(shards, searcher); ClientQueryParamsBuilder> queryBuilder = ClientQueryParams.builder(); queryBuilder.query(new MatchAllDocsQuery()); queryBuilder.snapshot(null); - queryBuilder.scoreMode(scoreMode); + queryBuilder.complete(true); queryBuilder.sort(multiSort); var query = queryBuilder.build(); try (var results = run(luceneIndex.search(query)).receive()) { var hits = results.totalHitsCount(); - assertHitsIfPossible(0, hits); + assertHitsIfPossible(ELEMENTS.size(), hits); var keys = getResults(results); - assertResults(elements.keySet().stream().toList(), keys, false); + assertResults(ELEMENTS.keySet().stream().toList(), keys, false); } }); } @@ -329,19 +312,6 @@ public class TestLuceneSearches { } } - private boolean isSorted(MultiSort> multiSort) { - return !(multiSort.getQuerySort() instanceof NoSort); - } - - private boolean isScored(LLScoreMode scoreMode, MultiSort> multiSort) { - var needsScores = LLUtils.toScoreMode(scoreMode).needsScores(); - var sort =QueryParser.toSort(multiSort.getQuerySort()); - if (sort != null) { - needsScores |= sort.needsScores(); - } - return needsScores; - } - private List getResults(SearchResultKeys results) { return run(results .results()