diff --git a/src/main/data-generator/lucene-query.yaml b/src/main/data-generator/lucene-query.yaml index 78c2f8b..08c215e 100644 --- a/src/main/data-generator/lucene-query.yaml +++ b/src/main/data-generator/lucene-query.yaml @@ -196,3 +196,7 @@ versions: data: onlyTopScores: boolean computeScores: boolean + TotalHitsCount: + data: + value: long + exact: boolean diff --git a/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java b/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java index 086bc59..e313868 100644 --- a/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java @@ -2,6 +2,7 @@ package it.cavallium.dbengine.client; import it.cavallium.dbengine.client.query.ClientQueryParams; import it.cavallium.dbengine.client.query.current.data.Query; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.database.Delta; import it.cavallium.dbengine.database.LLSnapshottable; import it.cavallium.dbengine.database.collections.ValueGetter; @@ -80,7 +81,7 @@ public interface LuceneIndex extends LLSnapshottable { Mono> searchWithTransformer(ClientQueryParams> queryParams, ValueTransformer valueTransformer); - Mono count(@Nullable CompositeSnapshot snapshot, Query query); + Mono count(@Nullable CompositeSnapshot snapshot, Query query); boolean isLowMemoryMode(); diff --git a/src/main/java/it/cavallium/dbengine/client/LuceneIndexImpl.java b/src/main/java/it/cavallium/dbengine/client/LuceneIndexImpl.java index 2805ee0..a9d82af 100644 --- a/src/main/java/it/cavallium/dbengine/client/LuceneIndexImpl.java +++ b/src/main/java/it/cavallium/dbengine/client/LuceneIndexImpl.java @@ -2,6 +2,7 @@ package it.cavallium.dbengine.client; import it.cavallium.dbengine.client.query.ClientQueryParams; import it.cavallium.dbengine.client.query.current.data.Query; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.database.LLKeyScore; import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.LLSearchResultShard; @@ -190,7 +191,7 @@ public class LuceneIndexImpl implements LuceneIndex { } @Override - public Mono count(@Nullable CompositeSnapshot snapshot, Query query) { + public Mono count(@Nullable CompositeSnapshot snapshot, Query query) { return this .search(ClientQueryParams.>builder().snapshot(snapshot).query(query).limit(0).build()) .flatMap(tSearchResultKeys -> tSearchResultKeys.release().thenReturn(tSearchResultKeys.totalHitsCount())); diff --git a/src/main/java/it/cavallium/dbengine/client/SearchResult.java b/src/main/java/it/cavallium/dbengine/client/SearchResult.java index 225957b..6436102 100644 --- a/src/main/java/it/cavallium/dbengine/client/SearchResult.java +++ b/src/main/java/it/cavallium/dbengine/client/SearchResult.java @@ -1,12 +1,13 @@ package it.cavallium.dbengine.client; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public record SearchResult(Flux> results, long totalHitsCount, Mono release) { +public record SearchResult(Flux> results, TotalHitsCount totalHitsCount, Mono release) { public static SearchResult empty() { - return new SearchResult<>(Flux.empty(), 0L, Mono.empty()); + return new SearchResult<>(Flux.empty(), TotalHitsCount.of(0, true), Mono.empty()); } public Flux> resultsThenRelease() { diff --git a/src/main/java/it/cavallium/dbengine/client/SearchResultKeys.java b/src/main/java/it/cavallium/dbengine/client/SearchResultKeys.java index d0ce1f3..42b6649 100644 --- a/src/main/java/it/cavallium/dbengine/client/SearchResultKeys.java +++ b/src/main/java/it/cavallium/dbengine/client/SearchResultKeys.java @@ -1,15 +1,16 @@ package it.cavallium.dbengine.client; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.database.collections.ValueGetter; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @SuppressWarnings("unused") -public record SearchResultKeys(Flux> results, long totalHitsCount, Mono release) { +public record SearchResultKeys(Flux> results, TotalHitsCount totalHitsCount, Mono release) { public static SearchResultKeys empty() { - return new SearchResultKeys<>(Flux.empty(), 0L, Mono.empty()); + return new SearchResultKeys<>(Flux.empty(), TotalHitsCount.of(0, true), Mono.empty()); } public SearchResult withValues(ValueGetter valuesGetter) { diff --git a/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java index 0f38032..4180025 100644 --- a/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java @@ -5,6 +5,7 @@ import it.cavallium.dbengine.client.query.current.data.NoSort; import it.cavallium.dbengine.client.query.current.data.Query; import it.cavallium.dbengine.client.query.current.data.QueryParams; import it.cavallium.dbengine.client.query.current.data.ScoreMode; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.lucene.LuceneUtils; import java.util.List; import java.util.Map; @@ -51,11 +52,11 @@ public interface LLLuceneIndex extends LLSnapshottable { */ Mono search(@Nullable LLSnapshot snapshot, QueryParams queryParams, String keyFieldName); - default Mono count(@Nullable LLSnapshot snapshot, Query query) { + default Mono count(@Nullable LLSnapshot snapshot, Query query) { QueryParams params = QueryParams.of(query, 0, 0, Nullablefloat.empty(), NoSort.of(), ScoreMode.of(false, false)); return Mono.from(this.search(snapshot, params, null) .flatMap(llSearchResultShard -> llSearchResultShard.release().thenReturn(llSearchResultShard.totalHitsCount())) - .defaultIfEmpty(0L)); + .defaultIfEmpty(TotalHitsCount.of(0, true))); } boolean isLowMemoryMode(); diff --git a/src/main/java/it/cavallium/dbengine/database/LLSearchResultShard.java b/src/main/java/it/cavallium/dbengine/database/LLSearchResultShard.java index 81e6349..c326fa4 100644 --- a/src/main/java/it/cavallium/dbengine/database/LLSearchResultShard.java +++ b/src/main/java/it/cavallium/dbengine/database/LLSearchResultShard.java @@ -1,6 +1,7 @@ package it.cavallium.dbengine.database; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public record LLSearchResultShard (Flux results, long totalHitsCount, Mono release) {} +public record LLSearchResultShard (Flux results, TotalHitsCount totalHitsCount, Mono release) {} diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java index 6bf34be..4301984 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java @@ -469,8 +469,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase { private ColumnFamilyHandle getCfh(byte[] columnName) throws RocksDBException { ColumnFamilyHandle cfh = handles.get(Column.special(Column.toString(columnName))); - //noinspection RedundantIfStatement if (databaseOptions.enableDbAssertionsWhenUsingAssertions()) { + //noinspection RedundantIfStatement if (!enableColumnsBug) { assert Arrays.equals(cfh.getName(), columnName); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java index 71a30e6..70a7487 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java +++ b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java @@ -6,6 +6,7 @@ import it.cavallium.dbengine.client.IndicizerSimilarities; import it.cavallium.dbengine.client.query.BasicType; import it.cavallium.dbengine.client.query.QueryParser; import it.cavallium.dbengine.client.query.current.data.QueryParams; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.database.LLKeyScore; import it.cavallium.dbengine.database.LLScoreMode; import it.cavallium.dbengine.database.collections.DatabaseMapDictionary; @@ -50,6 +51,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; @@ -463,4 +465,17 @@ public class LuceneUtils { public static int totalHitsThreshold() { return 1; } + + public static TotalHitsCount convertTotalHitsCount(TotalHits totalHits) { + return switch (totalHits.relation) { + case EQUAL_TO -> TotalHitsCount.of(totalHits.value, true); + case GREATER_THAN_OR_EQUAL_TO -> TotalHitsCount.of(totalHits.value, false); + }; + } + + public static TotalHitsCount sum(TotalHitsCount totalHitsCount, TotalHitsCount totalHitsCount1) { + return TotalHitsCount.of(totalHitsCount.value() + totalHitsCount1.value(), + totalHitsCount.exact() && totalHitsCount1.exact() + ); + } } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java index 3ba49e9..e5766a4 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java @@ -2,6 +2,7 @@ package it.cavallium.dbengine.lucene.searcher; import it.cavallium.dbengine.client.query.QueryParser; import it.cavallium.dbengine.client.query.current.data.QueryParams; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import org.apache.lucene.search.IndexSearcher; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -18,7 +19,7 @@ public class CountLuceneLocalSearcher implements LuceneLocalSearcher { //noinspection BlockingMethodInNonBlockingContext return Mono .fromCallable(() -> new LuceneSearchResult( - indexSearcher.count(queryParams.query()), + TotalHitsCount.of(indexSearcher.count(queryParams.query()), true), Flux.empty(), releaseIndexSearcher) ) diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneMultiSearcher.java index dd70c2a..d1e9330 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneMultiSearcher.java @@ -2,7 +2,9 @@ package it.cavallium.dbengine.lucene.searcher; import it.cavallium.dbengine.client.query.QueryParser; import it.cavallium.dbengine.client.query.current.data.QueryParams; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -36,7 +38,7 @@ public class CountLuceneMultiSearcher implements LuceneMultiSearcher { @Override public Mono collect(LocalQueryParams queryParams, String keyFieldName, Scheduler scheduler) { - return Mono.fromCallable(() -> new LuceneSearchResult(totalHits.get(), Flux.empty(), Mono.when(release))); + return Mono.fromCallable(() -> new LuceneSearchResult(TotalHitsCount.of(totalHits.get(), true), Flux.empty(), Mono.when(release))); } }; }); diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneSearchResult.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneSearchResult.java index 0bc2504..a926e9f 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneSearchResult.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneSearchResult.java @@ -1,5 +1,6 @@ package it.cavallium.dbengine.lucene.searcher; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.database.LLKeyScore; import it.cavallium.dbengine.database.disk.LLLocalKeyValueDatabase; import java.io.IOException; @@ -14,11 +15,11 @@ public final class LuceneSearchResult { protected static final Logger logger = LoggerFactory.getLogger(LuceneSearchResult.class); private volatile boolean releaseCalled; - private final long totalHitsCount; + private final TotalHitsCount totalHitsCount; private final Flux results; private final Mono release; - public LuceneSearchResult(long totalHitsCount, Flux results, Mono release) { + public LuceneSearchResult(TotalHitsCount totalHitsCount, Flux results, Mono release) { this.totalHitsCount = totalHitsCount; this.results = results; this.release = Mono.fromRunnable(() -> { @@ -38,7 +39,7 @@ public final class LuceneSearchResult { super.finalize(); } - public long totalHitsCount() { + public TotalHitsCount totalHitsCount() { return totalHitsCount; } diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneShardSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneShardSearcher.java index 0d1e9aa..b3f721e 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneShardSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneShardSearcher.java @@ -140,7 +140,7 @@ class ScoredSimpleLuceneShardSearcher implements LuceneShardSearcher { ); }); - return new LuceneSearchResult(result.totalHits.value, + return new LuceneSearchResult(LuceneUtils.convertTotalHitsCount(result.totalHits), firstPageHits .concatWith(nextHits), //.transform(flux -> LuceneUtils.filterTopDoc(flux, queryParams)), diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java index 516c9f6..3d30412 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java @@ -111,7 +111,7 @@ public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher { combinedFlux = firstPageMono; } - return new LuceneSearchResult(firstPageTopDocs.totalHits.value, combinedFlux, + return new LuceneSearchResult(LuceneUtils.convertTotalHitsCount(firstPageTopDocs.totalHits), combinedFlux, //.transform(flux -> LuceneUtils.filterTopDoc(flux, queryParams)), releaseIndexSearcher ); diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredPagedLuceneShardSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredPagedLuceneShardSearcher.java index f1f6e91..dda79d8 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredPagedLuceneShardSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredPagedLuceneShardSearcher.java @@ -128,7 +128,7 @@ class UnscoredPagedLuceneShardSearcher implements LuceneShardSearcher { ); }); - return new LuceneSearchResult(result.totalHits.value, firstPageHits + return new LuceneSearchResult(LuceneUtils.convertTotalHitsCount(result.totalHits), firstPageHits .concatWith(nextHits), //.transform(flux -> LuceneUtils.filterTopDoc(flux, queryParams)), release diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredUnsortedContinuousLuceneMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredUnsortedContinuousLuceneMultiSearcher.java index 1d83b12..537c4e1 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredUnsortedContinuousLuceneMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnscoredUnsortedContinuousLuceneMultiSearcher.java @@ -1,5 +1,6 @@ package it.cavallium.dbengine.lucene.searcher; +import it.cavallium.dbengine.client.query.current.data.TotalHitsCount; import it.cavallium.dbengine.lucene.LuceneUtils; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import java.io.IOException; @@ -159,7 +160,7 @@ public class UnscoredUnsortedContinuousLuceneMultiSearcher implements LuceneMult false )); - return new LuceneSearchResult(1, resultsFlux, release); + return new LuceneSearchResult(TotalHitsCount.of(0, false), resultsFlux, release); }); } };