From 90aa7a2522f553397984ab2f7eaa263818d6d072 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Fri, 22 Oct 2021 01:26:22 +0200 Subject: [PATCH] Use in-memory database if the limit is lower than 8192 --- .../dbengine/lucene/searcher/AdaptiveMultiSearcher.java | 6 ++++-- .../cavallium/dbengine/lucene/searcher/MultiSearcher.java | 5 +++++ .../lucene/searcher/SortedScoredFullMultiSearcher.java | 4 ++++ .../lucene/searcher/UnsortedScoredFullMultiSearcher.java | 3 +++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java index 23e6368..74269d2 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java @@ -53,12 +53,14 @@ public class AdaptiveMultiSearcher implements MultiSearcher { LLSearchTransformer transformer) { // offset + limit long realLimit = queryParams.offsetLong() + queryParams.limitLong(); + long maxAllowedInMemoryLimit + = Math.max(MultiSearcher.MAX_IN_MEMORY_SIZE, (long) queryParams.pageLimits().getPageLimit(0)); return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> { if (queryParams.limitLong() == 0) { return count.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); } else if (queryParams.isSorted() || queryParams.needsScores()) { - if (realLimit <= (long) queryParams.pageLimits().getPageLimit(0)) { + if (realLimit <= maxAllowedInMemoryLimit) { return scoredPaged.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); } else { if ((queryParams.isSorted() && !queryParams.isSortedByScore())) { @@ -67,7 +69,7 @@ public class AdaptiveMultiSearcher implements MultiSearcher { return unsortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); } } - } else if (realLimit <= (long) queryParams.pageLimits().getPageLimit(0)) { + } else if (realLimit <= maxAllowedInMemoryLimit) { // Run single-page searches using the paged multi searcher return unsortedUnscoredPaged.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer); } else { diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java index 00b3fbe..280d01c 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java @@ -7,6 +7,11 @@ import reactor.core.publisher.Mono; public interface MultiSearcher extends LocalSearcher { + /** + * Use in-memory collectors if the expected results count is lower or equal than this limit + */ + int MAX_IN_MEMORY_SIZE = 8192; + /** * @param indexSearchersMono Lucene index searcher * @param queryParams the query parameters diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/SortedScoredFullMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/SortedScoredFullMultiSearcher.java index 762d5d8..3e1063c 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/SortedScoredFullMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/SortedScoredFullMultiSearcher.java @@ -16,6 +16,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.ServiceLoader; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TopFieldCollector; import org.warp.commonutils.log.Logger; import org.warp.commonutils.log.LoggerFactory; import reactor.core.publisher.Flux; @@ -64,6 +65,9 @@ public class SortedScoredFullMultiSearcher implements MultiSearcher { .fromCallable(() -> { LLUtils.ensureBlocking(); var totalHitsThreshold = queryParams.getTotalHitsThresholdLong(); + if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) { + throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater"); + } return LMDBFullFieldDocCollector.createSharedManager(env, queryParams.sort(), queryParams.limitInt(), totalHitsThreshold); }) diff --git a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java index 88958df..5093774 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java +++ b/src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java @@ -69,6 +69,9 @@ public class UnsortedScoredFullMultiSearcher implements MultiSearcher { return Mono .fromCallable(() -> { LLUtils.ensureBlocking(); + if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) { + throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater"); + } var totalHitsThreshold = queryParams.getTotalHitsThresholdLong(); return LMDBFullScoreDocCollector.createSharedManager(env, queryParams.limitLong(), totalHitsThreshold); })