From 575bde94f046fa0020dba899991c60f05da7ba92 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Mon, 22 Feb 2021 01:37:17 +0100 Subject: [PATCH] Convert limit to long, add new query type. --- .../dbengine/client/LuceneIndex.java | 4 +-- .../dbengine/database/LLLuceneIndex.java | 4 +-- .../database/disk/LLLocalLuceneIndex.java | 23 ++++++++++------ .../disk/LLLocalMultiLuceneIndex.java | 4 +-- .../dbengine/lucene/LuceneUtils.java | 6 ++--- .../serializer/DocValuesFieldExistsQuery.java | 26 +++++++++++++++++++ .../serializer/QueryConstructorType.java | 1 + .../lucene/serializer/QueryParser.java | 5 ++++ 8 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 src/main/java/it/cavallium/dbengine/lucene/serializer/DocValuesFieldExistsQuery.java diff --git a/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java b/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java index f163121..41bc608 100644 --- a/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/client/LuceneIndex.java @@ -95,7 +95,7 @@ public class LuceneIndex implements LLSnapshottable { private SearchResult transformLuceneResultWithValues(LLSearchResult llSearchResult, @Nullable MultiSort> sort, LLScoreMode scoreMode, - @Nullable Integer limit, + @Nullable Long limit, ValueGetter valueGetter) { var mappedKeys = llSearchResult .results() @@ -182,7 +182,7 @@ public class LuceneIndex implements LLSnapshottable { */ public Mono> searchWithValues(@Nullable CompositeSnapshot snapshot, Query query, - int limit, + long limit, @Nullable MultiSort> sort, LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, diff --git a/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java index 63537a5..cb7c1bb 100644 --- a/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java @@ -33,7 +33,7 @@ public interface LLLuceneIndex extends LLSnapshottable { */ Mono moreLikeThis(@Nullable LLSnapshot snapshot, Flux>> mltDocumentFields, - int limit, + long limit, @Nullable Float minCompetitiveScore, String keyFieldName); @@ -46,7 +46,7 @@ public interface LLLuceneIndex extends LLSnapshottable { */ Mono search(@Nullable LLSnapshot snapshot, Query query, - int limit, + long limit, @Nullable LLSort sort, LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java index dd1b01d..e653e1d 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java @@ -33,6 +33,7 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.math3.exception.NumberIsTooLargeException; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -338,7 +339,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { @Override public Mono moreLikeThis(@Nullable LLSnapshot snapshot, Flux>> mltDocumentFieldsFlux, - int limit, + long limit, @Nullable Float minCompetitiveScore, String keyFieldName) { return moreLikeThis(snapshot, mltDocumentFieldsFlux, limit, minCompetitiveScore, keyFieldName, false, 0, 1); @@ -346,7 +347,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { public Mono distributedMoreLikeThis(@Nullable LLSnapshot snapshot, Flux>> mltDocumentFieldsFlux, - int limit, + long limit, @Nullable Float minCompetitiveScore, String keyFieldName, long actionId, @@ -365,7 +366,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { @SuppressWarnings({"Convert2MethodRef", "unchecked", "rawtypes"}) private Mono moreLikeThis(@Nullable LLSnapshot snapshot, Flux>> mltDocumentFieldsFlux, - int limit, + long limit, @Nullable Float minCompetitiveScore, String keyFieldName, boolean doDistributedPre, @@ -408,9 +409,12 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { allowOnlyQueryParsingCollectorStreamSearcher.search(indexSearcher, query); totalHitsCountSink.tryEmitValue(0L); } else { + if (limit > Integer.MAX_VALUE) { + throw new NumberIsTooLargeException(limit, Integer.MAX_VALUE, true); + } streamSearcher.search(indexSearcher, query, - limit, + (int) limit, null, ScoreMode.TOP_SCORES, minCompetitiveScore, @@ -458,13 +462,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { } @Override - public Mono search(@Nullable LLSnapshot snapshot, it.cavallium.dbengine.lucene.serializer.Query query, int limit, + public Mono search(@Nullable LLSnapshot snapshot, it.cavallium.dbengine.lucene.serializer.Query query, long limit, @Nullable LLSort sort, @NotNull LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, String keyFieldName) { return search(snapshot, query, limit, sort, scoreMode, minCompetitiveScore, keyFieldName, false, 0, 1); } - public Mono distributedSearch(@Nullable LLSnapshot snapshot, it.cavallium.dbengine.lucene.serializer.Query query, int limit, + public Mono distributedSearch(@Nullable LLSnapshot snapshot, it.cavallium.dbengine.lucene.serializer.Query query, long limit, @Nullable LLSort sort, @NotNull LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, String keyFieldName, long actionId, int scoreDivisor) { return search(snapshot, query, limit, sort, scoreMode, minCompetitiveScore, keyFieldName, false, actionId, scoreDivisor); @@ -480,7 +484,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { @SuppressWarnings("Convert2MethodRef") private Mono search(@Nullable LLSnapshot snapshot, - it.cavallium.dbengine.lucene.serializer.Query query, int limit, + it.cavallium.dbengine.lucene.serializer.Query query, long limit, @Nullable LLSort sort, @NotNull LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, String keyFieldName, boolean doDistributedPre, long actionId, int scoreDivisor) { return acquireSearcherWrapper(snapshot, doDistributedPre, actionId) @@ -511,9 +515,12 @@ public class LLLocalLuceneIndex implements LLLuceneIndex { allowOnlyQueryParsingCollectorStreamSearcher.search(indexSearcher, luceneQuery); totalHitsCountSink.tryEmitValue(0L); } else { + if (limit > Integer.MAX_VALUE) { + throw new NumberIsTooLargeException(limit, Integer.MAX_VALUE, true); + } streamSearcher.search(indexSearcher, luceneQuery, - limit, + (int) limit, luceneSort, luceneScoreMode, minCompetitiveScore, diff --git a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java index 3a1b6c4..aee594a 100644 --- a/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java +++ b/src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java @@ -203,7 +203,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex { @Override public Mono moreLikeThis(@Nullable LLSnapshot snapshot, Flux>> mltDocumentFields, - int limit, + long limit, @Nullable Float minCompetitiveScore, String keyFieldName) { long actionId; @@ -272,7 +272,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex { @Override public Mono search(@Nullable LLSnapshot snapshot, Query query, - int limit, + long limit, @Nullable LLSort sort, LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, diff --git a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java index c365a61..5c475e5 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java +++ b/src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java @@ -171,7 +171,7 @@ public class LuceneUtils { */ public static Flux mergeStream(Flux> mappedMultiResults, @Nullable MultiSort sort, - @Nullable Integer limit) { + @Nullable Long limit) { if (limit != null && limit == 0) { return mappedMultiResults.flatMap(f -> f).ignoreElements().flux(); } @@ -183,10 +183,10 @@ public class LuceneUtils { //noinspection unchecked mergedFlux = Flux.mergeOrdered(32, sort.getResultSort(), mappedMultiResultsList.toArray(Flux[]::new)); } - if (limit == null) { + if (limit == null || limit == Long.MAX_VALUE) { return mergedFlux; } else { - return mergedFlux.take(limit); + return mergedFlux.limitRequest(limit); } }); } diff --git a/src/main/java/it/cavallium/dbengine/lucene/serializer/DocValuesFieldExistsQuery.java b/src/main/java/it/cavallium/dbengine/lucene/serializer/DocValuesFieldExistsQuery.java new file mode 100644 index 0000000..4195152 --- /dev/null +++ b/src/main/java/it/cavallium/dbengine/lucene/serializer/DocValuesFieldExistsQuery.java @@ -0,0 +1,26 @@ +package it.cavallium.dbengine.lucene.serializer; + +public class DocValuesFieldExistsQuery implements Query { + + private final String field; + + public DocValuesFieldExistsQuery(String field) { + this.field = field; + } + + public String getField() { + return field; + } + + @Override + public void stringify(StringBuilder output) { + StringBuilder data = new StringBuilder(); + StringifyUtils.stringifyString(data, field); + StringifyUtils.writeHeader(output, QueryConstructorType.DOC_VALUES_FIELD_EXISTS_QUERY, data); + } + + @Override + public String toString() { + return "(existence of field " + field + ")"; + } +} diff --git a/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryConstructorType.java b/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryConstructorType.java index c306765..3e3c7c8 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryConstructorType.java +++ b/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryConstructorType.java @@ -8,6 +8,7 @@ public enum QueryConstructorType { INT_POINT_EXACT_QUERY, LONG_POINT_EXACT_QUERY, TERM_QUERY, + DOC_VALUES_FIELD_EXISTS_QUERY, FUZZY_QUERY, PHRASE_QUERY, SYNONYM_QUERY, diff --git a/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryParser.java b/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryParser.java index 2b881cf..a9838b9 100644 --- a/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryParser.java +++ b/src/main/java/it/cavallium/dbengine/lucene/serializer/QueryParser.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PhraseQuery; @@ -185,6 +186,10 @@ public class QueryParser { Term term2 = (Term) parse(completeText, position); assert term2 != null; return new TermQuery(term2); + case DOC_VALUES_FIELD_EXISTS_QUERY: + String fieldKey = (String) parse(completeText, position); + assert fieldKey != null; + return new DocValuesFieldExistsQuery(fieldKey); case FLOAT: position.addAndGet(toParse.length()); return Float.parseFloat(toParse);