andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Generated serializable queries

This commit is contained in:
Andrea Cavalli 2021-03-02 01:53:36 +01:00
parent 89f20b449b
commit 1fc6ab2e4a
37 changed files with 640 additions and 1265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
pom.xml
View File

@ -178,6 +178,11 @@
<artifactId>lucene-relevance</artifactId>
<version>8.0.0.0.5</version>
</dependency>
<dependency>
<groupId>it.cavallium</groupId>
<artifactId>data-generator</artifactId>
<version>0.9.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<testSourceDirectory>src/test/java</testSourceDirectory>
@ -230,6 +235,24 @@
<useIncrementalCompilation>false</useIncrementalCompilation>
</configuration>
</plugin>
<plugin>
<groupId>it.cavallium</groupId>
<artifactId>data-generator</artifactId>
<version>0.9.0-SNAPSHOT</version>
<executions>
<execution>
<id>generate-lucene-query-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<basePackageName>it.cavallium.dbengine.client.query</basePackageName>
<configPath>${basedir}/src/main/data-generator/lucene-query.yaml</configPath>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

33
src/example/java/it.cavallium.dbengine.client/IndicizationExample.java
View File

@ -1,16 +1,17 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.client.query.QueryUtils;
import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.ScoreMode;
import it.cavallium.dbengine.client.query.current.data.ScoreSort;
import it.cavallium.dbengine.database.LLDocument;
import it.cavallium.dbengine.database.LLItem;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLSort;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
import it.cavallium.dbengine.lucene.serializer.Query;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -36,11 +37,13 @@ public class IndicizationExample {
)
.then(index.refresh())
.then(index.search(null,
Query.exactSearch(TextFieldsAnalyzer.N4GramPartialString, "name", "Mario"),
1,
LLSort.newSortScore(),
LLScoreMode.COMPLETE,
null,
QueryParams
.builder()
.query(QueryUtils.exactSearch(TextFieldsAnalyzer.N4GramPartialString, "name", "Mario"))
.limit(1)
.sort(ScoreSort.of())
.scoreMode(ScoreMode.of(false, true))
.build(),
"id"
))
.flatMap(results -> results
@ -106,8 +109,16 @@ public class IndicizationExample {
})
))
.then(index.refresh())
.then(index.search(null, Query.exactSearch(TextFieldsAnalyzer.N4GramPartialString,"name", "Mario"), 10, MultiSort.topScore()
.getQuerySort(), LLScoreMode.COMPLETE, null, "id"))
.then(index.search(null,
QueryParams
.builder()
.query(QueryUtils.exactSearch(TextFieldsAnalyzer.N4GramPartialString, "name", "Mario"))
.limit(10)
.sort(MultiSort.topScore().getQuerySort())
.scoreMode(ScoreMode.of(false, true))
.build(),
"id"
))
.flatMap(results -> LuceneUtils.mergeStream(results
.results(), MultiSort.topScoreRaw(), 10L)
.doOnNext(value -> System.out.println("Value: " + value))

198
src/main/data-generator/lucene-query.yaml Normal file
View File

@ -0,0 +1,198 @@
# A type that starts with "-" is an optional type, otherwise it can't be null
currentVersion: "0.0.0"
interfacesData:
Query: []
# versions must have only numbers, lowercase letters, dots, dashes. Maximum: 99.999.9999
versions:
0.0.0:
details:
changelog: "First version"
superTypes:
Query: [
BoxedQuery, TermQuery, PhraseQuery, WildcardQuery, SynonymQuery, FuzzyQuery, MatchAllDocsQuery,
MatchNoDocsQuery, BooleanQuery, SortedNumericDocValuesFieldSlowRangeQuery, SortedDocFieldExistsQuery,
ConstantScoreQuery, BoostQuery, IntPointRangeQuery, LongPointRangeQuery, IntPointExactQuery,
LongPointExactQuery
]
Occur: [OccurMust, OccurMustNot, OccurShould, OccurFilter]
Sort: [NoSort, NumericSort, ScoreSort, DocSort, RandomSort]
customTypes: {}
classes:
# Basic data
# ==========
# Wrapper for type Query
BoxedQuery:
data:
query: Query
# A term consists in a field that is exactly equal to the value string
Term:
data:
field: String
value: String
# A Term with a position relative to the start of the query. Used internally in some specific queries
TermPosition:
data:
term: Term
position: int
# A Term with a specified boost. Used internally in some specific queries
TermAndBoost:
data:
term: Term
boost: float
# Occur options used for boolean queries
OccurMust:
data: { }
OccurMustNot:
data: { }
OccurShould:
data: { }
OccurFilter:
data: { }
# Text queries
# ============
# Query that matches a term.
TermQuery:
data:
term: Term
# Query that matches a phrase.
PhraseQuery:
data:
# A phrase is a collection of positioned terms, with absolute positions,
# counted as characters from the beginning of the phrase.
phrase: TermPosition[]
slop: int
# Advanced query that matches text allowing asterisks in the query
WildcardQuery:
data:
field: String
pattern: String # Example: "*ing"
# Advanced query that matches different exact values (synonyms)
SynonymQuery:
data:
field: String
parts: TermAndBoost[] # Each term has a boost. The preferred synonym has the highest boost value.
# Advanced query. todo: document it
FuzzyQuery:
data:
term: Term
maxEdits: int
prefixLength: int
maxExpansions: int
transpositions: boolean
# Combination queries
# ===================
# Query that matches everything
MatchAllDocsQuery:
data: {}
# Query that matches nothing
MatchNoDocsQuery:
data: {}
# Query that matches if the document satisfies all the required parts
BooleanQuery:
data:
# Each part can be:
# - "MUST"
# - "MUST_NOT"
# - "SHOULD"
# - "FILTER" (advanced, ignore this)
# "SHOULD" is like "MUST" but it's not necessary.
parts: BooleanQueryPart[]
minShouldMatch: int # If set, it specifies how many "SHOULD" parts must be matched. 0 if not set
# Part of a boolean query
BooleanQueryPart:
data:
query: Query
occur: Occur
# Number queries
# ==============
# Advanced query that matches only a range of a sorted field, from "min" to "max".
SortedNumericDocValuesFieldSlowRangeQuery:
data:
field: String
min: long
max: long
# Query that matches if the sorted field exist in the document
SortedDocFieldExistsQuery:
data:
field: String
# Score modifying queries
# ============
# Query that fixes the score of a query
ConstantScoreQuery:
data:
query: Query
score: float
# Query that boosts the query score
BoostQuery:
data:
query: Query
scoreBoost: float
# Sorted fields queries
# =====================
# Query that matches an int point field, from "min", to "max"
IntPointRangeQuery:
data:
field: String
min: int
max: int
# Query that matches a long point field, from "min", to "max"
LongPointRangeQuery:
data:
field: String
min: long
max: long
# Query that matches an int point field
IntPointExactQuery:
data:
field: String
value: int
# Query that matches a long point field
LongPointExactQuery:
data:
field: String
value: long
# Extra data used for parameters and the client
# =============================================
# Query parameters
QueryParams:
data:
query: Query
limit: long
minCompetitiveScore: -float
sort: Sort
scoreMode: ScoreMode
NoSort:
data: { }
NumericSort:
data:
field: String
reverse: boolean
RandomSort:
data: { }
ScoreSort:
data: { }
DocSort:
data: { }
ScoreMode:
data:
onlyTopScores: boolean
computeScores: boolean

95
src/main/java/it/cavallium/dbengine/client/LuceneIndex.java
View File

@ -1,15 +1,15 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLSearchResult;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLSnapshottable;
import it.cavallium.dbengine.database.LLSort;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.database.collections.Joiner.ValueGetter;
import it.cavallium.dbengine.lucene.serializer.Query;
import it.cavallium.dbengine.lucene.LuceneUtils;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@ -112,96 +112,95 @@ public class LuceneIndex<T, U> implements LLSnapshottable {
/**
*
* @param limit the limit is valid for each lucene instance.
* @param queryParams the limit is valid for each lucene instance.
* If you have 15 instances, the number of elements returned
* can be at most <code>limit * 15</code>
* @return the collection has one or more flux
*/
public Mono<SearchResultKeys<T>> moreLikeThis(@Nullable CompositeSnapshot snapshot,
public Mono<SearchResultKeys<T>> moreLikeThis(
ClientQueryParams<SearchResultKey<T>> queryParams,
T key,
U mltDocumentValue,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
long limit,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore) {
U mltDocumentValue) {
Flux<Tuple2<String, Set<String>>> mltDocumentFields
= indicizer.getMoreLikeThisDocumentFields(key, mltDocumentValue);
return luceneIndex
.moreLikeThis(resolveSnapshot(snapshot), mltDocumentFields, additionalQuery, limit,
minCompetitiveScore, enableScoring, sortByScore, indicizer.getKeyFieldName())
.map(llSearchResult -> this.transformLuceneResult(llSearchResult, null, LLScoreMode.TOP_SCORES, limit));
.moreLikeThis(resolveSnapshot(queryParams.getSnapshot()), queryParams.toQueryParams(), indicizer.getKeyFieldName(), mltDocumentFields)
.map(llSearchResult -> this.transformLuceneResult(llSearchResult,
queryParams.getSort(),
queryParams.getScoreMode(),
queryParams.getLimit()
));
}
/**
*
* @param limit the limit is valid for each lucene instance.
* @param queryParams the limit is valid for each lucene instance.
* If you have 15 instances, the number of elements returned
* can be at most <code>limit * 15</code>
* @return the collection has one or more flux
*/
public Mono<SearchResult<T, U>> moreLikeThisWithValues(@Nullable CompositeSnapshot snapshot,
public Mono<SearchResult<T, U>> moreLikeThisWithValues(
ClientQueryParams<SearchResultItem<T, U>> queryParams,
T key,
U mltDocumentValue,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
long limit,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore,
ValueGetter<T, U> valueGetter) {
Flux<Tuple2<String, Set<String>>> mltDocumentFields
= indicizer.getMoreLikeThisDocumentFields(key, mltDocumentValue);
return luceneIndex
.moreLikeThis(resolveSnapshot(snapshot), mltDocumentFields, additionalQuery, limit,
minCompetitiveScore, enableScoring, sortByScore, indicizer.getKeyFieldName())
.map(llSearchResult ->
this.transformLuceneResultWithValues(llSearchResult, null, LLScoreMode.TOP_SCORES, limit, valueGetter));
.moreLikeThis(resolveSnapshot(queryParams.getSnapshot()),
queryParams.toQueryParams(),
indicizer.getKeyFieldName(),
mltDocumentFields
)
.map(llSearchResult -> this.transformLuceneResultWithValues(llSearchResult,
queryParams.getSort(),
queryParams.getScoreMode(),
queryParams.getLimit(),
valueGetter
));
}
/**
*
* @param limit the limit is valid for each lucene instance.
* @param queryParams the limit is valid for each lucene instance.
* If you have 15 instances, the number of elements returned
* can be at most <code>limit * 15</code>
* @return the collection has one or more flux
*/
public Mono<SearchResultKeys<T>> search(@Nullable CompositeSnapshot snapshot,
Query query,
long limit,
@Nullable MultiSort<SearchResultKey<T>> sort,
LLScoreMode scoreMode,
@Nullable Float minCompetitiveScore) {
LLSort querySort = sort != null ? sort.getQuerySort() : null;
public Mono<SearchResultKeys<T>> search(
ClientQueryParams<SearchResultKey<T>> queryParams) {
return luceneIndex
.search(resolveSnapshot(snapshot), query, limit, querySort, scoreMode, minCompetitiveScore,
indicizer.getKeyFieldName())
.map(llSearchResult -> this.transformLuceneResult(llSearchResult, sort, scoreMode, limit));
.search(resolveSnapshot(queryParams.getSnapshot()), queryParams.toQueryParams(), indicizer.getKeyFieldName())
.map(llSearchResult -> this.transformLuceneResult(llSearchResult,
queryParams.getSort(),
queryParams.getScoreMode(),
queryParams.getLimit()
));
}
/**
*
* @param limit the limit is valid for each lucene instance.
* @param queryParams the limit is valid for each lucene instance.
* If you have 15 instances, the number of elements returned
* can be at most <code>limit * 15</code>
* @return the collection has one or more flux
*/
public Mono<SearchResult<T, U>> searchWithValues(@Nullable CompositeSnapshot snapshot,
Query query,
long limit,
@Nullable MultiSort<SearchResultItem<T, U>> sort,
LLScoreMode scoreMode,
@Nullable Float minCompetitiveScore,
public Mono<SearchResult<T, U>> searchWithValues(
ClientQueryParams<SearchResultItem<T, U>> queryParams,
ValueGetter<T, U> valueGetter) {
LLSort querySort = sort != null ? sort.getQuerySort() : null;
return luceneIndex
.search(resolveSnapshot(snapshot), query, limit, querySort, scoreMode, minCompetitiveScore,
indicizer.getKeyFieldName())
.map(llSearchResult -> this.transformLuceneResultWithValues(llSearchResult, sort, scoreMode, limit, valueGetter));
.search(resolveSnapshot(queryParams.getSnapshot()), queryParams.toQueryParams(), indicizer.getKeyFieldName())
.map(llSearchResult -> this.transformLuceneResultWithValues(llSearchResult,
queryParams.getSort(),
queryParams.getScoreMode(),
queryParams.getLimit(),
valueGetter
));
}
public Mono<Long> count(@Nullable CompositeSnapshot snapshot, Query query) {
return this.search(snapshot, query, 0, null, LLScoreMode.COMPLETE_NO_SCORES, null)
return this.search(ClientQueryParams.<SearchResultKey<T>>builder().snapshot(snapshot).query(query).limit(0).build())
.flatMap(SearchResultKeys::totalHitsCount)
.single();
}

23
src/main/java/it/cavallium/dbengine/client/MultiSort.java
View File

@ -1,17 +1,20 @@
package it.cavallium.dbengine.client;
import it.cavallium.dbengine.client.query.current.data.NumericSort;
import it.cavallium.dbengine.client.query.current.data.RandomSort;
import it.cavallium.dbengine.client.query.current.data.ScoreSort;
import it.cavallium.dbengine.client.query.current.data.Sort;
import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLSort;
import java.util.Comparator;
import java.util.function.ToIntFunction;
import java.util.function.ToLongFunction;
public class MultiSort<T> {
private final LLSort querySort;
private final Sort querySort;
private final Comparator<T> resultSort;
public MultiSort(LLSort querySort, Comparator<T> resultSort) {
public MultiSort(Sort querySort, Comparator<T> resultSort) {
this.querySort = querySort;
this.resultSort = resultSort;
}
@ -26,7 +29,7 @@ public class MultiSort<T> {
*/
public static <T> MultiSort<T> sortedNumericInt(String fieldName, ToIntFunction<T> toIntFunction, boolean reverse) {
// Create lucene sort
LLSort querySort = LLSort.newSortedNumericSortField(fieldName, reverse);
Sort querySort = NumericSort.of(fieldName, reverse);
// Create result sort
Comparator<T> resultSort = Comparator.comparingInt(toIntFunction);
@ -48,7 +51,7 @@ public class MultiSort<T> {
*/
public static <T> MultiSort<T> sortedNumericLong(String fieldName, ToLongFunction<T> toLongFunction, boolean reverse) {
// Create lucene sort
LLSort querySort = LLSort.newSortedNumericSortField(fieldName, reverse);
Sort querySort = NumericSort.of(fieldName, reverse);
// Create result sort
Comparator<T> resultSort = Comparator.comparingLong(toLongFunction);
@ -61,22 +64,22 @@ public class MultiSort<T> {
}
public static <T> MultiSort<T> randomSortField() {
return new MultiSort<>(LLSort.newRandomSortField(), (a, b) -> 0);
return new MultiSort<>(RandomSort.of(), (a, b) -> 0);
}
public static MultiSort<LLKeyScore> topScoreRaw() {
return new MultiSort<>(LLSort.newSortScore(), Comparator.comparingDouble(LLKeyScore::getScore).reversed());
return new MultiSort<>(ScoreSort.of(), Comparator.comparingDouble(LLKeyScore::getScore).reversed());
}
public static <T> MultiSort<SearchResultKey<T>> topScore() {
return new MultiSort<>(LLSort.newSortScore(), Comparator.<SearchResultKey<T>>comparingDouble(SearchResultKey::getScore).reversed());
return new MultiSort<>(ScoreSort.of(), Comparator.<SearchResultKey<T>>comparingDouble(SearchResultKey::getScore).reversed());
}
public static <T, U> MultiSort<SearchResultItem<T, U>> topScoreWithValues() {
return new MultiSort<>(LLSort.newSortScore(), Comparator.<SearchResultItem<T, U>>comparingDouble(SearchResultItem::getScore).reversed());
return new MultiSort<>(ScoreSort.of(), Comparator.<SearchResultItem<T, U>>comparingDouble(SearchResultItem::getScore).reversed());
}
public LLSort getQuerySort() {
public Sort getQuerySort() {
return querySort;
}

82
src/main/java/it/cavallium/dbengine/client/query/ClientQueryParams.java Normal file
View File

@ -0,0 +1,82 @@
package it.cavallium.dbengine.client.query;
import it.cavallium.data.generator.nativedata.Nullablefloat;
import it.cavallium.dbengine.client.CompositeSnapshot;
import it.cavallium.dbengine.client.MultiSort;
import it.cavallium.dbengine.client.query.current.data.NoSort;
import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.ScoreMode;
import it.cavallium.dbengine.database.LLScoreMode;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Builder.Default;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NonNull;
import lombok.ToString;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@EqualsAndHashCode
@AllArgsConstructor(
staticName = "of"
)
@Data
@Builder
@ToString
public final class ClientQueryParams<T> {
@Nullable
@Default
private CompositeSnapshot snapshot = null;
@NotNull
@NonNull
private Query query;
@Default
private long limit = Long.MAX_VALUE;
@Nullable
@Default
private Float minCompetitiveScore = null;
@Nullable
@Default
private MultiSort<T> sort = null;
@NotNull
@NonNull
@Default
private LLScoreMode scoreMode = LLScoreMode.COMPLETE;
public ScoreMode toScoreMode() {
ScoreMode scoreMode;
switch (getScoreMode()) {
case COMPLETE:
scoreMode = ScoreMode.of(false, true);
break;
case COMPLETE_NO_SCORES:
scoreMode = ScoreMode.of(false, false);
break;
case TOP_SCORES:
scoreMode = ScoreMode.of(true, true);
break;
default:
throw new IllegalArgumentException();
}
return scoreMode;
}
public QueryParams toQueryParams() {
return QueryParams
.builder()
.query(getQuery())
.sort(getSort() != null ? getSort().getQuerySort() : NoSort.of())
.minCompetitiveScore(Nullablefloat.ofNullable(getMinCompetitiveScore()))
.limit(getLimit())
.scoreMode(toScoreMode())
.build();
}
}

88
src/main/java/it/cavallium/dbengine/client/query/QueryGson.java Normal file
View File

@ -0,0 +1,88 @@
package it.cavallium.dbengine.client.query;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import com.google.gson.JsonSyntaxException;
import it.cavallium.dbengine.client.query.current.CurrentVersion;
import it.cavallium.dbengine.client.query.current.data.IBasicType;
import it.cavallium.dbengine.client.query.current.data.IType;
import java.lang.reflect.Modifier;
import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Set;
public class QueryGson {
private static final HashMap<Class<? extends IType>, Set<Class<? extends IBasicType>>> implementationClassesSerializers = new HashMap<>();
private static final JsonElement EMPTY_JSON_OBJECT = new JsonObject();
static {
for (var superTypeClass : CurrentVersion.getSuperTypeClasses()) {
implementationClassesSerializers.put(superTypeClass, CurrentVersion.getSuperTypeSubtypesClasses(superTypeClass));
}
}
public static GsonBuilder registerAdapters(GsonBuilder gsonBuilder) {
implementationClassesSerializers.forEach((interfaceClass, implementationClasses) -> {
gsonBuilder.registerTypeAdapter(interfaceClass, new DbClassesGenericSerializer<>(implementationClasses));
});
return gsonBuilder;
}
public static class DbClassesGenericSerializer<T extends IType> implements JsonSerializer<T>, JsonDeserializer<T> {
private final BiMap<String, Class<? extends IBasicType>> subTypes;
public DbClassesGenericSerializer(Set<Class<? extends IBasicType>> implementationClasses) {
subTypes = HashBiMap.create(implementationClasses.size());
for (Class<? extends IBasicType> implementationClass : implementationClasses) {
var name = implementationClass.getSimpleName();
this.subTypes.put(name, implementationClass);
}
}
@Override
public JsonElement serialize(T src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject result = new JsonObject();
Class<?> type = src.getClass();
if (!subTypes.inverse().containsKey(type)) {
throw new JsonSyntaxException("Unknown element type: " + type.getCanonicalName());
}
result.add("type", new JsonPrimitive(subTypes.inverse().get(src.getClass())));
if (Arrays
.stream(src.getClass().getDeclaredFields())
.anyMatch(field -> !Modifier.isStatic(field.getModifiers()) && !Modifier.isTransient(field.getModifiers()))) {
result.add("properties", context.serialize(src, src.getClass()));
}
return result;
}
@Override
public T deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
JsonObject jsonObject = json.getAsJsonObject();
String type = jsonObject.get("type").getAsString();
JsonElement element;
if (jsonObject.has("properties")) {
element = jsonObject.get("properties");
} else {
element = EMPTY_JSON_OBJECT;
}
if (!subTypes.containsKey(type)) {
throw new JsonParseException("Unknown element type: " + type);
}
return context.deserialize(element, subTypes.get(type));
}
}
}

58
src/main/java/it/cavallium/dbengine/client/query/QueryParser.java Normal file
View File

@ -0,0 +1,58 @@
package it.cavallium.dbengine.client.query;
import it.cavallium.dbengine.client.query.current.data.NumericSort;
import it.cavallium.dbengine.client.query.current.data.QueryParams;
import org.apache.commons.lang.NotImplementedException;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.SortedNumericSortField;
public class QueryParser {
public static Query toQuery(it.cavallium.dbengine.client.query.current.data.Query query) {
if (query == null) return null;
throw new NotImplementedException();
}
public static boolean isScoringEnabled(QueryParams queryParams) {
return queryParams.getScoreMode().getComputeScores();
}
public static Sort toSort(it.cavallium.dbengine.client.query.current.data.Sort sort) {
switch (sort.getBasicType$()) {
case NoSort:
return null;
case ScoreSort:
return new Sort(SortField.FIELD_SCORE);
case DocSort:
return new Sort(SortField.FIELD_DOC);
case NumericSort:
NumericSort numericSort = (NumericSort) sort;
return new Sort(new SortedNumericSortField(numericSort.getField(), Type.LONG, numericSort.getReverse()));
default:
throw new IllegalStateException("Unexpected value: " + sort.getBasicType$());
}
}
@SuppressWarnings("ConstantConditions")
public static ScoreMode toScoreMode(it.cavallium.dbengine.client.query.current.data.ScoreMode scoreMode) {
if (scoreMode.getComputeScores() && scoreMode.getOnlyTopScores()) {
return ScoreMode.TOP_SCORES;
} else if (scoreMode.getComputeScores() && !scoreMode.getOnlyTopScores()) {
return ScoreMode.COMPLETE;
} else if (!scoreMode.getComputeScores() && scoreMode.getOnlyTopScores()) {
throw new IllegalStateException("Conflicting score mode options: [computeScores = false, onlyTopScore = true]");
} else if (!scoreMode.getComputeScores() && !scoreMode.getOnlyTopScores()) {
return ScoreMode.COMPLETE_NO_SCORES;
} else {
throw new IllegalStateException("Unexpected value: " + scoreMode);
}
}
public static it.cavallium.dbengine.client.query.current.data.Term toQueryTerm(Term term) {
return it.cavallium.dbengine.client.query.current.data.Term.of(term.field(), term.text());
}
}

101
src/main/java/it/cavallium/dbengine/lucene/serializer/Query.java → src/main/java/it/cavallium/dbengine/client/query/QueryUtils.java
View File

@ -1,8 +1,18 @@
package it.cavallium.dbengine.lucene.serializer;
import static it.cavallium.dbengine.lucene.serializer.QueryParser.USE_PHRASE_QUERY;
import static it.cavallium.dbengine.lucene.serializer.QueryParser.USE_QUERY_BUILDER;
package it.cavallium.dbengine.client.query;
import it.cavallium.dbengine.client.query.current.data.BooleanQuery;
import it.cavallium.dbengine.client.query.current.data.BooleanQueryPart;
import it.cavallium.dbengine.client.query.current.data.Occur;
import it.cavallium.dbengine.client.query.current.data.OccurFilter;
import it.cavallium.dbengine.client.query.current.data.OccurMust;
import it.cavallium.dbengine.client.query.current.data.OccurMustNot;
import it.cavallium.dbengine.client.query.current.data.OccurShould;
import it.cavallium.dbengine.client.query.current.data.PhraseQuery;
import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.client.query.current.data.SynonymQuery;
import it.cavallium.dbengine.client.query.current.data.TermAndBoost;
import it.cavallium.dbengine.client.query.current.data.TermPosition;
import it.cavallium.dbengine.client.query.current.data.TermQuery;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import java.io.IOException;
@ -19,62 +29,27 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.util.QueryBuilder;
import org.jetbrains.annotations.NotNull;
public interface Query extends SerializedQueryObject {
public class QueryUtils {
static Query approximateSearch(TextFieldsAnalyzer preferredAnalyzer, String field, String text) {
if (USE_QUERY_BUILDER) {
var qb = new QueryBuilder(LuceneUtils.getAnalyzer(preferredAnalyzer));
var luceneQuery = qb.createMinShouldMatchQuery(field, text, 0.75f);
return transformQuery(field, luceneQuery);
}
try {
var terms = getTerms(preferredAnalyzer, field, text);
List<BooleanQueryPart> booleanQueryParts = new LinkedList<>();
for (TermPosition term : terms) {
booleanQueryParts.add(new BooleanQueryPart(new TermQuery(term.getTerm()), Occur.MUST));
booleanQueryParts.add(new BooleanQueryPart(new PhraseQuery(terms.toArray(TermPosition[]::new)), Occur.SHOULD));
}
return new BooleanQuery(booleanQueryParts);
} catch (IOException e) {
e.printStackTrace();
return exactSearch(preferredAnalyzer, field, text);
}
public static Query approximateSearch(TextFieldsAnalyzer preferredAnalyzer, String field, String text) {
var qb = new QueryBuilder(LuceneUtils.getAnalyzer(preferredAnalyzer));
var luceneQuery = qb.createMinShouldMatchQuery(field, text, 0.75f);
return transformQuery(field, luceneQuery);
}
static Query exactSearch(TextFieldsAnalyzer preferredAnalyzer, String field, String text) {
if (USE_QUERY_BUILDER) {
var qb = new QueryBuilder(LuceneUtils.getAnalyzer(preferredAnalyzer));
var luceneQuery = qb.createPhraseQuery(field, text);
return transformQuery(field, luceneQuery);
}
try {
var terms = getTerms(preferredAnalyzer, field, text);
if (USE_PHRASE_QUERY) {
return new PhraseQuery(terms.toArray(TermPosition[]::new));
} else {
List<BooleanQueryPart> booleanQueryParts = new LinkedList<>();
for (TermPosition term : terms) {
booleanQueryParts.add(new BooleanQueryPart(new TermQuery(term.getTerm()), Occur.MUST));
}
booleanQueryParts.add(new BooleanQueryPart(new PhraseQuery(terms.toArray(TermPosition[]::new)), Occur.FILTER));
return new BooleanQuery(booleanQueryParts);
}
} catch (IOException exception) {
throw new RuntimeException(exception);
}
public static Query exactSearch(TextFieldsAnalyzer preferredAnalyzer, String field, String text) {
var qb = new QueryBuilder(LuceneUtils.getAnalyzer(preferredAnalyzer));
var luceneQuery = qb.createPhraseQuery(field, text);
return transformQuery(field, luceneQuery);
}
@NotNull
private static Query transformQuery(String field, org.apache.lucene.search.Query luceneQuery) {
if (luceneQuery == null) {
return new TermQuery(field, "");
return TermQuery.of(it.cavallium.dbengine.client.query.current.data.Term.of(field, ""));
}
if (luceneQuery instanceof org.apache.lucene.search.TermQuery) {
return new TermQuery(((org.apache.lucene.search.TermQuery) luceneQuery).getTerm());
return TermQuery.of(QueryParser.toQueryTerm(((org.apache.lucene.search.TermQuery) luceneQuery).getTerm()));
}
if (luceneQuery instanceof org.apache.lucene.search.BooleanQuery) {
var booleanQuery = (org.apache.lucene.search.BooleanQuery) luceneQuery;
@ -85,23 +60,23 @@ public interface Query extends SerializedQueryObject {
Occur occur;
switch (booleanClause.getOccur()) {
case MUST:
occur = Occur.MUST;
occur = OccurMust.of();
break;
case FILTER:
occur = Occur.FILTER;
occur = OccurFilter.of();
break;
case SHOULD:
occur = Occur.SHOULD;
occur = OccurShould.of();
break;
case MUST_NOT:
occur = Occur.MUST_NOT;
occur = OccurMustNot.of();
break;
default:
throw new IllegalArgumentException();
}
queryParts.add(new BooleanQueryPart(transformQuery(field, queryPartQuery), occur));
queryParts.add(BooleanQueryPart.of(transformQuery(field, queryPartQuery), occur));
}
return new BooleanQuery(queryParts).setMinShouldMatch(booleanQuery.getMinimumNumberShouldMatch());
return BooleanQuery.of(queryParts.toArray(BooleanQueryPart[]::new), booleanQuery.getMinimumNumberShouldMatch());
}
if (luceneQuery instanceof org.apache.lucene.search.PhraseQuery) {
var phraseQuery = (org.apache.lucene.search.PhraseQuery) luceneQuery;
@ -112,13 +87,17 @@ public interface Query extends SerializedQueryObject {
for (int i = 0; i < terms.length; i++) {
var term = terms[i];
var position = positions[i];
termPositions[i] = new TermPosition(term, position);
termPositions[i] = TermPosition.of(QueryParser.toQueryTerm(term), position);
}
return new PhraseQuery(termPositions).setSlop(slop);
return PhraseQuery.of(termPositions, slop);
}
org.apache.lucene.search.SynonymQuery synonymQuery = (org.apache.lucene.search.SynonymQuery) luceneQuery;
return new SynonymQuery(field,
synonymQuery.getTerms().stream().map(TermQuery::new).toArray(TermQuery[]::new)
return SynonymQuery.of(field,
synonymQuery
.getTerms()
.stream()
.map(term -> TermAndBoost.of(QueryParser.toQueryTerm(term), 1))
.toArray(TermAndBoost[]::new)
);
}
@ -138,7 +117,7 @@ public interface Query extends SerializedQueryObject {
while (ts.incrementToken()) {
var tokenPositionIncrement = positionIncrementTermAttr.getPositionIncrement();
termPosition += tokenPositionIncrement;
terms.add(new TermPosition(new Term(field, charTermAttr.getBytesRef()), termPosition));
terms.add(TermPosition.of(QueryParser.toQueryTerm(new Term(field, charTermAttr.getBytesRef())), termPosition));
}
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
}

41
src/main/java/it/cavallium/dbengine/database/LLLuceneIndex.java
View File

@ -1,6 +1,10 @@
package it.cavallium.dbengine.database;
import it.cavallium.dbengine.lucene.serializer.Query;
import it.cavallium.data.generator.nativedata.Nullablefloat;
import it.cavallium.dbengine.client.query.current.data.NoSort;
import it.cavallium.dbengine.client.query.current.data.Query;
import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.ScoreMode;
import java.util.Set;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Flux;
@ -25,38 +29,27 @@ public interface LLLuceneIndex extends LLSnapshottable {
Mono<Void> deleteAll();
/**
* @param additionalQuery An additional query that will be used with the moreLikeThis query: "mltQuery AND additionalQuery"
* @param limit the limit is valid for each lucene instance.
* If you have 15 instances, the number of elements returned
* can be at most <code>limit * 15</code>
* @param queryParams the limit is valid for each lucene instance. If you have 15 instances, the number of elements
* returned can be at most <code>limit * 15</code>.
* <p>
* The additional query will be used with the moreLikeThis query: "mltQuery AND additionalQuery"
* @return the collection has one or more flux
*/
Mono<LLSearchResult> moreLikeThis(@Nullable LLSnapshot snapshot,
Flux<Tuple2<String, Set<String>>> mltDocumentFields,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
long limit,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore,
String keyFieldName);
QueryParams queryParams,
String keyFieldName,
Flux<Tuple2<String, Set<String>>> mltDocumentFields);
/**
*
* @param limit the limit is valid for each lucene instance.
* If you have 15 instances, the number of elements returned
* can be at most <code>limit * 15</code>
* @param queryParams the limit is valid for each lucene instance. If you have 15 instances, the number of elements
* returned can be at most <code>limit * 15</code>
* @return the collection has one or more flux
*/
Mono<LLSearchResult> search(@Nullable LLSnapshot snapshot,
Query query,
long limit,
@Nullable LLSort sort,
LLScoreMode scoreMode,
@Nullable Float minCompetitiveScore,
String keyFieldName);
Mono<LLSearchResult> search(@Nullable LLSnapshot snapshot, QueryParams queryParams, String keyFieldName);
default Mono<Long> count(@Nullable LLSnapshot snapshot, Query query) {
return this.search(snapshot, query, 0, null, null, null, null)
QueryParams params = QueryParams.of(query, 0, Nullablefloat.empty(), NoSort.of(), ScoreMode.of(false, false));
return this.search(snapshot, params, null)
.flatMap(LLSearchResult::totalHitsCount)
.single();
}

122
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -1,17 +1,15 @@
package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.lucene.LuceneUtils.checkScoringArgumentsValidity;
import com.google.common.base.Suppliers;
import it.cavallium.dbengine.client.query.QueryParser;
import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.database.EnglishItalianStopFilter;
import it.cavallium.dbengine.database.LLDocument;
import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLSearchCollectionStatisticsGetter;
import it.cavallium.dbengine.database.LLSearchResult;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLSort;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.lucene.LuceneUtils;
@ -21,8 +19,6 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.lucene.searcher.AdaptiveStreamSearcher;
import it.cavallium.dbengine.lucene.searcher.AllowOnlyQueryParsingCollectorStreamSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneStreamSearcher;
import it.cavallium.dbengine.lucene.serializer.ParseException;
import it.cavallium.dbengine.lucene.serializer.QueryParser;
import java.io.IOException;
import java.nio.file.Path;
import java.time.Duration;
@ -49,12 +45,10 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory;
@ -104,7 +98,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
*/
private final AtomicLong lastSnapshotSeqNo = new AtomicLong(0);
/**
* Snapshot seq no to index commit point
* LLSnapshot seq no to index commit point
*/
private final ConcurrentHashMap<Long, LuceneIndexSnapshot> snapshots = new ConcurrentHashMap<>();
private final boolean lowMemory;
@ -229,7 +223,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
return Mono.<Void>fromCallable(() -> {
var indexSnapshot = this.snapshots.remove(snapshot.getSequenceNumber());
if (indexSnapshot == null) {
throw new IOException("Snapshot " + snapshot.getSequenceNumber() + " not found!");
throw new IOException("LLSnapshot " + snapshot.getSequenceNumber() + " not found!");
}
//noinspection BlockingMethodInNonBlockingContext
@ -358,21 +352,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
@Override
public Mono<LLSearchResult> moreLikeThis(@Nullable LLSnapshot snapshot,
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsFlux,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
long limit,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore,
String keyFieldName) {
QueryParams queryParams,
String keyFieldName,
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsFlux) {
return moreLikeThis(snapshot,
mltDocumentFieldsFlux,
additionalQuery,
limit,
minCompetitiveScore,
enableScoring,
sortByScore,
queryParams,
keyFieldName,
mltDocumentFieldsFlux,
false,
0,
1
@ -380,23 +366,15 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
}
public Mono<LLSearchResult> distributedMoreLikeThis(@Nullable LLSnapshot snapshot,
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsFlux,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
long limit,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore,
QueryParams queryParams,
String keyFieldName,
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsFlux,
long actionId,
int scoreDivisor) {
return moreLikeThis(snapshot,
mltDocumentFieldsFlux,
additionalQuery,
limit,
minCompetitiveScore,
enableScoring,
sortByScore,
queryParams,
keyFieldName,
mltDocumentFieldsFlux,
false,
actionId,
scoreDivisor
@ -404,20 +382,14 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
}
public Mono<Void> distributedPreMoreLikeThis(@Nullable LLSnapshot snapshot,
QueryParams queryParams,
String keyFieldName,
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsFlux,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore,
String keyFieldName, long actionId) {
long actionId) {
return moreLikeThis(snapshot,
mltDocumentFieldsFlux,
additionalQuery,
-1,
minCompetitiveScore,
enableScoring,
sortByScore,
queryParams,
keyFieldName,
mltDocumentFieldsFlux,
true,
actionId,
1
@ -427,20 +399,16 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
@SuppressWarnings({"unchecked", "rawtypes"})
private Mono<LLSearchResult> moreLikeThis(@Nullable LLSnapshot snapshot,
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsFlux,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
long limit,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore,
QueryParams queryParams,
String keyFieldName,
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsFlux,
boolean doDistributedPre,
long actionId,
int scoreDivisor) {
Query luceneAdditionalQuery;
try {
luceneAdditionalQuery = additionalQuery != null ? QueryParser.parse(additionalQuery) : null;
} catch (ParseException e) {
luceneAdditionalQuery = QueryParser.toQuery(queryParams.getQuery());
} catch (Exception e) {
return Mono.error(e);
}
return mltDocumentFieldsFlux
@ -459,7 +427,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
mlt.setMinTermFreq(1);
mlt.setMinDocFreq(3);
mlt.setMaxDocFreqPct(20);
mlt.setBoost(enableScoring);
mlt.setBoost(QueryParser.isScoringEnabled(queryParams));
mlt.setStopWords(EnglishItalianStopFilter.getStopWordsString());
var similarity = getSimilarity();
if (similarity instanceof TFIDFSimilarity) {
@ -483,15 +451,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
return luceneSearch(doDistributedPre,
indexSearcher,
limit,
minCompetitiveScore,
queryParams.getLimit(),
queryParams.getMinCompetitiveScore().getNullable(),
keyFieldName,
scoreDivisor,
luceneQuery,
(enableScoring && sortByScore) ? new Sort(SortField.FIELD_SCORE) : null,
(enableScoring && minCompetitiveScore != null) ?
ScoreMode.TOP_SCORES :
(enableScoring ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES)
QueryParser.toSort(queryParams.getSort()),
QueryParser.toScoreMode(queryParams.getScoreMode())
);
})
.subscribeOn(luceneQueryScheduler)
@ -511,38 +477,30 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
}
@Override
public Mono<LLSearchResult> search(@Nullable LLSnapshot snapshot, it.cavallium.dbengine.lucene.serializer.Query query, long limit,
@Nullable LLSort sort, @NotNull LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, String keyFieldName) {
return search(snapshot, query, limit, sort, scoreMode, minCompetitiveScore,
keyFieldName, false, 0, 1);
public Mono<LLSearchResult> search(@Nullable LLSnapshot snapshot, QueryParams queryParams, String keyFieldName) {
return search(snapshot, queryParams, keyFieldName, false, 0, 1);
}
public Mono<LLSearchResult> distributedSearch(@Nullable LLSnapshot snapshot, it.cavallium.dbengine.lucene.serializer.Query query, long limit,
@Nullable LLSort sort, @NotNull LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, String keyFieldName, long actionId, int scoreDivisor) {
return search(snapshot, query, limit, sort, scoreMode, minCompetitiveScore,
keyFieldName, false, actionId, scoreDivisor);
public Mono<LLSearchResult> distributedSearch(@Nullable LLSnapshot snapshot, QueryParams queryParams, String keyFieldName, long actionId, int scoreDivisor) {
return search(snapshot, queryParams, keyFieldName, false, actionId, scoreDivisor);
}
public Mono<Void> distributedPreSearch(@Nullable LLSnapshot snapshot, it.cavallium.dbengine.lucene.serializer.Query query,
@Nullable LLSort sort, @NotNull LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, String keyFieldName, long actionId) {
public Mono<Void> distributedPreSearch(@Nullable LLSnapshot snapshot, QueryParams queryParams, String keyFieldName, long actionId) {
return this
.search(snapshot, query, -1, sort, scoreMode,
minCompetitiveScore, keyFieldName, true, actionId, 1)
.search(snapshot, queryParams, keyFieldName, true, actionId, 1)
.flatMap(LLSearchResult::completion);
}
private Mono<LLSearchResult> search(@Nullable LLSnapshot snapshot,
it.cavallium.dbengine.lucene.serializer.Query query, long limit,
@Nullable LLSort sort, @NotNull LLScoreMode scoreMode, @Nullable Float minCompetitiveScore, String keyFieldName,
QueryParams queryParams, String keyFieldName,
boolean doDistributedPre, long actionId, int scoreDivisor) {
return acquireSearcherWrapper(snapshot, doDistributedPre, actionId)
.flatMap(indexSearcher -> Mono
.fromCallable(() -> {
Objects.requireNonNull(scoreMode, "ScoreMode must not be null");
checkScoringArgumentsValidity(sort, scoreMode);
Query luceneQuery = QueryParser.parse(query);
Sort luceneSort = LLUtils.toSort(sort);
org.apache.lucene.search.ScoreMode luceneScoreMode = LLUtils.toScoreMode(scoreMode);
Objects.requireNonNull(queryParams.getScoreMode(), "ScoreMode must not be null");
Query luceneQuery = QueryParser.toQuery(queryParams.getQuery());
Sort luceneSort = QueryParser.toSort(queryParams.getSort());
org.apache.lucene.search.ScoreMode luceneScoreMode = QueryParser.toScoreMode(queryParams.getScoreMode());
return Tuples.of(luceneQuery, Optional.ofNullable(luceneSort), luceneScoreMode);
})
.subscribeOn(luceneQueryScheduler)
@ -554,8 +512,8 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
return luceneSearch(doDistributedPre,
indexSearcher,
limit,
minCompetitiveScore,
queryParams.getLimit(),
queryParams.getMinCompetitiveScore().getNullable(),
keyFieldName,
scoreDivisor,
luceneQuery,

48
src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java
View File

@ -2,16 +2,14 @@ package it.cavallium.dbengine.database.disk;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.database.LLDocument;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLSearchResult;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLSort;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.lucene.serializer.Query;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import java.io.IOException;
@ -202,13 +200,9 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
@Override
public Mono<LLSearchResult> moreLikeThis(@Nullable LLSnapshot snapshot,
Flux<Tuple2<String, Set<String>>> mltDocumentFields,
@Nullable it.cavallium.dbengine.lucene.serializer.Query additionalQuery,
long limit,
@Nullable Float minCompetitiveScore,
boolean enableScoring,
boolean sortByScore,
String keyFieldName) {
QueryParams queryParams,
String keyFieldName,
Flux<Tuple2<String, Set<String>>> mltDocumentFields) {
long actionId;
int scoreDivisor;
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsShared;
@ -227,12 +221,9 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
.flatMap(tuple -> tuple
.getT1()
.distributedPreMoreLikeThis(tuple.getT2().orElse(null),
mltDocumentFieldsShared,
additionalQuery,
minCompetitiveScore,
enableScoring,
sortByScore,
queryParams,
keyFieldName,
mltDocumentFieldsShared,
actionId
)
)
@ -253,13 +244,9 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
.flatMap(tuple -> tuple
.getT1()
.distributedMoreLikeThis(tuple.getT2().orElse(null),
mltDocumentFieldsShared,
additionalQuery,
limit,
minCompetitiveScore,
enableScoring,
sortByScore,
queryParams,
keyFieldName,
mltDocumentFieldsShared,
actionId,
scoreDivisor
)
@ -280,16 +267,12 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
@Override
public Mono<LLSearchResult> search(@Nullable LLSnapshot snapshot,
Query query,
long limit,
@Nullable LLSort sort,
LLScoreMode scoreMode,
@Nullable Float minCompetitiveScore,
QueryParams queryParams,
String keyFieldName) {
long actionId;
int scoreDivisor;
Mono<Void> distributedPre;
if (luceneIndices.length <= 1 || scoreMode == LLScoreMode.COMPLETE_NO_SCORES) {
if (luceneIndices.length <= 1 || !queryParams.getScoreMode().getComputeScores()) {
actionId = -1;
scoreDivisor = 1;
distributedPre = Mono.empty();
@ -306,10 +289,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
.flatMap(tuple -> tuple
.getT1()
.distributedPreSearch(tuple.getT2().orElse(null),
query,
sort,
scoreMode,
minCompetitiveScore,
queryParams,
keyFieldName,
actionId
)
@ -327,11 +307,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
.flatMap(tuple -> tuple
.getT1()
.distributedSearch(tuple.getT2().orElse(null),
query,
limit,
sort,
scoreMode,
minCompetitiveScore,
queryParams,
keyFieldName,
actionId,
scoreDivisor

9
src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java
View File

@ -2,9 +2,6 @@ package it.cavallium.dbengine.lucene;
import it.cavallium.dbengine.client.MultiSort;
import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLSort;
import it.cavallium.dbengine.database.LLSortType;
import it.cavallium.dbengine.lucene.analyzer.NCharGramAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.NCharGramEdgeAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
@ -191,12 +188,6 @@ public class LuceneUtils {
});
}
public static void checkScoringArgumentsValidity(LLSort sort, LLScoreMode scoreMode) {
if ((sort == null || sort.getType() != LLSortType.SCORE) && scoreMode != LLScoreMode.COMPLETE_NO_SCORES) {
throw new IllegalArgumentException("You must sort by score if the scores are enabled");
}
}
public static void collectTopDoc(Logger logger,
int docId,
float score,

43
src/main/java/it/cavallium/dbengine/lucene/serializer/BooleanQuery.java
View File

@ -1,43 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import java.util.Arrays;
import java.util.Collection;
import java.util.stream.Collectors;
@SuppressWarnings("unused")
public class BooleanQuery implements Query {
private final BooleanQueryPart[] parts;
private int minShouldMatch;
public BooleanQuery(BooleanQueryPart... parts) {
this.parts = parts;
}
public BooleanQuery(Collection<BooleanQueryPart> parts) {
this.parts = parts.toArray(BooleanQueryPart[]::new);
}
public BooleanQuery setMinShouldMatch(int minShouldMatch) {
this.minShouldMatch = minShouldMatch;
return this;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyInt(data, minShouldMatch);
StringBuilder listData = new StringBuilder();
listData.append(parts.length).append('|');
for (BooleanQueryPart part : parts) {
part.stringify(listData);
}
StringifyUtils.writeHeader(data, QueryConstructorType.BOOLEAN_QUERY_INFO_LIST, listData);
StringifyUtils.writeHeader(output, QueryConstructorType.BOOLEAN_QUERY, data);
}
@Override
public String toString() {
return Arrays.stream(parts).map(Object::toString).collect(Collectors.joining(" || ", "((", ") Minimum should matches:" + minShouldMatch + ")"));
}
}

42
src/main/java/it/cavallium/dbengine/lucene/serializer/BooleanQueryInfo.java
View File

@ -1,42 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import java.util.Objects;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Query;
public class BooleanQueryInfo {
public final Query query;
public final BooleanClause.Occur occur;
public BooleanQueryInfo(Query query, BooleanClause.Occur occur) {
this.query = query;
this.occur = occur;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
BooleanQueryInfo that = (BooleanQueryInfo) o;
return Objects.equals(query, that.query) &&
occur == that.occur;
}
@Override
public int hashCode() {
return Objects.hash(query, occur);
}
@Override
public String toString() {
return "BooleanQueryInfo{" +
"query=" + query +
", occur=" + occur +
'}';
}
}

25
src/main/java/it/cavallium/dbengine/lucene/serializer/BooleanQueryPart.java
View File

@ -1,25 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class BooleanQueryPart implements SerializedQueryObject {
private final Query query;
private final Occur occur;
public BooleanQueryPart(Query query, Occur occur) {
this.query = query;
this.occur = occur;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
query.stringify(data);
occur.stringify(data);
StringifyUtils.writeHeader(output, QueryConstructorType.BOOLEAN_QUERY_INFO, data);
}
@Override
public String toString() {
return occur + ":" + query;
}
}

25
src/main/java/it/cavallium/dbengine/lucene/serializer/BoostQuery.java
View File

@ -1,25 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class BoostQuery implements Query {
private final Query query;
private final float boostValue;
public BoostQuery(Query query, float boostValue) {
this.query = query;
this.boostValue = boostValue;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
query.stringify(data);
StringifyUtils.stringifyFloat(data, boostValue);
StringifyUtils.writeHeader(output, QueryConstructorType.BOOST_QUERY, data);
}
@Override
public String toString() {
return "(" + query + " *" + boostValue + ")";
}
}

22
src/main/java/it/cavallium/dbengine/lucene/serializer/ConstantScoreQuery.java
View File

@ -1,22 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class ConstantScoreQuery implements Query {
private final Query query;
public ConstantScoreQuery(Query query) {
this.query = query;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
query.stringify(data);
StringifyUtils.writeHeader(output, QueryConstructorType.CONSTANT_SCORE_QUERY, data);
}
@Override
public String toString() {
return "(" + query + " *1)";
}
}

26
src/main/java/it/cavallium/dbengine/lucene/serializer/DocValuesFieldExistsQuery.java
View File

@ -1,26 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class DocValuesFieldExistsQuery implements Query {
private final String field;
public DocValuesFieldExistsQuery(String field) {
this.field = field;
}
public String getField() {
return field;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyString(data, field);
StringifyUtils.writeHeader(output, QueryConstructorType.DOC_VALUES_FIELD_EXISTS_QUERY, data);
}
@Override
public String toString() {
return "(existence of field " + field + ")";
}
}

50
src/main/java/it/cavallium/dbengine/lucene/serializer/FuzzyQuery.java
View File

@ -1,50 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
@SuppressWarnings("unused")
public class FuzzyQuery implements Query {
private final Term term;
private final int val1;
private final int val2;
private final int val3;
private final boolean bool;
/**
* Create a new FuzzyQuery that will match terms with an edit distance of at most
* <code>maxEdits</code> to <code>term</code>. If a <code>prefixLength</code> &gt; 0 is
* specified, a common prefix of that length is also required.
*
* @param term the term to search for
* @param maxEdits must be {@code >= 0} and {@code <=} {@link LevenshteinAutomata#MAXIMUM_SUPPORTED_DISTANCE}.
* @param prefixLength length of common (non-fuzzy) prefix
* @param maxExpansions the maximum number of terms to match. If this number is greater than
* {@link BooleanQuery#getMaxClauseCount} when the query is rewritten, then
* the maxClauseCount will be used instead.
* @param transpositions true if transpositions should be treated as a primitive edit operation.
* If this is false, comparisons will implement the classic Levenshtein
* algorithm.
*/
public FuzzyQuery(Term term, int maxEdits, int prefixLength, int maxExpansions,
boolean transpositions) {
this.term = term;
this.val1 = maxEdits;
this.val2 = prefixLength;
this.val3 = maxExpansions;
this.bool = transpositions;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyTerm(data, term);
StringifyUtils.stringifyInt(data, val1);
StringifyUtils.stringifyInt(data, val2);
StringifyUtils.stringifyInt(data, val3);
StringifyUtils.stringifyBool(data, bool);
StringifyUtils.writeHeader(output, QueryConstructorType.FUZZY_QUERY, data);
}
}

20
src/main/java/it/cavallium/dbengine/lucene/serializer/IntPointExactQuery.java
View File

@ -1,20 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class IntPointExactQuery implements Query {
private final String name;
private final int value;
public IntPointExactQuery(String name, int value) {
this.name = name;
this.value = value;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyString(data, name);
StringifyUtils.stringifyInt(data, value);
StringifyUtils.writeHeader(output, QueryConstructorType.INT_POINT_EXACT_QUERY, data);
}
}

23
src/main/java/it/cavallium/dbengine/lucene/serializer/IntPointRangeQuery.java
View File

@ -1,23 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class IntPointRangeQuery implements Query {
private final String name;
private final int min;
private final int max;
public IntPointRangeQuery(String name, int min, int max) {
this.name = name;
this.min = min;
this.max = max;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyString(data, name);
StringifyUtils.stringifyInt(data, min);
StringifyUtils.stringifyInt(data, max);
StringifyUtils.writeHeader(output, QueryConstructorType.INT_POINT_RANGE_QUERY, data);
}
}

20
src/main/java/it/cavallium/dbengine/lucene/serializer/LongPointExactQuery.java
View File

@ -1,20 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class LongPointExactQuery implements Query {
private final String name;
private final long value;
public LongPointExactQuery(String name, long value) {
this.name = name;
this.value = value;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyString(data, name);
StringifyUtils.stringifyLong(data, value);
StringifyUtils.writeHeader(output, QueryConstructorType.LONG_POINT_EXACT_QUERY, data);
}
}

23
src/main/java/it/cavallium/dbengine/lucene/serializer/LongPointRangeQuery.java
View File

@ -1,23 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class LongPointRangeQuery implements Query {
private final String name;
private final long min;
private final long max;
public LongPointRangeQuery(String name, long min, long max) {
this.name = name;
this.min = min;
this.max = max;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyString(data, name);
StringifyUtils.stringifyLong(data, min);
StringifyUtils.stringifyLong(data, max);
StringifyUtils.writeHeader(output, QueryConstructorType.LONG_POINT_RANGE_QUERY, data);
}
}

13
src/main/java/it/cavallium/dbengine/lucene/serializer/MatchAllDocsQuery.java
View File

@ -1,13 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class MatchAllDocsQuery implements Query {
public MatchAllDocsQuery() {
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.writeHeader(output, QueryConstructorType.MATCH_ALL_DOCS_QUERY, data);
}
}

41
src/main/java/it/cavallium/dbengine/lucene/serializer/Occur.java
View File

@ -1,41 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import org.apache.lucene.search.BooleanClause;
@SuppressWarnings("unused")
public class Occur implements SerializedQueryObject {
private final BooleanClause.Occur occur;
public Occur(BooleanClause.Occur occur) {
this.occur = occur;
}
public static Occur MUST = new Occur(BooleanClause.Occur.MUST);
public static Occur FILTER = new Occur(BooleanClause.Occur.FILTER);
public static Occur SHOULD = new Occur(BooleanClause.Occur.SHOULD);
public static Occur MUST_NOT = new Occur(BooleanClause.Occur.MUST_NOT);
@Override
public void stringify(StringBuilder output) {
switch (occur) {
case MUST:
StringifyUtils.writeHeader(output, QueryConstructorType.OCCUR_MUST, new StringBuilder());
break;
case FILTER:
StringifyUtils.writeHeader(output, QueryConstructorType.OCCUR_FILTER, new StringBuilder());
break;
case SHOULD:
StringifyUtils.writeHeader(output, QueryConstructorType.OCCUR_SHOULD, new StringBuilder());
break;
case MUST_NOT:
StringifyUtils.writeHeader(output, QueryConstructorType.OCCUR_MUST_NOT, new StringBuilder());
break;
}
}
@Override
public String toString() {
return occur.name();
}
}

8
src/main/java/it/cavallium/dbengine/lucene/serializer/ParseException.java
View File

@ -1,8 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public class ParseException extends Exception {
public ParseException(Exception e) {
super(e);
}
}

39
src/main/java/it/cavallium/dbengine/lucene/serializer/PhraseQuery.java
View File

@ -1,39 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import java.util.Arrays;
import java.util.stream.Collectors;
public class PhraseQuery implements Query {
// some terms can be null
private final TermPosition[] parts;
private int slop;
public PhraseQuery(TermPosition... parts) {
this.parts = parts;
this.slop = 0;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringBuilder listData = new StringBuilder();
listData.append(parts.length).append('|');
for (TermPosition part : parts) {
StringifyUtils.stringifyTermPosition(listData, part);
}
StringifyUtils.writeHeader(data, QueryConstructorType.TERM_POSITION_LIST, listData);
StringifyUtils.stringifyInt(data, slop);
StringifyUtils.writeHeader(output, QueryConstructorType.PHRASE_QUERY, data);
}
@Override
public String toString() {
return Arrays.stream(parts).map(Object::toString).collect(Collectors.joining(", ", "(", ")"));
}
public PhraseQuery setSlop(int slop) {
this.slop = slop;
return this;
}
}

35
src/main/java/it/cavallium/dbengine/lucene/serializer/QueryConstructorType.java
View File

@ -1,35 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public enum QueryConstructorType {
BOOLEAN_QUERY,
BOOLEAN_QUERY_INFO,
BOOST_QUERY,
CONSTANT_SCORE_QUERY,
SORTED_SLOW_RANGE_QUERY,
INT_POINT_EXACT_QUERY,
LONG_POINT_EXACT_QUERY,
TERM_QUERY,
WILDCARD_QUERY,
DOC_VALUES_FIELD_EXISTS_QUERY,
FUZZY_QUERY,
PHRASE_QUERY,
SYNONYM_QUERY,
STRING,
OCCUR_MUST,
OCCUR_SHOULD,
OCCUR_FILTER,
OCCUR_MUST_NOT,
FLOAT,
LONG,
INT,
TERM,
NULL,
BOOLEAN,
TERM_POSITION_LIST,
TERM_QUERY_LIST,
TERM_POSITION,
BOOLEAN_QUERY_INFO_LIST,
LONG_POINT_RANGE_QUERY,
INT_POINT_RANGE_QUERY,
MATCH_ALL_DOCS_QUERY
}

295
src/main/java/it/cavallium/dbengine/lucene/serializer/QueryParser.java
View File

@ -1,295 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Objects;
import java.util.PrimitiveIterator;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
public class QueryParser {
static final boolean USE_PHRASE_QUERY = true;
static final boolean USE_QUERY_BUILDER = true;
public static Query parse(String text) throws ParseException {
try {
return (Query) parse(text, new AtomicInteger(0));
} catch (Exception e) {
throw new ParseException(e);
}
}
public static Query parse(it.cavallium.dbengine.lucene.serializer.Query query) throws ParseException {
StringBuilder sb = new StringBuilder();
query.stringify(sb);
return parse(sb.toString());
}
private static Object parse(String completeText, AtomicInteger position) {
String text = completeText.substring(position.get());
if (text.length() <= 2) {
return null;
}
PrimitiveIterator.OfInt iterator = text.chars().iterator();
StringBuilder numberBuilder = new StringBuilder();
int index = 0;
while (iterator.hasNext()) {
char character = (char) iterator.nextInt();
index++;
if (character == '|') {
break;
} else {
numberBuilder.append(character);
}
}
int len = Integer.parseInt(numberBuilder.toString(), 16);
StringBuilder typeBuilder = new StringBuilder();
while (iterator.hasNext()) {
char character = (char) iterator.nextInt();
index++;
if (character == '|') {
break;
} else {
typeBuilder.append(character);
}
}
QueryConstructorType type = QueryConstructorType.values()[Integer.parseInt(typeBuilder.toString())];
position.addAndGet(index);
String toParse = text.substring(index, index + len);
switch (type) {
case BOOST_QUERY:
Query query = (Query) parse(completeText, position);
Float numb = (Float) parse(completeText, position);
assert query != null;
assert numb != null;
return new BoostQuery(query, numb);
case CONSTANT_SCORE_QUERY:
Query queryC = (Query) parse(completeText, position);
assert queryC != null;
return new ConstantScoreQuery(queryC);
case FUZZY_QUERY:
Term fqTerm = (Term) parse(completeText, position);
Integer numb1 = (Integer) parse(completeText, position);
Integer numb2 = (Integer) parse(completeText, position);
Integer numb3 = (Integer) parse(completeText, position);
Boolean bool1 = (Boolean) parse(completeText, position);
assert fqTerm != null;
assert numb1 != null;
assert numb2 != null;
assert numb3 != null;
assert bool1 != null;
return new FuzzyQuery(fqTerm, numb1, numb2, numb3, bool1);
case PHRASE_QUERY:
TermPosition[] pqTerms = (TermPosition[]) parse(completeText, position);
var pqB = new PhraseQuery.Builder();
assert pqTerms != null;
for (TermPosition pqTerm : pqTerms) {
if (pqTerm != null) {
pqB.add(pqTerm.getTerm(), pqTerm.getPosition());
}
}
Integer slops = Objects.requireNonNull((Integer) parse(completeText, position));
pqB.setSlop(slops);
return pqB.build();
case SYNONYM_QUERY:
var fieldName = (String) parse(completeText, position);
var pqB2 = new SynonymQuery.Builder(fieldName);
TermQuery[] pqTerms2
= (TermQuery[]) parse(completeText, position);
assert pqTerms2 != null;
for (TermQuery pqTerm : pqTerms2) {
if (pqTerm != null) {
pqB2.addTerm(pqTerm.getTerm());
}
}
return pqB2.build();
case BOOLEAN_QUERY:
var bqB = new BooleanQuery.Builder();
//noinspection ConstantConditions
int minShouldMatch = (Integer) parse(completeText, position);
bqB.setMinimumNumberShouldMatch(minShouldMatch);
BooleanQueryInfo[] bqTerms = (BooleanQueryInfo[]) parse(completeText, position);
assert bqTerms != null;
for (BooleanQueryInfo bqTerm : bqTerms) {
bqB.add(bqTerm.query, bqTerm.occur);
}
return bqB.build();
case BOOLEAN_QUERY_INFO:
Query query1 = (Query) parse(completeText, position);
BooleanClause.Occur occur = (BooleanClause.Occur) parse(completeText, position);
return new BooleanQueryInfo(query1, occur);
case INT_POINT_EXACT_QUERY:
String string1 = (String) parse(completeText, position);
Integer int1 = (Integer) parse(completeText, position);
assert string1 != null;
assert int1 != null;
return IntPoint.newExactQuery(string1, int1);
case LONG_POINT_EXACT_QUERY:
String string5 = (String) parse(completeText, position);
Long long3 = (Long) parse(completeText, position);
assert string5 != null;
assert long3 != null;
return LongPoint.newExactQuery(string5, long3);
case SORTED_SLOW_RANGE_QUERY:
String string2 = (String) parse(completeText, position);
Long long1 = (Long) parse(completeText, position);
Long long2 = (Long) parse(completeText, position);
assert string2 != null;
assert long1 != null;
assert long2 != null;
return SortedNumericDocValuesField.newSlowRangeQuery(string2, long1, long2);
case LONG_POINT_RANGE_QUERY:
String stringX2 = (String) parse(completeText, position);
Long longX1 = (Long) parse(completeText, position);
Long longX2 = (Long) parse(completeText, position);
assert stringX2 != null;
assert longX1 != null;
assert longX2 != null;
return LongPoint.newRangeQuery(stringX2, longX1, longX2);
case INT_POINT_RANGE_QUERY:
String stringX3 = (String) parse(completeText, position);
Integer intX1 = (Integer) parse(completeText, position);
Integer intX2 = (Integer) parse(completeText, position);
assert stringX3 != null;
assert intX1 != null;
assert intX2 != null;
return IntPoint.newRangeQuery(stringX3, intX1, intX2);
case INT:
position.addAndGet(toParse.length());
return Integer.parseInt(toParse);
case LONG:
position.addAndGet(toParse.length());
return Long.parseLong(toParse);
case TERM:
String string3 = (String) parse(completeText, position);
String string4 = (String) parse(completeText, position);
assert string4 != null;
return new Term(string3, string4);
case TERM_POSITION:
Term term1 = (Term) parse(completeText, position);
Integer intX3 = (Integer) parse(completeText, position);
assert intX3 != null;
return new TermPosition(term1, intX3);
case TERM_QUERY:
Term term2 = (Term) parse(completeText, position);
assert term2 != null;
return new TermQuery(term2);
case WILDCARD_QUERY:
Term term3 = (Term) parse(completeText, position);
assert term3 != null;
return new WildcardQuery(term3);
case DOC_VALUES_FIELD_EXISTS_QUERY:
String fieldKey = (String) parse(completeText, position);
assert fieldKey != null;
return new DocValuesFieldExistsQuery(fieldKey);
case FLOAT:
position.addAndGet(toParse.length());
return Float.parseFloat(toParse);
case STRING:
position.addAndGet(toParse.length());
return new String(Base64.getDecoder().decode(toParse), StandardCharsets.UTF_8);
case BOOLEAN:
position.addAndGet(toParse.length());
return Boolean.parseBoolean(toParse);
case NULL:
position.addAndGet(toParse.length());
return null;
case TERM_POSITION_LIST:
int termsCount;
StringBuilder termsCountBuilder = new StringBuilder();
var it = toParse.chars().iterator();
while (it.hasNext()) {
char character = (char) it.nextInt();
position.incrementAndGet();
if (character == '|') {
break;
} else {
termsCountBuilder.append(character);
}
}
termsCount = Integer.parseInt(termsCountBuilder.toString());
var result1 = new TermPosition[termsCount];
for (int i = 0; i < termsCount; i++) {
result1[i] = (TermPosition) parse(completeText, position);
}
return result1;
case TERM_QUERY_LIST:
int termsCount2;
StringBuilder termsCountBuilder2 = new StringBuilder();
var it2 = toParse.chars().iterator();
while (it2.hasNext()) {
char character = (char) it2.nextInt();
position.incrementAndGet();
if (character == '|') {
break;
} else {
termsCountBuilder2.append(character);
}
}
termsCount2 = Integer.parseInt(termsCountBuilder2.toString());
var result2 = new TermQuery[termsCount2];
for (int i = 0; i < termsCount2; i++) {
result2[i] = (TermQuery) parse(completeText, position);
}
return result2;
case BOOLEAN_QUERY_INFO_LIST:
int termsCount3;
StringBuilder termsCountBuilder3 = new StringBuilder();
var it3 = toParse.chars().iterator();
while (it3.hasNext()) {
char character = (char) it3.nextInt();
position.incrementAndGet();
if (character == '|') {
break;
} else {
termsCountBuilder3.append(character);
}
}
termsCount3 = Integer.parseInt(termsCountBuilder3.toString());
var result3 = new BooleanQueryInfo[termsCount3];
for (int i = 0; i < termsCount3; i++) {
result3[i] = (BooleanQueryInfo) parse(completeText, position);
}
return result3;
case OCCUR_MUST:
return BooleanClause.Occur.MUST;
case OCCUR_FILTER:
return BooleanClause.Occur.FILTER;
case OCCUR_SHOULD:
return BooleanClause.Occur.SHOULD;
case OCCUR_MUST_NOT:
return BooleanClause.Occur.MUST_NOT;
case MATCH_ALL_DOCS_QUERY:
return new MatchAllDocsQuery();
default:
throw new UnsupportedOperationException("Unknown query constructor type: " + type);
}
}
public static String stringify(SerializedQueryObject query) {
StringBuilder sb = new StringBuilder();
query.stringify(sb);
return sb.toString();
}
}

9
src/main/java/it/cavallium/dbengine/lucene/serializer/SerializedQueryObject.java
View File

@ -1,9 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
public interface SerializedQueryObject {
/**
* returns length|type|---data---
*/
void stringify(StringBuilder output);
}

24
src/main/java/it/cavallium/dbengine/lucene/serializer/SortedNumericDocValuesFieldSlowRangeQuery.java
View File

@ -1,24 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
@SuppressWarnings("unused")
public class SortedNumericDocValuesFieldSlowRangeQuery implements Query {
private final String name;
private final long min;
private final long max;
public SortedNumericDocValuesFieldSlowRangeQuery(String name, long min, long max) {
this.name = name;
this.min = min;
this.max = max;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyString(data, name);
StringifyUtils.stringifyLong(data, min);
StringifyUtils.stringifyLong(data, max);
StringifyUtils.writeHeader(output, QueryConstructorType.SORTED_SLOW_RANGE_QUERY, data);
}
}

61
src/main/java/it/cavallium/dbengine/lucene/serializer/StringifyUtils.java
View File

@ -1,61 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import org.apache.lucene.index.Term;
@SuppressWarnings("unused")
public class StringifyUtils {
public static void stringifyFloat(StringBuilder output, float value) {
writeHeader(output, QueryConstructorType.FLOAT, new StringBuilder().append(value));
}
public static void stringifyInt(StringBuilder output, int value) {
writeHeader(output, QueryConstructorType.INT, new StringBuilder().append(value));
}
public static void stringifyLong(StringBuilder output, long value) {
writeHeader(output, QueryConstructorType.LONG, new StringBuilder().append(value));
}
public static void stringifyBool(StringBuilder output, boolean value) {
writeHeader(output, QueryConstructorType.BOOLEAN, new StringBuilder().append(value));
}
public static void stringifyString(StringBuilder output, String value) {
writeHeader(output, QueryConstructorType.STRING, new StringBuilder()
.append(Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8))));
}
public static void stringifyTerm(StringBuilder output, Term value) {
var data = new StringBuilder();
stringifyString(data, value.field());
stringifyString(data, value.text());
writeHeader(output, QueryConstructorType.TERM, data);
}
public static void stringifyTermQuery(StringBuilder output, TermQuery value) {
var data = new StringBuilder();
stringifyTerm(data, value.getTerm());
writeHeader(output, QueryConstructorType.TERM_QUERY, data);
}
public static void stringifyTermPosition(StringBuilder output, TermPosition value) {
var data = new StringBuilder();
stringifyTerm(data, value.getTerm());
stringifyInt(data, value.getPosition());
writeHeader(output, QueryConstructorType.TERM_POSITION, data);
}
public static void stringifyNullTerm(StringBuilder output) {
writeHeader(output, QueryConstructorType.NULL, new StringBuilder());
}
public static void writeHeader(StringBuilder output, QueryConstructorType type,
StringBuilder data) {
output.append(Integer.toHexString(data.length())).append('|');
output.append(type.ordinal()).append('|');
output.append(data);
}
}

34
src/main/java/it/cavallium/dbengine/lucene/serializer/SynonymQuery.java
View File

@ -1,34 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import java.util.Arrays;
import java.util.stream.Collectors;
public class SynonymQuery implements Query {
private final String field;
// some terms can be null
private final TermQuery[] parts;
public SynonymQuery(String field, TermQuery... parts) {
this.field = field;
this.parts = parts;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyString(data, field);
StringBuilder listData = new StringBuilder();
listData.append(parts.length).append('|');
for (TermQuery part : parts) {
StringifyUtils.stringifyTermQuery(listData, part);
}
StringifyUtils.writeHeader(data, QueryConstructorType.TERM_QUERY_LIST, listData);
StringifyUtils.writeHeader(output, QueryConstructorType.SYNONYM_QUERY, data);
}
@Override
public String toString() {
return Arrays.stream(parts).map(Object::toString).collect(Collectors.joining(", ", "(", ")"));
}
}

32
src/main/java/it/cavallium/dbengine/lucene/serializer/TermPosition.java
View File

@ -1,32 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import org.apache.lucene.index.Term;
public class TermPosition implements Query {
private final Term term;
private final int position;
public TermPosition(Term term, int position) {
this.term = term;
this.position = position;
}
public Term getTerm() {
return term;
}
public int getPosition() {
return position;
}
@Override
public void stringify(StringBuilder output) {
}
@Override
public String toString() {
return "(" + term.field() + ":" + term.text() + " at " + position + ")";
}
}

37
src/main/java/it/cavallium/dbengine/lucene/serializer/TermQuery.java
View File

@ -1,37 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import it.cavallium.dbengine.database.LLTerm;
import org.apache.lucene.index.Term;
public class TermQuery implements Query {
private final Term term;
public TermQuery(Term term) {
this.term = term;
}
public TermQuery(LLTerm term) {
this.term = new Term(term.getKey(), term.getValue());
}
public TermQuery(String name, String val) {
this.term = new Term(name, val);
}
public Term getTerm() {
return term;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyTerm(data, term);
StringifyUtils.writeHeader(output, QueryConstructorType.TERM_QUERY, data);
}
@Override
public String toString() {
return "(" + term.field() + ":" + term.text() + ")";
}
}

37
src/main/java/it/cavallium/dbengine/lucene/serializer/WildcardQuery.java
View File

@ -1,37 +0,0 @@
package it.cavallium.dbengine.lucene.serializer;
import it.cavallium.dbengine.database.LLTerm;
import org.apache.lucene.index.Term;
public class WildcardQuery implements Query {
private final Term term;
public WildcardQuery(Term term) {
this.term = term;
}
public WildcardQuery(LLTerm term) {
this.term = new Term(term.getKey(), term.getValue());
}
public WildcardQuery(String name, String val) {
this.term = new Term(name, val);
}
public Term getTerm() {
return term;
}
@Override
public void stringify(StringBuilder output) {
StringBuilder data = new StringBuilder();
StringifyUtils.stringifyTerm(data, term);
StringifyUtils.writeHeader(output, QueryConstructorType.WILDCARD_QUERY, data);
}
@Override
public String toString() {
return "(" + term.field() + ":" + term.text() + ")";
}
}