andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve reactive searchers performance

Browse Source
This commit is contained in:
Andrea Cavalli 2021-07-05 15:43:19 +02:00
parent a5d4584a11
commit 00ff36836e
5 changed files with 66 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/it/cavallium/dbengine/client/LuceneIndexImpl.java
View File

@ -87,7 +87,7 @@ public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
if (luceneIndex.supportsOffset()) {
return queryParams;
} else {
return queryParams.setOffset(0);
return queryParams.setOffset(0).setLimit(queryParams.limit() + queryParams.offset());
}
}

6
src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java
View File

@ -262,7 +262,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
QueryParams queryParams,
String keyFieldName,
Flux<Tuple2<String, Set<String>>> mltDocumentFields) {
if (queryParams.getOffset() != 0) {
if (queryParams.offset() != 0) {
return Mono.error(new IllegalArgumentException("MultiLuceneIndex requires an offset equal to 0"));
}
Flux<Tuple2<String, Set<String>>> mltDocumentFieldsShared;
@ -341,12 +341,12 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
public Mono<LLSearchResult> search(@Nullable LLSnapshot snapshot,
QueryParams queryParams,
String keyFieldName) {
if (queryParams.getOffset() != 0) {
if (queryParams.offset() != 0) {
return Mono.error(new IllegalArgumentException("MultiLuceneIndex requires an offset equal to 0"));
}
Mono<DistributedSearch> distributedSearchMono;
if (luceneIndices.length <= 1 || !queryParams.getScoreMode().getComputeScores()) {
if (luceneIndices.length <= 1 || !queryParams.scoreMode().computeScores()) {
distributedSearchMono = Mono.just(new DistributedSearch(-1, 1));
} else {
var actionId = newAction();

57
src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLuceneReactiveSearcher.java
View File

@ -21,9 +21,9 @@ import reactor.core.scheduler.Scheduler;
public class PagedLuceneReactiveSearcher implements LuceneReactiveSearcher {
private static final int FIRST_PAGE_HITS_MAX_COUNT = 10;
private static final long MIN_HITS_PER_PAGE = 20;
private static final long MAX_HITS_PER_PAGE = 1000;
private static final int FIRST_PAGE_HITS_MAX_COUNT = 1;
private static final long MIN_HITS_PER_PAGE = 5;
private static final long MAX_HITS_PER_PAGE = 100;
@SuppressWarnings("BlockingMethodInNonBlockingContext")
@Override
@ -36,34 +36,26 @@ public class PagedLuceneReactiveSearcher implements LuceneReactiveSearcher {
@Nullable Float minCompetitiveScore,
String keyFieldName,
Scheduler scheduler) {
// todo: check if offset and limit play well together.
// check especially these cases:
// - offset > limit
// - offset > FIRST_PAGE_HITS_MAX_COUNT
// - offset > MAX_HITS_PER_PAGE
return Mono
.fromCallable(() -> {
// Run the first page search
TopDocs firstTopDocsVal;
if (offset == 0) {
if (luceneSort != null) {
firstTopDocsVal = indexSearcher.search(query,
FIRST_PAGE_HITS_MAX_COUNT,
luceneSort,
scoreMode != ScoreMode.COMPLETE_NO_SCORES
);
} else {
firstTopDocsVal = indexSearcher.search(query,
FIRST_PAGE_HITS_MAX_COUNT
);
}
} else {
firstTopDocsVal = TopDocsSearcher.getTopDocs(indexSearcher,
query,
luceneSort,
FIRST_PAGE_HITS_MAX_COUNT,
null,
scoreMode != ScoreMode.COMPLETE_NO_SCORES,
1000
);
} else {
firstTopDocsVal = TopDocsSearcher.getTopDocs(indexSearcher,
query,
luceneSort,
offset + FIRST_PAGE_HITS_MAX_COUNT,
null,
scoreMode != ScoreMode.COMPLETE_NO_SCORES,
1000,
offset, FIRST_PAGE_HITS_MAX_COUNT);
}
@ -86,20 +78,14 @@ public class PagedLuceneReactiveSearcher implements LuceneReactiveSearcher {
}
try {
TopDocs lastTopDocs;
if (luceneSort != null) {
lastTopDocs = indexSearcher.searchAfter(s.lastItem(),
query,
s.hitsPerPage(),
luceneSort,
scoreMode != ScoreMode.COMPLETE_NO_SCORES
);
} else {
lastTopDocs = indexSearcher.searchAfter(s.lastItem(),
query,
s.hitsPerPage()
);
}
var lastTopDocs = TopDocsSearcher.getTopDocs(indexSearcher,
query,
luceneSort,
s.hitsPerPage(),
s.lastItem(),
scoreMode != ScoreMode.COMPLETE_NO_SCORES,
1000
);
if (lastTopDocs.scoreDocs.length > 0) {
ScoreDoc lastItem = getLastItem(lastTopDocs.scoreDocs);
var hitsList = LuceneReactiveSearcher.convertHits(
@ -139,6 +125,9 @@ public class PagedLuceneReactiveSearcher implements LuceneReactiveSearcher {
}
private static ScoreDoc getLastItem(ScoreDoc[] scoreDocs) {
if (scoreDocs.length == 0) {
return null;
}
return scoreDocs[scoreDocs.length - 1];
}

22
src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneReactiveSearcher.java
View File

@ -32,20 +32,14 @@ public class SimpleLuceneReactiveSearcher implements LuceneReactiveSearcher {
Scheduler scheduler) {
return Mono
.fromCallable(() -> {
TopDocs topDocs;
if (luceneSort == null) {
//noinspection BlockingMethodInNonBlockingContext
topDocs = indexSearcher.search(query,
offset + limit
);
} else {
//noinspection BlockingMethodInNonBlockingContext
topDocs = indexSearcher.search(query,
offset + limit,
luceneSort,
scoreMode != ScoreMode.COMPLETE_NO_SCORES
);
}
TopDocs topDocs = TopDocsSearcher.getTopDocs(indexSearcher,
query,
luceneSort,
offset + limit,
null,
scoreMode != ScoreMode.COMPLETE_NO_SCORES,
1000,
offset, limit);
Flux<LLKeyScore> hitsMono = LuceneReactiveSearcher
.convertHits(
topDocs.scoreDocs,

45
src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsSearcher.java
View File

@ -1,9 +1,12 @@
package it.cavallium.dbengine.lucene.searcher;
import java.io.IOException;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiCollectorManager.Collectors;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
@ -57,18 +60,13 @@ class TopDocsSearcher {
Query query,
Sort luceneSort,
int limit,
FieldDoc after,
ScoreDoc after,
boolean doDocScores,
int totalHitsThreshold,
int topDocsStartOffset,
int topDocsHowMany) throws IOException {
TopDocsCollector<?> collector;
if (luceneSort == null) {
collector = TopScoreDocCollector.create(limit, after, totalHitsThreshold);
} else {
collector = TopFieldCollector.create(luceneSort, limit, after, totalHitsThreshold);
}
TopDocsCollector<?> collector = getTopDocsCollector(luceneSort, limit, after, totalHitsThreshold);
TopDocs topDocs = collector.topDocs(topDocsStartOffset, topDocsHowMany);
if (doDocScores) {
TopFieldCollector.populateScores(topDocs.scoreDocs, indexSearcher, query);
@ -80,19 +78,38 @@ class TopDocsSearcher {
Query query,
Sort luceneSort,
int limit,
FieldDoc after,
ScoreDoc after,
boolean doDocScores,
int totalHitsThreshold) throws IOException {
TopDocsCollector<?> collector;
if (luceneSort == null) {
collector = TopScoreDocCollector.create(limit, after, totalHitsThreshold);
} else {
collector = TopFieldCollector.create(luceneSort, limit, after, totalHitsThreshold);
}
TopDocsCollector<?> collector = getTopDocsCollector(luceneSort, limit, after, totalHitsThreshold);
indexSearcher.search(query, collector);
TopDocs topDocs = collector.topDocs();
if (doDocScores) {
TopFieldCollector.populateScores(topDocs.scoreDocs, indexSearcher, query);
}
return topDocs;
}
public static TopDocsCollector<?> getTopDocsCollector(Sort luceneSort,
int limit,
ScoreDoc after,
int totalHitsThreshold) {
TopDocsCollector<?> collector;
if (luceneSort == null) {
if (after == null) {
collector = TopScoreDocCollector.create(limit, totalHitsThreshold);
} else {
collector = TopScoreDocCollector.create(limit, after, totalHitsThreshold);
}
} else {
if (after == null) {
collector = TopFieldCollector.create(luceneSort, limit, totalHitsThreshold);
} else if (after instanceof FieldDoc afterFieldDoc) {
collector = TopFieldCollector.create(luceneSort, limit, afterFieldDoc, totalHitsThreshold);
} else {
throw new UnsupportedOperationException("GetTopDocs with \"luceneSort\" != null requires \"after\" to be a FieldDoc");
}
}
return collector;
}
}