2021-07-06 00:30:14 +02:00
|
|
|
package it.cavallium.dbengine.lucene.searcher;
|
|
|
|
|
2021-07-06 01:30:37 +02:00
|
|
|
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS;
|
|
|
|
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
|
2023-02-26 21:41:20 +01:00
|
|
|
import static it.cavallium.dbengine.utils.StreamUtils.fastListing;
|
2023-02-22 22:31:36 +01:00
|
|
|
import static it.cavallium.dbengine.utils.StreamUtils.streamWhileNonNull;
|
2021-07-06 01:30:37 +02:00
|
|
|
|
2021-07-06 00:30:14 +02:00
|
|
|
import it.cavallium.dbengine.database.LLKeyScore;
|
2021-09-18 18:34:21 +02:00
|
|
|
import it.cavallium.dbengine.database.LLUtils;
|
2021-09-20 12:51:27 +02:00
|
|
|
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
|
|
|
|
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
2021-07-06 00:30:14 +02:00
|
|
|
import it.cavallium.dbengine.lucene.LuceneUtils;
|
2021-11-09 01:54:24 +01:00
|
|
|
import it.cavallium.dbengine.lucene.collector.TopDocsCollectorMultiManager;
|
2023-02-09 23:34:25 +01:00
|
|
|
import it.cavallium.dbengine.utils.DBException;
|
2023-02-26 21:41:20 +01:00
|
|
|
import it.cavallium.dbengine.utils.StreamUtils;
|
2021-07-06 01:30:37 +02:00
|
|
|
import java.io.IOException;
|
2021-09-22 11:03:39 +02:00
|
|
|
import java.util.List;
|
2023-02-09 23:34:25 +01:00
|
|
|
import java.util.concurrent.atomic.AtomicReference;
|
2023-02-26 21:41:20 +01:00
|
|
|
import java.util.function.Function;
|
2023-02-09 23:34:25 +01:00
|
|
|
import java.util.stream.Stream;
|
2022-06-14 13:10:38 +02:00
|
|
|
import org.apache.logging.log4j.LogManager;
|
|
|
|
import org.apache.logging.log4j.Logger;
|
2021-09-22 11:03:39 +02:00
|
|
|
import org.apache.lucene.search.IndexSearcher;
|
2021-07-17 23:06:26 +02:00
|
|
|
import org.apache.lucene.search.ScoreDoc;
|
2021-07-06 00:30:14 +02:00
|
|
|
import org.apache.lucene.search.TopDocs;
|
2021-10-13 00:23:56 +02:00
|
|
|
import org.apache.lucene.search.TotalHits;
|
|
|
|
import org.apache.lucene.search.TotalHits.Relation;
|
2022-02-26 03:28:20 +01:00
|
|
|
import org.jetbrains.annotations.Nullable;
|
2021-07-06 00:30:14 +02:00
|
|
|
|
2021-10-13 12:25:32 +02:00
|
|
|
public class PagedLocalSearcher implements LocalSearcher {
|
2021-07-06 00:30:14 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
private static final Logger LOG = LogManager.getLogger(PagedLocalSearcher.class);
|
|
|
|
|
2021-07-06 00:30:14 +02:00
|
|
|
@Override
|
2023-02-09 23:34:25 +01:00
|
|
|
public LuceneSearchResult collect(LLIndexSearcher indexSearcher,
|
2021-07-06 01:30:37 +02:00
|
|
|
LocalQueryParams queryParams,
|
2022-02-26 03:28:20 +01:00
|
|
|
@Nullable String keyFieldName,
|
2023-02-26 21:41:20 +01:00
|
|
|
GlobalQueryRewrite transformer,
|
|
|
|
Function<Stream<LLKeyScore>, Stream<LLKeyScore>> filterer) {
|
2022-07-02 11:44:13 +02:00
|
|
|
if (transformer != GlobalQueryRewrite.NO_REWRITE) {
|
2023-02-26 21:41:20 +01:00
|
|
|
return LuceneUtils.rewrite(this, indexSearcher, queryParams, keyFieldName, transformer, filterer);
|
2022-07-02 11:44:13 +02:00
|
|
|
}
|
2021-09-19 12:01:11 +02:00
|
|
|
PaginationInfo paginationInfo = getPaginationInfo(queryParams);
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
var indexSearchers = LLIndexSearchers.unsharded(indexSearcher);
|
|
|
|
|
|
|
|
// Search first page results
|
|
|
|
var firstPageTopDocs = this.searchFirstPage(indexSearchers.shards(), queryParams, paginationInfo);
|
|
|
|
// Compute the results of the first page
|
|
|
|
var firstResult = this.computeFirstPageResults(firstPageTopDocs,
|
|
|
|
indexSearchers.shards(),
|
|
|
|
keyFieldName,
|
|
|
|
queryParams
|
|
|
|
);
|
2023-02-26 21:41:20 +01:00
|
|
|
return this.computeOtherResults(firstResult, indexSearchers.shards(), queryParams, keyFieldName, filterer);
|
2021-09-19 12:01:11 +02:00
|
|
|
}
|
|
|
|
|
2021-10-13 00:23:56 +02:00
|
|
|
@Override
|
|
|
|
public String getName() {
|
2021-10-13 12:25:32 +02:00
|
|
|
return "paged local";
|
2021-10-13 00:23:56 +02:00
|
|
|
}
|
|
|
|
|
2021-09-19 12:01:11 +02:00
|
|
|
/**
|
|
|
|
* Get the pagination info
|
|
|
|
*/
|
|
|
|
private PaginationInfo getPaginationInfo(LocalQueryParams queryParams) {
|
2021-10-15 22:03:53 +02:00
|
|
|
if (queryParams.limitInt() <= MAX_SINGLE_SEARCH_LIMIT) {
|
|
|
|
return new PaginationInfo(queryParams.limitInt(), queryParams.offsetInt(), queryParams.pageLimits(), true);
|
2021-09-18 18:34:21 +02:00
|
|
|
} else {
|
2021-10-15 22:03:53 +02:00
|
|
|
return new PaginationInfo(queryParams.limitInt(), queryParams.offsetInt(), queryParams.pageLimits(), false);
|
2021-09-18 18:34:21 +02:00
|
|
|
}
|
2021-09-19 12:01:11 +02:00
|
|
|
}
|
2021-07-06 01:30:37 +02:00
|
|
|
|
2021-09-19 12:01:11 +02:00
|
|
|
/**
|
|
|
|
* Search effectively the raw results of the first page
|
|
|
|
*/
|
2023-02-09 23:34:25 +01:00
|
|
|
private PageData searchFirstPage(List<IndexSearcher> indexSearchers,
|
2021-09-19 12:01:11 +02:00
|
|
|
LocalQueryParams queryParams,
|
|
|
|
PaginationInfo paginationInfo) {
|
2021-09-20 18:20:59 +02:00
|
|
|
var limit = paginationInfo.totalLimit();
|
2021-09-19 12:01:11 +02:00
|
|
|
var pagination = !paginationInfo.forceSinglePage();
|
|
|
|
var resultsOffset = LuceneUtils.safeLongToInt(paginationInfo.firstPageOffset());
|
2021-10-13 00:23:56 +02:00
|
|
|
var currentPageInfo = new CurrentPageInfo(null, limit, 0);
|
2023-02-09 23:34:25 +01:00
|
|
|
return this.searchPageSync(queryParams, indexSearchers, pagination, resultsOffset, currentPageInfo).pageData();
|
2021-09-19 12:01:11 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Compute the results of the first page, extracting useful data
|
|
|
|
*/
|
2023-02-09 23:34:25 +01:00
|
|
|
private FirstPageResults computeFirstPageResults(PageData firstPageData,
|
2021-09-22 11:03:39 +02:00
|
|
|
List<IndexSearcher> indexSearchers,
|
2021-09-19 12:01:11 +02:00
|
|
|
String keyFieldName,
|
|
|
|
LocalQueryParams queryParams) {
|
2023-02-09 23:34:25 +01:00
|
|
|
var totalHitsCount = LuceneUtils.convertTotalHitsCount(firstPageData.topDocs().totalHits);
|
|
|
|
var scoreDocs = firstPageData.topDocs().scoreDocs;
|
|
|
|
assert LLUtils.isSet(scoreDocs);
|
2021-09-19 12:01:11 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
Stream<LLKeyScore> firstPageHitsFlux = LuceneUtils.convertHits(Stream.of(scoreDocs),
|
|
|
|
indexSearchers, keyFieldName
|
|
|
|
)
|
|
|
|
.limit(queryParams.limitLong());
|
2021-09-19 12:01:11 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
CurrentPageInfo nextPageInfo = firstPageData.nextPageInfo();
|
2021-09-19 12:01:11 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
return new FirstPageResults(totalHitsCount, firstPageHitsFlux, nextPageInfo);
|
2021-09-19 12:01:11 +02:00
|
|
|
}
|
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
private LuceneSearchResult computeOtherResults(FirstPageResults firstResult,
|
2021-09-22 11:03:39 +02:00
|
|
|
List<IndexSearcher> indexSearchers,
|
2021-09-19 12:01:11 +02:00
|
|
|
LocalQueryParams queryParams,
|
2021-09-22 11:03:39 +02:00
|
|
|
String keyFieldName,
|
2023-02-26 21:41:20 +01:00
|
|
|
Function<Stream<LLKeyScore>, Stream<LLKeyScore>> filterer) {
|
2023-02-09 23:34:25 +01:00
|
|
|
var totalHitsCount = firstResult.totalHitsCount();
|
|
|
|
var firstPageHitsStream = firstResult.firstPageHitsStream();
|
|
|
|
var secondPageInfo = firstResult.nextPageInfo();
|
2021-09-19 12:01:11 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
Stream<LLKeyScore> nextHitsFlux = searchOtherPages(indexSearchers, queryParams, keyFieldName, secondPageInfo);
|
2021-09-19 12:01:11 +02:00
|
|
|
|
2023-02-09 23:34:25 +01:00
|
|
|
Stream<LLKeyScore> combinedFlux = Stream.concat(firstPageHitsStream, nextHitsFlux);
|
2023-02-26 21:41:20 +01:00
|
|
|
return new LuceneSearchResult(totalHitsCount, StreamUtils.collect(filterer.apply(combinedFlux), fastListing()));
|
2021-09-19 12:01:11 +02:00
|
|
|
}
|
|
|
|
|
2021-09-19 19:59:37 +02:00
|
|
|
/**
|
|
|
|
* Search effectively the merged raw results of the next pages
|
|
|
|
*/
|
2023-02-09 23:34:25 +01:00
|
|
|
private Stream<LLKeyScore> searchOtherPages(List<IndexSearcher> indexSearchers,
|
2021-09-19 19:59:37 +02:00
|
|
|
LocalQueryParams queryParams, String keyFieldName, CurrentPageInfo secondPageInfo) {
|
2023-02-09 23:34:25 +01:00
|
|
|
AtomicReference<CurrentPageInfo> pageInfo = new AtomicReference<>(secondPageInfo);
|
|
|
|
Object lock = new Object();
|
2023-02-22 22:31:36 +01:00
|
|
|
Stream<ScoreDoc> topFieldDocFlux = streamWhileNonNull(() -> {
|
2023-02-09 23:34:25 +01:00
|
|
|
synchronized (lock) {
|
|
|
|
var currentPageInfo = pageInfo.getPlain();
|
|
|
|
var result = searchPageSync(queryParams, indexSearchers, true, 0, currentPageInfo);
|
|
|
|
pageInfo.setPlain(result.nextPageToIterate());
|
|
|
|
return result.pageData();
|
|
|
|
}
|
2023-02-22 22:31:36 +01:00
|
|
|
}).flatMap(pd -> Stream.of(pd.topDocs().scoreDocs));
|
2023-02-09 23:34:25 +01:00
|
|
|
return LuceneUtils.convertHits(topFieldDocFlux, indexSearchers, keyFieldName);
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
2021-09-19 12:01:11 +02:00
|
|
|
|
|
|
|
/**
|
|
|
|
*
|
|
|
|
* @param resultsOffset offset of the resulting topDocs. Useful if you want to
|
|
|
|
* skip the first n results in the first page
|
|
|
|
*/
|
2023-02-09 23:34:25 +01:00
|
|
|
private PageIterationStepResult searchPageSync(LocalQueryParams queryParams,
|
2021-09-22 11:03:39 +02:00
|
|
|
List<IndexSearcher> indexSearchers,
|
2021-09-19 12:01:11 +02:00
|
|
|
boolean allowPagination,
|
|
|
|
int resultsOffset,
|
2023-02-09 23:34:25 +01:00
|
|
|
CurrentPageInfo s) {
|
2021-09-19 12:01:11 +02:00
|
|
|
if (resultsOffset < 0) {
|
|
|
|
throw new IndexOutOfBoundsException(resultsOffset);
|
|
|
|
}
|
2023-02-09 23:34:25 +01:00
|
|
|
PageData result = null;
|
2021-09-20 18:20:59 +02:00
|
|
|
var currentPageLimit = queryParams.pageLimits().getPageLimit(s.pageIndex());
|
2021-10-13 00:23:56 +02:00
|
|
|
if (s.pageIndex() == 0 && s.remainingLimit() == 0) {
|
|
|
|
int count;
|
|
|
|
try {
|
|
|
|
count = indexSearchers.get(0).count(queryParams.query());
|
|
|
|
} catch (IOException e) {
|
2023-02-09 23:34:25 +01:00
|
|
|
throw new DBException(e);
|
2021-10-13 00:23:56 +02:00
|
|
|
}
|
|
|
|
var nextPageInfo = new CurrentPageInfo(null, 0, 1);
|
2023-02-09 23:34:25 +01:00
|
|
|
return new PageIterationStepResult(EMPTY_STATUS, new PageData(new TopDocs(new TotalHits(count, Relation.EQUAL_TO), new ScoreDoc[0]), nextPageInfo));
|
2021-10-13 00:23:56 +02:00
|
|
|
} else if (s.pageIndex() == 0 || (s.last() != null && s.remainingLimit() > 0)) {
|
2021-09-19 12:01:11 +02:00
|
|
|
TopDocs pageTopDocs;
|
|
|
|
try {
|
2021-11-09 01:54:24 +01:00
|
|
|
var cmm = new TopDocsCollectorMultiManager(queryParams.sort(),
|
2021-10-15 22:03:53 +02:00
|
|
|
currentPageLimit, s.last(), queryParams.getTotalHitsThresholdInt(),
|
2021-11-09 01:13:47 +01:00
|
|
|
allowPagination, queryParams.needsScores(), resultsOffset, currentPageLimit);
|
|
|
|
|
|
|
|
pageTopDocs = cmm.reduce(List.of(indexSearchers
|
|
|
|
.get(0)
|
|
|
|
.search(queryParams.query(), cmm.get(queryParams.query(), indexSearchers.get(0)))));
|
2021-09-19 12:01:11 +02:00
|
|
|
} catch (IOException e) {
|
2023-02-09 23:34:25 +01:00
|
|
|
throw new DBException(e);
|
2021-09-19 12:01:11 +02:00
|
|
|
}
|
|
|
|
var pageLastDoc = LuceneUtils.getLastScoreDoc(pageTopDocs.scoreDocs);
|
|
|
|
long nextRemainingLimit;
|
|
|
|
if (allowPagination) {
|
2021-09-20 18:20:59 +02:00
|
|
|
nextRemainingLimit = s.remainingLimit() - currentPageLimit;
|
2021-09-19 12:01:11 +02:00
|
|
|
} else {
|
|
|
|
nextRemainingLimit = 0L;
|
|
|
|
}
|
|
|
|
var nextPageIndex = s.pageIndex() + 1;
|
|
|
|
var nextPageInfo = new CurrentPageInfo(pageLastDoc, nextRemainingLimit, nextPageIndex);
|
2023-02-09 23:34:25 +01:00
|
|
|
return new PageIterationStepResult(nextPageInfo, new PageData(pageTopDocs, nextPageInfo));
|
2021-09-19 12:01:11 +02:00
|
|
|
} else {
|
2023-02-09 23:34:25 +01:00
|
|
|
return new PageIterationStepResult(EMPTY_STATUS, null);
|
2021-09-19 12:01:11 +02:00
|
|
|
}
|
2021-07-06 00:30:14 +02:00
|
|
|
}
|
|
|
|
}
|