2021-09-25 13:07:52 +02:00
|
|
|
package it.cavallium.dbengine.lucene.searcher;
|
|
|
|
|
|
|
|
import io.net5.buffer.api.Send;
|
|
|
|
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
|
|
|
import it.cavallium.dbengine.database.LLKeyScore;
|
|
|
|
import it.cavallium.dbengine.database.LLUtils;
|
|
|
|
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
|
|
|
import it.cavallium.dbengine.lucene.LuceneUtils;
|
2021-09-25 18:31:41 +02:00
|
|
|
import it.cavallium.dbengine.lucene.collector.ReactiveCollectorManager;
|
2021-10-08 11:50:28 +02:00
|
|
|
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
|
2021-09-25 13:07:52 +02:00
|
|
|
import java.util.Queue;
|
2021-11-08 18:52:52 +01:00
|
|
|
import java.util.concurrent.CancellationException;
|
|
|
|
import java.util.concurrent.ExecutorService;
|
|
|
|
import java.util.concurrent.Executors;
|
|
|
|
import java.util.concurrent.atomic.AtomicBoolean;
|
2021-09-25 13:07:52 +02:00
|
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
import java.util.concurrent.locks.LockSupport;
|
|
|
|
import java.util.function.Supplier;
|
|
|
|
import org.apache.lucene.search.IndexSearcher;
|
|
|
|
import org.apache.lucene.search.ScoreDoc;
|
2021-11-08 18:52:52 +01:00
|
|
|
import org.warp.commonutils.type.ShortNamedThreadFactory;
|
2021-09-25 13:07:52 +02:00
|
|
|
import reactor.core.publisher.Flux;
|
2021-11-08 16:33:41 +01:00
|
|
|
import reactor.core.publisher.FluxSink.OverflowStrategy;
|
2021-09-25 13:07:52 +02:00
|
|
|
import reactor.core.publisher.Mono;
|
|
|
|
import reactor.core.publisher.Sinks;
|
|
|
|
import reactor.core.publisher.Sinks.EmitResult;
|
|
|
|
import reactor.core.publisher.Sinks.Many;
|
|
|
|
import reactor.core.scheduler.Scheduler;
|
|
|
|
import reactor.core.scheduler.Schedulers;
|
|
|
|
import reactor.util.concurrent.Queues;
|
|
|
|
|
2021-10-13 12:25:32 +02:00
|
|
|
public class UnsortedUnscoredStreamingMultiSearcher implements MultiSearcher {
|
2021-09-25 13:07:52 +02:00
|
|
|
|
2021-11-08 18:52:52 +01:00
|
|
|
private static final ExecutorService EXECUTOR_SERVICE
|
|
|
|
= Executors.newCachedThreadPool(new ShortNamedThreadFactory("StreamingExecutor"));
|
2021-09-25 13:07:52 +02:00
|
|
|
|
|
|
|
@Override
|
2021-11-08 11:17:52 +01:00
|
|
|
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
2021-09-25 13:07:52 +02:00
|
|
|
LocalQueryParams queryParams,
|
|
|
|
String keyFieldName,
|
|
|
|
LLSearchTransformer transformer) {
|
|
|
|
|
2021-10-08 11:50:28 +02:00
|
|
|
return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> {
|
|
|
|
Mono<LocalQueryParams> queryParamsMono;
|
|
|
|
if (transformer == LLSearchTransformer.NO_TRANSFORMATION) {
|
|
|
|
queryParamsMono = Mono.just(queryParams);
|
|
|
|
} else {
|
|
|
|
queryParamsMono = transformer.transform(Mono
|
|
|
|
.fromCallable(() -> new TransformerInput(indexSearchers, queryParams)));
|
|
|
|
}
|
2021-09-25 13:07:52 +02:00
|
|
|
|
2021-10-08 11:50:28 +02:00
|
|
|
return queryParamsMono
|
|
|
|
.flatMap(queryParams2 -> {
|
|
|
|
var localQueryParams = getLocalQueryParams(queryParams2);
|
2021-10-15 22:03:53 +02:00
|
|
|
if (queryParams2.isSorted() && queryParams2.limitLong() > 0) {
|
2021-10-08 11:50:28 +02:00
|
|
|
return Mono.error(new UnsupportedOperationException("Sorted queries are not supported"
|
|
|
|
+ " by UnsortedUnscoredContinuousLuceneMultiSearcher"));
|
|
|
|
}
|
2021-10-15 22:03:53 +02:00
|
|
|
if (queryParams2.needsScores() && queryParams2.limitLong() > 0) {
|
2021-10-08 11:50:28 +02:00
|
|
|
return Mono.error(new UnsupportedOperationException("Scored queries are not supported"
|
|
|
|
+ " by UnsortedUnscoredContinuousLuceneMultiSearcher"));
|
|
|
|
}
|
|
|
|
return Mono.fromCallable(() -> {
|
|
|
|
LLUtils.ensureBlocking();
|
2021-09-25 13:07:52 +02:00
|
|
|
|
2021-10-08 11:50:28 +02:00
|
|
|
var shards = indexSearchers.shards();
|
|
|
|
|
2021-11-08 16:33:41 +01:00
|
|
|
Flux<ScoreDoc> scoreDocsFlux = Flux.create(scoreDocsSink -> {
|
|
|
|
var cm = new ReactiveCollectorManager(scoreDocsSink);
|
|
|
|
|
|
|
|
AtomicInteger runningTasks = new AtomicInteger(0);
|
|
|
|
|
|
|
|
runningTasks.addAndGet(shards.size());
|
|
|
|
int mutableShardIndex = 0;
|
|
|
|
for (IndexSearcher shard : shards) {
|
|
|
|
int shardIndex = mutableShardIndex++;
|
2021-11-08 18:52:52 +01:00
|
|
|
EXECUTOR_SERVICE.execute(() -> {
|
2021-11-08 16:33:41 +01:00
|
|
|
try {
|
|
|
|
var collector = cm.newCollector();
|
|
|
|
assert queryParams.complete() == collector.scoreMode().isExhaustive();
|
|
|
|
assert queryParams
|
|
|
|
.getScoreModeOptional()
|
|
|
|
.map(scoreMode -> scoreMode == collector.scoreMode())
|
|
|
|
.orElse(true);
|
|
|
|
|
|
|
|
collector.setShardIndex(shardIndex);
|
|
|
|
|
|
|
|
shard.search(localQueryParams.query(), collector);
|
|
|
|
} catch (Throwable e) {
|
2021-11-08 18:52:52 +01:00
|
|
|
if (!(e instanceof CancellationException)) {
|
|
|
|
scoreDocsSink.error(e);
|
|
|
|
}
|
2021-11-08 16:33:41 +01:00
|
|
|
} finally {
|
|
|
|
if (runningTasks.decrementAndGet() <= 0) {
|
|
|
|
scoreDocsSink.complete();
|
2021-10-08 11:50:28 +02:00
|
|
|
}
|
|
|
|
}
|
2021-11-08 16:33:41 +01:00
|
|
|
});
|
|
|
|
}
|
|
|
|
}, OverflowStrategy.BUFFER);
|
|
|
|
|
2021-09-25 13:07:52 +02:00
|
|
|
|
2021-11-08 16:33:41 +01:00
|
|
|
Flux<LLKeyScore> resultsFlux = LuceneUtils.convertHits(scoreDocsFlux, shards, keyFieldName, false);
|
2021-09-25 13:07:52 +02:00
|
|
|
|
2021-10-08 11:50:28 +02:00
|
|
|
var totalHitsCount = new TotalHitsCount(0, false);
|
|
|
|
Flux<LLKeyScore> mergedFluxes = resultsFlux
|
2021-10-15 22:03:53 +02:00
|
|
|
.skip(queryParams2.offsetLong())
|
|
|
|
.take(queryParams2.limitLong(), true);
|
2021-09-25 13:07:52 +02:00
|
|
|
|
2021-11-08 11:17:52 +01:00
|
|
|
return new LuceneSearchResult(totalHitsCount, mergedFluxes, indexSearchers::close);
|
2021-10-08 11:50:28 +02:00
|
|
|
});
|
|
|
|
});
|
|
|
|
}, false);
|
2021-09-25 13:07:52 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
private LocalQueryParams getLocalQueryParams(LocalQueryParams queryParams) {
|
|
|
|
return new LocalQueryParams(queryParams.query(),
|
2021-10-15 22:03:53 +02:00
|
|
|
0L,
|
|
|
|
queryParams.offsetLong() + queryParams.limitLong(),
|
2021-09-25 13:07:52 +02:00
|
|
|
queryParams.pageLimits(),
|
|
|
|
queryParams.minCompetitiveScore(),
|
|
|
|
queryParams.sort(),
|
2021-10-14 00:49:21 +02:00
|
|
|
queryParams.complete()
|
2021-09-25 13:07:52 +02:00
|
|
|
);
|
|
|
|
}
|
2021-10-13 00:23:56 +02:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getName() {
|
2021-10-13 12:25:32 +02:00
|
|
|
return "unsorted unscored streaming multi";
|
2021-10-13 00:23:56 +02:00
|
|
|
}
|
2021-09-25 13:07:52 +02:00
|
|
|
}
|