Bugfixes and updated reactor
This commit is contained in:
parent
505de18ecb
commit
df84562bb9
6
pom.xml
6
pom.xml
@ -178,17 +178,17 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.projectreactor</groupId>
|
<groupId>io.projectreactor</groupId>
|
||||||
<artifactId>reactor-core</artifactId>
|
<artifactId>reactor-core</artifactId>
|
||||||
<version>3.4.3</version>
|
<version>3.4.5</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.projectreactor</groupId>
|
<groupId>io.projectreactor</groupId>
|
||||||
<artifactId>reactor-tools</artifactId>
|
<artifactId>reactor-tools</artifactId>
|
||||||
<version>3.4.3</version>
|
<version>3.4.5</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.projectreactor</groupId>
|
<groupId>io.projectreactor</groupId>
|
||||||
<artifactId>reactor-test</artifactId>
|
<artifactId>reactor-test</artifactId>
|
||||||
<version>3.4.3</version>
|
<version>3.4.5</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.novasearch</groupId>
|
<groupId>org.novasearch</groupId>
|
||||||
|
@ -47,6 +47,9 @@ public class CountedStream<T> {
|
|||||||
|
|
||||||
public static <T> Mono<CountedStream<T>> counted(Flux<T> flux) {
|
public static <T> Mono<CountedStream<T>> counted(Flux<T> flux) {
|
||||||
var publishedFlux = flux.cache();
|
var publishedFlux = flux.cache();
|
||||||
return publishedFlux.count().map(count -> new CountedStream<>(publishedFlux, count));
|
return publishedFlux
|
||||||
|
.count()
|
||||||
|
.map(count -> new CountedStream<>(publishedFlux, count))
|
||||||
|
.switchIfEmpty(Mono.fromSupplier(() -> new CountedStream<>(Flux.empty(), 0)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
package it.cavallium.dbengine.database.disk;
|
package it.cavallium.dbengine.database.disk;
|
||||||
|
|
||||||
import com.google.common.base.Suppliers;
|
|
||||||
import it.cavallium.dbengine.database.Column;
|
import it.cavallium.dbengine.database.Column;
|
||||||
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
||||||
import it.cavallium.dbengine.database.LLSnapshot;
|
import it.cavallium.dbengine.database.LLSnapshot;
|
||||||
@ -24,7 +23,6 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
import java.util.concurrent.ThreadLocalRandom;
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
import java.util.function.Supplier;
|
|
||||||
import org.apache.commons.lang3.time.StopWatch;
|
import org.apache.commons.lang3.time.StopWatch;
|
||||||
import org.rocksdb.BlockBasedTableConfig;
|
import org.rocksdb.BlockBasedTableConfig;
|
||||||
import org.rocksdb.BloomFilter;
|
import org.rocksdb.BloomFilter;
|
||||||
@ -58,8 +56,6 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
|||||||
protected static final Logger logger = LoggerFactory.getLogger(LLLocalKeyValueDatabase.class);
|
protected static final Logger logger = LoggerFactory.getLogger(LLLocalKeyValueDatabase.class);
|
||||||
private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY = new ColumnFamilyDescriptor(
|
private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY = new ColumnFamilyDescriptor(
|
||||||
RocksDB.DEFAULT_COLUMN_FAMILY);
|
RocksDB.DEFAULT_COLUMN_FAMILY);
|
||||||
private static final Supplier<Scheduler> lowMemorySupplier = Suppliers.memoize(() ->
|
|
||||||
Schedulers.newBoundedElastic(1, Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, "db-low-memory", Integer.MAX_VALUE))::get;
|
|
||||||
|
|
||||||
private final Scheduler dbScheduler;
|
private final Scheduler dbScheduler;
|
||||||
private final Path dbPath;
|
private final Path dbPath;
|
||||||
@ -88,16 +84,13 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
|
|||||||
Path dbPath = Paths.get(dbPathString);
|
Path dbPath = Paths.get(dbPathString);
|
||||||
this.dbPath = dbPath;
|
this.dbPath = dbPath;
|
||||||
this.name = name;
|
this.name = name;
|
||||||
if (lowMemory) {
|
this.dbScheduler = Schedulers.newBoundedElastic(lowMemory ? Runtime.getRuntime().availableProcessors()
|
||||||
this.dbScheduler = lowMemorySupplier.get();
|
: Math.max(8, Runtime.getRuntime().availableProcessors()),
|
||||||
} else {
|
|
||||||
this.dbScheduler = Schedulers.newBoundedElastic(Math.max(8, Runtime.getRuntime().availableProcessors()),
|
|
||||||
Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
|
Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
|
||||||
"db-" + name,
|
"db-" + name,
|
||||||
60,
|
60,
|
||||||
true
|
true
|
||||||
);
|
);
|
||||||
}
|
|
||||||
|
|
||||||
createIfNotExists(descriptors, options, dbPath, dbPathString);
|
createIfNotExists(descriptors, options, dbPath, dbPathString);
|
||||||
// Create all column families that don't exist
|
// Create all column families that don't exist
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
package it.cavallium.dbengine.database.disk;
|
package it.cavallium.dbengine.database.disk;
|
||||||
|
|
||||||
import com.google.common.base.Suppliers;
|
|
||||||
import it.cavallium.dbengine.client.query.QueryParser;
|
import it.cavallium.dbengine.client.query.QueryParser;
|
||||||
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
||||||
import it.cavallium.dbengine.database.EnglishItalianStopFilter;
|
import it.cavallium.dbengine.database.EnglishItalianStopFilter;
|
||||||
@ -31,12 +30,11 @@ import java.util.Objects;
|
|||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.Semaphore;
|
import java.util.concurrent.Semaphore;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.function.Supplier;
|
|
||||||
import org.apache.lucene.index.IndexCommit;
|
import org.apache.lucene.index.IndexCommit;
|
||||||
import org.apache.lucene.index.IndexWriter;
|
import org.apache.lucene.index.IndexWriter;
|
||||||
import org.apache.lucene.index.IndexWriterConfig;
|
import org.apache.lucene.index.IndexWriterConfig;
|
||||||
@ -58,6 +56,7 @@ import org.apache.lucene.store.FSDirectory;
|
|||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import org.warp.commonutils.log.Logger;
|
import org.warp.commonutils.log.Logger;
|
||||||
import org.warp.commonutils.log.LoggerFactory;
|
import org.warp.commonutils.log.LoggerFactory;
|
||||||
|
import org.warp.commonutils.type.ShortNamedThreadFactory;
|
||||||
import reactor.core.publisher.Flux;
|
import reactor.core.publisher.Flux;
|
||||||
import reactor.core.publisher.FluxSink.OverflowStrategy;
|
import reactor.core.publisher.FluxSink.OverflowStrategy;
|
||||||
import reactor.core.publisher.GroupedFlux;
|
import reactor.core.publisher.GroupedFlux;
|
||||||
@ -69,7 +68,6 @@ import reactor.util.function.Tuples;
|
|||||||
|
|
||||||
public class LLLocalLuceneIndex implements LLLuceneIndex {
|
public class LLLocalLuceneIndex implements LLLuceneIndex {
|
||||||
|
|
||||||
private static final boolean USE_STANDARD_SCHEDULERS = true;
|
|
||||||
protected static final Logger logger = LoggerFactory.getLogger(LLLocalLuceneIndex.class);
|
protected static final Logger logger = LoggerFactory.getLogger(LLLocalLuceneIndex.class);
|
||||||
private static final LuceneStreamSearcher streamSearcher = new AdaptiveStreamSearcher();
|
private static final LuceneStreamSearcher streamSearcher = new AdaptiveStreamSearcher();
|
||||||
private static final AllowOnlyQueryParsingCollectorStreamSearcher allowOnlyQueryParsingCollectorStreamSearcher
|
private static final AllowOnlyQueryParsingCollectorStreamSearcher allowOnlyQueryParsingCollectorStreamSearcher
|
||||||
@ -85,30 +83,10 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
Integer.MAX_VALUE,
|
Integer.MAX_VALUE,
|
||||||
true
|
true
|
||||||
);
|
);
|
||||||
private final Scheduler luceneBlockingScheduler;
|
// Scheduler used to get callback values of LuceneStreamSearcher without creating deadlocks
|
||||||
private static final Function<String, Scheduler> boundedSchedulerSupplier = name ->
|
private static final Scheduler luceneSearcherScheduler = Schedulers
|
||||||
Schedulers.newBoundedElastic(Runtime.getRuntime().availableProcessors(),
|
.fromExecutorService(Executors
|
||||||
Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
|
.newCachedThreadPool(new ShortNamedThreadFactory("lucene-searcher")));
|
||||||
"lucene-" + name,
|
|
||||||
60
|
|
||||||
);
|
|
||||||
private static final Supplier<Scheduler> lowMemorySchedulerSupplier = Suppliers.memoize(() ->
|
|
||||||
Schedulers.newBoundedElastic(1, Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
|
|
||||||
"lucene-low-memory", Integer.MAX_VALUE))::get;
|
|
||||||
@SuppressWarnings("FieldCanBeLocal")
|
|
||||||
private final Supplier<Scheduler> querySchedulerSupplier = USE_STANDARD_SCHEDULERS ?
|
|
||||||
Schedulers::boundedElastic : Suppliers.memoize(() -> boundedSchedulerSupplier.apply("query"))::get;
|
|
||||||
@SuppressWarnings("FieldCanBeLocal")
|
|
||||||
private final Supplier<Scheduler> blockingSchedulerSupplier = USE_STANDARD_SCHEDULERS ?
|
|
||||||
Schedulers::boundedElastic : Suppliers.memoize(() -> boundedSchedulerSupplier.apply("blocking"))::get;
|
|
||||||
@SuppressWarnings("FieldCanBeLocal")
|
|
||||||
private final Supplier<Scheduler> blockingLuceneSearchSchedulerSupplier = USE_STANDARD_SCHEDULERS ?
|
|
||||||
Schedulers::boundedElastic : Suppliers.memoize(() -> boundedSchedulerSupplier.apply("search-blocking"))::get;
|
|
||||||
/**
|
|
||||||
* Lucene query scheduler.
|
|
||||||
*/
|
|
||||||
private final Scheduler luceneQueryScheduler;
|
|
||||||
private final Scheduler blockingLuceneSearchScheduler;
|
|
||||||
|
|
||||||
private final String luceneIndexName;
|
private final String luceneIndexName;
|
||||||
private final SnapshotDeletionPolicy snapshotter;
|
private final SnapshotDeletionPolicy snapshotter;
|
||||||
@ -162,14 +140,6 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
this.indexWriter = new IndexWriter(directory, indexWriterConfig);
|
this.indexWriter = new IndexWriter(directory, indexWriterConfig);
|
||||||
this.searcherManager
|
this.searcherManager
|
||||||
= new SearcherManager(indexWriter, false, false, null);
|
= new SearcherManager(indexWriter, false, false, null);
|
||||||
if (lowMemory) {
|
|
||||||
this.luceneQueryScheduler = this.luceneBlockingScheduler = blockingLuceneSearchScheduler
|
|
||||||
= lowMemorySchedulerSupplier.get();
|
|
||||||
} else {
|
|
||||||
this.luceneBlockingScheduler = blockingSchedulerSupplier.get();
|
|
||||||
this.luceneQueryScheduler = querySchedulerSupplier.get();
|
|
||||||
this.blockingLuceneSearchScheduler = blockingLuceneSearchSchedulerSupplier.get();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create scheduled tasks lifecycle manager
|
// Create scheduled tasks lifecycle manager
|
||||||
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();
|
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();
|
||||||
@ -208,7 +178,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
this.snapshots.put(snapshotSeqNo, new LuceneIndexSnapshot(snapshot));
|
this.snapshots.put(snapshotSeqNo, new LuceneIndexSnapshot(snapshot));
|
||||||
return new LLSnapshot(snapshotSeqNo);
|
return new LLSnapshot(snapshotSeqNo);
|
||||||
})
|
})
|
||||||
.subscribeOn(luceneBlockingScheduler)
|
.subscribeOn(Schedulers.boundedElastic())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -219,7 +189,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
private Mono<IndexCommit> takeLuceneSnapshot() {
|
private Mono<IndexCommit> takeLuceneSnapshot() {
|
||||||
return Mono
|
return Mono
|
||||||
.fromCallable(snapshotter::snapshot)
|
.fromCallable(snapshotter::snapshot)
|
||||||
.subscribeOn(luceneBlockingScheduler)
|
.subscribeOn(Schedulers.boundedElastic())
|
||||||
.onErrorResume(ex -> Mono
|
.onErrorResume(ex -> Mono
|
||||||
.defer(() -> {
|
.defer(() -> {
|
||||||
if (ex instanceof IllegalStateException && "No index commit to snapshot".equals(ex.getMessage())) {
|
if (ex instanceof IllegalStateException && "No index commit to snapshot".equals(ex.getMessage())) {
|
||||||
@ -244,26 +214,22 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
throw new IOException("LLSnapshot " + snapshot.getSequenceNumber() + " not found!");
|
throw new IOException("LLSnapshot " + snapshot.getSequenceNumber() + " not found!");
|
||||||
}
|
}
|
||||||
|
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexSnapshot.close();
|
indexSnapshot.close();
|
||||||
|
|
||||||
var luceneIndexSnapshot = indexSnapshot.getSnapshot();
|
var luceneIndexSnapshot = indexSnapshot.getSnapshot();
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
snapshotter.release(luceneIndexSnapshot);
|
snapshotter.release(luceneIndexSnapshot);
|
||||||
// Delete unused files after releasing the snapshot
|
// Delete unused files after releasing the snapshot
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexWriter.deleteUnusedFiles();
|
indexWriter.deleteUnusedFiles();
|
||||||
return null;
|
return null;
|
||||||
}).subscribeOn(luceneBlockingScheduler);
|
}).subscribeOn(Schedulers.boundedElastic());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Void> addDocument(LLTerm key, LLDocument doc) {
|
public Mono<Void> addDocument(LLTerm key, LLDocument doc) {
|
||||||
return Mono.<Void>fromCallable(() -> {
|
return Mono.<Void>fromCallable(() -> {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexWriter.addDocument(LLUtils.toDocument(doc));
|
indexWriter.addDocument(LLUtils.toDocument(doc));
|
||||||
return null;
|
return null;
|
||||||
}).subscribeOn(luceneBlockingScheduler);
|
}).subscribeOn(Schedulers.boundedElastic());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -273,11 +239,10 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
.collectList()
|
.collectList()
|
||||||
.flatMap(docs -> Mono
|
.flatMap(docs -> Mono
|
||||||
.<Void>fromCallable(() -> {
|
.<Void>fromCallable(() -> {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexWriter.addDocuments(LLUtils.toDocuments(docs));
|
indexWriter.addDocuments(LLUtils.toDocuments(docs));
|
||||||
return null;
|
return null;
|
||||||
})
|
})
|
||||||
.subscribeOn(luceneBlockingScheduler))
|
.subscribeOn(Schedulers.boundedElastic()))
|
||||||
)
|
)
|
||||||
.then();
|
.then();
|
||||||
}
|
}
|
||||||
@ -286,19 +251,17 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
@Override
|
@Override
|
||||||
public Mono<Void> deleteDocument(LLTerm id) {
|
public Mono<Void> deleteDocument(LLTerm id) {
|
||||||
return Mono.<Void>fromCallable(() -> {
|
return Mono.<Void>fromCallable(() -> {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexWriter.deleteDocuments(LLUtils.toTerm(id));
|
indexWriter.deleteDocuments(LLUtils.toTerm(id));
|
||||||
return null;
|
return null;
|
||||||
}).subscribeOn(luceneBlockingScheduler);
|
}).subscribeOn(Schedulers.boundedElastic());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Void> updateDocument(LLTerm id, LLDocument document) {
|
public Mono<Void> updateDocument(LLTerm id, LLDocument document) {
|
||||||
return Mono.<Void>fromCallable(() -> {
|
return Mono.<Void>fromCallable(() -> {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexWriter.updateDocument(LLUtils.toTerm(id), LLUtils.toDocument(document));
|
indexWriter.updateDocument(LLUtils.toTerm(id), LLUtils.toDocument(document));
|
||||||
return null;
|
return null;
|
||||||
}).subscribeOn(luceneBlockingScheduler);
|
}).subscribeOn(Schedulers.boundedElastic());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -312,11 +275,10 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
.collectList()
|
.collectList()
|
||||||
.flatMap(luceneDocuments -> Mono
|
.flatMap(luceneDocuments -> Mono
|
||||||
.<Void>fromCallable(() -> {
|
.<Void>fromCallable(() -> {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexWriter.updateDocuments(LLUtils.toTerm(documents.key()), luceneDocuments);
|
indexWriter.updateDocuments(LLUtils.toTerm(documents.key()), luceneDocuments);
|
||||||
return null;
|
return null;
|
||||||
})
|
})
|
||||||
.subscribeOn(luceneBlockingScheduler)
|
.subscribeOn(Schedulers.boundedElastic())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -337,7 +299,6 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
return Mono.fromCallable(() -> {
|
return Mono.fromCallable(() -> {
|
||||||
IndexSearcher indexSearcher;
|
IndexSearcher indexSearcher;
|
||||||
if (snapshot == null) {
|
if (snapshot == null) {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
indexSearcher = searcherManager.acquire();
|
indexSearcher = searcherManager.acquire();
|
||||||
indexSearcher.setSimilarity(getSimilarity());
|
indexSearcher.setSimilarity(getSimilarity());
|
||||||
} else {
|
} else {
|
||||||
@ -352,20 +313,19 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
} else {
|
} else {
|
||||||
return indexSearcher;
|
return indexSearcher;
|
||||||
}
|
}
|
||||||
}).subscribeOn(luceneBlockingScheduler);
|
}).subscribeOn(Schedulers.boundedElastic());
|
||||||
}
|
}
|
||||||
|
|
||||||
private Mono<Void> releaseSearcherWrapper(LLSnapshot snapshot, IndexSearcher indexSearcher) {
|
private Mono<Void> releaseSearcherWrapper(LLSnapshot snapshot, IndexSearcher indexSearcher) {
|
||||||
return Mono.<Void>fromRunnable(() -> {
|
return Mono.<Void>fromRunnable(() -> {
|
||||||
if (snapshot == null) {
|
if (snapshot == null) {
|
||||||
try {
|
try {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
searcherManager.release(indexSearcher);
|
searcherManager.release(indexSearcher);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}).subscribeOn(luceneBlockingScheduler);
|
}).subscribeOn(Schedulers.boundedElastic());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -449,7 +409,6 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get the reference doc and apply it to MoreLikeThis, to generate the query
|
// Get the reference doc and apply it to MoreLikeThis, to generate the query
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
var mltQuery = mlt.like((Map) mltDocumentFields);
|
var mltQuery = mlt.like((Map) mltDocumentFields);
|
||||||
Query luceneQuery;
|
Query luceneQuery;
|
||||||
if (luceneAdditionalQuery != null) {
|
if (luceneAdditionalQuery != null) {
|
||||||
@ -463,7 +422,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
|
|
||||||
return luceneQuery;
|
return luceneQuery;
|
||||||
})
|
})
|
||||||
.subscribeOn(luceneQueryScheduler)
|
.subscribeOn(Schedulers.boundedElastic())
|
||||||
.map(luceneQuery -> luceneSearch(doDistributedPre,
|
.map(luceneQuery -> luceneSearch(doDistributedPre,
|
||||||
indexSearcher,
|
indexSearcher,
|
||||||
queryParams.getOffset(),
|
queryParams.getOffset(),
|
||||||
@ -520,7 +479,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
org.apache.lucene.search.ScoreMode luceneScoreMode = QueryParser.toScoreMode(queryParams.getScoreMode());
|
org.apache.lucene.search.ScoreMode luceneScoreMode = QueryParser.toScoreMode(queryParams.getScoreMode());
|
||||||
return Tuples.of(luceneQuery, Optional.ofNullable(luceneSort), luceneScoreMode);
|
return Tuples.of(luceneQuery, Optional.ofNullable(luceneSort), luceneScoreMode);
|
||||||
})
|
})
|
||||||
.subscribeOn(luceneQueryScheduler)
|
.subscribeOn(Schedulers.boundedElastic())
|
||||||
.<LLSearchResult>flatMap(tuple -> Mono
|
.<LLSearchResult>flatMap(tuple -> Mono
|
||||||
.fromSupplier(() -> {
|
.fromSupplier(() -> {
|
||||||
Query luceneQuery = tuple.getT1();
|
Query luceneQuery = tuple.getT1();
|
||||||
@ -560,19 +519,16 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
ScoreMode luceneScoreMode,
|
ScoreMode luceneScoreMode,
|
||||||
Mono<Void> successCleanup) {
|
Mono<Void> successCleanup) {
|
||||||
return new LLSearchResult(Mono.<LLSearchResultShard>create(monoSink -> {
|
return new LLSearchResult(Mono.<LLSearchResultShard>create(monoSink -> {
|
||||||
|
|
||||||
LuceneSearchInstance luceneSearchInstance;
|
LuceneSearchInstance luceneSearchInstance;
|
||||||
long totalHitsCount;
|
long totalHitsCount;
|
||||||
try {
|
try {
|
||||||
if (doDistributedPre) {
|
if (doDistributedPre) {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
allowOnlyQueryParsingCollectorStreamSearcher.search(indexSearcher, luceneQuery);
|
allowOnlyQueryParsingCollectorStreamSearcher.search(indexSearcher, luceneQuery);
|
||||||
monoSink.success(new LLSearchResultShard(successCleanup.thenMany(Flux.empty()), 0));
|
monoSink.success(new LLSearchResultShard(successCleanup.thenMany(Flux.empty()), 0));
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
int boundedOffset = Math.max(0, offset > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) offset);
|
int boundedOffset = Math.max(0, offset > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) offset);
|
||||||
int boundedLimit = Math.max(0, limit > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) limit);
|
int boundedLimit = Math.max(0, limit > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) limit);
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
luceneSearchInstance = streamSearcher.search(indexSearcher,
|
luceneSearchInstance = streamSearcher.search(indexSearcher,
|
||||||
luceneQuery,
|
luceneQuery,
|
||||||
boundedOffset,
|
boundedOffset,
|
||||||
@ -582,7 +538,6 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
minCompetitiveScore,
|
minCompetitiveScore,
|
||||||
keyFieldName
|
keyFieldName
|
||||||
);
|
);
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
totalHitsCount = luceneSearchInstance.getTotalHitsCount();
|
totalHitsCount = luceneSearchInstance.getTotalHitsCount();
|
||||||
}
|
}
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
@ -604,8 +559,9 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
sink.onCancel(() -> cancelled.set(true));
|
sink.onCancel(() -> cancelled.set(true));
|
||||||
sink.onRequest(delta -> requests.release((int) Math.min(delta, Integer.MAX_VALUE)));
|
sink.onRequest(delta -> requests.release((int) Math.min(delta, Integer.MAX_VALUE)));
|
||||||
|
|
||||||
|
luceneSearcherScheduler
|
||||||
|
.schedule(() -> {
|
||||||
try {
|
try {
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
|
||||||
luceneSearchInstance.getResults(keyScore -> {
|
luceneSearchInstance.getResults(keyScore -> {
|
||||||
try {
|
try {
|
||||||
if (cancelled.get()) {
|
if (cancelled.get()) {
|
||||||
@ -633,8 +589,9 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
sink.error(ex);
|
sink.error(ex);
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
|
||||||
}, OverflowStrategy.ERROR).subscribeOn(blockingLuceneSearchScheduler).publishOn(luceneQueryScheduler);
|
}, OverflowStrategy.ERROR).subscribeOn(Schedulers.boundedElastic());
|
||||||
|
|
||||||
monoSink.success(new LLSearchResultShard(Flux
|
monoSink.success(new LLSearchResultShard(Flux
|
||||||
.usingWhen(
|
.usingWhen(
|
||||||
@ -642,7 +599,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
b -> resultsFlux,
|
b -> resultsFlux,
|
||||||
b -> successCleanup),
|
b -> successCleanup),
|
||||||
totalHitsCount));
|
totalHitsCount));
|
||||||
}).subscribeOn(blockingLuceneSearchScheduler).publishOn(luceneQueryScheduler).flux());
|
}).subscribeOn(Schedulers.boundedElastic()).flux());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -650,7 +607,6 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
return Mono
|
return Mono
|
||||||
.<Void>fromCallable(() -> {
|
.<Void>fromCallable(() -> {
|
||||||
logger.debug("Closing IndexWriter...");
|
logger.debug("Closing IndexWriter...");
|
||||||
this.blockingLuceneSearchScheduler.dispose();
|
|
||||||
scheduledTasksLifecycle.cancelAndWait();
|
scheduledTasksLifecycle.cancelAndWait();
|
||||||
//noinspection BlockingMethodInNonBlockingContext
|
//noinspection BlockingMethodInNonBlockingContext
|
||||||
indexWriter.close();
|
indexWriter.close();
|
||||||
|
Loading…
x
Reference in New Issue
Block a user