Avoid indexsearcher leaks
This commit is contained in:
parent
cc6071a4de
commit
8e47c15809
@ -9,6 +9,7 @@ import io.netty5.buffer.api.Send;
|
|||||||
import it.cavallium.dbengine.database.LLSnapshot;
|
import it.cavallium.dbengine.database.LLSnapshot;
|
||||||
import it.cavallium.dbengine.database.LLUtils;
|
import it.cavallium.dbengine.database.LLUtils;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.lang.ref.Cleaner;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
@ -52,13 +53,15 @@ public class CachedIndexSearcherManager implements IndexSearcherManager {
|
|||||||
private final AtomicLong activeSearchers = new AtomicLong(0);
|
private final AtomicLong activeSearchers = new AtomicLong(0);
|
||||||
private final AtomicLong activeRefreshes = new AtomicLong(0);
|
private final AtomicLong activeRefreshes = new AtomicLong(0);
|
||||||
|
|
||||||
private final LoadingCache<LLSnapshot, Mono<Send<LLIndexSearcher>>> cachedSnapshotSearchers;
|
private final LoadingCache<LLSnapshot, Mono<LLIndexSearcher>> cachedSnapshotSearchers;
|
||||||
private final Mono<Send<LLIndexSearcher>> cachedMainSearcher;
|
private final Mono<LLIndexSearcher> cachedMainSearcher;
|
||||||
|
|
||||||
private final AtomicBoolean closeRequested = new AtomicBoolean();
|
private final AtomicBoolean closeRequested = new AtomicBoolean();
|
||||||
private final Empty<Void> closeRequestedMono = Sinks.empty();
|
private final Empty<Void> closeRequestedMono = Sinks.empty();
|
||||||
private final Mono<Void> closeMono;
|
private final Mono<Void> closeMono;
|
||||||
|
|
||||||
|
private final Cleaner cleaner = Cleaner.create();
|
||||||
|
|
||||||
public CachedIndexSearcherManager(IndexWriter indexWriter,
|
public CachedIndexSearcherManager(IndexWriter indexWriter,
|
||||||
SnapshotsManager snapshotsManager,
|
SnapshotsManager snapshotsManager,
|
||||||
Scheduler luceneHeavyTasksScheduler,
|
Scheduler luceneHeavyTasksScheduler,
|
||||||
@ -96,7 +99,7 @@ public class CachedIndexSearcherManager implements IndexSearcherManager {
|
|||||||
.maximumSize(3)
|
.maximumSize(3)
|
||||||
.build(new CacheLoader<>() {
|
.build(new CacheLoader<>() {
|
||||||
@Override
|
@Override
|
||||||
public Mono<Send<LLIndexSearcher>> load(@NotNull LLSnapshot snapshot) {
|
public Mono<LLIndexSearcher> load(@NotNull LLSnapshot snapshot) {
|
||||||
return CachedIndexSearcherManager.this.generateCachedSearcher(snapshot);
|
return CachedIndexSearcherManager.this.generateCachedSearcher(snapshot);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@ -141,24 +144,41 @@ public class CachedIndexSearcherManager implements IndexSearcherManager {
|
|||||||
.cache();
|
.cache();
|
||||||
}
|
}
|
||||||
|
|
||||||
private Mono<Send<LLIndexSearcher>> generateCachedSearcher(@Nullable LLSnapshot snapshot) {
|
private Mono<LLIndexSearcher> generateCachedSearcher(@Nullable LLSnapshot snapshot) {
|
||||||
return Mono.fromCallable(() -> {
|
return Mono.fromCallable(() -> {
|
||||||
if (closeRequested.get()) {
|
if (closeRequested.get()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
activeSearchers.incrementAndGet();
|
activeSearchers.incrementAndGet();
|
||||||
IndexSearcher indexSearcher;
|
IndexSearcher indexSearcher;
|
||||||
boolean decRef;
|
boolean fromSnapshot;
|
||||||
if (snapshot == null) {
|
if (snapshot == null) {
|
||||||
indexSearcher = searcherManager.acquire();
|
indexSearcher = searcherManager.acquire();
|
||||||
decRef = true;
|
fromSnapshot = false;
|
||||||
} else {
|
} else {
|
||||||
indexSearcher = snapshotsManager.resolveSnapshot(snapshot).getIndexSearcher(searchExecutor);
|
indexSearcher = snapshotsManager.resolveSnapshot(snapshot).getIndexSearcher(searchExecutor);
|
||||||
decRef = false;
|
fromSnapshot = true;
|
||||||
}
|
}
|
||||||
indexSearcher.setSimilarity(similarity);
|
indexSearcher.setSimilarity(similarity);
|
||||||
assert indexSearcher.getIndexReader().getRefCount() > 0;
|
assert indexSearcher.getIndexReader().getRefCount() > 0;
|
||||||
return new LLIndexSearcher(indexSearcher, decRef, this::dropCachedIndexSearcher).send();
|
LLIndexSearcher llIndexSearcher;
|
||||||
|
if (fromSnapshot) {
|
||||||
|
llIndexSearcher = new SnapshotIndexSearcher(indexSearcher);
|
||||||
|
} else {
|
||||||
|
var released = new AtomicBoolean();
|
||||||
|
llIndexSearcher = new MainIndexSearcher(indexSearcher, released);
|
||||||
|
cleaner.register(llIndexSearcher, () -> {
|
||||||
|
if (released.compareAndSet(false, true)) {
|
||||||
|
logger.warn("An index searcher was not closed!");
|
||||||
|
try {
|
||||||
|
searcherManager.release(indexSearcher);
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error("Failed to release the index searcher", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return llIndexSearcher;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -192,7 +212,7 @@ public class CachedIndexSearcherManager implements IndexSearcherManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<Send<LLIndexSearcher>> retrieveSearcher(@Nullable LLSnapshot snapshot) {
|
public Mono<LLIndexSearcher> retrieveSearcher(@Nullable LLSnapshot snapshot) {
|
||||||
if (snapshot == null) {
|
if (snapshot == null) {
|
||||||
return this.cachedMainSearcher;
|
return this.cachedMainSearcher;
|
||||||
} else {
|
} else {
|
||||||
@ -212,4 +232,34 @@ public class CachedIndexSearcherManager implements IndexSearcherManager {
|
|||||||
public long getActiveRefreshes() {
|
public long getActiveRefreshes() {
|
||||||
return activeRefreshes.get();
|
return activeRefreshes.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private class MainIndexSearcher extends LLIndexSearcher {
|
||||||
|
|
||||||
|
private final AtomicBoolean released;
|
||||||
|
|
||||||
|
public MainIndexSearcher(IndexSearcher indexSearcher, AtomicBoolean released) {
|
||||||
|
super(indexSearcher);
|
||||||
|
this.released = released;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onClose() throws IOException {
|
||||||
|
dropCachedIndexSearcher();
|
||||||
|
if (released.compareAndSet(false, true)) {
|
||||||
|
searcherManager.release(indexSearcher);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class SnapshotIndexSearcher extends LLIndexSearcher {
|
||||||
|
|
||||||
|
public SnapshotIndexSearcher(IndexSearcher indexSearcher) {
|
||||||
|
super(indexSearcher);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onClose() {
|
||||||
|
dropCachedIndexSearcher();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -15,7 +15,7 @@ public interface IndexSearcherManager {
|
|||||||
|
|
||||||
void maybeRefresh() throws IOException;
|
void maybeRefresh() throws IOException;
|
||||||
|
|
||||||
Mono<Send<LLIndexSearcher>> retrieveSearcher(@Nullable LLSnapshot snapshot);
|
Mono<LLIndexSearcher> retrieveSearcher(@Nullable LLSnapshot snapshot);
|
||||||
|
|
||||||
Mono<Void> close();
|
Mono<Void> close();
|
||||||
}
|
}
|
||||||
|
@ -3,79 +3,42 @@ package it.cavallium.dbengine.database.disk;
|
|||||||
import io.netty5.buffer.api.Drop;
|
import io.netty5.buffer.api.Drop;
|
||||||
import io.netty5.buffer.api.Owned;
|
import io.netty5.buffer.api.Owned;
|
||||||
import io.netty5.buffer.api.internal.ResourceSupport;
|
import io.netty5.buffer.api.internal.ResourceSupport;
|
||||||
|
import it.cavallium.dbengine.database.SafeCloseable;
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.apache.lucene.index.IndexReader;
|
import org.apache.lucene.index.IndexReader;
|
||||||
import org.apache.lucene.search.IndexSearcher;
|
import org.apache.lucene.search.IndexSearcher;
|
||||||
|
|
||||||
public class LLIndexSearcher extends ResourceSupport<LLIndexSearcher, LLIndexSearcher> {
|
public abstract class LLIndexSearcher implements Closeable {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(LLIndexSearcher.class);
|
protected static final Logger LOG = LogManager.getLogger(LLIndexSearcher.class);
|
||||||
|
|
||||||
private static final Drop<LLIndexSearcher> DROP = new Drop<>() {
|
protected final IndexSearcher indexSearcher;
|
||||||
@Override
|
private final AtomicBoolean closed = new AtomicBoolean();
|
||||||
public void drop(LLIndexSearcher obj) {
|
|
||||||
try {
|
|
||||||
if (obj.onClose != null) {
|
|
||||||
obj.onClose.run();
|
|
||||||
}
|
|
||||||
} catch (Throwable ex) {
|
|
||||||
logger.error("Failed to close onClose", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
public LLIndexSearcher(IndexSearcher indexSearcher) {
|
||||||
public Drop<LLIndexSearcher> fork() {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void attach(LLIndexSearcher obj) {
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
private IndexSearcher indexSearcher;
|
|
||||||
private final boolean decRef;
|
|
||||||
|
|
||||||
private Runnable onClose;
|
|
||||||
|
|
||||||
public LLIndexSearcher(IndexSearcher indexSearcher, boolean decRef, Runnable onClose) {
|
|
||||||
super(DROP);
|
|
||||||
this.indexSearcher = indexSearcher;
|
this.indexSearcher = indexSearcher;
|
||||||
this.decRef = decRef;
|
|
||||||
this.onClose = onClose;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public IndexReader getIndexReader() {
|
public IndexReader getIndexReader() {
|
||||||
if (!isOwned()) {
|
if (closed.get()) throw new IllegalStateException("Closed");
|
||||||
throw attachTrace(new IllegalStateException("LLIndexSearcher must be owned to be used"));
|
|
||||||
}
|
|
||||||
return indexSearcher.getIndexReader();
|
return indexSearcher.getIndexReader();
|
||||||
}
|
}
|
||||||
|
|
||||||
public IndexSearcher getIndexSearcher() {
|
public IndexSearcher getIndexSearcher() {
|
||||||
if (!isOwned()) {
|
if (closed.get()) throw new IllegalStateException("Closed");
|
||||||
throw attachTrace(new IllegalStateException("LLIndexSearcher must be owned to be used"));
|
|
||||||
}
|
|
||||||
return indexSearcher;
|
return indexSearcher;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected RuntimeException createResourceClosedException() {
|
public final void close() throws IOException {
|
||||||
return new IllegalStateException("Closed");
|
if (closed.compareAndSet(false, true)) {
|
||||||
}
|
onClose();
|
||||||
|
}
|
||||||
@Override
|
|
||||||
protected Owned<LLIndexSearcher> prepareSend() {
|
|
||||||
var indexSearcher = this.indexSearcher;
|
|
||||||
var onClose = this.onClose;
|
|
||||||
return drop -> new LLIndexSearcher(indexSearcher, decRef, onClose);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void makeInaccessible() {
|
|
||||||
this.indexSearcher = null;
|
|
||||||
this.onClose = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected abstract void onClose() throws IOException;
|
||||||
}
|
}
|
||||||
|
@ -6,6 +6,7 @@ import io.netty5.buffer.api.Resource;
|
|||||||
import io.netty5.buffer.api.Send;
|
import io.netty5.buffer.api.Send;
|
||||||
import io.netty5.buffer.api.internal.ResourceSupport;
|
import io.netty5.buffer.api.internal.ResourceSupport;
|
||||||
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
|
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
|
||||||
|
import java.io.Closeable;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.UncheckedIOException;
|
import java.io.UncheckedIOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@ -18,64 +19,32 @@ import org.apache.lucene.index.IndexReader;
|
|||||||
import org.apache.lucene.index.MultiReader;
|
import org.apache.lucene.index.MultiReader;
|
||||||
import org.apache.lucene.search.IndexSearcher;
|
import org.apache.lucene.search.IndexSearcher;
|
||||||
|
|
||||||
public interface LLIndexSearchers extends Resource<LLIndexSearchers> {
|
public interface LLIndexSearchers extends Closeable {
|
||||||
|
|
||||||
static LLIndexSearchers of(List<Send<LLIndexSearcher>> indexSearchers) {
|
static LLIndexSearchers of(List<LLIndexSearcher> indexSearchers) {
|
||||||
return new ShardedIndexSearchers(indexSearchers, null);
|
return new ShardedIndexSearchers(indexSearchers);
|
||||||
}
|
}
|
||||||
|
|
||||||
static UnshardedIndexSearchers unsharded(Send<LLIndexSearcher> indexSearcher) {
|
static UnshardedIndexSearchers unsharded(LLIndexSearcher indexSearcher) {
|
||||||
return new UnshardedIndexSearchers(indexSearcher, null);
|
return new UnshardedIndexSearchers(indexSearcher);
|
||||||
}
|
}
|
||||||
|
|
||||||
List<IndexSearcher> shards();
|
List<IndexSearcher> shards();
|
||||||
|
|
||||||
|
List<LLIndexSearcher> llShards();
|
||||||
|
|
||||||
IndexSearcher shard(int shardIndex);
|
IndexSearcher shard(int shardIndex);
|
||||||
|
|
||||||
|
LLIndexSearcher llShard(int shardIndex);
|
||||||
|
|
||||||
IndexReader allShards();
|
IndexReader allShards();
|
||||||
|
|
||||||
class UnshardedIndexSearchers extends ResourceSupport<LLIndexSearchers, UnshardedIndexSearchers>
|
class UnshardedIndexSearchers implements LLIndexSearchers {
|
||||||
implements LLIndexSearchers {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(UnshardedIndexSearchers.class);
|
private final LLIndexSearcher indexSearcher;
|
||||||
|
|
||||||
private static final Drop<UnshardedIndexSearchers> DROP = new Drop<>() {
|
public UnshardedIndexSearchers(LLIndexSearcher indexSearcher) {
|
||||||
@Override
|
this.indexSearcher = indexSearcher;
|
||||||
public void drop(UnshardedIndexSearchers obj) {
|
|
||||||
try {
|
|
||||||
if (obj.indexSearcher != null) {
|
|
||||||
obj.indexSearcher.close();
|
|
||||||
}
|
|
||||||
} catch (Throwable ex) {
|
|
||||||
logger.error("Failed to close indexSearcher", ex);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
if (obj.onClose != null) {
|
|
||||||
obj.onClose.run();
|
|
||||||
}
|
|
||||||
} catch (Throwable ex) {
|
|
||||||
logger.error("Failed to close onClose", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Drop<UnshardedIndexSearchers> fork() {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void attach(UnshardedIndexSearchers obj) {
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
private LLIndexSearcher indexSearcher;
|
|
||||||
private Runnable onClose;
|
|
||||||
|
|
||||||
public UnshardedIndexSearchers(Send<LLIndexSearcher> indexSearcher, Runnable onClose) {
|
|
||||||
super(DROP);
|
|
||||||
this.indexSearcher = indexSearcher.receive();
|
|
||||||
this.onClose = onClose;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -84,16 +53,26 @@ public interface LLIndexSearchers extends Resource<LLIndexSearchers> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IndexSearcher shard(int shardIndex) {
|
public List<LLIndexSearcher> llShards() {
|
||||||
if (!isOwned()) {
|
return Collections.singletonList(indexSearcher);
|
||||||
throw attachTrace(new IllegalStateException("UnshardedIndexSearchers must be owned to be used"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IndexSearcher shard(int shardIndex) {
|
||||||
if (shardIndex != -1) {
|
if (shardIndex != -1) {
|
||||||
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid, this is a unsharded index");
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid, this is a unsharded index");
|
||||||
}
|
}
|
||||||
return indexSearcher.getIndexSearcher();
|
return indexSearcher.getIndexSearcher();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public LLIndexSearcher llShard(int shardIndex) {
|
||||||
|
if (shardIndex != -1) {
|
||||||
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid, this is a unsharded index");
|
||||||
|
}
|
||||||
|
return indexSearcher;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IndexReader allShards() {
|
public IndexReader allShards() {
|
||||||
return indexSearcher.getIndexReader();
|
return indexSearcher.getIndexReader();
|
||||||
@ -103,92 +82,42 @@ public interface LLIndexSearchers extends Resource<LLIndexSearchers> {
|
|||||||
return this.shard(-1);
|
return this.shard(-1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
public LLIndexSearcher llShard() {
|
||||||
protected RuntimeException createResourceClosedException() {
|
return this.llShard(-1);
|
||||||
return new IllegalStateException("Closed");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Owned<UnshardedIndexSearchers> prepareSend() {
|
public void close() throws IOException {
|
||||||
Send<LLIndexSearcher> indexSearcher = this.indexSearcher.send();
|
|
||||||
var onClose = this.onClose;
|
|
||||||
return drop -> {
|
|
||||||
var instance = new UnshardedIndexSearchers(indexSearcher, onClose);
|
|
||||||
drop.attach(instance);
|
|
||||||
return instance;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void makeInaccessible() {
|
|
||||||
this.indexSearcher = null;
|
|
||||||
this.onClose = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class ShardedIndexSearchers extends ResourceSupport<LLIndexSearchers, ShardedIndexSearchers>
|
|
||||||
implements LLIndexSearchers {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ShardedIndexSearchers.class);
|
|
||||||
|
|
||||||
private static final Drop<ShardedIndexSearchers> DROP = new Drop<>() {
|
|
||||||
@Override
|
|
||||||
public void drop(ShardedIndexSearchers obj) {
|
|
||||||
try {
|
|
||||||
for (LLIndexSearcher indexSearcher : obj.indexSearchers) {
|
|
||||||
indexSearcher.close();
|
indexSearcher.close();
|
||||||
}
|
}
|
||||||
} catch (Throwable ex) {
|
|
||||||
logger.error("Failed to close indexSearcher", ex);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
if (obj.onClose != null) {
|
|
||||||
obj.onClose.run();
|
|
||||||
}
|
|
||||||
} catch (Throwable ex) {
|
|
||||||
logger.error("Failed to close onClose", ex);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
class ShardedIndexSearchers implements LLIndexSearchers {
|
||||||
public Drop<ShardedIndexSearchers> fork() {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
private final List<LLIndexSearcher> indexSearchers;
|
||||||
public void attach(ShardedIndexSearchers obj) {
|
private final List<IndexSearcher> indexSearchersVals;
|
||||||
|
|
||||||
}
|
public ShardedIndexSearchers(List<LLIndexSearcher> indexSearchers) {
|
||||||
};
|
|
||||||
|
|
||||||
private List<LLIndexSearcher> indexSearchers;
|
|
||||||
private List<IndexSearcher> indexSearchersVals;
|
|
||||||
private Runnable onClose;
|
|
||||||
|
|
||||||
public ShardedIndexSearchers(List<Send<LLIndexSearcher>> indexSearchers, Runnable onClose) {
|
|
||||||
super(DROP);
|
|
||||||
this.indexSearchers = new ArrayList<>(indexSearchers.size());
|
this.indexSearchers = new ArrayList<>(indexSearchers.size());
|
||||||
this.indexSearchersVals = new ArrayList<>(indexSearchers.size());
|
this.indexSearchersVals = new ArrayList<>(indexSearchers.size());
|
||||||
for (Send<LLIndexSearcher> llIndexSearcher : indexSearchers) {
|
for (LLIndexSearcher indexSearcher : indexSearchers) {
|
||||||
var indexSearcher = llIndexSearcher.receive();
|
|
||||||
this.indexSearchers.add(indexSearcher);
|
this.indexSearchers.add(indexSearcher);
|
||||||
this.indexSearchersVals.add(indexSearcher.getIndexSearcher());
|
this.indexSearchersVals.add(indexSearcher.getIndexSearcher());
|
||||||
}
|
}
|
||||||
this.onClose = onClose;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<IndexSearcher> shards() {
|
public List<IndexSearcher> shards() {
|
||||||
if (!isOwned()) {
|
|
||||||
throw attachTrace(new IllegalStateException("ShardedIndexSearchers must be owned to be used"));
|
|
||||||
}
|
|
||||||
return Collections.unmodifiableList(indexSearchersVals);
|
return Collections.unmodifiableList(indexSearchersVals);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IndexSearcher shard(int shardIndex) {
|
public List<LLIndexSearcher> llShards() {
|
||||||
if (!isOwned()) {
|
return Collections.unmodifiableList(indexSearchers);
|
||||||
throw attachTrace(new IllegalStateException("ShardedIndexSearchers must be owned to be used"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IndexSearcher shard(int shardIndex) {
|
||||||
if (shardIndex < 0) {
|
if (shardIndex < 0) {
|
||||||
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid");
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid");
|
||||||
}
|
}
|
||||||
@ -196,10 +125,15 @@ public interface LLIndexSearchers extends Resource<LLIndexSearchers> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IndexReader allShards() {
|
public LLIndexSearcher llShard(int shardIndex) {
|
||||||
if (!isOwned()) {
|
if (shardIndex < 0) {
|
||||||
throw attachTrace(new IllegalStateException("ShardedIndexSearchers must be owned to be used"));
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid");
|
||||||
}
|
}
|
||||||
|
return indexSearchers.get(shardIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IndexReader allShards() {
|
||||||
var irs = new IndexReader[indexSearchersVals.size()];
|
var irs = new IndexReader[indexSearchersVals.size()];
|
||||||
for (int i = 0, s = indexSearchersVals.size(); i < s; i++) {
|
for (int i = 0, s = indexSearchersVals.size(); i < s; i++) {
|
||||||
irs[i] = indexSearchersVals.get(i).getIndexReader();
|
irs[i] = indexSearchersVals.get(i).getIndexReader();
|
||||||
@ -217,28 +151,10 @@ public interface LLIndexSearchers extends Resource<LLIndexSearchers> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected RuntimeException createResourceClosedException() {
|
public void close() throws IOException {
|
||||||
return new IllegalStateException("Closed");
|
for (LLIndexSearcher indexSearcher : indexSearchers) {
|
||||||
}
|
indexSearcher.close();
|
||||||
|
}
|
||||||
@Override
|
|
||||||
protected Owned<ShardedIndexSearchers> prepareSend() {
|
|
||||||
List<Send<LLIndexSearcher>> indexSearchers = new ArrayList<>(this.indexSearchers.size());
|
|
||||||
for (LLIndexSearcher indexSearcher : this.indexSearchers) {
|
|
||||||
indexSearchers.add(indexSearcher.send());
|
|
||||||
}
|
|
||||||
var onClose = this.onClose;
|
|
||||||
return drop -> {
|
|
||||||
var instance = new ShardedIndexSearchers(indexSearchers, onClose);
|
|
||||||
drop.attach(instance);
|
|
||||||
return instance;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void makeInaccessible() {
|
|
||||||
this.indexSearchers = null;
|
|
||||||
this.indexSearchersVals = null;
|
|
||||||
this.onClose = null;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -27,6 +27,7 @@ import it.cavallium.dbengine.database.LLTerm;
|
|||||||
import it.cavallium.dbengine.database.LLUpdateDocument;
|
import it.cavallium.dbengine.database.LLUpdateDocument;
|
||||||
import it.cavallium.dbengine.database.LLUpdateFields;
|
import it.cavallium.dbengine.database.LLUpdateFields;
|
||||||
import it.cavallium.dbengine.database.LLUtils;
|
import it.cavallium.dbengine.database.LLUtils;
|
||||||
|
import it.cavallium.dbengine.database.disk.LLIndexSearchers.UnshardedIndexSearchers;
|
||||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||||
import it.cavallium.dbengine.lucene.LuceneRocksDBManager;
|
import it.cavallium.dbengine.lucene.LuceneRocksDBManager;
|
||||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||||
@ -510,14 +511,14 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||||||
localQueries.add(QueryParser.toQuery(query, luceneAnalyzer));
|
localQueries.add(QueryParser.toQuery(query, luceneAnalyzer));
|
||||||
}
|
}
|
||||||
var localNormalizationQuery = QueryParser.toQuery(normalizationQuery, luceneAnalyzer);
|
var localNormalizationQuery = QueryParser.toQuery(normalizationQuery, luceneAnalyzer);
|
||||||
var searchers = searcherManager
|
Mono<LLIndexSearchers> searchers = searcherManager
|
||||||
.retrieveSearcher(snapshot)
|
.retrieveSearcher(snapshot)
|
||||||
.map(indexSearcher -> LLIndexSearchers.unsharded(indexSearcher).send());
|
.map(LLIndexSearchers::unsharded);
|
||||||
|
|
||||||
return decimalBucketMultiSearcher.collectMulti(searchers, bucketParams, localQueries, localNormalizationQuery);
|
return decimalBucketMultiSearcher.collectMulti(searchers, bucketParams, localQueries, localNormalizationQuery);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Mono<Send<LLIndexSearcher>> retrieveSearcher(@Nullable LLSnapshot snapshot) {
|
public Mono<LLIndexSearcher> retrieveSearcher(@Nullable LLSnapshot snapshot) {
|
||||||
return searcherManager.retrieveSearcher(snapshot);
|
return searcherManager.retrieveSearcher(snapshot);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,7 +146,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
|
|||||||
return clusterName;
|
return clusterName;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Mono<Send<LLIndexSearchers>> getIndexSearchers(LLSnapshot snapshot) {
|
private Mono<LLIndexSearchers> getIndexSearchers(LLSnapshot snapshot) {
|
||||||
return luceneIndicesFlux
|
return luceneIndicesFlux
|
||||||
.index()
|
.index()
|
||||||
// Resolve the snapshot of each shard
|
// Resolve the snapshot of each shard
|
||||||
@ -155,7 +155,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
|
|||||||
.flatMap(luceneSnapshot -> tuple.getT2().retrieveSearcher(luceneSnapshot.orElse(null)))
|
.flatMap(luceneSnapshot -> tuple.getT2().retrieveSearcher(luceneSnapshot.orElse(null)))
|
||||||
)
|
)
|
||||||
.collectList()
|
.collectList()
|
||||||
.map(searchers -> LLIndexSearchers.of(searchers).send());
|
.map(LLIndexSearchers::of);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -12,6 +12,7 @@ import java.io.IOException;
|
|||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
import reactor.core.scheduler.Schedulers;
|
import reactor.core.scheduler.Schedulers;
|
||||||
|
import reactor.util.function.Tuples;
|
||||||
|
|
||||||
public class AdaptiveLocalSearcher implements LocalSearcher {
|
public class AdaptiveLocalSearcher implements LocalSearcher {
|
||||||
|
|
||||||
@ -41,28 +42,22 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collect(Mono<Send<LLIndexSearcher>> indexSearcher,
|
public Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
Mono<Send<LLIndexSearchers>> indexSearchersMono = indexSearcher
|
return indexSearcherMono.flatMap(indexSearcher -> {
|
||||||
.map(LLIndexSearchers::unsharded)
|
var indexSearchers = LLIndexSearchers.unsharded(indexSearcher);
|
||||||
.map(ResourceSupport::send);
|
|
||||||
|
|
||||||
if (transformer == NO_REWRITE) {
|
if (transformer == NO_REWRITE) {
|
||||||
return transformedCollect(indexSearcher, queryParams, keyFieldName, transformer);
|
return transformedCollect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
return indexSearchersMono
|
return Mono
|
||||||
.publishOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
.fromCallable(() -> transformer.rewrite(indexSearchers, queryParams))
|
||||||
.<LocalQueryParams>handle((indexSearchers, sink) -> {
|
.subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
||||||
try {
|
|
||||||
sink.next(transformer.rewrite(indexSearchers.receive(), queryParams));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
sink.error(ex);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.flatMap(queryParams2 -> transformedCollect(indexSearcher, queryParams2, keyFieldName, NO_REWRITE));
|
.flatMap(queryParams2 -> transformedCollect(indexSearcher, queryParams2, keyFieldName, NO_REWRITE));
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -71,7 +66,7 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Remember to change also AdaptiveMultiSearcher
|
// Remember to change also AdaptiveMultiSearcher
|
||||||
public Mono<LuceneSearchResult> transformedCollect(Mono<Send<LLIndexSearcher>> indexSearcher,
|
public Mono<LuceneSearchResult> transformedCollect(LLIndexSearcher indexSearcher,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
String keyFieldName,
|
String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
@ -81,36 +76,36 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
|
|||||||
= Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0));
|
= Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0));
|
||||||
|
|
||||||
if (queryParams.limitLong() == 0) {
|
if (queryParams.limitLong() == 0) {
|
||||||
return countSearcher.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return countSearcher.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
} else if (realLimit <= maxInMemoryResultEntries) {
|
} else if (realLimit <= maxInMemoryResultEntries) {
|
||||||
return standardSearcher.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return standardSearcher.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
} else if (queryParams.isSorted()) {
|
} else if (queryParams.isSorted()) {
|
||||||
if (realLimit <= maxAllowedInMemoryLimit) {
|
if (realLimit <= maxAllowedInMemoryLimit) {
|
||||||
return scoredPaged.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return scoredPaged.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
if (queryParams.isSortedByScore()) {
|
if (queryParams.isSortedByScore()) {
|
||||||
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
||||||
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
||||||
}
|
}
|
||||||
if (sortedByScoreFull != null) {
|
if (sortedByScoreFull != null) {
|
||||||
return sortedByScoreFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return sortedByScoreFull.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
return scoredPaged.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return scoredPaged.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
||||||
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
||||||
}
|
}
|
||||||
if (sortedScoredFull != null) {
|
if (sortedScoredFull != null) {
|
||||||
return sortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return sortedScoredFull.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
return scoredPaged.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return scoredPaged.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Run large/unbounded searches using the continuous multi searcher
|
// Run large/unbounded searches using the continuous multi searcher
|
||||||
return unsortedUnscoredContinuous.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
return unsortedUnscoredContinuous.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -40,28 +40,23 @@ public class AdaptiveMultiSearcher implements MultiSearcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
if (transformer == NO_REWRITE) {
|
if (transformer == NO_REWRITE) {
|
||||||
return transformedCollectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return transformedCollectMulti(indexSearchers, queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
return indexSearchersMono
|
return Mono.fromCallable(() -> transformer.rewrite(indexSearchers, queryParams))
|
||||||
.publishOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
.subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
||||||
.<LocalQueryParams>handle((indexSearchers, sink) -> {
|
.flatMap(queryParams2 -> transformedCollectMulti(indexSearchers, queryParams2, keyFieldName, NO_REWRITE));
|
||||||
try {
|
|
||||||
sink.next(transformer.rewrite(indexSearchers.receive(), queryParams));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
sink.error(ex);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.flatMap(queryParams2 -> transformedCollectMulti(indexSearchersMono, queryParams2, keyFieldName, NO_REWRITE));
|
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remember to change also AdaptiveLocalSearcher
|
// Remember to change also AdaptiveLocalSearcher
|
||||||
public Mono<LuceneSearchResult> transformedCollectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> transformedCollectMulti(LLIndexSearchers indexSearchers,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
@ -70,40 +65,38 @@ public class AdaptiveMultiSearcher implements MultiSearcher {
|
|||||||
long maxAllowedInMemoryLimit
|
long maxAllowedInMemoryLimit
|
||||||
= Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0));
|
= Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0));
|
||||||
|
|
||||||
return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> {
|
|
||||||
if (queryParams.limitLong() == 0) {
|
if (queryParams.limitLong() == 0) {
|
||||||
return count.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return count.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
} else if (realLimit <= maxInMemoryResultEntries) {
|
} else if (realLimit <= maxInMemoryResultEntries) {
|
||||||
return standardSearcher.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return standardSearcher.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
} else if (queryParams.isSorted()) {
|
} else if (queryParams.isSorted()) {
|
||||||
if (realLimit <= maxAllowedInMemoryLimit) {
|
if (realLimit <= maxAllowedInMemoryLimit) {
|
||||||
return scoredPaged.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return scoredPaged.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
if (queryParams.isSortedByScore()) {
|
if (queryParams.isSortedByScore()) {
|
||||||
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
||||||
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
||||||
}
|
}
|
||||||
if (sortedByScoreFull != null) {
|
if (sortedByScoreFull != null) {
|
||||||
return sortedByScoreFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return sortedByScoreFull.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
return scoredPaged.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return scoredPaged.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
if (queryParams.limitLong() < maxInMemoryResultEntries) {
|
||||||
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
|
||||||
}
|
}
|
||||||
if (sortedScoredFull != null) {
|
if (sortedScoredFull != null) {
|
||||||
return sortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return sortedScoredFull.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
} else {
|
} else {
|
||||||
return scoredPaged.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return scoredPaged.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Run large/unbounded searches using the continuous multi searcher
|
// Run large/unbounded searches using the continuous multi searcher
|
||||||
return unsortedUnscoredContinuous.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
return unsortedUnscoredContinuous.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
|
||||||
}
|
}
|
||||||
}, true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -11,6 +11,8 @@ import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import reactor.core.publisher.Flux;
|
import reactor.core.publisher.Flux;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
@ -18,30 +20,27 @@ import reactor.core.scheduler.Schedulers;
|
|||||||
|
|
||||||
public class CountMultiSearcher implements MultiSearcher {
|
public class CountMultiSearcher implements MultiSearcher {
|
||||||
|
|
||||||
|
protected static final Logger LOG = LogManager.getLogger(CountMultiSearcher.class);
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
String keyFieldName,
|
String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
} else {
|
} else {
|
||||||
queryParamsMono = indexSearchersMono
|
queryParamsMono = Mono
|
||||||
.publishOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
.fromCallable(() -> transformer.rewrite(indexSearchers, queryParams))
|
||||||
.handle((indexSearchers, sink) -> {
|
.subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic()));
|
||||||
try {
|
|
||||||
sink.next(transformer.rewrite(indexSearchers.receive(), queryParams));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
sink.error(ex);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return queryParamsMono.flatMap(queryParams2 -> LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> {
|
return queryParamsMono.flatMap(queryParams2 -> {
|
||||||
var localQueryParams = getLocalQueryParams(queryParams2);
|
var localQueryParams = getLocalQueryParams(queryParams2);
|
||||||
return Mono.fromRunnable(() -> {
|
return Mono
|
||||||
|
.fromRunnable(() -> {
|
||||||
if (queryParams2.isSorted() && queryParams2.limitLong() > 0) {
|
if (queryParams2.isSorted() && queryParams2.limitLong() > 0) {
|
||||||
throw new UnsupportedOperationException(
|
throw new UnsupportedOperationException(
|
||||||
"Sorted queries are not supported by SimpleUnsortedUnscoredLuceneMultiSearcher");
|
"Sorted queries are not supported by SimpleUnsortedUnscoredLuceneMultiSearcher");
|
||||||
@ -50,10 +49,11 @@ public class CountMultiSearcher implements MultiSearcher {
|
|||||||
throw new UnsupportedOperationException(
|
throw new UnsupportedOperationException(
|
||||||
"Scored queries are not supported by SimpleUnsortedUnscoredLuceneMultiSearcher");
|
"Scored queries are not supported by SimpleUnsortedUnscoredLuceneMultiSearcher");
|
||||||
}
|
}
|
||||||
}).thenMany(Flux.fromIterable(indexSearchers.shards())).flatMap(searcher -> {
|
})
|
||||||
var llSearcher = Mono.fromCallable(() -> new LLIndexSearcher(searcher, false, null).send());
|
.thenMany(Flux.fromIterable(indexSearchers.llShards()))
|
||||||
return this.collect(llSearcher, localQueryParams, keyFieldName, transformer);
|
.flatMap(searcher -> this.collect(Mono.just(searcher), localQueryParams, keyFieldName, transformer))
|
||||||
}).collectList().map(results -> {
|
.collectList()
|
||||||
|
.map(results -> {
|
||||||
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
|
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
|
||||||
List<Flux<LLKeyScore>> resultsFluxes = new ArrayList<>(results.size());
|
List<Flux<LLKeyScore>> resultsFluxes = new ArrayList<>(results.size());
|
||||||
boolean exactTotalHitsCount = true;
|
boolean exactTotalHitsCount = true;
|
||||||
@ -77,12 +77,15 @@ public class CountMultiSearcher implements MultiSearcher {
|
|||||||
luceneSearchResult.close();
|
luceneSearchResult.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearchers.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}, false));
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private LocalQueryParams getLocalQueryParams(LocalQueryParams queryParams) {
|
private LocalQueryParams getLocalQueryParams(LocalQueryParams queryParams) {
|
||||||
@ -97,14 +100,11 @@ public class CountMultiSearcher implements MultiSearcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
|
public Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
return Mono
|
return indexSearcherMono.flatMap(indexSearcher -> {
|
||||||
.usingWhen(
|
|
||||||
indexSearcherMono,
|
|
||||||
indexSearcher -> {
|
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
@ -116,18 +116,19 @@ public class CountMultiSearcher implements MultiSearcher {
|
|||||||
|
|
||||||
return queryParamsMono
|
return queryParamsMono
|
||||||
.flatMap(queryParams2 -> Mono.fromCallable(() -> {
|
.flatMap(queryParams2 -> Mono.fromCallable(() -> {
|
||||||
try (var is = indexSearcher.receive()) {
|
|
||||||
LLUtils.ensureBlocking();
|
LLUtils.ensureBlocking();
|
||||||
|
return (long) indexSearcher.getIndexSearcher().count(queryParams2.query());
|
||||||
return (long) is.getIndexSearcher().count(queryParams2.query());
|
|
||||||
}
|
|
||||||
}).subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic())))
|
}).subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic())))
|
||||||
.publishOn(Schedulers.parallel())
|
.publishOn(Schedulers.parallel())
|
||||||
.transform(TimeoutUtil.timeoutMono(queryParams.timeout()));
|
.transform(TimeoutUtil.timeoutMono(queryParams.timeout()))
|
||||||
},
|
.map(count -> new LuceneSearchResult(TotalHitsCount.of(count, true), Flux.empty(), () -> {
|
||||||
is -> Mono.empty()
|
try {
|
||||||
)
|
indexSearcher.close();
|
||||||
.map(count -> new LuceneSearchResult(TotalHitsCount.of(count, true), Flux.empty(), null));
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -14,22 +14,25 @@ import org.jetbrains.annotations.NotNull;
|
|||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import reactor.core.publisher.Flux;
|
import reactor.core.publisher.Flux;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
|
import reactor.core.scheduler.Schedulers;
|
||||||
|
|
||||||
public class DecimalBucketMultiSearcher {
|
public class DecimalBucketMultiSearcher {
|
||||||
|
|
||||||
protected static final Logger logger = LogManager.getLogger(DecimalBucketMultiSearcher.class);
|
protected static final Logger logger = LogManager.getLogger(DecimalBucketMultiSearcher.class);
|
||||||
|
|
||||||
public Mono<Buckets> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<Buckets> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
BucketParams bucketParams,
|
BucketParams bucketParams,
|
||||||
@NotNull List<Query> queries,
|
@NotNull List<Query> queries,
|
||||||
@Nullable Query normalizationQuery) {
|
@Nullable Query normalizationQuery) {
|
||||||
|
|
||||||
return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> this
|
return Mono.usingWhen(indexSearchersMono, indexSearchers -> this
|
||||||
// Search results
|
// Search results
|
||||||
.search(indexSearchers.shards(), bucketParams, queries, normalizationQuery)
|
.search(indexSearchers.shards(), bucketParams, queries, normalizationQuery)
|
||||||
// Ensure that one result is always returned
|
// Ensure that one result is always returned
|
||||||
.single(),
|
.single(), indexSearchers -> Mono.fromCallable(() -> {
|
||||||
true);
|
indexSearchers.close();
|
||||||
|
return null;
|
||||||
|
}).subscribeOn(Schedulers.boundedElastic()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private Mono<Buckets> search(Iterable<IndexSearcher> indexSearchers,
|
private Mono<Buckets> search(Iterable<IndexSearcher> indexSearchers,
|
||||||
|
@ -13,7 +13,7 @@ public interface LocalSearcher {
|
|||||||
* @param keyFieldName the name of the key field
|
* @param keyFieldName the name of the key field
|
||||||
* @param transformer the search query transformer
|
* @param transformer the search query transformer
|
||||||
*/
|
*/
|
||||||
Mono<LuceneSearchResult> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
|
Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer);
|
GlobalQueryRewrite transformer);
|
||||||
|
@ -14,7 +14,7 @@ public interface MultiSearcher extends LocalSearcher {
|
|||||||
* @param keyFieldName the name of the key field
|
* @param keyFieldName the name of the key field
|
||||||
* @param transformer the search query transformer
|
* @param transformer the search query transformer
|
||||||
*/
|
*/
|
||||||
Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer);
|
GlobalQueryRewrite transformer);
|
||||||
@ -26,11 +26,11 @@ public interface MultiSearcher extends LocalSearcher {
|
|||||||
* @param transformer the search query transformer
|
* @param transformer the search query transformer
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
default Mono<LuceneSearchResult> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
|
default Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
var searchers = indexSearcherMono.map(a -> LLIndexSearchers.unsharded(a).send());
|
Mono<LLIndexSearchers> searchers = indexSearcherMono.map(LLIndexSearchers::unsharded);
|
||||||
return this.collectMulti(searchers, queryParams, keyFieldName, transformer);
|
return this.collectMulti(searchers, queryParams, keyFieldName, transformer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -15,6 +15,8 @@ import it.cavallium.dbengine.lucene.collector.TopDocsCollectorMultiManager;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.apache.lucene.search.IndexSearcher;
|
import org.apache.lucene.search.IndexSearcher;
|
||||||
import org.apache.lucene.search.ScoreDoc;
|
import org.apache.lucene.search.ScoreDoc;
|
||||||
import org.apache.lucene.search.TopDocs;
|
import org.apache.lucene.search.TopDocs;
|
||||||
@ -28,16 +30,18 @@ import reactor.core.scheduler.Schedulers;
|
|||||||
|
|
||||||
public class PagedLocalSearcher implements LocalSearcher {
|
public class PagedLocalSearcher implements LocalSearcher {
|
||||||
|
|
||||||
|
private static final Logger LOG = LogManager.getLogger(PagedLocalSearcher.class);
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
|
public Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
PaginationInfo paginationInfo = getPaginationInfo(queryParams);
|
PaginationInfo paginationInfo = getPaginationInfo(queryParams);
|
||||||
|
|
||||||
var indexSearchersMono = indexSearcherMono.map(LLIndexSearchers::unsharded).map(ResourceSupport::send);
|
return indexSearcherMono.flatMap(indexSearcher -> {
|
||||||
|
var indexSearchers = LLIndexSearchers.unsharded(indexSearcher);
|
||||||
|
|
||||||
return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> {
|
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
@ -51,24 +55,27 @@ public class PagedLocalSearcher implements LocalSearcher {
|
|||||||
// Search first page results
|
// Search first page results
|
||||||
.searchFirstPage(indexSearchers.shards(), queryParams2, paginationInfo)
|
.searchFirstPage(indexSearchers.shards(), queryParams2, paginationInfo)
|
||||||
// Compute the results of the first page
|
// Compute the results of the first page
|
||||||
.transform(firstPageTopDocsMono -> this.computeFirstPageResults(firstPageTopDocsMono, indexSearchers.shards(),
|
.transform(firstPageTopDocsMono -> this.computeFirstPageResults(firstPageTopDocsMono,
|
||||||
keyFieldName, queryParams2))
|
indexSearchers.shards(),
|
||||||
|
keyFieldName,
|
||||||
|
queryParams2
|
||||||
|
))
|
||||||
// Compute other results
|
// Compute other results
|
||||||
.transform(firstResult -> this.computeOtherResults(firstResult,
|
.transform(firstResult -> this.computeOtherResults(firstResult,
|
||||||
indexSearchers.shards(),
|
indexSearchers.shards(),
|
||||||
queryParams2,
|
queryParams2,
|
||||||
keyFieldName,
|
keyFieldName,
|
||||||
() -> {
|
() -> {
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearcher.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
))
|
))
|
||||||
// Ensure that one LuceneSearchResult is always returned
|
// Ensure that one LuceneSearchResult is always returned
|
||||||
.single()
|
.single());
|
||||||
);
|
});
|
||||||
},
|
|
||||||
false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -27,54 +27,54 @@ import reactor.core.scheduler.Schedulers;
|
|||||||
|
|
||||||
public class ScoredPagedMultiSearcher implements MultiSearcher {
|
public class ScoredPagedMultiSearcher implements MultiSearcher {
|
||||||
|
|
||||||
protected static final Logger logger = LogManager.getLogger(ScoredPagedMultiSearcher.class);
|
protected static final Logger LOG = LogManager.getLogger(ScoredPagedMultiSearcher.class);
|
||||||
|
|
||||||
public ScoredPagedMultiSearcher() {
|
public ScoredPagedMultiSearcher() {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
} else {
|
} else {
|
||||||
queryParamsMono = indexSearchersMono
|
queryParamsMono = Mono
|
||||||
.publishOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
.fromCallable(() -> transformer.rewrite(indexSearchers, queryParams))
|
||||||
.handle((indexSearchers, sink) -> {
|
.subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic()));
|
||||||
try {
|
|
||||||
sink.next(transformer.rewrite(indexSearchers.receive(), queryParams));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
sink.error(ex);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return queryParamsMono.flatMap(queryParams2 -> {
|
return queryParamsMono.flatMap(queryParams2 -> {
|
||||||
PaginationInfo paginationInfo = getPaginationInfo(queryParams2);
|
PaginationInfo paginationInfo = getPaginationInfo(queryParams2);
|
||||||
|
|
||||||
return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> this
|
return this
|
||||||
// Search first page results
|
// Search first page results
|
||||||
.searchFirstPage(indexSearchers.shards(), queryParams2, paginationInfo)
|
.searchFirstPage(indexSearchers.shards(), queryParams2, paginationInfo)
|
||||||
// Compute the results of the first page
|
// Compute the results of the first page
|
||||||
.transform(firstPageTopDocsMono -> this.computeFirstPageResults(firstPageTopDocsMono, indexSearchers,
|
.transform(firstPageTopDocsMono -> this.computeFirstPageResults(firstPageTopDocsMono,
|
||||||
keyFieldName, queryParams2))
|
indexSearchers,
|
||||||
|
keyFieldName,
|
||||||
|
queryParams2
|
||||||
|
))
|
||||||
// Compute other results
|
// Compute other results
|
||||||
.map(firstResult -> this.computeOtherResults(firstResult,
|
.map(firstResult -> this.computeOtherResults(firstResult,
|
||||||
indexSearchers.shards(),
|
indexSearchers.shards(),
|
||||||
queryParams2,
|
queryParams2,
|
||||||
keyFieldName,
|
keyFieldName,
|
||||||
() -> {
|
() -> {
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearchers.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
))
|
))
|
||||||
// Ensure that one LuceneSearchResult is always returned
|
// Ensure that one LuceneSearchResult is always returned
|
||||||
.single(),
|
.single();
|
||||||
false);
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -155,10 +155,10 @@ public class ScoredPagedMultiSearcher implements MultiSearcher {
|
|||||||
AtomicReference<CurrentPageInfo> currentPageInfoRef = new AtomicReference<>(secondPageInfo);
|
AtomicReference<CurrentPageInfo> currentPageInfoRef = new AtomicReference<>(secondPageInfo);
|
||||||
return Mono
|
return Mono
|
||||||
.fromSupplier(currentPageInfoRef::get)
|
.fromSupplier(currentPageInfoRef::get)
|
||||||
.doOnNext(s -> logger.trace("Current page info: {}", s))
|
.doOnNext(s -> LOG.trace("Current page info: {}", s))
|
||||||
.flatMap(currentPageInfo -> this.searchPage(queryParams, indexSearchers, true,
|
.flatMap(currentPageInfo -> this.searchPage(queryParams, indexSearchers, true,
|
||||||
queryParams.pageLimits(), 0, currentPageInfo))
|
queryParams.pageLimits(), 0, currentPageInfo))
|
||||||
.doOnNext(s -> logger.trace("Next page info: {}", s.nextPageInfo()))
|
.doOnNext(s -> LOG.trace("Next page info: {}", s.nextPageInfo()))
|
||||||
.doOnNext(s -> currentPageInfoRef.set(s.nextPageInfo()))
|
.doOnNext(s -> currentPageInfoRef.set(s.nextPageInfo()))
|
||||||
.repeatWhen(s -> s.takeWhile(n -> n > 0));
|
.repeatWhen(s -> s.takeWhile(n -> n > 0));
|
||||||
})
|
})
|
||||||
|
@ -22,7 +22,7 @@ import reactor.core.scheduler.Schedulers;
|
|||||||
|
|
||||||
public class SortedByScoreFullMultiSearcher implements MultiSearcher {
|
public class SortedByScoreFullMultiSearcher implements MultiSearcher {
|
||||||
|
|
||||||
protected static final Logger logger = LogManager.getLogger(SortedByScoreFullMultiSearcher.class);
|
protected static final Logger LOG = LogManager.getLogger(SortedByScoreFullMultiSearcher.class);
|
||||||
|
|
||||||
private final LLTempHugePqEnv env;
|
private final LLTempHugePqEnv env;
|
||||||
|
|
||||||
@ -31,23 +31,18 @@ public class SortedByScoreFullMultiSearcher implements MultiSearcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
} else {
|
} else {
|
||||||
queryParamsMono = indexSearchersMono
|
queryParamsMono = Mono
|
||||||
.publishOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
.fromCallable(() -> transformer.rewrite(indexSearchers, queryParams))
|
||||||
.handle((indexSearchers, sink) -> {
|
.subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic()));
|
||||||
try {
|
|
||||||
sink.next(transformer.rewrite(indexSearchers.receive(), queryParams));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
sink.error(ex);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return queryParamsMono.flatMap(queryParams2 -> {
|
return queryParamsMono.flatMap(queryParams2 -> {
|
||||||
@ -56,15 +51,14 @@ public class SortedByScoreFullMultiSearcher implements MultiSearcher {
|
|||||||
+ " doesn't support sorted queries");
|
+ " doesn't support sorted queries");
|
||||||
}
|
}
|
||||||
|
|
||||||
return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> this
|
return this
|
||||||
// Search results
|
// Search results
|
||||||
.search(indexSearchers.shards(), queryParams2)
|
.search(indexSearchers.shards(), queryParams2)
|
||||||
// Compute the results
|
// Compute the results
|
||||||
.transform(fullDocsMono -> this.computeResults(fullDocsMono, indexSearchers,
|
.transform(fullDocsMono -> this.computeResults(fullDocsMono, indexSearchers, keyFieldName, queryParams2))
|
||||||
keyFieldName, queryParams2))
|
|
||||||
// Ensure that one LuceneSearchResult is always returned
|
// Ensure that one LuceneSearchResult is always returned
|
||||||
.single(),
|
.single();
|
||||||
false);
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -127,13 +121,15 @@ public class SortedByScoreFullMultiSearcher implements MultiSearcher {
|
|||||||
.take(queryParams.limitLong(), true);
|
.take(queryParams.limitLong(), true);
|
||||||
|
|
||||||
return new LuceneSearchResult(totalHitsCount, hitsFlux, () -> {
|
return new LuceneSearchResult(totalHitsCount, hitsFlux, () -> {
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearchers.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
data.close();
|
data.close();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("Failed to discard data", e);
|
LOG.error("Failed to discard data", e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -22,7 +22,7 @@ import reactor.core.scheduler.Schedulers;
|
|||||||
|
|
||||||
public class SortedScoredFullMultiSearcher implements MultiSearcher {
|
public class SortedScoredFullMultiSearcher implements MultiSearcher {
|
||||||
|
|
||||||
protected static final Logger logger = LogManager.getLogger(SortedScoredFullMultiSearcher.class);
|
protected static final Logger LOG = LogManager.getLogger(SortedScoredFullMultiSearcher.class);
|
||||||
|
|
||||||
private final LLTempHugePqEnv env;
|
private final LLTempHugePqEnv env;
|
||||||
|
|
||||||
@ -31,34 +31,28 @@ public class SortedScoredFullMultiSearcher implements MultiSearcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
} else {
|
} else {
|
||||||
queryParamsMono = indexSearchersMono
|
queryParamsMono = Mono
|
||||||
.publishOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
.fromCallable(() -> transformer.rewrite(indexSearchers, queryParams))
|
||||||
.handle((indexSearchers, sink) -> {
|
.subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic()));
|
||||||
try {
|
|
||||||
sink.next(transformer.rewrite(indexSearchers.receive(), queryParams));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
sink.error(ex);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return queryParamsMono.flatMap(queryParams2 -> LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> this
|
return queryParamsMono.flatMap(queryParams2 -> this
|
||||||
// Search results
|
// Search results
|
||||||
.search(indexSearchers.shards(), queryParams2)
|
.search(indexSearchers.shards(), queryParams2)
|
||||||
// Compute the results
|
// Compute the results
|
||||||
.transform(fullDocsMono -> this.computeResults(fullDocsMono, indexSearchers,
|
.transform(fullDocsMono -> this.computeResults(fullDocsMono, indexSearchers, keyFieldName, queryParams2))
|
||||||
keyFieldName, queryParams2))
|
|
||||||
// Ensure that one LuceneSearchResult is always returned
|
// Ensure that one LuceneSearchResult is always returned
|
||||||
.single(),
|
.single());
|
||||||
false));
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -121,8 +115,10 @@ public class SortedScoredFullMultiSearcher implements MultiSearcher {
|
|||||||
.take(queryParams.limitLong(), true);
|
.take(queryParams.limitLong(), true);
|
||||||
|
|
||||||
return new LuceneSearchResult(totalHitsCount, hitsFlux, () -> {
|
return new LuceneSearchResult(totalHitsCount, hitsFlux, () -> {
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearchers.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
}
|
}
|
||||||
data.close();
|
data.close();
|
||||||
});
|
});
|
||||||
|
@ -26,40 +26,34 @@ import reactor.core.scheduler.Schedulers;
|
|||||||
|
|
||||||
public class StandardSearcher implements MultiSearcher {
|
public class StandardSearcher implements MultiSearcher {
|
||||||
|
|
||||||
protected static final Logger logger = LogManager.getLogger(StandardSearcher.class);
|
protected static final Logger LOG = LogManager.getLogger(StandardSearcher.class);
|
||||||
|
|
||||||
public StandardSearcher() {
|
public StandardSearcher() {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
} else {
|
} else {
|
||||||
queryParamsMono = indexSearchersMono
|
queryParamsMono = Mono
|
||||||
.publishOn(uninterruptibleScheduler(Schedulers.boundedElastic()))
|
.fromCallable(() -> transformer.rewrite(indexSearchers, queryParams))
|
||||||
.handle((indexSearchers, sink) -> {
|
.subscribeOn(uninterruptibleScheduler(Schedulers.boundedElastic()));
|
||||||
try {
|
|
||||||
sink.next(transformer.rewrite(indexSearchers.receive(), queryParams));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
sink.error(ex);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return queryParamsMono.flatMap(queryParams2 -> LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> this
|
return queryParamsMono.flatMap(queryParams2 -> this
|
||||||
// Search results
|
// Search results
|
||||||
.search(indexSearchers.shards(), queryParams2)
|
.search(indexSearchers.shards(), queryParams2)
|
||||||
// Compute the results
|
// Compute the results
|
||||||
.transform(fullDocsMono -> this.computeResults(fullDocsMono, indexSearchers,
|
.transform(fullDocsMono -> this.computeResults(fullDocsMono, indexSearchers, keyFieldName, queryParams2))
|
||||||
keyFieldName, queryParams2))
|
|
||||||
// Ensure that one LuceneSearchResult is always returned
|
// Ensure that one LuceneSearchResult is always returned
|
||||||
.single(),
|
.single());
|
||||||
false));
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -144,8 +138,10 @@ public class StandardSearcher implements MultiSearcher {
|
|||||||
.take(queryParams.limitLong(), true);
|
.take(queryParams.limitLong(), true);
|
||||||
|
|
||||||
return new LuceneSearchResult(totalHitsCount, hitsFlux, () -> {
|
return new LuceneSearchResult(totalHitsCount, hitsFlux, () -> {
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearchers.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -9,8 +9,11 @@ import it.cavallium.dbengine.database.LLUtils;
|
|||||||
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
||||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||||
import it.cavallium.dbengine.lucene.MaxScoreAccumulator;
|
import it.cavallium.dbengine.lucene.MaxScoreAccumulator;
|
||||||
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import it.cavallium.dbengine.lucene.hugepq.search.CustomHitsThresholdChecker;
|
import it.cavallium.dbengine.lucene.hugepq.search.CustomHitsThresholdChecker;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.apache.lucene.search.IndexSearcher;
|
import org.apache.lucene.search.IndexSearcher;
|
||||||
import org.apache.lucene.search.ScoreDoc;
|
import org.apache.lucene.search.ScoreDoc;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
@ -20,13 +23,15 @@ import reactor.core.scheduler.Schedulers;
|
|||||||
|
|
||||||
public class UnsortedStreamingMultiSearcher implements MultiSearcher {
|
public class UnsortedStreamingMultiSearcher implements MultiSearcher {
|
||||||
|
|
||||||
|
|
||||||
|
protected static final Logger LOG = LogManager.getLogger(UnsortedStreamingMultiSearcher.class);
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
return LLUtils.usingSendResource(indexSearchersMono, indexSearchers -> {
|
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
if (transformer == GlobalQueryRewrite.NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
@ -54,12 +59,14 @@ public class UnsortedStreamingMultiSearcher implements MultiSearcher {
|
|||||||
.take(queryParams2.limitLong(), true);
|
.take(queryParams2.limitLong(), true);
|
||||||
|
|
||||||
return new LuceneSearchResult(totalHitsCount, mergedFluxes, () -> {
|
return new LuceneSearchResult(totalHitsCount, mergedFluxes, () -> {
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearchers.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}, false);
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private Flux<ScoreDoc> getScoreDocs(LocalQueryParams localQueryParams, List<IndexSearcher> shards) {
|
private Flux<ScoreDoc> getScoreDocs(LocalQueryParams localQueryParams, List<IndexSearcher> shards) {
|
||||||
|
@ -27,7 +27,7 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
|
public Mono<LuceneSearchResult> collect(Mono<LLIndexSearcher> indexSearcherMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
@Nullable String keyFieldName,
|
@Nullable String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
@ -52,7 +52,7 @@ public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Cl
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
String keyFieldName,
|
String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
@ -14,14 +14,19 @@ import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
|
|||||||
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
|
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
|
||||||
import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult;
|
import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult;
|
||||||
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
|
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
|
||||||
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
import reactor.core.publisher.Flux;
|
import reactor.core.publisher.Flux;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
import reactor.core.scheduler.Schedulers;
|
import reactor.core.scheduler.Schedulers;
|
||||||
|
|
||||||
public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
||||||
|
|
||||||
|
private static final Logger LOG = LogManager.getLogger(UnsortedUnscoredSimpleMultiSearcher.class);
|
||||||
|
|
||||||
private final LocalSearcher localSearcher;
|
private final LocalSearcher localSearcher;
|
||||||
|
|
||||||
public UnsortedUnscoredSimpleMultiSearcher(LocalSearcher localSearcher) {
|
public UnsortedUnscoredSimpleMultiSearcher(LocalSearcher localSearcher) {
|
||||||
@ -29,13 +34,12 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
public Mono<LuceneSearchResult> collectMulti(Mono<LLIndexSearchers> indexSearchersMono,
|
||||||
LocalQueryParams queryParams,
|
LocalQueryParams queryParams,
|
||||||
String keyFieldName,
|
String keyFieldName,
|
||||||
GlobalQueryRewrite transformer) {
|
GlobalQueryRewrite transformer) {
|
||||||
|
|
||||||
return LLUtils.usingSendResource(indexSearchersMono,
|
return indexSearchersMono.flatMap(indexSearchers -> {
|
||||||
indexSearchers -> {
|
|
||||||
Mono<LocalQueryParams> queryParamsMono;
|
Mono<LocalQueryParams> queryParamsMono;
|
||||||
if (transformer == NO_REWRITE) {
|
if (transformer == NO_REWRITE) {
|
||||||
queryParamsMono = Mono.just(queryParams);
|
queryParamsMono = Mono.just(queryParams);
|
||||||
@ -48,11 +52,9 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
|||||||
return queryParamsMono.flatMap(queryParams2 -> {
|
return queryParamsMono.flatMap(queryParams2 -> {
|
||||||
var localQueryParams = getLocalQueryParams(queryParams2);
|
var localQueryParams = getLocalQueryParams(queryParams2);
|
||||||
return Flux
|
return Flux
|
||||||
.fromIterable(indexSearchers.shards())
|
.fromIterable(indexSearchers.llShards())
|
||||||
.flatMap(searcher -> {
|
.flatMap(searcher ->
|
||||||
var llSearcher = Mono.fromCallable(() -> new LLIndexSearcher(searcher, false, null).send());
|
localSearcher.collect(Mono.just(searcher), localQueryParams, keyFieldName, transformer))
|
||||||
return localSearcher.collect(llSearcher, localQueryParams, keyFieldName, transformer);
|
|
||||||
})
|
|
||||||
.collectList()
|
.collectList()
|
||||||
.map(results -> {
|
.map(results -> {
|
||||||
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
|
List<LuceneSearchResult> resultsToDrop = new ArrayList<>(results.size());
|
||||||
@ -78,8 +80,10 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
|||||||
luceneSearchResult.close();
|
luceneSearchResult.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (indexSearchers.isAccessible()) {
|
try {
|
||||||
indexSearchers.close();
|
indexSearchers.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Can't close index searchers", e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
})
|
})
|
||||||
@ -96,9 +100,7 @@ public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
},
|
});
|
||||||
false
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private LocalQueryParams getLocalQueryParams(LocalQueryParams queryParams) {
|
private LocalQueryParams getLocalQueryParams(LocalQueryParams queryParams) {
|
||||||
|
Loading…
Reference in New Issue
Block a user