2021-09-19 19:59:37 +02:00
|
|
|
package it.cavallium.dbengine.database.disk;
|
|
|
|
|
2022-03-16 13:47:56 +01:00
|
|
|
import io.netty5.buffer.api.Drop;
|
|
|
|
import io.netty5.buffer.api.Owned;
|
|
|
|
import io.netty5.buffer.api.Resource;
|
|
|
|
import io.netty5.buffer.api.Send;
|
|
|
|
import io.netty5.buffer.api.internal.ResourceSupport;
|
2022-06-30 15:06:10 +02:00
|
|
|
import it.cavallium.dbengine.database.DiscardingCloseable;
|
2022-06-30 13:54:55 +02:00
|
|
|
import it.cavallium.dbengine.database.SafeCloseable;
|
2022-06-14 21:58:26 +02:00
|
|
|
import it.cavallium.dbengine.lucene.searcher.ShardIndexSearcher;
|
2022-06-30 13:54:55 +02:00
|
|
|
import it.cavallium.dbengine.utils.SimpleResource;
|
2021-09-19 19:59:37 +02:00
|
|
|
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
|
2022-06-14 13:10:38 +02:00
|
|
|
import java.io.Closeable;
|
2021-09-19 19:59:37 +02:00
|
|
|
import java.io.IOException;
|
|
|
|
import java.io.UncheckedIOException;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.Comparator;
|
|
|
|
import java.util.List;
|
2022-06-14 21:58:26 +02:00
|
|
|
import java.util.concurrent.atomic.AtomicBoolean;
|
2021-12-17 01:48:49 +01:00
|
|
|
import org.apache.logging.log4j.LogManager;
|
|
|
|
import org.apache.logging.log4j.Logger;
|
2021-09-19 19:59:37 +02:00
|
|
|
import org.apache.lucene.index.IndexReader;
|
|
|
|
import org.apache.lucene.index.MultiReader;
|
2021-09-22 11:03:39 +02:00
|
|
|
import org.apache.lucene.search.IndexSearcher;
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-30 15:06:10 +02:00
|
|
|
public interface LLIndexSearchers extends DiscardingCloseable {
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
static LLIndexSearchers of(List<LLIndexSearcher> indexSearchers) {
|
|
|
|
return new ShardedIndexSearchers(indexSearchers);
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
static UnshardedIndexSearchers unsharded(LLIndexSearcher indexSearcher) {
|
|
|
|
return new UnshardedIndexSearchers(indexSearcher);
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
2021-09-22 11:03:39 +02:00
|
|
|
List<IndexSearcher> shards();
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
List<LLIndexSearcher> llShards();
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
IndexSearcher shard(int shardIndex);
|
2021-10-01 19:17:33 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
LLIndexSearcher llShard(int shardIndex);
|
2021-10-01 19:17:33 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
IndexReader allShards();
|
2021-10-01 19:17:33 +02:00
|
|
|
|
2022-06-30 13:54:55 +02:00
|
|
|
class UnshardedIndexSearchers extends SimpleResource implements LLIndexSearchers {
|
2021-10-01 19:17:33 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
private final LLIndexSearcher indexSearcher;
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
public UnshardedIndexSearchers(LLIndexSearcher indexSearcher) {
|
|
|
|
this.indexSearcher = indexSearcher;
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2021-09-22 11:03:39 +02:00
|
|
|
public List<IndexSearcher> shards() {
|
|
|
|
return List.of(indexSearcher.getIndexSearcher());
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
@Override
|
|
|
|
public List<LLIndexSearcher> llShards() {
|
|
|
|
return Collections.singletonList(indexSearcher);
|
|
|
|
}
|
|
|
|
|
2021-09-19 19:59:37 +02:00
|
|
|
@Override
|
2021-09-22 11:03:39 +02:00
|
|
|
public IndexSearcher shard(int shardIndex) {
|
2021-09-19 19:59:37 +02:00
|
|
|
if (shardIndex != -1) {
|
|
|
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid, this is a unsharded index");
|
|
|
|
}
|
2021-09-22 11:03:39 +02:00
|
|
|
return indexSearcher.getIndexSearcher();
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
@Override
|
|
|
|
public LLIndexSearcher llShard(int shardIndex) {
|
|
|
|
if (shardIndex != -1) {
|
|
|
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid, this is a unsharded index");
|
|
|
|
}
|
|
|
|
return indexSearcher;
|
|
|
|
}
|
|
|
|
|
2021-09-19 19:59:37 +02:00
|
|
|
@Override
|
|
|
|
public IndexReader allShards() {
|
|
|
|
return indexSearcher.getIndexReader();
|
|
|
|
}
|
|
|
|
|
2021-09-22 11:03:39 +02:00
|
|
|
public IndexSearcher shard() {
|
2021-09-20 18:20:59 +02:00
|
|
|
return this.shard(-1);
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
public LLIndexSearcher llShard() {
|
|
|
|
return this.llShard(-1);
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-06-30 13:54:55 +02:00
|
|
|
protected void onClose() {
|
2022-06-14 13:10:38 +02:00
|
|
|
indexSearcher.close();
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
2021-10-01 19:17:33 +02:00
|
|
|
}
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-30 13:54:55 +02:00
|
|
|
class ShardedIndexSearchers extends SimpleResource implements LLIndexSearchers {
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
private final List<LLIndexSearcher> indexSearchers;
|
|
|
|
private final List<IndexSearcher> indexSearchersVals;
|
2021-09-19 19:59:37 +02:00
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
public ShardedIndexSearchers(List<LLIndexSearcher> indexSearchers) {
|
2022-06-14 21:58:26 +02:00
|
|
|
var shardedIndexSearchers = new ArrayList<LLIndexSearcher>(indexSearchers.size());
|
|
|
|
List<IndexSearcher> shardedIndexSearchersVals = new ArrayList<>(indexSearchers.size());
|
2022-06-14 13:10:38 +02:00
|
|
|
for (LLIndexSearcher indexSearcher : indexSearchers) {
|
2022-06-14 21:58:26 +02:00
|
|
|
shardedIndexSearchersVals.add(indexSearcher.getIndexSearcher());
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
2022-06-14 21:58:26 +02:00
|
|
|
shardedIndexSearchersVals = ShardIndexSearcher.create(shardedIndexSearchersVals);
|
|
|
|
int i = 0;
|
|
|
|
for (IndexSearcher shardedIndexSearcher : shardedIndexSearchersVals) {
|
|
|
|
shardedIndexSearchers.add(new WrappedLLIndexSearcher(shardedIndexSearcher, indexSearchers.get(i)));
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
this.indexSearchers = shardedIndexSearchers;
|
|
|
|
this.indexSearchersVals = shardedIndexSearchersVals;
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2021-09-22 11:03:39 +02:00
|
|
|
public List<IndexSearcher> shards() {
|
|
|
|
return Collections.unmodifiableList(indexSearchersVals);
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
2022-06-14 13:10:38 +02:00
|
|
|
@Override
|
|
|
|
public List<LLIndexSearcher> llShards() {
|
|
|
|
return Collections.unmodifiableList(indexSearchers);
|
|
|
|
}
|
|
|
|
|
2021-09-19 19:59:37 +02:00
|
|
|
@Override
|
2021-09-22 11:03:39 +02:00
|
|
|
public IndexSearcher shard(int shardIndex) {
|
2021-09-19 19:59:37 +02:00
|
|
|
if (shardIndex < 0) {
|
|
|
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid");
|
|
|
|
}
|
2021-09-22 11:03:39 +02:00
|
|
|
return indexSearchersVals.get(shardIndex);
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-06-14 13:10:38 +02:00
|
|
|
public LLIndexSearcher llShard(int shardIndex) {
|
|
|
|
if (shardIndex < 0) {
|
|
|
|
throw new IndexOutOfBoundsException("Shard index " + shardIndex + " is invalid");
|
2021-09-22 11:03:39 +02:00
|
|
|
}
|
2022-06-14 13:10:38 +02:00
|
|
|
return indexSearchers.get(shardIndex);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public IndexReader allShards() {
|
2021-09-22 11:03:39 +02:00
|
|
|
var irs = new IndexReader[indexSearchersVals.size()];
|
|
|
|
for (int i = 0, s = indexSearchersVals.size(); i < s; i++) {
|
|
|
|
irs[i] = indexSearchersVals.get(i).getIndexReader();
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
Object2IntOpenHashMap<IndexReader> indexes = new Object2IntOpenHashMap<>();
|
|
|
|
for (int i = 0; i < irs.length; i++) {
|
|
|
|
indexes.put(irs[i], i);
|
|
|
|
}
|
|
|
|
try {
|
2021-09-22 11:03:39 +02:00
|
|
|
return new MultiReader(irs, Comparator.comparingInt(indexes::getInt), false);
|
2021-09-19 19:59:37 +02:00
|
|
|
} catch (IOException ex) {
|
|
|
|
// This shouldn't happen
|
|
|
|
throw new UncheckedIOException(ex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-06-30 13:54:55 +02:00
|
|
|
protected void onClose() {
|
2022-06-14 13:10:38 +02:00
|
|
|
for (LLIndexSearcher indexSearcher : indexSearchers) {
|
|
|
|
indexSearcher.close();
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
}
|
2022-06-14 21:58:26 +02:00
|
|
|
|
|
|
|
private static class WrappedLLIndexSearcher extends LLIndexSearcher {
|
|
|
|
|
|
|
|
private final LLIndexSearcher parent;
|
|
|
|
|
|
|
|
public WrappedLLIndexSearcher(IndexSearcher indexSearcher, LLIndexSearcher parent) {
|
|
|
|
super(indexSearcher, parent.getClosed());
|
|
|
|
this.parent = parent;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public IndexSearcher getIndexSearcher() {
|
|
|
|
return indexSearcher;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public IndexReader getIndexReader() {
|
|
|
|
return indexSearcher.getIndexReader();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2022-06-30 13:54:55 +02:00
|
|
|
protected void onClose() {
|
|
|
|
parent.close();
|
2022-06-14 21:58:26 +02:00
|
|
|
}
|
|
|
|
}
|
2021-09-19 19:59:37 +02:00
|
|
|
}
|
|
|
|
}
|