andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Rename classes and improve readonly tests performance

This commit is contained in:
Andrea Cavalli 2021-10-13 12:25:32 +02:00
parent 09f60a3a99
commit 584115c5cc
42 changed files with 463 additions and 384 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pom.xml
View File

@ -224,7 +224,6 @@
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.novasearch</groupId>

16
src/main/java/it/cavallium/dbengine/SwappableLuceneSearcher.java
View File

@ -6,22 +6,20 @@ import static java.util.Objects.requireNonNullElseGet;
import io.net5.buffer.api.Send;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLLocalSingleton;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult;
import java.io.Closeable;
import java.io.IOException;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference;
import reactor.core.publisher.Mono;
public class SwappableLuceneSearcher implements LuceneLocalSearcher, LuceneMultiSearcher, Closeable {
public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Closeable {
private final AtomicReference<LuceneLocalSearcher> single = new AtomicReference<>(null);
private final AtomicReference<LuceneMultiSearcher> multi = new AtomicReference<>(null);
private final AtomicReference<LocalSearcher> single = new AtomicReference<>(null);
private final AtomicReference<MultiSearcher> multi = new AtomicReference<>(null);
public SwappableLuceneSearcher() {
@ -61,11 +59,11 @@ public class SwappableLuceneSearcher implements LuceneLocalSearcher, LuceneMulti
return multi.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
}
public void setSingle(LuceneLocalSearcher single) {
public void setSingle(LocalSearcher single) {
this.single.set(single);
}
public void setMulti(LuceneMultiSearcher multi) {
public void setMulti(MultiSearcher multi) {
this.multi.set(multi);
}

2
src/main/java/it/cavallium/dbengine/database/LLDatabaseConnection.java
View File

@ -5,7 +5,7 @@ import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.client.IndicizerAnalyzers;
import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.client.LuceneOptions;
import it.cavallium.dbengine.database.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneHacks;
import java.util.List;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;

1
src/main/java/it/cavallium/dbengine/database/disk/CachedIndexSearcherManager.java
View File

@ -5,7 +5,6 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import io.net5.buffer.api.Send;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.lucene.searcher.ExecutorSearcherFactory;
import java.io.IOException;
import java.time.Duration;
import java.util.concurrent.Executor;

2
src/main/java/it/cavallium/dbengine/lucene/searcher/ExecutorSearcherFactory.java → src/main/java/it/cavallium/dbengine/database/disk/ExecutorSearcherFactory.java
View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher;
package it.cavallium.dbengine.database.disk;
import java.util.concurrent.Executor;
import org.apache.lucene.index.IndexReader;

3
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDatabaseConnection.java
View File

@ -8,8 +8,7 @@ import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.client.DatabaseOptions;
import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.netty.JMXNettyMonitoringManager;
import java.nio.file.Files;
import java.nio.file.Path;

12
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -16,13 +16,12 @@ import it.cavallium.dbengine.database.LLSearchResultShard;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.AlwaysDirectIOFSDirectory;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
import java.io.IOException;
import java.nio.file.Path;
@ -50,7 +49,6 @@ import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.Constants;
import org.jetbrains.annotations.Nullable;
import org.reactivestreams.Publisher;
import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux;
@ -62,7 +60,7 @@ import reactor.util.function.Tuple2;
public class LLLocalLuceneIndex implements LLLuceneIndex {
protected static final Logger logger = LoggerFactory.getLogger(LLLocalLuceneIndex.class);
private final LuceneLocalSearcher localSearcher;
private final LocalSearcher localSearcher;
/**
* Global lucene index scheduler.
* There is only a single thread globally to not overwhelm the disk with
@ -170,7 +168,7 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
if (luceneHacks != null && luceneHacks.customLocalSearcher() != null) {
localSearcher = luceneHacks.customLocalSearcher().get();
} else {
localSearcher = new AdaptiveLuceneLocalSearcher();
localSearcher = new AdaptiveLocalSearcher();
}
var indexWriterConfig = new IndexWriterConfig(luceneAnalyzer);

11
src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java
View File

@ -10,16 +10,15 @@ import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLSearchResultShard;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLTerm;
import it.cavallium.dbengine.database.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -46,7 +45,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
private final PerFieldAnalyzerWrapper luceneAnalyzer;
private final PerFieldSimilarityWrapper luceneSimilarity;
private final LuceneMultiSearcher multiSearcher;
private final MultiSearcher multiSearcher;
public LLLocalMultiLuceneIndex(Path lucene,
String name,
@ -83,7 +82,7 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
if (luceneHacks != null && luceneHacks.customMultiSearcher() != null) {
multiSearcher = luceneHacks.customMultiSearcher().get();
} else {
multiSearcher = new AdaptiveLuceneMultiSearcher();
multiSearcher = new AdaptiveMultiSearcher();
}
}

10
src/main/java/it/cavallium/dbengine/database/lucene/LuceneHacks.java
View File

@ -1,10 +0,0 @@
package it.cavallium.dbengine.database.lucene;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher;
import java.util.function.Supplier;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public record LuceneHacks(@Nullable Supplier<@NotNull LuceneLocalSearcher> customLocalSearcher,
@Nullable Supplier<@NotNull LuceneMultiSearcher> customMultiSearcher) {}

2
src/main/java/it/cavallium/dbengine/database/memory/LLMemoryDatabaseConnection.java
View File

@ -10,7 +10,7 @@ import it.cavallium.dbengine.database.LLDatabaseConnection;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.disk.LLLocalLuceneIndex;
import it.cavallium.dbengine.database.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.netty.JMXNettyMonitoringManager;
import java.util.List;
import org.jetbrains.annotations.Nullable;

4
src/main/java/it/cavallium/dbengine/lucene/searcher/ExponentialPageLimits.java → src/main/java/it/cavallium/dbengine/lucene/ExponentialPageLimits.java
View File

@ -1,6 +1,4 @@
package it.cavallium.dbengine.lucene.searcher;
import it.cavallium.dbengine.lucene.LuceneUtils;
package it.cavallium.dbengine.lucene;
/**
* <pre>y = 2 ^ (x + pageIndexOffset) + firstPageLimit</pre>

4
src/main/java/it/cavallium/dbengine/lucene/searcher/LinearPageLimits.java → src/main/java/it/cavallium/dbengine/lucene/LinearPageLimits.java
View File

@ -1,6 +1,4 @@
package it.cavallium.dbengine.lucene.searcher;
import it.cavallium.dbengine.lucene.LuceneUtils;
package it.cavallium.dbengine.lucene;
/**
* <pre>y = (x * factor) + firstPageLimit</pre>

10
src/main/java/it/cavallium/dbengine/lucene/LuceneHacks.java Normal file
View File

@ -0,0 +1,10 @@
package it.cavallium.dbengine.lucene;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import java.util.function.Supplier;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public record LuceneHacks(@Nullable Supplier<@NotNull LocalSearcher> customLocalSearcher,
@Nullable Supplier<@NotNull MultiSearcher> customMultiSearcher) {}

7
src/main/java/it/cavallium/dbengine/lucene/LuceneUtils.java
View File

@ -8,12 +8,10 @@ import it.cavallium.dbengine.client.query.current.data.QueryParams;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.EnglishItalianStopFilter;
import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLSnapshot;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionary;
import it.cavallium.dbengine.database.collections.DatabaseMapDictionaryDeep;
import it.cavallium.dbengine.database.collections.ValueGetter;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.analyzer.NCharGramAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.NCharGramEdgeAnalyzer;
@ -22,9 +20,7 @@ import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.lucene.analyzer.WordAnalyzer;
import it.cavallium.dbengine.lucene.mlt.BigCompositeReader;
import it.cavallium.dbengine.lucene.mlt.MultiMoreLikeThis;
import it.cavallium.dbengine.lucene.searcher.ExponentialPageLimits;
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
import it.cavallium.dbengine.lucene.searcher.PageLimits;
import it.cavallium.dbengine.lucene.similarity.NGramSimilarity;
import java.io.EOFException;
import java.io.IOException;
@ -50,10 +46,8 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanQuery.Builder;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
@ -79,7 +73,6 @@ import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.util.concurrent.Queues;
import reactor.util.function.Tuple2;

2
src/main/java/it/cavallium/dbengine/lucene/searcher/PageLimits.java → src/main/java/it/cavallium/dbengine/lucene/PageLimits.java
View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher;
package it.cavallium.dbengine.lucene;
import it.cavallium.dbengine.lucene.LuceneUtils;

2
src/main/java/it/cavallium/dbengine/lucene/searcher/SinglePageLimits.java → src/main/java/it/cavallium/dbengine/lucene/SinglePageLimits.java
View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher;
package it.cavallium.dbengine.lucene;
public class SinglePageLimits implements PageLimits {

2
src/main/java/it/cavallium/dbengine/lucene/searcher/ScoringShardsCollectorManager.java → src/main/java/it/cavallium/dbengine/lucene/collector/ScoringShardsCollectorManager.java
View File

@ -1,4 +1,4 @@
package it.cavallium.dbengine.lucene.searcher;
package it.cavallium.dbengine.lucene.collector;
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.TIE_BREAKER;

8
src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneLocalSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLocalSearcher.java
View File

@ -5,16 +5,14 @@ import io.net5.buffer.api.internal.ResourceSupport;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLIndexSearchers.UnshardedIndexSearchers;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import org.apache.lucene.search.IndexSearcher;
import reactor.core.publisher.Mono;
public class AdaptiveLuceneLocalSearcher implements LuceneLocalSearcher {
public class AdaptiveLocalSearcher implements LocalSearcher {
private static final LuceneLocalSearcher localSearcher = new SimpleLuceneLocalSearcher();
private static final LocalSearcher localSearcher = new PagedLocalSearcher();
private static final LuceneLocalSearcher countSearcher = new CountLuceneLocalSearcher();
private static final LocalSearcher countSearcher = new CountLocalSearcher();
@Override
public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcher,

24
src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLuceneMultiSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java
View File

@ -8,23 +8,23 @@ import java.io.Closeable;
import java.io.IOException;
import reactor.core.publisher.Mono;
public class AdaptiveLuceneMultiSearcher implements LuceneMultiSearcher, Closeable {
public class AdaptiveMultiSearcher implements MultiSearcher, Closeable {
private static final LuceneMultiSearcher count
= new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher());
private static final MultiSearcher count
= new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher());
private static final LuceneMultiSearcher scoredSimple = new ScoredSimpleLuceneMultiSearcher();
private static final MultiSearcher scoredSimple = new ScoredPagedMultiSearcher();
private static final LuceneMultiSearcher unsortedUnscoredPaged
= new SimpleUnsortedUnscoredLuceneMultiSearcher(new SimpleLuceneLocalSearcher());
private static final MultiSearcher unsortedUnscoredPaged
= new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher());
private static final LuceneMultiSearcher unsortedUnscoredContinuous
= new UnsortedUnscoredContinuousLuceneMultiSearcher();
private static final MultiSearcher unsortedUnscoredContinuous
= new UnsortedUnscoredStreamingMultiSearcher();
private final UnsortedScoredFullLuceneMultiSearcher scoredFull;
private final UnsortedScoredFullMultiSearcher scoredFull;
public AdaptiveLuceneMultiSearcher() throws IOException {
scoredFull = new UnsortedScoredFullLuceneMultiSearcher();
public AdaptiveMultiSearcher() throws IOException {
scoredFull = new UnsortedScoredFullMultiSearcher();
}
@Override
@ -76,6 +76,6 @@ public class AdaptiveLuceneMultiSearcher implements LuceneMultiSearcher, Closeab
@Override
public String getName() {
return "adaptivemulti";
return "adaptive local";
}
}

7
src/main/java/it/cavallium/dbengine/lucene/searcher/CalculatedResults.java
View File

@ -1,7 +0,0 @@
package it.cavallium.dbengine.lucene.searcher;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLKeyScore;
import reactor.core.publisher.Flux;
record CalculatedResults(TotalHitsCount totalHitsCount, Flux<LLKeyScore> firstPageHitsFlux) {}

5
src/main/java/it/cavallium/dbengine/lucene/searcher/CountLuceneLocalSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/CountLocalSearcher.java
View File

@ -6,12 +6,11 @@ import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import org.apache.lucene.search.IndexSearcher;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class CountLuceneLocalSearcher implements LuceneLocalSearcher {
public class CountLocalSearcher implements LocalSearcher {
@Override
public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
@ -45,6 +44,6 @@ public class CountLuceneLocalSearcher implements LuceneLocalSearcher {
@Override
public String getName() {
return "count";
return "count local";
}
}

6
src/main/java/it/cavallium/dbengine/lucene/searcher/CurrentPageInfo.java
View File

@ -6,8 +6,10 @@ import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.ScoreDoc;
import org.jetbrains.annotations.Nullable;
record CurrentPageInfo(@Nullable ScoreDoc last, long remainingLimit, int pageIndex) {
public record CurrentPageInfo(@Nullable ScoreDoc last, long remainingLimit, int pageIndex) {
public static final Comparator<ScoreDoc> TIE_BREAKER = Comparator.comparingInt((d) -> d.shardIndex);
public static final Comparator<ScoreDoc> TIE_BREAKER = Comparator
.<ScoreDoc>comparingInt((d) -> d.shardIndex)
.thenComparingInt(d -> -d.doc);
public static final CurrentPageInfo EMPTY_STATUS = new CurrentPageInfo(null, 0, 0);
}

1
src/main/java/it/cavallium/dbengine/lucene/searcher/LocalQueryParams.java
View File

@ -1,5 +1,6 @@
package it.cavallium.dbengine.lucene.searcher;
import it.cavallium.dbengine.lucene.PageLimits;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Sort;

2
src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneLocalSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/LocalSearcher.java
View File

@ -4,7 +4,7 @@ import io.net5.buffer.api.Send;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import reactor.core.publisher.Mono;
public interface LuceneLocalSearcher {
public interface LocalSearcher {
/**
* @param indexSearcherMono Lucene index searcher

2
src/main/java/it/cavallium/dbengine/lucene/searcher/LuceneMultiSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/MultiSearcher.java
View File

@ -5,7 +5,7 @@ import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import reactor.core.publisher.Mono;
public interface LuceneMultiSearcher extends LuceneLocalSearcher {
public interface MultiSearcher extends LocalSearcher {
/**
* @param indexSearchersMono Lucene index searcher

9
src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleLuceneLocalSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/PagedLocalSearcher.java
View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.FIRST_PAGE_LIMIT;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
import io.net5.buffer.api.Send;
@ -10,13 +9,11 @@ import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLIndexSearchers.UnshardedIndexSearchers;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
@ -29,7 +26,7 @@ import reactor.core.publisher.Mono;
import reactor.core.publisher.SynchronousSink;
import reactor.core.scheduler.Schedulers;
public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher {
public class PagedLocalSearcher implements LocalSearcher {
@Override
public Mono<Send<LuceneSearchResult>> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
@ -69,7 +66,7 @@ public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher {
@Override
public String getName() {
return "simplelocal";
return "paged local";
}
/**
@ -187,7 +184,7 @@ public class SimpleLuceneLocalSearcher implements LuceneLocalSearcher {
} else if (s.pageIndex() == 0 || (s.last() != null && s.remainingLimit() > 0)) {
TopDocs pageTopDocs;
try {
TopDocsCollector<ScoreDoc> collector = TopDocsSearcher.getTopDocsCollector(queryParams.sort(),
TopDocsCollector<ScoreDoc> collector = TopDocsCollectorUtils.getTopDocsCollector(queryParams.sort(),
currentPageLimit, s.last(), LuceneUtils.totalHitsThreshold(),
allowPagination, queryParams.isScored());
indexSearchers.get(0).search(queryParams.query(), collector);

3
src/main/java/it/cavallium/dbengine/lucene/searcher/PaginationInfo.java
View File

@ -1,7 +1,6 @@
package it.cavallium.dbengine.lucene.searcher;
import java.util.Comparator;
import org.apache.lucene.search.ScoreDoc;
import it.cavallium.dbengine.lucene.PageLimits;
public record PaginationInfo(long totalLimit, long firstPageOffset, PageLimits pageLimits, boolean forceSinglePage) {

16
src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredSimpleLuceneMultiSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/ScoredPagedMultiSearcher.java
View File

@ -1,6 +1,5 @@
package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.lucene.searcher.CurrentPageInfo.EMPTY_STATUS;
import static it.cavallium.dbengine.lucene.searcher.PaginationInfo.MAX_SINGLE_SEARCH_LIMIT;
import io.net5.buffer.api.Send;
@ -8,19 +7,16 @@ import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.lucene.LuceneUtils;
import it.cavallium.dbengine.lucene.PageLimits;
import it.cavallium.dbengine.lucene.collector.ScoringShardsCollectorManager;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.TotalHits.Relation;
import org.jetbrains.annotations.Nullable;
import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory;
@ -28,11 +24,11 @@ import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class ScoredSimpleLuceneMultiSearcher implements LuceneMultiSearcher {
public class ScoredPagedMultiSearcher implements MultiSearcher {
protected static final Logger logger = LoggerFactory.getLogger(ScoredSimpleLuceneMultiSearcher.class);
protected static final Logger logger = LoggerFactory.getLogger(ScoredPagedMultiSearcher.class);
public ScoredSimpleLuceneMultiSearcher() {
public ScoredPagedMultiSearcher() {
}
@Override
@ -214,6 +210,6 @@ public class ScoredSimpleLuceneMultiSearcher implements LuceneMultiSearcher {
@Override
public String getName() {
return "scoredsimplemulti";
return "scored paged multi";
}
}

2
src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/TopDocsCollectorUtils.java
View File

@ -10,7 +10,7 @@ import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector;
class TopDocsSearcher {
class TopDocsCollectorUtils {
@SuppressWarnings({"unchecked", "rawtypes"})
public static TopDocsCollector<ScoreDoc> getTopDocsCollector(Sort luceneSort,

10
src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullLuceneMultiSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedScoredFullMultiSearcher.java
View File

@ -20,13 +20,13 @@ import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearcher, Closeable {
public class UnsortedScoredFullMultiSearcher implements MultiSearcher, Closeable {
protected static final Logger logger = LoggerFactory.getLogger(UnsortedScoredFullLuceneMultiSearcher.class);
protected static final Logger logger = LoggerFactory.getLogger(UnsortedScoredFullMultiSearcher.class);
private final LLTempLMDBEnv env;
public UnsortedScoredFullLuceneMultiSearcher() throws IOException {
public UnsortedScoredFullMultiSearcher() throws IOException {
this.env = new LLTempLMDBEnv();
}
@ -46,7 +46,7 @@ public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearche
return queryParamsMono.flatMap(queryParams2 -> {
Objects.requireNonNull(queryParams2.scoreMode(), "ScoreMode must not be null");
if (queryParams2.sort() != null && queryParams2.sort() != Sort.RELEVANCE) {
throw new IllegalArgumentException(UnsortedScoredFullLuceneMultiSearcher.this.getClass().getSimpleName()
throw new IllegalArgumentException(UnsortedScoredFullMultiSearcher.this.getClass().getSimpleName()
+ " doesn't support sorted queries");
}
@ -115,6 +115,6 @@ public class UnsortedScoredFullLuceneMultiSearcher implements LuceneMultiSearche
@Override
public String getName() {
return "scoredfullmulti";
return "unsorted scored full multi";
}
}

8
src/main/java/it/cavallium/dbengine/lucene/searcher/SimpleUnsortedUnscoredLuceneMultiSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredSimpleMultiSearcher.java
View File

@ -13,11 +13,11 @@ import java.util.List;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class SimpleUnsortedUnscoredLuceneMultiSearcher implements LuceneMultiSearcher {
public class UnsortedUnscoredSimpleMultiSearcher implements MultiSearcher {
private final LuceneLocalSearcher localSearcher;
private final LocalSearcher localSearcher;
public SimpleUnsortedUnscoredLuceneMultiSearcher(LuceneLocalSearcher localSearcher) {
public UnsortedUnscoredSimpleMultiSearcher(LocalSearcher localSearcher) {
this.localSearcher = localSearcher;
}
@ -103,6 +103,6 @@ public class SimpleUnsortedUnscoredLuceneMultiSearcher implements LuceneMultiSea
@Override
public String getName() {
return "simpleunsortedunscoredmulti";
return "unsorted unscored simple multi";
}
}

4
src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredContinuousLuceneMultiSearcher.java → src/main/java/it/cavallium/dbengine/lucene/searcher/UnsortedUnscoredStreamingMultiSearcher.java
View File

@ -23,7 +23,7 @@ import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.util.concurrent.Queues;
public class UnsortedUnscoredContinuousLuceneMultiSearcher implements LuceneMultiSearcher {
public class UnsortedUnscoredStreamingMultiSearcher implements MultiSearcher {
private static final Scheduler UNSCORED_UNSORTED_EXECUTOR = Schedulers.newBoundedElastic(
Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE,
@ -117,6 +117,6 @@ public class UnsortedUnscoredContinuousLuceneMultiSearcher implements LuceneMult
@Override
public String getName() {
return "unsortedunscoredcontinuousmulti";
return "unsorted unscored streaming multi";
}
}

32
src/test/java/it/cavallium/dbengine/DbTestUtils.java
View File

@ -47,38 +47,6 @@ public class DbTestUtils {
return "0123456789".repeat(1024);
}
public static void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
public static void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
public static void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
public static record TestAllocator(PooledBufferAllocator allocator) {}
public static TestAllocator newAllocator() {

6
src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java
View File

@ -12,11 +12,9 @@ import it.cavallium.dbengine.client.NRTCachingOptions;
import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
import it.cavallium.dbengine.database.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -36,7 +34,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
private static final Optional<NRTCachingOptions> NRT = Optional.empty();
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
false, true, Optional.empty(), true, NRT, -1, true, true);
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
@Override
public Mono<TempDb> openTempDb(TestAllocator allocator) {

4
src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java
View File

@ -8,7 +8,7 @@ import it.cavallium.dbengine.client.IndicizerSimilarities;
import it.cavallium.dbengine.client.LuceneOptions;
import it.cavallium.dbengine.client.NRTCachingOptions;
import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.lucene.LuceneHacks;
import it.cavallium.dbengine.lucene.LuceneHacks;
import it.cavallium.dbengine.database.memory.LLMemoryDatabaseConnection;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
@ -22,7 +22,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
private static final Optional<NRTCachingOptions> NRT = Optional.empty();
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
false, true, Optional.empty(), true, NRT, -1, true, true);
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
@Override
public Mono<TempDb> openTempDb(TestAllocator allocator) {

40
src/test/java/it/cavallium/dbengine/SyncUtils.java Normal file
View File

@ -0,0 +1,40 @@
package it.cavallium.dbengine;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
public class SyncUtils {
public static void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
public static void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
public static <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
public static void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
}

1
src/test/java/it/cavallium/dbengine/TestDictionaryMap.java
View File

@ -1,6 +1,7 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.*;
import static it.cavallium.dbengine.SyncUtils.*;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;

3
src/test/java/it/cavallium/dbengine/TestDictionaryMapDeep.java
View File

@ -5,12 +5,11 @@ import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.isCIMode;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.run;
import static it.cavallium.dbengine.DbTestUtils.runVoid;
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryDeepMap;
import static it.cavallium.dbengine.DbTestUtils.tempDatabaseMapDictionaryMap;
import static it.cavallium.dbengine.DbTestUtils.tempDb;
import static it.cavallium.dbengine.DbTestUtils.tempDictionary;
import static it.cavallium.dbengine.SyncUtils.*;
import io.net5.buffer.api.internal.ResourceSupport;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;

33
src/test/java/it/cavallium/dbengine/TestLLDictionary.java
View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import io.net5.buffer.api.Buffer;
@ -115,38 +116,6 @@ public abstract class TestLLDictionary {
}
}
private void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
private void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
private void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
@Test
public void testNoOp() {
}

27
src/test/java/it/cavallium/dbengine/TestLLDictionaryLeaks.java
View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@ -131,9 +132,9 @@ public abstract class TestLLDictionaryLeaks {
public void testGet(UpdateMode updateMode) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test"));
DbTestUtils.runVoid(dict.get(null, key).then().transform(LLUtils::handleDiscard));
DbTestUtils.runVoid(dict.get(null, key, true).then().transform(LLUtils::handleDiscard));
DbTestUtils.runVoid(dict.get(null, key, false).then().transform(LLUtils::handleDiscard));
runVoid(dict.get(null, key).then().transform(LLUtils::handleDiscard));
runVoid(dict.get(null, key, true).then().transform(LLUtils::handleDiscard));
runVoid(dict.get(null, key, false).then().transform(LLUtils::handleDiscard));
}
@ParameterizedTest
@ -142,14 +143,14 @@ public abstract class TestLLDictionaryLeaks {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
var value = Mono.fromCallable(() -> fromString("test-value"));
DbTestUtils.runVoid(dict.put(key, value, resultType).then().doOnDiscard(Send.class, Send::close));
runVoid(dict.put(key, value, resultType).then().doOnDiscard(Send.class, Send::close));
}
@ParameterizedTest
@MethodSource("provideArguments")
public void testGetUpdateMode(UpdateMode updateMode) {
var dict = getDict(updateMode);
assertEquals(updateMode, DbTestUtils.run(dict.getUpdateMode()));
assertEquals(updateMode, run(dict.getUpdateMode()));
}
@ParameterizedTest
@ -157,13 +158,13 @@ public abstract class TestLLDictionaryLeaks {
public void testUpdate(UpdateMode updateMode, UpdateReturnMode updateReturnMode) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW,
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, old -> old, updateReturnMode, true).then().transform(LLUtils::handleDiscard)
);
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW,
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, old -> old, updateReturnMode, false).then().transform(LLUtils::handleDiscard)
);
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW,
runVoid(updateMode == UpdateMode.DISALLOW,
dict.update(key, old -> old, updateReturnMode).then().transform(LLUtils::handleDiscard)
);
}
@ -173,13 +174,13 @@ public abstract class TestLLDictionaryLeaks {
public void testUpdateAndGetDelta(UpdateMode updateMode) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW,
runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, old -> old, true).then().transform(LLUtils::handleDiscard)
);
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW,
runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, old -> old, false).then().transform(LLUtils::handleDiscard)
);
DbTestUtils.runVoid(updateMode == UpdateMode.DISALLOW,
runVoid(updateMode == UpdateMode.DISALLOW,
dict.updateAndGetDelta(key, old -> old).then().transform(LLUtils::handleDiscard)
);
}
@ -188,7 +189,7 @@ public abstract class TestLLDictionaryLeaks {
@MethodSource("provideArguments")
public void testClear(UpdateMode updateMode) {
var dict = getDict(updateMode);
DbTestUtils.runVoid(dict.clear());
runVoid(dict.clear());
}
@ParameterizedTest
@ -196,6 +197,6 @@ public abstract class TestLLDictionaryLeaks {
public void testRemove(UpdateMode updateMode, LLDictionaryResultType resultType) {
var dict = getDict(updateMode);
var key = Mono.fromCallable(() -> fromString("test-key"));
DbTestUtils.runVoid(dict.remove(key, resultType).then().doOnDiscard(Send.class, Send::close));
runVoid(dict.remove(key, resultType).then().doOnDiscard(Send.class, Send::close));
}
}

196
src/test/java/it/cavallium/dbengine/TestLuceneIndex.java
View File

@ -3,6 +3,7 @@ package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
@ -11,29 +12,16 @@ import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.MultiSort;
import it.cavallium.dbengine.client.SearchResultKey;
import it.cavallium.dbengine.client.SearchResultKeys;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder;
import it.cavallium.dbengine.client.query.QueryParser;
import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery;
import it.cavallium.dbengine.client.query.current.data.MatchNoDocsQuery;
import it.cavallium.dbengine.client.query.current.data.NoSort;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.CountLuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedScoredFullLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.ScoredSimpleLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.SimpleLuceneLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.SimpleUnsortedUnscoredLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredContinuousLuceneMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.CountLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredSimpleMultiSearcher;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import org.jetbrains.annotations.Nullable;
@ -47,9 +35,6 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink.OverflowStrategy;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuples;
public class TestLuceneIndex {
@ -92,38 +77,6 @@ public class TestLuceneIndex {
MultiSort.numericSort("longsort", true)
);
private static Flux<LuceneLocalSearcher> getSearchers(ExpectedQueryType info) {
return Flux.push(sink -> {
try {
if (info.shard()) {
sink.next(new AdaptiveLuceneMultiSearcher());
if (info.onlyCount()) {
sink.next(new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher()));
} else {
sink.next(new ScoredSimpleLuceneMultiSearcher());
if (!info.sorted()) {
sink.next(new UnsortedScoredFullLuceneMultiSearcher());
}
if (!info.scored() && !info.sorted()) {
sink.next(new SimpleUnsortedUnscoredLuceneMultiSearcher(new SimpleLuceneLocalSearcher()));
sink.next(new UnsortedUnscoredContinuousLuceneMultiSearcher());
}
}
} else {
sink.next(new AdaptiveLuceneLocalSearcher());
if (info.onlyCount()) {
sink.next(new CountLuceneLocalSearcher());
} else {
sink.next(new SimpleLuceneLocalSearcher());
}
}
sink.complete();
} catch (IOException e) {
sink.error(e);
}
}, OverflowStrategy.BUFFER);
}
public static Stream<Arguments> provideQueryArgumentsScoreMode() {
return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
@ -153,7 +106,7 @@ public class TestLuceneIndex {
destroyAllocator(allocator);
}
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LuceneLocalSearcher customSearcher) {
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
index.updateDocument("test-key-1", "0123456789").block();
index.updateDocument("test-key-2", "test 0123456789 test word").block();
@ -171,22 +124,22 @@ public class TestLuceneIndex {
index.updateDocument("test-key-14", "2999").block();
index.updateDocument("test-key-15", "3902").block();
Flux.range(1, 1000).concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i)).blockLast();
tempDb.swappableLuceneSearcher().setSingle(new CountLuceneLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new SimpleUnsortedUnscoredLuceneMultiSearcher(new CountLuceneLocalSearcher()));
tempDb.swappableLuceneSearcher().setSingle(new CountLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher()));
assertCount(index, 1000 + 15);
try {
if (customSearcher != null) {
tempDb.swappableLuceneSearcher().setSingle(customSearcher);
if (shards) {
if (customSearcher instanceof LuceneMultiSearcher multiSearcher) {
if (customSearcher instanceof MultiSearcher multiSearcher) {
tempDb.swappableLuceneSearcher().setMulti(multiSearcher);
} else {
throw new IllegalArgumentException("Expected a LuceneMultiSearcher, got a LuceneLocalSearcher: " + customSearcher.getName());
}
}
} else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLuceneLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveLuceneMultiSearcher());
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher());
}
} catch (IOException e) {
fail(e);
@ -194,38 +147,6 @@ public class TestLuceneIndex {
return index;
}
private void run(Flux<?> publisher) {
publisher.subscribeOn(Schedulers.immediate()).blockLast();
}
private void runVoid(Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).block();
}
private <T> T run(boolean shouldFail, Mono<T> publisher) {
return publisher.subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
private void runVoid(boolean shouldFail, Mono<Void> publisher) {
publisher.then().subscribeOn(Schedulers.immediate()).transform(mono -> {
if (shouldFail) {
return mono.onErrorResume(ex -> Mono.empty());
} else {
return mono;
}
}).block();
}
private void assertCount(LuceneIndex<String, String> luceneIndex, long expected) {
Assertions.assertEquals(expected, getCount(luceneIndex));
}
@ -291,95 +212,4 @@ public class TestLuceneIndex {
assertCount(luceneIndex, prevCount + 1);
}
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchNoDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LuceneLocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean supportsPreciseHitsCount(LuceneLocalSearcher searcher,
ClientQueryParams<SearchResultKey<String>> query) {
if (searcher instanceof UnsortedUnscoredContinuousLuceneMultiSearcher) {
return false;
}
var scored = isScored(query.scoreMode(), Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
var sorted = isSorted(Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
if (!sorted && !scored) {
if (searcher instanceof AdaptiveLuceneMultiSearcher || searcher instanceof AdaptiveLuceneLocalSearcher) {
return false;
}
}
return true;
}
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchAllDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LuceneLocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean isSorted(MultiSort<SearchResultKey<String>> multiSort) {
return !(multiSort.getQuerySort() instanceof NoSort);
}
private boolean isScored(LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var needsScores = LLUtils.toScoreMode(scoreMode).needsScores();
var sort =QueryParser.toSort(multiSort.getQuerySort());
if (sort != null) {
needsScores |= sort.needsScores();
}
return needsScores;
}
private List<Scored> getResults(SearchResultKeys<String> results) {
return run(results
.results()
.flatMapSequential(searchResultKey -> searchResultKey
.key()
.single()
.map(key -> new Scored(key, searchResultKey.score()))
)
.collectList());
}
}

307
src/test/java/it/cavallium/dbengine/TestLuceneSearches.java Normal file
View File

@ -0,0 +1,307 @@
package it.cavallium.dbengine;
import static it.cavallium.dbengine.DbTestUtils.destroyAllocator;
import static it.cavallium.dbengine.DbTestUtils.ensureNoLeaks;
import static it.cavallium.dbengine.DbTestUtils.newAllocator;
import static it.cavallium.dbengine.SyncUtils.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import it.cavallium.dbengine.DbTestUtils.TempDb;
import it.cavallium.dbengine.DbTestUtils.TestAllocator;
import it.cavallium.dbengine.client.LuceneIndex;
import it.cavallium.dbengine.client.MultiSort;
import it.cavallium.dbengine.client.SearchResultKey;
import it.cavallium.dbengine.client.SearchResultKeys;
import it.cavallium.dbengine.client.query.ClientQueryParams;
import it.cavallium.dbengine.client.query.ClientQueryParamsBuilder;
import it.cavallium.dbengine.client.query.QueryParser;
import it.cavallium.dbengine.client.query.current.data.MatchAllDocsQuery;
import it.cavallium.dbengine.client.query.current.data.MatchNoDocsQuery;
import it.cavallium.dbengine.client.query.current.data.NoSort;
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
import it.cavallium.dbengine.database.LLLuceneIndex;
import it.cavallium.dbengine.database.LLScoreMode;
import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.AdaptiveMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.CountLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
import it.cavallium.dbengine.lucene.searcher.ScoredPagedMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.PagedLocalSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredSimpleMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedScoredFullMultiSearcher;
import it.cavallium.dbengine.lucene.searcher.UnsortedUnscoredStreamingMultiSearcher;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.warp.commonutils.log.Logger;
import org.warp.commonutils.log.LoggerFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink.OverflowStrategy;
import reactor.util.function.Tuples;
public class TestLuceneSearches {
private static final Logger log = LoggerFactory.getLogger(TestLuceneSearches.class);
private static final MemoryTemporaryDbGenerator TEMP_DB_GENERATOR = new MemoryTemporaryDbGenerator();
private static TestAllocator allocator;
private static TempDb tempDb;
private static LLLuceneIndex luceneSingle;
private static LLLuceneIndex luceneMulti;
private static LuceneIndex<String, String> multiIndex;
private static LuceneIndex<String, String> localIndex;
@BeforeAll
public static void beforeAll() {
allocator = newAllocator();
ensureNoLeaks(allocator.allocator(), false, false);
tempDb = Objects.requireNonNull(TEMP_DB_GENERATOR.openTempDb(allocator).block(), "TempDB");
luceneSingle = tempDb.luceneSingle();
luceneMulti = tempDb.luceneMulti();
setUpIndex(true);
setUpIndex(false);
}
private static void setUpIndex(boolean shards) {
LuceneIndex<String, String> index = run(DbTestUtils.tempLuceneIndex(shards ? luceneSingle : luceneMulti));
index.updateDocument("test-key-1", "0123456789").block();
index.updateDocument("test-key-2", "test 0123456789 test word").block();
index.updateDocument("test-key-3", "0123456789 test example string").block();
index.updateDocument("test-key-4", "hello world the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-5", "hello the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-6", "hello the quick brown fox jumps over the world dog").block();
index.updateDocument("test-key-7", "the quick brown fox jumps over the world dog").block();
index.updateDocument("test-key-8", "the quick brown fox jumps over the lazy dog").block();
index.updateDocument("test-key-9", "Example1").block();
index.updateDocument("test-key-10", "Example2").block();
index.updateDocument("test-key-11", "Example3").block();
index.updateDocument("test-key-12", "-234").block();
index.updateDocument("test-key-13", "2111").block();
index.updateDocument("test-key-14", "2999").block();
index.updateDocument("test-key-15", "3902").block();
Flux.range(1, 1000).concatMap(i -> index.updateDocument("test-key-" + (15 + i), "" + i)).blockLast();
tempDb.swappableLuceneSearcher().setSingle(new CountLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher()));
assertCount(index, 1000 + 15);
if (shards) {
multiIndex = index;
} else {
localIndex = index;
}
}
public static Stream<Arguments> provideArguments() {
return Stream.of(false, true).map(Arguments::of);
}
private static final Flux<Boolean> multi = Flux.just(false, true);
private static final Flux<LLScoreMode> scoreModes = Flux.just(LLScoreMode.NO_SCORES,
LLScoreMode.TOP_SCORES,
LLScoreMode.COMPLETE_NO_SCORES,
LLScoreMode.COMPLETE
);
private static final Flux<MultiSort<SearchResultKey<String>>> multiSort = Flux.just(MultiSort.topScore(),
MultiSort.randomSortField(),
MultiSort.noSort(),
MultiSort.docSort(),
MultiSort.numericSort("longsort", false),
MultiSort.numericSort("longsort", true)
);
private static Flux<LocalSearcher> getSearchers(ExpectedQueryType info) {
return Flux.push(sink -> {
try {
if (info.shard()) {
sink.next(new AdaptiveMultiSearcher());
if (info.onlyCount()) {
sink.next(new UnsortedUnscoredSimpleMultiSearcher(new CountLocalSearcher()));
} else {
sink.next(new ScoredPagedMultiSearcher());
if (!info.sorted()) {
sink.next(new UnsortedScoredFullMultiSearcher());
}
if (!info.scored() && !info.sorted()) {
sink.next(new UnsortedUnscoredSimpleMultiSearcher(new PagedLocalSearcher()));
sink.next(new UnsortedUnscoredStreamingMultiSearcher());
}
}
} else {
sink.next(new AdaptiveLocalSearcher());
if (info.onlyCount()) {
sink.next(new CountLocalSearcher());
} else {
sink.next(new PagedLocalSearcher());
}
}
sink.complete();
} catch (IOException e) {
sink.error(e);
}
}, OverflowStrategy.BUFFER);
}
public static Stream<Arguments> provideQueryArgumentsScoreMode() {
return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
}
public static Stream<Arguments> provideQueryArgumentsSort() {
return multi
.concatMap(shard -> multiSort.map(multiSort -> Tuples.of(shard, multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
}
public static Stream<Arguments> provideQueryArgumentsScoreModeAndSort() {
return multi
.concatMap(shard -> scoreModes.map(scoreMode -> Tuples.of(shard, scoreMode)))
.concatMap(tuple -> multiSort.map(multiSort -> Tuples.of(tuple.getT1(), tuple.getT2(), multiSort)))
.map(tuple -> Arguments.of(tuple.toArray()))
.toStream();
}
@AfterAll
public static void afterAll() {
TEMP_DB_GENERATOR.closeTempDb(tempDb).block();
ensureNoLeaks(allocator.allocator(), true, false);
destroyAllocator(allocator);
}
private LuceneIndex<String, String> getLuceneIndex(boolean shards, @Nullable LocalSearcher customSearcher) {
try {
if (customSearcher != null) {
tempDb.swappableLuceneSearcher().setSingle(customSearcher);
if (shards) {
if (customSearcher instanceof MultiSearcher multiSearcher) {
tempDb.swappableLuceneSearcher().setMulti(multiSearcher);
} else {
throw new IllegalArgumentException("Expected a LuceneMultiSearcher, got a LuceneLocalSearcher: " + customSearcher.getName());
}
}
} else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher());
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher());
}
} catch (IOException e) {
fail(e);
}
return shards ? multiIndex : localIndex;
}
private static void assertCount(LuceneIndex<String, String> luceneIndex, long expected) {
Assertions.assertEquals(expected, getCount(luceneIndex));
}
private static long getCount(LuceneIndex<String, String> luceneIndex) {
luceneIndex.refresh(true).block();
var totalHitsCount = run(luceneIndex.count(null, new MatchAllDocsQuery()));
Assertions.assertTrue(totalHitsCount.exact(), "Can't get count because the total hits count is not exact");
return totalHitsCount.value();
}
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchNoDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean supportsPreciseHitsCount(LocalSearcher searcher,
ClientQueryParams<SearchResultKey<String>> query) {
if (searcher instanceof UnsortedUnscoredStreamingMultiSearcher) {
return false;
}
var scored = isScored(query.scoreMode(), Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
var sorted = isSorted(Objects.requireNonNullElse(query.sort(), MultiSort.noSort()));
if (!sorted && !scored) {
if (searcher instanceof AdaptiveMultiSearcher || searcher instanceof AdaptiveLocalSearcher) {
return false;
}
}
return true;
}
@ParameterizedTest
@MethodSource("provideQueryArgumentsScoreModeAndSort")
public void testSearchAllDocs(boolean shards, LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var searchers = run(getSearchers(new ExpectedQueryType(shards, isSorted(multiSort), isScored(scoreMode, multiSort), true, false)).collectList());
for (LocalSearcher searcher : searchers) {
log.info("Using searcher \"{}\"", searcher.getName());
var luceneIndex = getLuceneIndex(shards, searcher);
ClientQueryParamsBuilder<SearchResultKey<String>> queryBuilder = ClientQueryParams.builder();
queryBuilder.query(new MatchNoDocsQuery());
queryBuilder.snapshot(null);
queryBuilder.scoreMode(scoreMode);
queryBuilder.sort(multiSort);
var query = queryBuilder.build();
try (var results = run(luceneIndex.search(query)).receive()) {
var hits = results.totalHitsCount();
if (supportsPreciseHitsCount(searcher, query)) {
assertEquals(new TotalHitsCount(0, true), hits);
}
var keys = getResults(results);
assertEquals(List.of(), keys);
}
}
}
private boolean isSorted(MultiSort<SearchResultKey<String>> multiSort) {
return !(multiSort.getQuerySort() instanceof NoSort);
}
private boolean isScored(LLScoreMode scoreMode, MultiSort<SearchResultKey<String>> multiSort) {
var needsScores = LLUtils.toScoreMode(scoreMode).needsScores();
var sort =QueryParser.toSort(multiSort.getQuerySort());
if (sort != null) {
needsScores |= sort.needsScores();
}
return needsScores;
}
private List<Scored> getResults(SearchResultKeys<String> results) {
return run(results
.results()
.flatMapSequential(searchResultKey -> searchResultKey
.key()
.single()
.map(key -> new Scored(key, searchResultKey.score()))
)
.collectList());
}
}