Optionally disable LMDB
This commit is contained in:
Andrea Cavalli
parent
297c249243
commit
3cdafd748e
|
|
@ -17,4 +17,5 @@ public record LuceneOptions(Map<String, String> extraFlags,
|
|||
Optional<NRTCachingOptions> nrtCachingOptions,
|
||||
int indexWriterBufferSize,
|
||||
boolean applyAllDeletes,
|
||||
boolean writeAllDeletes) {}
|
||||
boolean writeAllDeletes,
|
||||
boolean allowNonVolatileCollection) {}
|
||||
|
|
|
|||
|
|
@ -194,10 +194,12 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
|
|||
this.lowMemory = lowMemory;
|
||||
this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers);
|
||||
this.luceneSimilarity = LuceneUtils.toPerFieldSimilarityWrapper(indicizerSimilarities);
|
||||
|
||||
var useLMDB = luceneOptions.allowNonVolatileCollection();
|
||||
if (luceneHacks != null && luceneHacks.customLocalSearcher() != null) {
|
||||
localSearcher = luceneHacks.customLocalSearcher().get();
|
||||
} else {
|
||||
localSearcher = new AdaptiveLocalSearcher(env);
|
||||
localSearcher = new AdaptiveLocalSearcher(env, useLMDB);
|
||||
}
|
||||
|
||||
var indexWriterConfig = new IndexWriterConfig(luceneAnalyzer);
|
||||
|
|
|
|||
|
|
@ -97,10 +97,11 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
|
|||
this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers);
|
||||
this.luceneSimilarity = LuceneUtils.toPerFieldSimilarityWrapper(indicizerSimilarities);
|
||||
|
||||
var useLMDB = luceneOptions.allowNonVolatileCollection();
|
||||
if (luceneHacks != null && luceneHacks.customMultiSearcher() != null) {
|
||||
multiSearcher = luceneHacks.customMultiSearcher().get();
|
||||
} else {
|
||||
multiSearcher = new AdaptiveMultiSearcher(env);
|
||||
multiSearcher = new AdaptiveMultiSearcher(env, useLMDB);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,23 +9,28 @@ import it.cavallium.dbengine.database.disk.LLIndexSearcher;
|
|||
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
||||
import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
|
||||
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class AdaptiveLocalSearcher implements LocalSearcher {
|
||||
|
||||
private static final OfficialSearcher officialSearcher = new OfficialSearcher();
|
||||
|
||||
private static final LocalSearcher localPagedSearcher = new PagedLocalSearcher();
|
||||
|
||||
private static final LocalSearcher countSearcher = new CountMultiSearcher();
|
||||
|
||||
private static final MultiSearcher unsortedUnscoredContinuous = new UnsortedUnscoredStreamingMultiSearcher();
|
||||
|
||||
@Nullable
|
||||
private final UnsortedScoredFullMultiSearcher unsortedScoredFull;
|
||||
|
||||
@Nullable
|
||||
private final SortedScoredFullMultiSearcher sortedScoredFull;
|
||||
|
||||
public AdaptiveLocalSearcher(LLTempLMDBEnv env) {
|
||||
unsortedScoredFull = new UnsortedScoredFullMultiSearcher(env);
|
||||
sortedScoredFull = new SortedScoredFullMultiSearcher(env);
|
||||
public AdaptiveLocalSearcher(LLTempLMDBEnv env, boolean useLMDB) {
|
||||
unsortedScoredFull = useLMDB ? new UnsortedScoredFullMultiSearcher(env) : null;
|
||||
sortedScoredFull = useLMDB ? new SortedScoredFullMultiSearcher(env) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -73,12 +78,20 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
|
|||
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
|
||||
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
|
||||
}
|
||||
return sortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||
if (sortedScoredFull != null) {
|
||||
return sortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||
} else {
|
||||
return officialSearcher.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
} else {
|
||||
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
|
||||
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
|
||||
}
|
||||
return unsortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||
if (unsortedScoredFull != null) {
|
||||
return unsortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||
} else {
|
||||
return officialSearcher.collect(indexSearcher, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -5,23 +5,28 @@ import it.cavallium.dbengine.database.LLUtils;
|
|||
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
||||
import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
|
||||
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class AdaptiveMultiSearcher implements MultiSearcher {
|
||||
|
||||
private static final OfficialSearcher officialSearcher = new OfficialSearcher();
|
||||
|
||||
private static final MultiSearcher count = new CountMultiSearcher();
|
||||
|
||||
private static final MultiSearcher scoredPaged = new ScoredPagedMultiSearcher();
|
||||
|
||||
private static final MultiSearcher unsortedUnscoredContinuous = new UnsortedUnscoredStreamingMultiSearcher();
|
||||
|
||||
@Nullable
|
||||
private final UnsortedScoredFullMultiSearcher unsortedScoredFull;
|
||||
|
||||
@Nullable
|
||||
private final SortedScoredFullMultiSearcher sortedScoredFull;
|
||||
|
||||
public AdaptiveMultiSearcher(LLTempLMDBEnv env) {
|
||||
unsortedScoredFull = new UnsortedScoredFullMultiSearcher(env);
|
||||
sortedScoredFull = new SortedScoredFullMultiSearcher(env);
|
||||
public AdaptiveMultiSearcher(LLTempLMDBEnv env, boolean useLMDB) {
|
||||
unsortedScoredFull = useLMDB ? new UnsortedScoredFullMultiSearcher(env) : null;
|
||||
sortedScoredFull = useLMDB ? new SortedScoredFullMultiSearcher(env) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -61,12 +66,20 @@ public class AdaptiveMultiSearcher implements MultiSearcher {
|
|||
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
|
||||
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
|
||||
}
|
||||
return sortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
if (sortedScoredFull != null) {
|
||||
return sortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
} else {
|
||||
return officialSearcher.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
} else {
|
||||
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
|
||||
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
|
||||
}
|
||||
return unsortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
if (unsortedScoredFull != null) {
|
||||
return unsortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
} else {
|
||||
return officialSearcher.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -21,10 +21,7 @@ public class OfficialSearcher implements MultiSearcher {
|
|||
|
||||
protected static final Logger logger = LoggerFactory.getLogger(OfficialSearcher.class);
|
||||
|
||||
private final LLTempLMDBEnv env;
|
||||
|
||||
public OfficialSearcher(LLTempLMDBEnv env) {
|
||||
this.env = env;
|
||||
public OfficialSearcher() {
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
|
|||
|
||||
private static final Optional<NRTCachingOptions> NRT = Optional.empty();
|
||||
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
|
||||
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
|
||||
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false, true);
|
||||
|
||||
@Override
|
||||
public Mono<TempDb> openTempDb(TestAllocator allocator) {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
|
|||
|
||||
private static final Optional<NRTCachingOptions> NRT = Optional.empty();
|
||||
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
|
||||
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
|
||||
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false, false);
|
||||
|
||||
@Override
|
||||
public Mono<TempDb> openTempDb(TestAllocator allocator) {
|
||||
|
|
|
|||
|
|
@ -149,8 +149,8 @@ public class TestLuceneIndex {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV));
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true));
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -167,14 +167,14 @@ public class TestLuceneSearches {
|
|||
sink.next(new UnsortedUnscoredStreamingMultiSearcher());
|
||||
}
|
||||
}
|
||||
sink.next(new AdaptiveMultiSearcher(ENV));
|
||||
sink.next(new AdaptiveMultiSearcher(ENV, true));
|
||||
} else {
|
||||
if (info.onlyCount()) {
|
||||
sink.next(new CountMultiSearcher());
|
||||
} else {
|
||||
sink.next(new PagedLocalSearcher());
|
||||
}
|
||||
sink.next(new AdaptiveLocalSearcher(ENV));
|
||||
sink.next(new AdaptiveLocalSearcher(ENV, true));
|
||||
}
|
||||
sink.complete();
|
||||
}, OverflowStrategy.BUFFER);
|
||||
|
|
@ -219,8 +219,8 @@ public class TestLuceneSearches {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV));
|
||||
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true));
|
||||
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true));
|
||||
}
|
||||
return shards ? multiIndex : localIndex;
|
||||
}
|
||||
|
|
@ -263,7 +263,7 @@ public class TestLuceneSearches {
|
|||
Assertions.assertTrue(keys.size() >= hits.value());
|
||||
}
|
||||
|
||||
var officialSearcher = new OfficialSearcher(ENV);
|
||||
var officialSearcher = new OfficialSearcher();
|
||||
luceneIndex = getLuceneIndex(expectedQueryType.shard(), officialSearcher);
|
||||
var officialQuery = queryParamsBuilder.limit(ELEMENTS.size() * 2L).build();
|
||||
try (var officialResults = run(luceneIndex.search(officialQuery))) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user