andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Optionally disable LMDB

Browse Source
This commit is contained in:
Andrea Cavalli 2021-12-12 16:41:49 +01:00
parent 297c249243
commit 3cdafd748e
11 changed files with 53 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/it/cavallium/dbengine/client/LuceneOptions.java
View File

@ -17,4 +17,5 @@ public record LuceneOptions(Map<String, String> extraFlags,
Optional<NRTCachingOptions> nrtCachingOptions,
int indexWriterBufferSize,
boolean applyAllDeletes,
boolean writeAllDeletes) {}
boolean writeAllDeletes,
boolean allowNonVolatileCollection) {}

4
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -194,10 +194,12 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
this.lowMemory = lowMemory;
this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers);
this.luceneSimilarity = LuceneUtils.toPerFieldSimilarityWrapper(indicizerSimilarities);
var useLMDB = luceneOptions.allowNonVolatileCollection();
if (luceneHacks != null && luceneHacks.customLocalSearcher() != null) {
localSearcher = luceneHacks.customLocalSearcher().get();
} else {
localSearcher = new AdaptiveLocalSearcher(env);
localSearcher = new AdaptiveLocalSearcher(env, useLMDB);
}
var indexWriterConfig = new IndexWriterConfig(luceneAnalyzer);

3
src/main/java/it/cavallium/dbengine/database/disk/LLLocalMultiLuceneIndex.java
View File

@ -97,10 +97,11 @@ public class LLLocalMultiLuceneIndex implements LLLuceneIndex {
this.luceneAnalyzer = LuceneUtils.toPerFieldAnalyzerWrapper(indicizerAnalyzers);
this.luceneSimilarity = LuceneUtils.toPerFieldSimilarityWrapper(indicizerSimilarities);
var useLMDB = luceneOptions.allowNonVolatileCollection();
if (luceneHacks != null && luceneHacks.customMultiSearcher() != null) {
multiSearcher = luceneHacks.customMultiSearcher().get();
} else {
multiSearcher = new AdaptiveMultiSearcher(env);
multiSearcher = new AdaptiveMultiSearcher(env, useLMDB);
}
}

23
src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLocalSearcher.java
View File

@ -9,23 +9,28 @@ import it.cavallium.dbengine.database.disk.LLIndexSearcher;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class AdaptiveLocalSearcher implements LocalSearcher {
private static final OfficialSearcher officialSearcher = new OfficialSearcher();
private static final LocalSearcher localPagedSearcher = new PagedLocalSearcher();
private static final LocalSearcher countSearcher = new CountMultiSearcher();
private static final MultiSearcher unsortedUnscoredContinuous = new UnsortedUnscoredStreamingMultiSearcher();
@Nullable
private final UnsortedScoredFullMultiSearcher unsortedScoredFull;
@Nullable
private final SortedScoredFullMultiSearcher sortedScoredFull;
public AdaptiveLocalSearcher(LLTempLMDBEnv env) {
unsortedScoredFull = new UnsortedScoredFullMultiSearcher(env);
sortedScoredFull = new SortedScoredFullMultiSearcher(env);
public AdaptiveLocalSearcher(LLTempLMDBEnv env, boolean useLMDB) {
unsortedScoredFull = useLMDB ? new UnsortedScoredFullMultiSearcher(env) : null;
sortedScoredFull = useLMDB ? new SortedScoredFullMultiSearcher(env) : null;
}
@Override
@ -73,12 +78,20 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
}
return sortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
if (sortedScoredFull != null) {
return sortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
} else {
return officialSearcher.collect(indexSearcher, queryParams, keyFieldName, transformer);
}
} else {
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
}
return unsortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
if (unsortedScoredFull != null) {
return unsortedScoredFull.collect(indexSearcher, queryParams, keyFieldName, transformer);
} else {
return officialSearcher.collect(indexSearcher, queryParams, keyFieldName, transformer);
}
}
}
} else {

23
src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java
View File

@ -5,23 +5,28 @@ import it.cavallium.dbengine.database.LLUtils;
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
import it.cavallium.dbengine.database.disk.LLTempLMDBEnv;
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer.TransformerInput;
import org.jetbrains.annotations.Nullable;
import reactor.core.publisher.Mono;
public class AdaptiveMultiSearcher implements MultiSearcher {
private static final OfficialSearcher officialSearcher = new OfficialSearcher();
private static final MultiSearcher count = new CountMultiSearcher();
private static final MultiSearcher scoredPaged = new ScoredPagedMultiSearcher();
private static final MultiSearcher unsortedUnscoredContinuous = new UnsortedUnscoredStreamingMultiSearcher();
@Nullable
private final UnsortedScoredFullMultiSearcher unsortedScoredFull;
@Nullable
private final SortedScoredFullMultiSearcher sortedScoredFull;
public AdaptiveMultiSearcher(LLTempLMDBEnv env) {
unsortedScoredFull = new UnsortedScoredFullMultiSearcher(env);
sortedScoredFull = new SortedScoredFullMultiSearcher(env);
public AdaptiveMultiSearcher(LLTempLMDBEnv env, boolean useLMDB) {
unsortedScoredFull = useLMDB ? new UnsortedScoredFullMultiSearcher(env) : null;
sortedScoredFull = useLMDB ? new SortedScoredFullMultiSearcher(env) : null;
}
@Override
@ -61,12 +66,20 @@ public class AdaptiveMultiSearcher implements MultiSearcher {
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
}
return sortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
if (sortedScoredFull != null) {
return sortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
} else {
return officialSearcher.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
}
} else {
if (queryParams.limitLong() < MAX_IN_MEMORY_SIZE) {
throw new UnsupportedOperationException("Allowed limit is " + MAX_IN_MEMORY_SIZE + " or greater");
}
return unsortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
if (unsortedScoredFull != null) {
return unsortedScoredFull.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
} else {
return officialSearcher.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
}
}
}
} else {

5
src/main/java/it/cavallium/dbengine/lucene/searcher/OfficialSearcher.java
View File

@ -21,10 +21,7 @@ public class OfficialSearcher implements MultiSearcher {
protected static final Logger logger = LoggerFactory.getLogger(OfficialSearcher.class);
private final LLTempLMDBEnv env;
public OfficialSearcher(LLTempLMDBEnv env) {
this.env = env;
public OfficialSearcher() {
}
@Override

2
src/test/java/it/cavallium/dbengine/LocalTemporaryDbGenerator.java
View File

@ -35,7 +35,7 @@ public class LocalTemporaryDbGenerator implements TemporaryDbGenerator {
private static final Optional<NRTCachingOptions> NRT = Optional.empty();
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false, true);
@Override
public Mono<TempDb> openTempDb(TestAllocator allocator) {

2
src/test/java/it/cavallium/dbengine/MemoryTemporaryDbGenerator.java
View File

@ -23,7 +23,7 @@ public class MemoryTemporaryDbGenerator implements TemporaryDbGenerator {
private static final Optional<NRTCachingOptions> NRT = Optional.empty();
private static final LuceneOptions LUCENE_OPTS = new LuceneOptions(Map.of(), Duration.ofSeconds(5), Duration.ofSeconds(5),
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false);
false, true, Optional.empty(), true, NRT, 16 * 1024 * 1024, true, false, false);
@Override
public Mono<TempDb> openTempDb(TestAllocator allocator) {

0
src/main/java/it/cavallium/dbengine/SwappableLuceneSearcher.java → src/test/java/it/cavallium/dbengine/SwappableLuceneSearcher.java
View File

4
src/test/java/it/cavallium/dbengine/TestLuceneIndex.java
View File

@ -149,8 +149,8 @@ public class TestLuceneIndex {
}
}
} else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV));
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true));
}
return index;
}

10
src/test/java/it/cavallium/dbengine/TestLuceneSearches.java
View File

@ -167,14 +167,14 @@ public class TestLuceneSearches {
sink.next(new UnsortedUnscoredStreamingMultiSearcher());
}
}
sink.next(new AdaptiveMultiSearcher(ENV));
sink.next(new AdaptiveMultiSearcher(ENV, true));
} else {
if (info.onlyCount()) {
sink.next(new CountMultiSearcher());
} else {
sink.next(new PagedLocalSearcher());
}
sink.next(new AdaptiveLocalSearcher(ENV));
sink.next(new AdaptiveLocalSearcher(ENV, true));
}
sink.complete();
}, OverflowStrategy.BUFFER);
@ -219,8 +219,8 @@ public class TestLuceneSearches {
}
}
} else {
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV));
tempDb.swappableLuceneSearcher().setSingle(new AdaptiveLocalSearcher(ENV, true));
tempDb.swappableLuceneSearcher().setMulti(new AdaptiveMultiSearcher(ENV, true));
}
return shards ? multiIndex : localIndex;
}
@ -263,7 +263,7 @@ public class TestLuceneSearches {
Assertions.assertTrue(keys.size() >= hits.value());
}
var officialSearcher = new OfficialSearcher(ENV);
var officialSearcher = new OfficialSearcher();
luceneIndex = getLuceneIndex(expectedQueryType.shard(), officialSearcher);
var officialQuery = queryParamsBuilder.limit(ELEMENTS.size() * 2L).build();
try (var officialResults = run(luceneIndex.search(officialQuery))) {