andreacavalli
/
CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add lucene options

This commit is contained in:
Andrea Cavalli 2021-07-06 14:33:47 +02:00
parent e7e25e646d
commit 2a47e6f3fd
2 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/it/cavallium/dbengine/client/LuceneOptions.java
View File

@ -14,4 +14,7 @@ public record LuceneOptions(Map<String, String> extraFlags,
boolean inMemory,
Optional<DirectIOOptions> directIOOptions,
boolean allowMemoryMapping,
Optional<NRTCachingOptions> nrtCachingOptions) {}
Optional<NRTCachingOptions> nrtCachingOptions,
int indexWriterBufferSize,
boolean applyAllDeletes,
boolean writeAllDeletes) {}

26
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -39,9 +39,11 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.misc.store.DirectIODirectory;
@ -54,6 +56,7 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.similarities.Similarity;
@ -197,21 +200,24 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
indexWriterConfig.setIndexDeletionPolicy(snapshotter);
indexWriterConfig.setCommitOnClose(true);
MergeScheduler mergeScheduler;
if (lowMemory) {
indexWriterConfig.setRAMBufferSizeMB(8);
indexWriterConfig.setRAMPerThreadHardLimitMB(32);
var mergeScheduler = new SerialMergeScheduler();
indexWriterConfig.setMergeScheduler(mergeScheduler);
mergeScheduler = new SerialMergeScheduler();
} else {
indexWriterConfig.setRAMBufferSizeMB(16);
//indexWriterConfig.setRAMPerThreadHardLimitMB(512);
var mergeScheduler = new ConcurrentMergeScheduler();
mergeScheduler.enableAutoIOThrottle();
indexWriterConfig.setMergeScheduler(mergeScheduler);
var concurrentMergeScheduler = new ConcurrentMergeScheduler();
concurrentMergeScheduler.enableAutoIOThrottle();
mergeScheduler = concurrentMergeScheduler;
}
indexWriterConfig.setMergeScheduler(mergeScheduler);
indexWriterConfig.setRAMBufferSizeMB(luceneOptions.indexWriterBufferSize() / 1024D / 1024D);
indexWriterConfig.setReaderPooling(false);
indexWriterConfig.setSimilarity(getSimilarity());
this.indexWriter = new IndexWriter(directory, indexWriterConfig);
this.searcherManager = new SearcherManager(indexWriter, false, false, null);
this.searcherManager = new SearcherManager(indexWriter,
luceneOptions.applyAllDeletes(),
luceneOptions.writeAllDeletes(),
new SearcherFactory()
);
// Create scheduled tasks lifecycle manager
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();