andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add lucene options

Browse Source
This commit is contained in:
Andrea Cavalli 2021-07-06 14:33:47 +02:00
parent e7e25e646d
commit 2a47e6f3fd
2 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/it/cavallium/dbengine/client/LuceneOptions.java
View File

@ -14,4 +14,7 @@ public record LuceneOptions(Map<String, String> extraFlags,
boolean inMemory, boolean inMemory,
Optional<DirectIOOptions> directIOOptions, Optional<DirectIOOptions> directIOOptions,
boolean allowMemoryMapping, boolean allowMemoryMapping,
Optional<NRTCachingOptions> nrtCachingOptions) {} Optional<NRTCachingOptions> nrtCachingOptions,
int indexWriterBufferSize,
boolean applyAllDeletes,
boolean writeAllDeletes) {}

26
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -39,9 +39,11 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.SnapshotDeletionPolicy; import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.misc.store.DirectIODirectory; import org.apache.lucene.misc.store.DirectIODirectory;
@ -54,6 +56,7 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager; import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort; import org.apache.lucene.search.Sort;
import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity;
@ -197,21 +200,24 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
indexWriterConfig.setIndexDeletionPolicy(snapshotter); indexWriterConfig.setIndexDeletionPolicy(snapshotter);
indexWriterConfig.setCommitOnClose(true); indexWriterConfig.setCommitOnClose(true);
MergeScheduler mergeScheduler;
if (lowMemory) { if (lowMemory) {
indexWriterConfig.setRAMBufferSizeMB(8); mergeScheduler = new SerialMergeScheduler();
indexWriterConfig.setRAMPerThreadHardLimitMB(32);
var mergeScheduler = new SerialMergeScheduler();
indexWriterConfig.setMergeScheduler(mergeScheduler);
} else { } else {
indexWriterConfig.setRAMBufferSizeMB(16); var concurrentMergeScheduler = new ConcurrentMergeScheduler();
//indexWriterConfig.setRAMPerThreadHardLimitMB(512); concurrentMergeScheduler.enableAutoIOThrottle();
var mergeScheduler = new ConcurrentMergeScheduler(); mergeScheduler = concurrentMergeScheduler;
mergeScheduler.enableAutoIOThrottle();
indexWriterConfig.setMergeScheduler(mergeScheduler);
} }
indexWriterConfig.setMergeScheduler(mergeScheduler);
indexWriterConfig.setRAMBufferSizeMB(luceneOptions.indexWriterBufferSize() / 1024D / 1024D);
indexWriterConfig.setReaderPooling(false);
indexWriterConfig.setSimilarity(getSimilarity()); indexWriterConfig.setSimilarity(getSimilarity());
this.indexWriter = new IndexWriter(directory, indexWriterConfig); this.indexWriter = new IndexWriter(directory, indexWriterConfig);
this.searcherManager = new SearcherManager(indexWriter, false, false, null); this.searcherManager = new SearcherManager(indexWriter,
luceneOptions.applyAllDeletes(),
luceneOptions.writeAllDeletes(),
new SearcherFactory()
);
// Create scheduled tasks lifecycle manager // Create scheduled tasks lifecycle manager
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle(); this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();