andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add low memory scheduler in lucene

Browse Source
This commit is contained in:
Andrea Cavalli 2021-02-28 10:57:16 +01:00
parent 8926916f67
commit a20dadfb39
2 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -1,6 +1,5 @@
package it.cavallium.dbengine.database.disk;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.LLKeyValueDatabase;
@ -23,6 +22,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter;
import org.rocksdb.ColumnFamilyDescriptor;
@ -49,7 +49,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY = new ColumnFamilyDescriptor(
RocksDB.DEFAULT_COLUMN_FAMILY);
private static Supplier<Scheduler> lowMemorySupplier = Suppliers.memoize(() -> Schedulers.newSingle("db-low-memory"));
private static final Supplier<Scheduler> lowMemorySupplier = Suppliers.memoize(() ->
Schedulers.newSingle("db-low-memory"))::get;
private final Scheduler dbScheduler;
private final Path dbPath;
@ -204,7 +205,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
;
}
final org.rocksdb.BloomFilter bloomFilter = new BloomFilter(10, false);
final BloomFilter bloomFilter = new BloomFilter(10, false);
final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
tableOptions.setFilterPolicy(bloomFilter);
options.setTableFormatConfig(tableOptions);

17
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.lucene.LuceneUtils.checkScoringArgumentsValidity;
import com.google.common.base.Suppliers;
import it.cavallium.dbengine.database.LLDocument;
import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLLuceneIndex;
@ -32,6 +33,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@ -79,16 +81,16 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
*/
private static final Scheduler luceneBlockingScheduler = Schedulers.newBoundedElastic(1,
Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
"Lucene",
"lucene",
120,
true
);
private static final Supplier<Scheduler> lowMemorySupplier = Suppliers.memoize(() ->
Schedulers.newSingle("lucene-low-memory"))::get;
/**
* Lucene query scheduler.
*/
private final Scheduler luceneQueryScheduler = Schedulers.newBoundedElastic(Runtime
.getRuntime()
.availableProcessors(), Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, "LuceneQuery", 120, true);
private final Scheduler luceneQueryScheduler;
private final String luceneIndexName;
private final SnapshotDeletionPolicy snapshotter;
@ -141,6 +143,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
indexWriterConfig.setSimilarity(getSimilarity());
this.indexWriter = new IndexWriter(directory, indexWriterConfig);
this.searcherManager = new SearcherManager(indexWriter, false, false, null);
if (lowMemory) {
this.luceneQueryScheduler = lowMemorySupplier.get();
} else {
this.luceneQueryScheduler = Schedulers.newBoundedElastic(Runtime
.getRuntime()
.availableProcessors(), Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, "lucene-query", 60, true);
}
// Create scheduled tasks lifecycle manager
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();