andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add low memory scheduler in lucene

Browse Source
This commit is contained in:
Andrea Cavalli 2021-02-28 10:57:16 +01:00
parent 8926916f67
commit a20dadfb39
2 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/it/cavallium/dbengine/database/disk/LLLocalKeyValueDatabase.java
View File

@ -1,6 +1,5 @@
package it.cavallium.dbengine.database.disk; package it.cavallium.dbengine.database.disk;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers; import com.google.common.base.Suppliers;
import it.cavallium.dbengine.database.Column; import it.cavallium.dbengine.database.Column;
import it.cavallium.dbengine.database.LLKeyValueDatabase; import it.cavallium.dbengine.database.LLKeyValueDatabase;
@ -23,6 +22,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter; import org.rocksdb.BloomFilter;
import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyDescriptor;
@ -49,7 +49,8 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY = new ColumnFamilyDescriptor( private static final ColumnFamilyDescriptor DEFAULT_COLUMN_FAMILY = new ColumnFamilyDescriptor(
RocksDB.DEFAULT_COLUMN_FAMILY); RocksDB.DEFAULT_COLUMN_FAMILY);
private static Supplier<Scheduler> lowMemorySupplier = Suppliers.memoize(() -> Schedulers.newSingle("db-low-memory")); private static final Supplier<Scheduler> lowMemorySupplier = Suppliers.memoize(() ->
Schedulers.newSingle("db-low-memory"))::get;
private final Scheduler dbScheduler; private final Scheduler dbScheduler;
private final Path dbPath; private final Path dbPath;
@ -204,7 +205,7 @@ public class LLLocalKeyValueDatabase implements LLKeyValueDatabase {
; ;
} }
final org.rocksdb.BloomFilter bloomFilter = new BloomFilter(10, false); final BloomFilter bloomFilter = new BloomFilter(10, false);
final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
tableOptions.setFilterPolicy(bloomFilter); tableOptions.setFilterPolicy(bloomFilter);
options.setTableFormatConfig(tableOptions); options.setTableFormatConfig(tableOptions);

17
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneIndex.java
View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.disk;
import static it.cavallium.dbengine.lucene.LuceneUtils.checkScoringArgumentsValidity; import static it.cavallium.dbengine.lucene.LuceneUtils.checkScoringArgumentsValidity;
import com.google.common.base.Suppliers;
import it.cavallium.dbengine.database.LLDocument; import it.cavallium.dbengine.database.LLDocument;
import it.cavallium.dbengine.database.LLKeyScore; import it.cavallium.dbengine.database.LLKeyScore;
import it.cavallium.dbengine.database.LLLuceneIndex; import it.cavallium.dbengine.database.LLLuceneIndex;
@ -32,6 +33,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig;
@ -79,16 +81,16 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
*/ */
private static final Scheduler luceneBlockingScheduler = Schedulers.newBoundedElastic(1, private static final Scheduler luceneBlockingScheduler = Schedulers.newBoundedElastic(1,
Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
"Lucene", "lucene",
120, 120,
true true
); );
private static final Supplier<Scheduler> lowMemorySupplier = Suppliers.memoize(() ->
Schedulers.newSingle("lucene-low-memory"))::get;
/** /**
* Lucene query scheduler. * Lucene query scheduler.
*/ */
private final Scheduler luceneQueryScheduler = Schedulers.newBoundedElastic(Runtime private final Scheduler luceneQueryScheduler;
.getRuntime()
.availableProcessors(), Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, "LuceneQuery", 120, true);
private final String luceneIndexName; private final String luceneIndexName;
private final SnapshotDeletionPolicy snapshotter; private final SnapshotDeletionPolicy snapshotter;
@ -141,6 +143,13 @@ public class LLLocalLuceneIndex implements LLLuceneIndex {
indexWriterConfig.setSimilarity(getSimilarity()); indexWriterConfig.setSimilarity(getSimilarity());
this.indexWriter = new IndexWriter(directory, indexWriterConfig); this.indexWriter = new IndexWriter(directory, indexWriterConfig);
this.searcherManager = new SearcherManager(indexWriter, false, false, null); this.searcherManager = new SearcherManager(indexWriter, false, false, null);
if (lowMemory) {
this.luceneQueryScheduler = lowMemorySupplier.get();
} else {
this.luceneQueryScheduler = Schedulers.newBoundedElastic(Runtime
.getRuntime()
.availableProcessors(), Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, "lucene-query", 60, true);
}
// Create scheduled tasks lifecycle manager // Create scheduled tasks lifecycle manager
this.scheduledTasksLifecycle = new ScheduledTaskLifecycle(); this.scheduledTasksLifecycle = new ScheduledTaskLifecycle();