andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Force huge pq with an option

Browse Source
This commit is contained in:
Andrea Cavalli 2022-06-15 13:09:45 +02:00
parent 86e48eab92
commit 4a08a876ca
6 changed files with 59 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/it/cavallium/dbengine/database/disk/LLTempHugePqEnv.java
View File

@ -15,6 +15,7 @@ import org.rocksdb.ChecksumType;
import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.CompressionType;
import org.rocksdb.DBOptions; import org.rocksdb.DBOptions;
import org.rocksdb.InfoLogLevel; import org.rocksdb.InfoLogLevel;
import org.rocksdb.RocksDB; import org.rocksdb.RocksDB;
@ -81,10 +82,13 @@ public class LLTempHugePqEnv implements Closeable {
var opts = new ColumnFamilyOptions() var opts = new ColumnFamilyOptions()
.setOptimizeFiltersForHits(true) .setOptimizeFiltersForHits(true)
.setParanoidFileChecks(false) .setParanoidFileChecks(false)
.setEnableBlobFiles(true)
.setBlobCompressionType(CompressionType.LZ4_COMPRESSION)
.optimizeLevelStyleCompaction() .optimizeLevelStyleCompaction()
.setLevelCompactionDynamicLevelBytes(true) .setLevelCompactionDynamicLevelBytes(true)
.setTableFormatConfig(new BlockBasedTableConfig() .setTableFormatConfig(new BlockBasedTableConfig()
.setOptimizeFiltersForMemory(true) .setOptimizeFiltersForMemory(true)
.setVerifyCompression(false)
.setChecksumType(ChecksumType.kNoChecksum)); .setChecksumType(ChecksumType.kNoChecksum));
if (comparator != null) { if (comparator != null) {
opts.setComparator(comparator); opts.setComparator(comparator);

5
src/main/java/it/cavallium/dbengine/database/disk/SnapshotsManager.java
View File

@ -88,12 +88,9 @@ public class SnapshotsManager {
throw new IOException("LLSnapshot " + snapshot.getSequenceNumber() + " not found!"); throw new IOException("LLSnapshot " + snapshot.getSequenceNumber() + " not found!");
} }
indexSnapshot.close();
var luceneIndexSnapshot = indexSnapshot.getSnapshot(); var luceneIndexSnapshot = indexSnapshot.getSnapshot();
snapshotter.release(luceneIndexSnapshot); snapshotter.release(luceneIndexSnapshot);
// Delete unused files after releasing the snapshot indexSnapshot.close();
indexWriter.deleteUnusedFiles();
return null; return null;
} finally { } finally {
activeTasks.arriveAndDeregister(); activeTasks.arriveAndDeregister();

19
src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveLocalSearcher.java
View File

@ -14,6 +14,9 @@ import reactor.core.scheduler.Schedulers;
public class AdaptiveLocalSearcher implements LocalSearcher { public class AdaptiveLocalSearcher implements LocalSearcher {
static final boolean FORCE_HUGE_PQ
= Boolean.parseBoolean(System.getProperty("it.cavallium.hugepq.force", "false"));
private static final StandardSearcher standardSearcher = new StandardSearcher(); private static final StandardSearcher standardSearcher = new StandardSearcher();
private static final LocalSearcher scoredPaged = new PagedLocalSearcher(); private static final LocalSearcher scoredPaged = new PagedLocalSearcher();
@ -34,8 +37,8 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
private final SortedScoredFullMultiSearcher sortedScoredFull; private final SortedScoredFullMultiSearcher sortedScoredFull;
public AdaptiveLocalSearcher(LLTempHugePqEnv env, boolean useHugePq, int maxInMemoryResultEntries) { public AdaptiveLocalSearcher(LLTempHugePqEnv env, boolean useHugePq, int maxInMemoryResultEntries) {
sortedByScoreFull = useHugePq ? new SortedByScoreFullMultiSearcher(env) : null; sortedByScoreFull = (FORCE_HUGE_PQ || useHugePq) ? new SortedByScoreFullMultiSearcher(env) : null;
sortedScoredFull = useHugePq ? new SortedScoredFullMultiSearcher(env) : null; sortedScoredFull = (FORCE_HUGE_PQ || useHugePq) ? new SortedScoredFullMultiSearcher(env) : null;
this.maxInMemoryResultEntries = maxInMemoryResultEntries; this.maxInMemoryResultEntries = maxInMemoryResultEntries;
} }
@ -73,16 +76,16 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
long maxAllowedInMemoryLimit long maxAllowedInMemoryLimit
= Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0)); = Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0));
if (queryParams.limitLong() == 0) { if (!FORCE_HUGE_PQ && queryParams.limitLong() == 0) {
return countSearcher.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer); return countSearcher.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
} else if (realLimit <= maxInMemoryResultEntries) { } else if (!FORCE_HUGE_PQ && realLimit <= maxInMemoryResultEntries) {
return standardSearcher.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer); return standardSearcher.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
} else if (queryParams.isSorted()) { } else if (FORCE_HUGE_PQ || queryParams.isSorted()) {
if (realLimit <= maxAllowedInMemoryLimit) { if (!FORCE_HUGE_PQ && realLimit <= maxAllowedInMemoryLimit) {
return scoredPaged.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer); return scoredPaged.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
} else { } else {
if (queryParams.isSortedByScore()) { if (queryParams.isSortedByScore()) {
if (queryParams.limitLong() < maxInMemoryResultEntries) { if (!FORCE_HUGE_PQ && queryParams.limitLong() < maxInMemoryResultEntries) {
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater"); throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
} }
if (sortedByScoreFull != null) { if (sortedByScoreFull != null) {
@ -91,7 +94,7 @@ public class AdaptiveLocalSearcher implements LocalSearcher {
return scoredPaged.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer); return scoredPaged.collect(Mono.just(indexSearcher), queryParams, keyFieldName, transformer);
} }
} else { } else {
if (queryParams.limitLong() < maxInMemoryResultEntries) { if (!FORCE_HUGE_PQ && queryParams.limitLong() < maxInMemoryResultEntries) {
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater"); throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
} }
if (sortedScoredFull != null) { if (sortedScoredFull != null) {

17
src/main/java/it/cavallium/dbengine/lucene/searcher/AdaptiveMultiSearcher.java
View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.lucene.searcher;
import static it.cavallium.dbengine.client.UninterruptibleScheduler.uninterruptibleScheduler; import static it.cavallium.dbengine.client.UninterruptibleScheduler.uninterruptibleScheduler;
import static it.cavallium.dbengine.database.LLUtils.singleOrClose; import static it.cavallium.dbengine.database.LLUtils.singleOrClose;
import static it.cavallium.dbengine.lucene.searcher.AdaptiveLocalSearcher.FORCE_HUGE_PQ;
import static it.cavallium.dbengine.lucene.searcher.GlobalQueryRewrite.NO_REWRITE; import static it.cavallium.dbengine.lucene.searcher.GlobalQueryRewrite.NO_REWRITE;
import io.netty5.buffer.api.Send; import io.netty5.buffer.api.Send;
@ -35,8 +36,8 @@ public class AdaptiveMultiSearcher implements MultiSearcher {
private final SortedScoredFullMultiSearcher sortedScoredFull; private final SortedScoredFullMultiSearcher sortedScoredFull;
public AdaptiveMultiSearcher(LLTempHugePqEnv env, boolean useHugePq, int maxInMemoryResultEntries) { public AdaptiveMultiSearcher(LLTempHugePqEnv env, boolean useHugePq, int maxInMemoryResultEntries) {
sortedByScoreFull = useHugePq ? new SortedByScoreFullMultiSearcher(env) : null; sortedByScoreFull = (FORCE_HUGE_PQ || useHugePq) ? new SortedByScoreFullMultiSearcher(env) : null;
sortedScoredFull = useHugePq ? new SortedScoredFullMultiSearcher(env) : null; sortedScoredFull = (FORCE_HUGE_PQ || useHugePq) ? new SortedScoredFullMultiSearcher(env) : null;
this.maxInMemoryResultEntries = maxInMemoryResultEntries; this.maxInMemoryResultEntries = maxInMemoryResultEntries;
} }
@ -66,16 +67,16 @@ public class AdaptiveMultiSearcher implements MultiSearcher {
long maxAllowedInMemoryLimit long maxAllowedInMemoryLimit
= Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0)); = Math.max(maxInMemoryResultEntries, (long) queryParams.pageLimits().getPageLimit(0));
if (queryParams.limitLong() == 0) { if (!FORCE_HUGE_PQ && queryParams.limitLong() == 0) {
return count.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer); return count.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
} else if (realLimit <= maxInMemoryResultEntries) { } else if (!FORCE_HUGE_PQ && realLimit <= maxInMemoryResultEntries) {
return standardSearcher.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer); return standardSearcher.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
} else if (queryParams.isSorted()) { } else if (FORCE_HUGE_PQ || queryParams.isSorted()) {
if (realLimit <= maxAllowedInMemoryLimit) { if (!FORCE_HUGE_PQ && realLimit <= maxAllowedInMemoryLimit) {
return scoredPaged.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer); return scoredPaged.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
} else { } else {
if (queryParams.isSortedByScore()) { if (queryParams.isSortedByScore()) {
if (queryParams.limitLong() < maxInMemoryResultEntries) { if (!FORCE_HUGE_PQ && queryParams.limitLong() < maxInMemoryResultEntries) {
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater"); throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
} }
if (sortedByScoreFull != null) { if (sortedByScoreFull != null) {
@ -84,7 +85,7 @@ public class AdaptiveMultiSearcher implements MultiSearcher {
return scoredPaged.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer); return scoredPaged.collectMulti(Mono.just(indexSearchers), queryParams, keyFieldName, transformer);
} }
} else { } else {
if (queryParams.limitLong() < maxInMemoryResultEntries) { if (!FORCE_HUGE_PQ && queryParams.limitLong() < maxInMemoryResultEntries) {
throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater"); throw new UnsupportedOperationException("Allowed limit is " + maxInMemoryResultEntries + " or greater");
} }
if (sortedScoredFull != null) { if (sortedScoredFull != null) {

16
src/test/java/it/cavallium/dbengine/TestHugePq.java
View File

@ -9,6 +9,8 @@ import it.cavallium.dbengine.lucene.HugePqPriorityQueue;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Function; import java.util.function.Function;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
@ -70,6 +72,20 @@ public class TestHugePq {
Assertions.assertEquals(0, queue.top()); Assertions.assertEquals(0, queue.top());
} }
@Test
public void testAddRandomMulti() {
var list = new ArrayList<Integer>(1000);
for (int i = 0; i < 1000; i++) {
var n = ThreadLocalRandom.current().nextInt(-20, 20);
queue.add(n);
list.add(n);
}
list.sort(Comparator.reverseOrder());
for (int i = 0; i < 1000; i++) {
Assertions.assertEquals(list.remove(list.size() - 1), queue.pop());
}
}
@Test @Test
public void testAddMultiClear() { public void testAddMultiClear() {
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {

18
src/test/java/it/cavallium/dbengine/TestHugePqHitQueue.java
View File

@ -11,8 +11,10 @@ import it.cavallium.dbengine.lucene.PriorityQueue;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Random; import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Function; import java.util.function.Function;
import org.apache.lucene.search.HitQueue; import org.apache.lucene.search.HitQueue;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
@ -148,6 +150,22 @@ public class TestHugePqHitQueue {
assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), new ScoreDoc(1, 0, -1), testingPriorityQueue.top()); assertEqualsScoreDoc(new TextDescription("top value of %s", testingPriorityQueue), new ScoreDoc(1, 0, -1), testingPriorityQueue.top());
} }
@Test
public void testAddMultiRandom() {
var list = new ArrayList<Integer>(1000);
for (int i = 0; i < 1000; i++) {
var ri = ThreadLocalRandom.current().nextInt(0, 20);
list.add(ri);
var item = new ScoreDoc(ri, ri << 1, ri % 4);
testingPriorityQueue.addUnsafe(item);
}
list.sort(Comparator.reverseOrder());
for (int i = 0; i < 1000; i++) {
var top = list.remove(list.size() - 1);
assertEqualsScoreDoc(new TextDescription("%d value of %s", i, testingPriorityQueue), new ScoreDoc(top, top << 1, top % 4), testingPriorityQueue.pop());
}
}
@Test @Test
public void testAddMultiClear() { public void testAddMultiClear() {
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {