andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update rocksdb

Browse Source
This commit is contained in:
Andrea Cavalli 2021-03-14 18:51:42 +01:00
parent 05e8c87015
commit 1aed618ca5
6 changed files with 16 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -138,7 +138,7 @@
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>6.15.2</version>
<version>6.15.5</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>

5
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java
View File

@ -249,10 +249,9 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implem
} else {
return dictionary
.getRangeKeyPrefixes(resolveSnapshot(snapshot), range, keyPrefix.length + keySuffixLength)
.flatMapSequential(groupKeyWithExt -> {
byte[] groupKeyWithoutExt = removeExtFromFullKey(groupKeyWithExt);
.flatMapSequential(groupKeyWithoutExt -> {
byte[] groupSuffix = this.stripPrefix(groupKeyWithoutExt);
assert subStageKeysConsistency(groupKeyWithExt.length);
assert subStageKeysConsistency(groupKeyWithoutExt.length + keyExtLength);
return this.subStageGetter
.subStage(dictionary,
snapshot,

18
src/main/java/it/cavallium/dbengine/database/disk/LLLocalDictionary.java
View File

@ -48,6 +48,8 @@ public class LLLocalDictionary implements LLDictionary {
static final int CAPPED_WRITE_BATCH_CAP = 50000; // 50K operations
static final int MULTI_GET_WINDOW = 500;
static final WriteOptions BATCH_WRITE_OPTIONS = new WriteOptions().setLowPri(true);
static final boolean PREFER_ALWAYS_SEEK_TO_FIRST = true;
static final boolean ALWAYS_VERIFY_CHECKSUMS = true;
private static final int STRIPES = 512;
private static final byte[] FIRST_KEY = new byte[]{};
@ -167,7 +169,7 @@ public class LLLocalDictionary implements LLDictionary {
return Mono
.fromCallable(() -> {
var readOpts = resolveSnapshot(snapshot);
readOpts.setVerifyChecksums(false);
readOpts.setVerifyChecksums(ALWAYS_VERIFY_CHECKSUMS);
readOpts.setFillCache(false);
if (range.hasMin()) {
readOpts.setIterateLowerBound(new Slice(range.getMin()));
@ -698,7 +700,7 @@ public class LLLocalDictionary implements LLDictionary {
return Mono
.<Void>fromCallable(() -> {
var readOpts = getReadOptions(null);
readOpts.setVerifyChecksums(false);
readOpts.setVerifyChecksums(ALWAYS_VERIFY_CHECKSUMS);
// readOpts.setIgnoreRangeDeletions(true);
readOpts.setFillCache(false);
@ -752,7 +754,7 @@ public class LLLocalDictionary implements LLDictionary {
.fromCallable(() -> {
var readOpts = resolveSnapshot(snapshot);
readOpts.setFillCache(false);
readOpts.setVerifyChecksums(false);
readOpts.setVerifyChecksums(ALWAYS_VERIFY_CHECKSUMS);
if (range.hasMin()) {
readOpts.setIterateLowerBound(new Slice(range.getMin()));
}
@ -840,12 +842,8 @@ public class LLLocalDictionary implements LLDictionary {
}
} else {
rocksdbSnapshot.setFillCache(false);
rocksdbSnapshot.setVerifyChecksums(false);
rocksdbSnapshot.setPinData(false);
rocksdbSnapshot.setVerifyChecksums(ALWAYS_VERIFY_CHECKSUMS);
rocksdbSnapshot.setIgnoreRangeDeletions(false);
if (snapshot == null) {
rocksdbSnapshot.setTailing(true);
}
long count = 0;
try (RocksIterator iter = db.newIterator(cfh, rocksdbSnapshot)) {
iter.seekToFirst();
@ -862,7 +860,7 @@ public class LLLocalDictionary implements LLDictionary {
private long exactSizeAll(@Nullable LLSnapshot snapshot) {
var readOpts = resolveSnapshot(snapshot);
readOpts.setFillCache(false);
readOpts.setVerifyChecksums(false);
readOpts.setVerifyChecksums(ALWAYS_VERIFY_CHECKSUMS);
long count = 0;
try (RocksIterator iter = db.newIterator(cfh, readOpts)) {
@ -887,7 +885,7 @@ public class LLLocalDictionary implements LLDictionary {
readOpts.setIterateUpperBound(new Slice(range.getMax()));
}
try (RocksIterator iter = db.newIterator(cfh, readOpts)) {
if (range.hasMin()) {
if (!LLLocalDictionary.PREFER_ALWAYS_SEEK_TO_FIRST && range.hasMin()) {
iter.seek(range.getMin());
} else {
iter.seekToFirst();

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneGroupedReactiveIterator.java
View File

@ -53,7 +53,7 @@ public abstract class LLLocalLuceneGroupedReactiveIterator<T> {
readOptions.setIterateUpperBound(new Slice(range.getMax()));
}
var rocksIterator = db.newIterator(cfh, readOptions);
if (range.hasMin()) {
if (!LLLocalDictionary.PREFER_ALWAYS_SEEK_TO_FIRST && range.hasMin()) {
rocksIterator.seek(range.getMin());
} else {
rocksIterator.seekToFirst();

4
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneKeyPrefixesReactiveIterator.java
View File

@ -39,6 +39,7 @@ public class LLLocalLuceneKeyPrefixesReactiveIterator {
return Flux
.generate(() -> {
synchronized (this) {
System.out.println(Thread.currentThread().getName());
var readOptions = new ReadOptions(this.readOptions);
readOptions.setFillCache(range.hasMin() && range.hasMax());
if (range.hasMin()) {
@ -48,7 +49,7 @@ public class LLLocalLuceneKeyPrefixesReactiveIterator {
readOptions.setIterateUpperBound(new Slice(range.getMax()));
}
var rocksIterator = db.newIterator(cfh, readOptions);
if (range.hasMin()) {
if (!LLLocalDictionary.PREFER_ALWAYS_SEEK_TO_FIRST && range.hasMin()) {
rocksIterator.seek(range.getMin());
} else {
rocksIterator.seekToFirst();
@ -57,6 +58,7 @@ public class LLLocalLuceneKeyPrefixesReactiveIterator {
}
}, (rocksIterator, sink) -> {
synchronized (this) {
System.out.println(Thread.currentThread().getName());
byte[] firstGroupKey = null;
while (rocksIterator.isValid()) {

2
src/main/java/it/cavallium/dbengine/database/disk/LLLocalLuceneReactiveIterator.java
View File

@ -43,7 +43,7 @@ public abstract class LLLocalLuceneReactiveIterator<T> {
readOptions.setIterateUpperBound(new Slice(range.getMax()));
}
var rocksIterator = db.newIterator(cfh, readOptions);
if (range.hasMin()) {
if (!LLLocalDictionary.PREFER_ALWAYS_SEEK_TO_FIRST && range.hasMin()) {
rocksIterator.seek(range.getMin());
} else {
rocksIterator.seekToFirst();