andreacavalli/CavalliumDBEngine
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve suffix performance

Browse Source
This commit is contained in:
Andrea Cavalli 2022-03-16 22:41:51 +01:00
parent ba3765eece
commit 59c37c0fc9
2 changed files with 71 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
src/main/java/it/cavallium/dbengine/database/collections/DatabaseMapDictionaryDeep.java
View File

@ -2,6 +2,7 @@ package it.cavallium.dbengine.database.collections;
import io.netty5.buffer.api.Buffer; import io.netty5.buffer.api.Buffer;
import io.netty5.buffer.api.BufferAllocator; import io.netty5.buffer.api.BufferAllocator;
import io.netty5.buffer.api.DefaultBufferAllocators;
import io.netty5.buffer.api.Drop; import io.netty5.buffer.api.Drop;
import io.netty5.buffer.api.Owned; import io.netty5.buffer.api.Owned;
import io.netty5.buffer.api.Resource; import io.netty5.buffer.api.Resource;
@ -53,6 +54,13 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
} catch (Throwable ex) { } catch (Throwable ex) {
LOG.error("Failed to close keyPrefix", ex); LOG.error("Failed to close keyPrefix", ex);
} }
try {
if (obj.keySuffixAndExtZeroBuffer != null) {
obj.keySuffixAndExtZeroBuffer.close();
}
} catch (Throwable ex) {
LOG.error("Failed to close keySuffixAndExtZeroBuffer", ex);
}
try { try {
if (obj.onClose != null) { if (obj.onClose != null) {
obj.onClose.run(); obj.onClose.run();
@ -85,6 +93,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
protected LLRange range; protected LLRange range;
protected Buffer keyPrefix; protected Buffer keyPrefix;
protected Buffer keySuffixAndExtZeroBuffer;
protected Runnable onClose; protected Runnable onClose;
private static void incrementPrefix(Buffer prefix, int prefixLength) { private static void incrementPrefix(Buffer prefix, int prefixLength) {
@ -119,26 +128,48 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
} }
} }
static void firstRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) { static void firstRangeKey(Buffer prefixKey, int prefixLength, Buffer suffixAndExtZeroes) {
zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixLength, extLength); zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
} }
static void nextRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) { static void nextRangeKey(Buffer prefixKey, int prefixLength, Buffer suffixAndExtZeroes) {
zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixLength, extLength); zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
incrementPrefix(prefixKey, prefixLength); incrementPrefix(prefixKey, prefixLength);
} }
@Deprecated
static void firstRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) {
try (var zeroBuf = DefaultBufferAllocators.offHeapAllocator().allocate(suffixLength + extLength)) {
zeroBuf.fill((byte) 0);
zeroBuf.writerOffset(suffixLength + extLength);
zeroFillKeySuffixAndExt(prefixKey, prefixLength, zeroBuf);
}
}
@Deprecated
static void nextRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) {
try (var zeroBuf = DefaultBufferAllocators.offHeapAllocator().allocate(suffixLength + extLength)) {
zeroBuf.fill((byte) 0);
zeroBuf.writerOffset(suffixLength + extLength);
zeroFillKeySuffixAndExt(prefixKey, prefixLength, zeroBuf);
incrementPrefix(prefixKey, prefixLength);
}
}
protected static void zeroFillKeySuffixAndExt(@NotNull Buffer prefixKey, protected static void zeroFillKeySuffixAndExt(@NotNull Buffer prefixKey,
int prefixLength, int suffixLength, int extLength) { int prefixLength, Buffer suffixAndExtZeroes) {
//noinspection UnnecessaryLocalVariable //noinspection UnnecessaryLocalVariable
var result = prefixKey; var result = prefixKey;
var suffixLengthAndExtLength = suffixAndExtZeroes.readableBytes();
assert result.readableBytes() == prefixLength; assert result.readableBytes() == prefixLength;
assert suffixLength > 0; assert suffixLengthAndExtLength > 0 : "Suffix length + ext length is < 0: " + suffixLengthAndExtLength;
assert extLength >= 0; prefixKey.ensureWritable(suffixLengthAndExtLength);
result.ensureWritable(suffixLength + extLength, suffixLength + extLength, true); suffixAndExtZeroes.copyInto(suffixAndExtZeroes.readerOffset(),
for (int i = 0; i < suffixLength + extLength; i++) { prefixKey,
result.writeByte((byte) 0x0); prefixKey.writerOffset(),
} suffixLengthAndExtLength
);
prefixKey.skipWritable(suffixLengthAndExtLength);
} }
/** /**
@ -180,14 +211,25 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
this.keyPrefixLength = prefixKey == null ? 0 : prefixKey.readableBytes(); this.keyPrefixLength = prefixKey == null ? 0 : prefixKey.readableBytes();
this.keySuffixLength = keySuffixSerializer.getSerializedBinaryLength(); this.keySuffixLength = keySuffixSerializer.getSerializedBinaryLength();
this.keyExtLength = keyExtLength; this.keyExtLength = keyExtLength;
this.keySuffixAndExtZeroBuffer = alloc
.allocate(keySuffixLength + keyExtLength)
.fill((byte) 0)
.writerOffset(keySuffixLength + keyExtLength)
.makeReadOnly();
assert keySuffixAndExtZeroBuffer.readableBytes() == keySuffixLength + keyExtLength :
"Key suffix and ext zero buffer readable length is not equal"
+ " to the key suffix length + key ext length. keySuffixAndExtZeroBuffer="
+ keySuffixAndExtZeroBuffer.readableBytes() + " keySuffixLength=" + keySuffixLength + " keyExtLength="
+ keyExtLength;
assert keySuffixAndExtZeroBuffer.readableBytes() > 0;
var firstKey = prefixKey == null ? alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength) var firstKey = prefixKey == null ? alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength)
: prefixKey.copy(); : prefixKey.copy();
try { try {
firstRangeKey(firstKey, keyPrefixLength, keySuffixLength, keyExtLength); firstRangeKey(firstKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
var nextRangeKey = prefixKey == null ? alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength) var nextRangeKey = prefixKey == null ? alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength)
: prefixKey.copy(); : prefixKey.copy();
try { try {
nextRangeKey(nextRangeKey, keyPrefixLength, keySuffixLength, keyExtLength); nextRangeKey(nextRangeKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
assert prefixKey == null || prefixKey.isAccessible(); assert prefixKey == null || prefixKey.isAccessible();
assert keyPrefixLength == 0 || !LLUtils.equals(firstKey, nextRangeKey); assert keyPrefixLength == 0 || !LLUtils.equals(firstKey, nextRangeKey);
if (keyPrefixLength == 0) { if (keyPrefixLength == 0) {
@ -211,6 +253,9 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
this.keyPrefix = prefixKey; this.keyPrefix = prefixKey;
this.onClose = onClose; this.onClose = onClose;
} catch (Throwable t) { } catch (Throwable t) {
if (this.keySuffixAndExtZeroBuffer != null && keySuffixAndExtZeroBuffer.isAccessible()) {
keySuffixAndExtZeroBuffer.close();
}
if (prefixKey != null && prefixKey.isAccessible()) { if (prefixKey != null && prefixKey.isAccessible()) {
prefixKey.close(); prefixKey.close();
} }
@ -229,6 +274,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
Mono<Send<LLRange>> rangeMono, Mono<Send<LLRange>> rangeMono,
Send<LLRange> range, Send<LLRange> range,
Send<Buffer> keyPrefix, Send<Buffer> keyPrefix,
Send<Buffer> keySuffixAndExtZeroBuffer,
Runnable onClose) { Runnable onClose) {
super((Drop<DatabaseMapDictionaryDeep<T,U,US>>) (Drop) DROP); super((Drop<DatabaseMapDictionaryDeep<T,U,US>>) (Drop) DROP);
this.dictionary = dictionary; this.dictionary = dictionary;
@ -242,6 +288,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
this.range = range.receive(); this.range = range.receive();
this.keyPrefix = keyPrefix.receive(); this.keyPrefix = keyPrefix.receive();
this.keySuffixAndExtZeroBuffer = keySuffixAndExtZeroBuffer.receive();
this.onClose = onClose; this.onClose = onClose;
} }
@ -323,7 +370,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
groupKeyWithoutExtSend_::receive, groupKeyWithoutExtSend_::receive,
groupKeyWithoutExtSend -> this.subStageGetter groupKeyWithoutExtSend -> this.subStageGetter
.subStage(dictionary, snapshot, Mono.fromCallable(() -> groupKeyWithoutExtSend.copy().send())) .subStage(dictionary, snapshot, Mono.fromCallable(() -> groupKeyWithoutExtSend.copy().send()))
.<Entry<T, US>>handle((us, sink) -> { .handle((us, sink) -> {
T deserializedSuffix; T deserializedSuffix;
try (var splittedGroupSuffix = splitGroupSuffix(groupKeyWithoutExtSend)) { try (var splittedGroupSuffix = splitGroupSuffix(groupKeyWithoutExtSend)) {
deserializedSuffix = this.deserializeSuffix(splittedGroupSuffix); deserializedSuffix = this.deserializeSuffix(splittedGroupSuffix);
@ -413,7 +460,8 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
@Override @Override
protected Owned<DatabaseMapDictionaryDeep<T, U, US>> prepareSend() { protected Owned<DatabaseMapDictionaryDeep<T, U, US>> prepareSend() {
var keyPrefix = this.keyPrefix == null ? null : this.keyPrefix.send(); var keyPrefix = this.keyPrefix.send();
var keySuffixAndExtZeroBuffer = this.keySuffixAndExtZeroBuffer.send();
var range = this.range.send(); var range = this.range.send();
var onClose = this.onClose; var onClose = this.onClose;
return drop -> { return drop -> {
@ -427,6 +475,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
rangeMono, rangeMono,
range, range,
keyPrefix, keyPrefix,
keySuffixAndExtZeroBuffer,
onClose onClose
); );
drop.attach(instance); drop.attach(instance);
@ -437,6 +486,7 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
@Override @Override
protected void makeInaccessible() { protected void makeInaccessible() {
this.keyPrefix = null; this.keyPrefix = null;
this.keySuffixAndExtZeroBuffer = null;
this.range = null; this.range = null;
this.onClose = null; this.onClose = null;
} }

8
src/test/java/it/cavallium/dbengine/database/collections/TestRanges.java
View File

@ -1,5 +1,7 @@
package it.cavallium.dbengine.database.collections; package it.cavallium.dbengine.database.collections;
import static io.netty5.buffer.Unpooled.wrappedBuffer;
import io.netty5.buffer.Unpooled; import io.netty5.buffer.Unpooled;
import io.netty5.buffer.api.Buffer; import io.netty5.buffer.api.Buffer;
import io.netty5.buffer.api.BufferAllocator; import io.netty5.buffer.api.BufferAllocator;
@ -9,7 +11,6 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static io.netty5.buffer.Unpooled.*;
public class TestRanges { public class TestRanges {
@ -63,7 +64,10 @@ public class TestRanges {
} }
if (Arrays.equals(prefixKey, new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF})) { if (Arrays.equals(prefixKey, new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF})) {
Assertions.assertArrayEquals(new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, 0}, nextRangeKey); org.assertj.core.api.Assertions
.assertThat(nextRangeKey)
.isEqualTo(new byte[]{(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF,
(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, 0});
} else { } else {
long biPrefix = 0; long biPrefix = 0;
var s = 0; var s = 0;