andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

jni: expose memtable_whole_key_filtering option (#9394)

Browse Source 
Summary:
refer to: https://github.com/facebook/rocksdb/wiki/Prefix-Seek#configure-prefix-bloom-filter

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9394

Reviewed By: mrambacher

Differential Revision: D33671533

Pulled By: ajkr

fbshipit-source-id: d90db1712efdd5dd65020329867381d6b3cf2626
This commit is contained in:
Jermy Li 2022-02-04 16:00:19 -08:00 committed by Facebook GitHub Bot
parent afc280fdfd
commit 83ff350ff2
10 changed files with 154 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/rocksdb/advanced_options.h
View File

@ -377,7 +377,7 @@ struct AdvancedColumnFamilyOptions {
//
// If this value is larger than 0.25, it is sanitized to 0.25.
//
// Default: 0 (disable)
// Default: 0 (disabled)
//
// Dynamically changeable through SetOptions() API
double memtable_prefix_bloom_size_ratio = 0.0;
@ -386,7 +386,7 @@ struct AdvancedColumnFamilyOptions {
// if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering
// can potentially reduce CPU usage for point-look-ups.
//
// Default: false (disable)
// Default: false (disabled)
//
// Dynamically changeable through SetOptions() API
bool memtable_whole_key_filtering = false;
@ -421,7 +421,7 @@ struct AdvancedColumnFamilyOptions {
// example would be updating the same key over and over again, in which case
// the prefix can be the key itself.
//
// Default: nullptr (disable)
// Default: nullptr (disabled)
std::shared_ptr<const SliceTransform>
memtable_insert_with_hint_prefix_extractor = nullptr;

7
java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java
View File

@ -569,6 +569,7 @@ public class DbBenchmark {
options.setMinWriteBufferNumberToMerge(
(Integer)flags_.get(Flag.min_write_buffer_number_to_merge));
options.setMemtablePrefixBloomSizeRatio((Double) flags_.get(Flag.memtable_bloom_size_ratio));
options.setMemtableWholeKeyFiltering((Boolean) flags_.get(Flag.memtable_whole_key_filtering));
options.setNumLevels(
(Integer)flags_.get(Flag.num_levels));
options.setTargetFileSizeBase(
@ -1190,6 +1191,12 @@ public class DbBenchmark {
return Double.parseDouble(value);
}
},
memtable_whole_key_filtering(false, "Enable whole key bloom filter in memtable.") {
@Override
public Object parseValue(String value) {
return parseBoolean(value);
}
},
cache_numshardbits(-1,"Number of shards for the block cache\n" +
"\tis 2 ** cache_numshardbits. Negative means use default settings.\n" +
"\tThis is applied only if FLAGS_cache_size is non-negative.") {

46
java/rocksjni/options.cc
View File

@ -3124,6 +3124,29 @@ void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio(
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*
* Class: org_rocksdb_Options
* Method: memtableWholeKeyFiltering
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_Options_memtableWholeKeyFiltering(JNIEnv*, jobject,
jlong jhandle) {
return reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)
->memtable_whole_key_filtering;
}
/*
* Class: org_rocksdb_Options
* Method: setMemtableWholeKeyFiltering
* Signature: (JZ)V
*/
void Java_org_rocksdb_Options_setMemtableWholeKeyFiltering(
JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) {
reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)
->memtable_whole_key_filtering =
static_cast<bool>(jmemtable_whole_key_filtering);
}
/*
* Class: org_rocksdb_Options
* Method: bloomLocality
@ -4909,6 +4932,29 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio(
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: memtableWholeKeyFiltering
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_ColumnFamilyOptions_memtableWholeKeyFiltering(
JNIEnv*, jobject, jlong jhandle) {
return reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
->memtable_whole_key_filtering;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtableWholeKeyFiltering
* Signature: (JZ)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering(
JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) {
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
->memtable_whole_key_filtering =
static_cast<bool>(jmemtable_whole_key_filtering);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: bloomLocality

33
java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java
View File

@ -60,9 +60,10 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
* write_buffer_size * memtable_prefix_bloom_size_ratio.
* If it is larger than 0.25, it is santinized to 0.25.
*
* Default: 0 (disable)
* Default: 0 (disabled)
*
* @param memtablePrefixBloomSizeRatio The ratio
* @param memtablePrefixBloomSizeRatio the ratio of memtable used by the
* bloom filter, 0 means no bloom filter
* @return the reference to the current options.
*/
T setMemtablePrefixBloomSizeRatio(
@ -74,12 +75,32 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
* write_buffer_size * memtable_prefix_bloom_size_ratio.
* If it is larger than 0.25, it is santinized to 0.25.
*
* Default: 0 (disable)
* Default: 0 (disabled)
*
* @return the ratio
* @return the ratio of memtable used by the bloom filter
*/
double memtablePrefixBloomSizeRatio();
/**
* Enable whole key bloom filter in memtable. Note this will only take effect
* if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering
* can potentially reduce CPU usage for point-look-ups.
*
* Default: false (disabled)
*
* @param memtableWholeKeyFiltering true if whole key bloom filter is enabled
* in memtable
* @return the reference to the current options.
*/
T setMemtableWholeKeyFiltering(boolean memtableWholeKeyFiltering);
/**
* Returns whether whole key bloom filter is enabled in memtable
*
* @return true if whole key bloom filter is enabled in memtable
*/
boolean memtableWholeKeyFiltering();
/**
* Page size for huge page TLB for bloom in memtable. If &le; 0, not allocate
* from huge page TLB but from malloc.
@ -546,7 +567,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
* Dynamically changeable through
* {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}.
*
* @return true iff blob files are enabled
* @return true if blob files are enabled
*/
boolean enableBlobFiles();
@ -656,7 +677,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
*
* Default: false
*
* @return true iff blob garbage collection is currently enabled.
* @return true if blob garbage collection is currently enabled.
*/
boolean enableBlobGarbageCollection();

13
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
View File

@ -683,6 +683,17 @@ public class ColumnFamilyOptions extends RocksObject
return memtablePrefixBloomSizeRatio(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) {
setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering);
return this;
}
@Override
public boolean memtableWholeKeyFiltering() {
return memtableWholeKeyFiltering(nativeHandle_);
}
@Override
public ColumnFamilyOptions setBloomLocality(int bloomLocality) {
setBloomLocality(nativeHandle_, bloomLocality);
@ -1319,6 +1330,8 @@ public class ColumnFamilyOptions extends RocksObject
private native void setMemtablePrefixBloomSizeRatio(
long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering);
private native boolean memtableWholeKeyFiltering(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

12
java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
View File

@ -66,6 +66,7 @@ public class MutableColumnFamilyOptions
write_buffer_size(ValueType.LONG),
arena_block_size(ValueType.LONG),
memtable_prefix_bloom_size_ratio(ValueType.DOUBLE),
memtable_whole_key_filtering(ValueType.BOOLEAN),
@Deprecated memtable_prefix_bloom_bits(ValueType.INT),
@Deprecated memtable_prefix_bloom_probes(ValueType.INT),
memtable_huge_page_size(ValueType.LONG),
@ -226,6 +227,17 @@ public class MutableColumnFamilyOptions
return getDouble(MemtableOption.memtable_prefix_bloom_size_ratio);
}
@Override
public MutableColumnFamilyOptionsBuilder setMemtableWholeKeyFiltering(
final boolean memtableWholeKeyFiltering) {
return setBoolean(MemtableOption.memtable_whole_key_filtering, memtableWholeKeyFiltering);
}
@Override
public boolean memtableWholeKeyFiltering() {
return getBoolean(MemtableOption.memtable_whole_key_filtering);
}
@Override
public MutableColumnFamilyOptionsBuilder setMemtableHugePageSize(
final long memtableHugePageSize) {

13
java/src/main/java/org/rocksdb/Options.java
View File

@ -1638,6 +1638,17 @@ public class Options extends RocksObject
return this;
}
@Override
public boolean memtableWholeKeyFiltering() {
return memtableWholeKeyFiltering(nativeHandle_);
}
@Override
public Options setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) {
setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering);
return this;
}
@Override
public int bloomLocality() {
return bloomLocality(nativeHandle_);
@ -2405,6 +2416,8 @@ public class Options extends RocksObject
private native void setMemtablePrefixBloomSizeRatio(
long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering);
private native boolean memtableWholeKeyFiltering(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

9
java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
View File

@ -331,6 +331,15 @@ public class ColumnFamilyOptionsTest {
}
}
@Test
public void memtableWholeKeyFiltering() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final boolean booleanValue = rand.nextBoolean();
opt.setMemtableWholeKeyFiltering(booleanValue);
assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue);
}
}
@Test
public void memtableHugePageSize() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {

20
java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java
View File

@ -48,6 +48,7 @@ public class MutableOptionsGetSetTest {
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableWholeKeyFiltering(false)
.setMemtableHugePageSize(3)
.setMaxSuccessiveMerges(4)
.setMaxWriteBufferNumber(12)
@ -70,6 +71,7 @@ public class MutableOptionsGetSetTest {
.setEnableBlobFiles(false)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.236)
.setMemtableWholeKeyFiltering(true)
.setMemtableHugePageSize(8)
.setMaxSuccessiveMerges(12)
.setMaxWriteBufferNumber(22)
@ -104,8 +106,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);
@ -127,8 +130,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder2.enableBlobFiles()).isEqualTo(false);
assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder2.arenaBlockSize()).isEqualTo(42);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236);
assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12);
assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22);
assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160);
@ -192,6 +196,7 @@ public class MutableOptionsGetSetTest {
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableWholeKeyFiltering(false)
.setMemtableHugePageSize(3)
.setMaxSuccessiveMerges(4)
.setMaxWriteBufferNumber(12)
@ -215,8 +220,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);
@ -237,6 +243,7 @@ public class MutableOptionsGetSetTest {
.setEnableBlobFiles(false)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.236)
.setMemtableWholeKeyFiltering(true)
.setMemtableHugePageSize(8)
.setMaxSuccessiveMerges(12)
.setMaxWriteBufferNumber(22)
@ -258,8 +265,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder2.enableBlobFiles()).isEqualTo(false);
assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder2.arenaBlockSize()).isEqualTo(42);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236);
assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12);
assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22);
assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160);
@ -306,6 +314,7 @@ public class MutableOptionsGetSetTest {
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableWholeKeyFiltering(false)
.setMemtableHugePageSize(3)
.setMaxSuccessiveMerges(4)
.setMaxWriteBufferNumber(12)
@ -328,8 +337,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);

9
java/src/test/java/org/rocksdb/OptionsTest.java
View File

@ -274,6 +274,15 @@ public class OptionsTest {
}
}
@Test
public void memtableWholeKeyFiltering() {
try (final Options opt = new Options()) {
final boolean booleanValue = rand.nextBoolean();
opt.setMemtableWholeKeyFiltering(booleanValue);
assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue);
}
}
@Test
public void memtableHugePageSize() {
try (final Options opt = new Options()) {