jni: expose memtable_whole_key_filtering option (#9394)

Summary:
refer to: https://github.com/facebook/rocksdb/wiki/Prefix-Seek#configure-prefix-bloom-filter

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9394

Reviewed By: mrambacher

Differential Revision: D33671533

Pulled By: ajkr

fbshipit-source-id: d90db1712efdd5dd65020329867381d6b3cf2626
This commit is contained in:
Jermy Li 2022-02-04 16:00:19 -08:00 committed by Facebook GitHub Bot
parent afc280fdfd
commit 83ff350ff2
10 changed files with 154 additions and 14 deletions

View File

@ -377,7 +377,7 @@ struct AdvancedColumnFamilyOptions {
//
// If this value is larger than 0.25, it is sanitized to 0.25.
//
// Default: 0 (disable)
// Default: 0 (disabled)
//
// Dynamically changeable through SetOptions() API
double memtable_prefix_bloom_size_ratio = 0.0;
@ -386,7 +386,7 @@ struct AdvancedColumnFamilyOptions {
// if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering
// can potentially reduce CPU usage for point-look-ups.
//
// Default: false (disable)
// Default: false (disabled)
//
// Dynamically changeable through SetOptions() API
bool memtable_whole_key_filtering = false;
@ -421,7 +421,7 @@ struct AdvancedColumnFamilyOptions {
// example would be updating the same key over and over again, in which case
// the prefix can be the key itself.
//
// Default: nullptr (disable)
// Default: nullptr (disabled)
std::shared_ptr<const SliceTransform>
memtable_insert_with_hint_prefix_extractor = nullptr;

View File

@ -569,6 +569,7 @@ public class DbBenchmark {
options.setMinWriteBufferNumberToMerge(
(Integer)flags_.get(Flag.min_write_buffer_number_to_merge));
options.setMemtablePrefixBloomSizeRatio((Double) flags_.get(Flag.memtable_bloom_size_ratio));
options.setMemtableWholeKeyFiltering((Boolean) flags_.get(Flag.memtable_whole_key_filtering));
options.setNumLevels(
(Integer)flags_.get(Flag.num_levels));
options.setTargetFileSizeBase(
@ -1190,6 +1191,12 @@ public class DbBenchmark {
return Double.parseDouble(value);
}
},
memtable_whole_key_filtering(false, "Enable whole key bloom filter in memtable.") {
@Override
public Object parseValue(String value) {
return parseBoolean(value);
}
},
cache_numshardbits(-1,"Number of shards for the block cache\n" +
"\tis 2 ** cache_numshardbits. Negative means use default settings.\n" +
"\tThis is applied only if FLAGS_cache_size is non-negative.") {

View File

@ -3124,6 +3124,29 @@ void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio(
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*
* Class: org_rocksdb_Options
* Method: memtableWholeKeyFiltering
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_Options_memtableWholeKeyFiltering(JNIEnv*, jobject,
jlong jhandle) {
return reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)
->memtable_whole_key_filtering;
}
/*
* Class: org_rocksdb_Options
* Method: setMemtableWholeKeyFiltering
* Signature: (JZ)V
*/
void Java_org_rocksdb_Options_setMemtableWholeKeyFiltering(
JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) {
reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)
->memtable_whole_key_filtering =
static_cast<bool>(jmemtable_whole_key_filtering);
}
/*
* Class: org_rocksdb_Options
* Method: bloomLocality
@ -4909,6 +4932,29 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio(
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: memtableWholeKeyFiltering
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_ColumnFamilyOptions_memtableWholeKeyFiltering(
JNIEnv*, jobject, jlong jhandle) {
return reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
->memtable_whole_key_filtering;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtableWholeKeyFiltering
* Signature: (JZ)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering(
JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) {
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
->memtable_whole_key_filtering =
static_cast<bool>(jmemtable_whole_key_filtering);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: bloomLocality

View File

@ -60,9 +60,10 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
* write_buffer_size * memtable_prefix_bloom_size_ratio.
* If it is larger than 0.25, it is santinized to 0.25.
*
* Default: 0 (disable)
* Default: 0 (disabled)
*
* @param memtablePrefixBloomSizeRatio The ratio
* @param memtablePrefixBloomSizeRatio the ratio of memtable used by the
* bloom filter, 0 means no bloom filter
* @return the reference to the current options.
*/
T setMemtablePrefixBloomSizeRatio(
@ -74,12 +75,32 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
* write_buffer_size * memtable_prefix_bloom_size_ratio.
* If it is larger than 0.25, it is santinized to 0.25.
*
* Default: 0 (disable)
* Default: 0 (disabled)
*
* @return the ratio
* @return the ratio of memtable used by the bloom filter
*/
double memtablePrefixBloomSizeRatio();
/**
* Enable whole key bloom filter in memtable. Note this will only take effect
* if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering
* can potentially reduce CPU usage for point-look-ups.
*
* Default: false (disabled)
*
* @param memtableWholeKeyFiltering true if whole key bloom filter is enabled
* in memtable
* @return the reference to the current options.
*/
T setMemtableWholeKeyFiltering(boolean memtableWholeKeyFiltering);
/**
* Returns whether whole key bloom filter is enabled in memtable
*
* @return true if whole key bloom filter is enabled in memtable
*/
boolean memtableWholeKeyFiltering();
/**
* Page size for huge page TLB for bloom in memtable. If &le; 0, not allocate
* from huge page TLB but from malloc.
@ -546,7 +567,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
* Dynamically changeable through
* {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}.
*
* @return true iff blob files are enabled
* @return true if blob files are enabled
*/
boolean enableBlobFiles();
@ -656,7 +677,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
*
* Default: false
*
* @return true iff blob garbage collection is currently enabled.
* @return true if blob garbage collection is currently enabled.
*/
boolean enableBlobGarbageCollection();

View File

@ -683,6 +683,17 @@ public class ColumnFamilyOptions extends RocksObject
return memtablePrefixBloomSizeRatio(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) {
setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering);
return this;
}
@Override
public boolean memtableWholeKeyFiltering() {
return memtableWholeKeyFiltering(nativeHandle_);
}
@Override
public ColumnFamilyOptions setBloomLocality(int bloomLocality) {
setBloomLocality(nativeHandle_, bloomLocality);
@ -1319,6 +1330,8 @@ public class ColumnFamilyOptions extends RocksObject
private native void setMemtablePrefixBloomSizeRatio(
long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering);
private native boolean memtableWholeKeyFiltering(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

View File

@ -66,6 +66,7 @@ public class MutableColumnFamilyOptions
write_buffer_size(ValueType.LONG),
arena_block_size(ValueType.LONG),
memtable_prefix_bloom_size_ratio(ValueType.DOUBLE),
memtable_whole_key_filtering(ValueType.BOOLEAN),
@Deprecated memtable_prefix_bloom_bits(ValueType.INT),
@Deprecated memtable_prefix_bloom_probes(ValueType.INT),
memtable_huge_page_size(ValueType.LONG),
@ -226,6 +227,17 @@ public class MutableColumnFamilyOptions
return getDouble(MemtableOption.memtable_prefix_bloom_size_ratio);
}
@Override
public MutableColumnFamilyOptionsBuilder setMemtableWholeKeyFiltering(
final boolean memtableWholeKeyFiltering) {
return setBoolean(MemtableOption.memtable_whole_key_filtering, memtableWholeKeyFiltering);
}
@Override
public boolean memtableWholeKeyFiltering() {
return getBoolean(MemtableOption.memtable_whole_key_filtering);
}
@Override
public MutableColumnFamilyOptionsBuilder setMemtableHugePageSize(
final long memtableHugePageSize) {

View File

@ -1638,6 +1638,17 @@ public class Options extends RocksObject
return this;
}
@Override
public boolean memtableWholeKeyFiltering() {
return memtableWholeKeyFiltering(nativeHandle_);
}
@Override
public Options setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) {
setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering);
return this;
}
@Override
public int bloomLocality() {
return bloomLocality(nativeHandle_);
@ -2405,6 +2416,8 @@ public class Options extends RocksObject
private native void setMemtablePrefixBloomSizeRatio(
long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering);
private native boolean memtableWholeKeyFiltering(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

View File

@ -331,6 +331,15 @@ public class ColumnFamilyOptionsTest {
}
}
@Test
public void memtableWholeKeyFiltering() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final boolean booleanValue = rand.nextBoolean();
opt.setMemtableWholeKeyFiltering(booleanValue);
assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue);
}
}
@Test
public void memtableHugePageSize() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {

View File

@ -48,6 +48,7 @@ public class MutableOptionsGetSetTest {
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableWholeKeyFiltering(false)
.setMemtableHugePageSize(3)
.setMaxSuccessiveMerges(4)
.setMaxWriteBufferNumber(12)
@ -70,6 +71,7 @@ public class MutableOptionsGetSetTest {
.setEnableBlobFiles(false)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.236)
.setMemtableWholeKeyFiltering(true)
.setMemtableHugePageSize(8)
.setMaxSuccessiveMerges(12)
.setMaxWriteBufferNumber(22)
@ -104,8 +106,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);
@ -127,8 +130,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder2.enableBlobFiles()).isEqualTo(false);
assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder2.arenaBlockSize()).isEqualTo(42);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236);
assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12);
assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22);
assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160);
@ -192,6 +196,7 @@ public class MutableOptionsGetSetTest {
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableWholeKeyFiltering(false)
.setMemtableHugePageSize(3)
.setMaxSuccessiveMerges(4)
.setMaxWriteBufferNumber(12)
@ -215,8 +220,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);
@ -237,6 +243,7 @@ public class MutableOptionsGetSetTest {
.setEnableBlobFiles(false)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.236)
.setMemtableWholeKeyFiltering(true)
.setMemtableHugePageSize(8)
.setMaxSuccessiveMerges(12)
.setMaxWriteBufferNumber(22)
@ -258,8 +265,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder2.enableBlobFiles()).isEqualTo(false);
assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder2.arenaBlockSize()).isEqualTo(42);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236);
assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true);
assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12);
assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22);
assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160);
@ -306,6 +314,7 @@ public class MutableOptionsGetSetTest {
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableWholeKeyFiltering(false)
.setMemtableHugePageSize(3)
.setMaxSuccessiveMerges(4)
.setMaxWriteBufferNumber(12)
@ -328,8 +337,9 @@ public class MutableOptionsGetSetTest {
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);

View File

@ -274,6 +274,15 @@ public class OptionsTest {
}
}
@Test
public void memtableWholeKeyFiltering() {
try (final Options opt = new Options()) {
final boolean booleanValue = rand.nextBoolean();
opt.setMemtableWholeKeyFiltering(booleanValue);
assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue);
}
}
@Test
public void memtableHugePageSize() {
try (final Options opt = new Options()) {