From 20699df8438d14568915bb3a8e7038ce224f1e1c Mon Sep 17 00:00:00 2001 From: sdong Date: Fri, 3 Jun 2016 17:02:10 -0700 Subject: [PATCH] memtable_prefix_bloom_bits -> memtable_prefix_bloom_bits_ratio and deprecate memtable_prefix_bloom_probes Summary: memtable_prefix_bloom_probes is not a critical option. Remove it to reduce number of options. It's easier for users to make mistakes with memtable_prefix_bloom_bits, turn it to memtable_prefix_bloom_bits_ratio Test Plan: Run all existing tests Reviewers: yhchiang, igor, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: gunnarku, yoshinorim, MarkCallaghan, leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D59199 --- HISTORY.md | 3 + db/c.cc | 11 +-- db/column_family.cc | 6 ++ db/db_bloom_filter_test.cc | 3 +- db/memtable.cc | 45 +++++---- db/memtable.h | 1 - db/prefix_test.cc | 7 +- db/version_set.h | 1 - include/rocksdb/options.h | 15 ++- java/rocksjni/options.cc | 91 +++++-------------- .../java/org/rocksdb/ColumnFamilyOptions.java | 30 ++---- .../rocksdb/ColumnFamilyOptionsInterface.java | 25 +---- java/src/main/java/org/rocksdb/Options.java | 30 ++---- .../org/rocksdb/ColumnFamilyOptionsTest.java | 17 +--- .../test/java/org/rocksdb/OptionsTest.java | 17 +--- tools/db_bench_tool.cc | 7 +- util/mutable_cf_options.cc | 6 +- util/mutable_cf_options.h | 10 +- util/options.cc | 13 +-- util/options_helper.cc | 12 +-- util/options_helper.h | 9 +- util/options_settable_test.cc | 3 +- util/options_test.cc | 12 +-- util/testutil.cc | 4 +- 24 files changed, 128 insertions(+), 250 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index daa165b8d..d5d4dcc40 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,8 @@ # Rocksdb Change Log ## Unreleased +### Public API Change +* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false +* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes ## 4.9.0 (6/9/2016) ### Public API changes diff --git a/db/c.cc b/db/c.cc index 373ccb7ac..66caf36e6 100644 --- a/db/c.cc +++ b/db/c.cc @@ -1799,14 +1799,9 @@ void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t *opt) { opt->rep.memtable_factory.reset(factory); } -void rocksdb_options_set_memtable_prefix_bloom_bits( - rocksdb_options_t* opt, uint32_t v) { - opt->rep.memtable_prefix_bloom_bits = v; -} - -void rocksdb_options_set_memtable_prefix_bloom_probes( - rocksdb_options_t* opt, uint32_t v) { - opt->rep.memtable_prefix_bloom_probes = v; +void rocksdb_options_set_memtable_prefix_bloom_size_ratio( + rocksdb_options_t* opt, double v) { + opt->rep.memtable_prefix_bloom_size_ratio = v; } void rocksdb_options_set_memtable_prefix_bloom_huge_page_tlb_size( diff --git a/db/column_family.cc b/db/column_family.cc index c531cc71e..7445fac34 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -177,6 +177,12 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options, if (result.max_write_buffer_number_to_maintain < 0) { result.max_write_buffer_number_to_maintain = result.max_write_buffer_number; } + // bloom filter size shouldn't exceed 1/4 of memtable size. + if (result.memtable_prefix_bloom_size_ratio > 0.25) { + result.memtable_prefix_bloom_size_ratio = 0.25; + } else if (result.memtable_prefix_bloom_size_ratio < 0) { + result.memtable_prefix_bloom_size_ratio = 0; + } XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions", xf_transaction_set_memtable_history1, xf_transaction_set_memtable_history, diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index 4bc9453ad..731d523e1 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -615,7 +615,8 @@ class BloomStatsTestWithParam options_.create_if_missing = true; options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4)); - options_.memtable_prefix_bloom_bits = 8 * 1024; + options_.memtable_prefix_bloom_size_ratio = + 8.0 * 1024.0 / static_cast(options_.write_buffer_size); if (use_block_table_) { BlockBasedTableOptions table_options; table_options.hash_index_allow_collision = false; diff --git a/db/memtable.cc b/db/memtable.cc index e899974b3..ef605de11 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -34,24 +34,25 @@ namespace rocksdb { -MemTableOptions::MemTableOptions( - const ImmutableCFOptions& ioptions, - const MutableCFOptions& mutable_cf_options) - : write_buffer_size(mutable_cf_options.write_buffer_size), - arena_block_size(mutable_cf_options.arena_block_size), - memtable_prefix_bloom_bits(mutable_cf_options.memtable_prefix_bloom_bits), - memtable_prefix_bloom_probes( - mutable_cf_options.memtable_prefix_bloom_probes), - memtable_prefix_bloom_huge_page_tlb_size( - mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size), - inplace_update_support(ioptions.inplace_update_support), - inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks), - inplace_callback(ioptions.inplace_callback), - max_successive_merges(mutable_cf_options.max_successive_merges), - filter_deletes(mutable_cf_options.filter_deletes), - statistics(ioptions.statistics), - merge_operator(ioptions.merge_operator), - info_log(ioptions.info_log) {} +MemTableOptions::MemTableOptions(const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options) + : write_buffer_size(mutable_cf_options.write_buffer_size), + arena_block_size(mutable_cf_options.arena_block_size), + memtable_prefix_bloom_bits( + static_cast( + static_cast(mutable_cf_options.write_buffer_size) * + mutable_cf_options.memtable_prefix_bloom_size_ratio) * + 8u), + memtable_prefix_bloom_huge_page_tlb_size( + mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size), + inplace_update_support(ioptions.inplace_update_support), + inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks), + inplace_callback(ioptions.inplace_callback), + max_successive_merges(mutable_cf_options.max_successive_merges), + filter_deletes(mutable_cf_options.filter_deletes), + statistics(ioptions.statistics), + merge_operator(ioptions.merge_operator), + info_log(ioptions.info_log) {} MemTable::MemTable(const InternalKeyComparator& cmp, const ImmutableCFOptions& ioptions, @@ -88,11 +89,9 @@ MemTable::MemTable(const InternalKeyComparator& cmp, if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) { prefix_bloom_.reset(new DynamicBloom( - &allocator_, - moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality, - moptions_.memtable_prefix_bloom_probes, nullptr, - moptions_.memtable_prefix_bloom_huge_page_tlb_size, - ioptions.info_log)); + &allocator_, moptions_.memtable_prefix_bloom_bits, + ioptions.bloom_locality, 6 /* hard coded 6 probes */, nullptr, + moptions_.memtable_prefix_bloom_huge_page_tlb_size, ioptions.info_log)); } } diff --git a/db/memtable.h b/db/memtable.h index 2585ac0c7..2c56e2ed8 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -42,7 +42,6 @@ struct MemTableOptions { size_t write_buffer_size; size_t arena_block_size; uint32_t memtable_prefix_bloom_bits; - uint32_t memtable_prefix_bloom_probes; size_t memtable_prefix_bloom_huge_page_tlb_size; bool inplace_update_support; size_t inplace_update_num_locks; diff --git a/db/prefix_test.cc b/db/prefix_test.cc index 47068f559..4a24a3b00 100644 --- a/db/prefix_test.cc +++ b/db/prefix_test.cc @@ -43,8 +43,7 @@ DEFINE_int64(write_buffer_size, 33554432, ""); DEFINE_int32(max_write_buffer_number, 2, ""); DEFINE_int32(min_write_buffer_number_to_merge, 1, ""); DEFINE_int32(skiplist_height, 4, ""); -DEFINE_int32(memtable_prefix_bloom_bits, 10000000, ""); -DEFINE_int32(memtable_prefix_bloom_probes, 10, ""); +DEFINE_double(memtable_prefix_bloom_size_ratio, 0.1, ""); DEFINE_int32(memtable_prefix_bloom_huge_page_tlb_size, 2 * 1024 * 1024, ""); DEFINE_int32(value_size, 40, ""); @@ -160,8 +159,8 @@ class PrefixTest : public testing::Test { options.min_write_buffer_number_to_merge = FLAGS_min_write_buffer_number_to_merge; - options.memtable_prefix_bloom_bits = FLAGS_memtable_prefix_bloom_bits; - options.memtable_prefix_bloom_probes = FLAGS_memtable_prefix_bloom_probes; + options.memtable_prefix_bloom_size_ratio = + FLAGS_memtable_prefix_bloom_size_ratio; options.memtable_prefix_bloom_huge_page_tlb_size = FLAGS_memtable_prefix_bloom_huge_page_tlb_size; diff --git a/db/version_set.h b/db/version_set.h index 37f645e49..e379d9cce 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -57,7 +57,6 @@ class Version; class VersionSet; class WriteBuffer; class MergeContext; -class ColumnFamilyData; class ColumnFamilySet; class TableCache; class MergeIteratorBuilder; diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index fa937e141..9eefd885e 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -745,16 +745,15 @@ struct ColumnFamilyOptions { Slice delta_value, std::string* merged_value); - // if prefix_extractor is set and bloom_bits is not 0, create prefix bloom - // for memtable + // if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0, + // create prefix bloom for memtable with the size of + // write_buffer_size * memtable_prefix_bloom_size_ratio. + // If it is larger than 0.25, it is santinized to 0.25. + // + // Default: 0 (disable) // // Dynamically changeable through SetOptions() API - uint32_t memtable_prefix_bloom_bits; - - // number of hash probes per key - // - // Dynamically changeable through SetOptions() API - uint32_t memtable_prefix_bloom_probes; + double memtable_prefix_bloom_size_ratio; // Page size for huge page TLB for bloom in memtable. If <=0, not allocate // from huge page TLB but from malloc. diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index 108c2e5ab..e605a6ce0 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -1732,50 +1732,27 @@ void Java_org_rocksdb_Options_setInplaceUpdateNumLocks( /* * Class: org_rocksdb_Options - * Method: memtablePrefixBloomBits + * Method: memtablePrefixBloomSizeRatio * Signature: (J)I */ -jint Java_org_rocksdb_Options_memtablePrefixBloomBits( - JNIEnv* env, jobject jobj, jlong jhandle) { - return reinterpret_cast( - jhandle)->memtable_prefix_bloom_bits; +jdouble Java_org_rocksdb_Options_memtablePrefixBloomSizeRatio(JNIEnv* env, + jobject jobj, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio; } /* * Class: org_rocksdb_Options - * Method: setMemtablePrefixBloomBits + * Method: setMemtablePrefixBloomSizeRatio * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMemtablePrefixBloomBits( +void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio( JNIEnv* env, jobject jobj, jlong jhandle, - jint jmemtable_prefix_bloom_bits) { - reinterpret_cast( - jhandle)->memtable_prefix_bloom_bits = - static_cast(jmemtable_prefix_bloom_bits); -} - -/* - * Class: org_rocksdb_Options - * Method: memtablePrefixBloomProbes - * Signature: (J)I - */ -jint Java_org_rocksdb_Options_memtablePrefixBloomProbes( - JNIEnv* env, jobject jobj, jlong jhandle) { - return reinterpret_cast( - jhandle)->memtable_prefix_bloom_probes; -} - -/* - * Class: org_rocksdb_Options - * Method: setMemtablePrefixBloomProbes - * Signature: (JI)V - */ -void Java_org_rocksdb_Options_setMemtablePrefixBloomProbes( - JNIEnv* env, jobject jobj, jlong jhandle, - jint jmemtable_prefix_bloom_probes) { - reinterpret_cast( - jhandle)->memtable_prefix_bloom_probes = - static_cast(jmemtable_prefix_bloom_probes); + jdouble jmemtable_prefix_bloom_size_ratio) { + reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio = + static_cast(jmemtable_prefix_bloom_size_ratio); } /* @@ -2915,50 +2892,26 @@ void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateNumLocks( /* * Class: org_rocksdb_ColumnFamilyOptions - * Method: memtablePrefixBloomBits + * Method: memtablePrefixBloomSizeRatio * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomBits( +jdouble Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio( JNIEnv* env, jobject jobj, jlong jhandle) { - return reinterpret_cast( - jhandle)->memtable_prefix_bloom_bits; + return reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio; } /* * Class: org_rocksdb_ColumnFamilyOptions - * Method: setMemtablePrefixBloomBits + * Method: setMemtablePrefixBloomSizeRatio * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomBits( +void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio( JNIEnv* env, jobject jobj, jlong jhandle, - jint jmemtable_prefix_bloom_bits) { - reinterpret_cast( - jhandle)->memtable_prefix_bloom_bits = - static_cast(jmemtable_prefix_bloom_bits); -} - -/* - * Class: org_rocksdb_ColumnFamilyOptions - * Method: memtablePrefixBloomProbes - * Signature: (J)I - */ -jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomProbes( - JNIEnv* env, jobject jobj, jlong jhandle) { - return reinterpret_cast( - jhandle)->memtable_prefix_bloom_probes; -} - -/* - * Class: org_rocksdb_ColumnFamilyOptions - * Method: setMemtablePrefixBloomProbes - * Signature: (JI)V - */ -void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomProbes( - JNIEnv* env, jobject jobj, jlong jhandle, - jint jmemtable_prefix_bloom_probes) { - reinterpret_cast( - jhandle)->memtable_prefix_bloom_probes = - static_cast(jmemtable_prefix_bloom_probes); + jdouble jmemtable_prefix_bloom_size_ratio) { + reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio = + static_cast(jmemtable_prefix_bloom_size_ratio); } /* diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java index 6a7b580bf..8b4fd2fa4 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -589,27 +589,15 @@ public class ColumnFamilyOptions extends RocksObject } @Override - public ColumnFamilyOptions setMemtablePrefixBloomBits( - final int memtablePrefixBloomBits) { - setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits); + public ColumnFamilyOptions setMemtablePrefixBloomSizeRatio( + final double memtablePrefixBloomSizeRatio) { + setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomSizeRatio); return this; } @Override - public int memtablePrefixBloomBits() { - return memtablePrefixBloomBits(nativeHandle_); - } - - @Override - public ColumnFamilyOptions setMemtablePrefixBloomProbes( - final int memtablePrefixBloomProbes) { - setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes); - return this; - } - - @Override - public int memtablePrefixBloomProbes() { - return memtablePrefixBloomProbes(nativeHandle_); + public double memtablePrefixBloomSizeRatio() { + return memtablePrefixBloomSizeRatio(nativeHandle_); } @Override @@ -787,12 +775,8 @@ public class ColumnFamilyOptions extends RocksObject long handle, long inplaceUpdateNumLocks) throws IllegalArgumentException; private native long inplaceUpdateNumLocks(long handle); - private native void setMemtablePrefixBloomBits( - long handle, int memtablePrefixBloomBits); - private native int memtablePrefixBloomBits(long handle); - private native void setMemtablePrefixBloomProbes( - long handle, int memtablePrefixBloomProbes); - private native int memtablePrefixBloomProbes(long handle); + private native void setMemtablePrefixBloomBits(long handle, double memtablePrefixBloomSizeRatio); + private native double memtablePrefixBloomSizeRatio(long handle); private native void setBloomLocality( long handle, int bloomLocality); private native int bloomLocality(long handle); diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java index 9856ec686..72f59b7d1 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java @@ -1011,15 +1011,15 @@ public interface ColumnFamilyOptionsInterface { long inplaceUpdateNumLocks(); /** - * Sets the number of bits used in the prefix bloom filter. + * Sets the size ratio of the memtable used in the prefix bloom filter. * * This value will be used only when a prefix-extractor is specified. * - * @param memtablePrefixBloomBits the number of bits used in the + * @param memtablePrefixBloomSizeRatio the number of bits used in the * prefix bloom filter. * @return the reference to the current option. */ - Object setMemtablePrefixBloomBits(int memtablePrefixBloomBits); + Object setMemtablePrefixBloomSizeRatio(double memtablePrefixBloomSizeRatio); /** * Returns the number of bits used in the prefix bloom filter. @@ -1029,22 +1029,7 @@ public interface ColumnFamilyOptionsInterface { * @return the number of bloom-bits. * @see #useFixedLengthPrefixExtractor(int) */ - int memtablePrefixBloomBits(); - - /** - * The number of hash probes per key used in the mem-table. - * - * @param memtablePrefixBloomProbes the number of hash probes per key. - * @return the reference to the current option. - */ - Object setMemtablePrefixBloomProbes(int memtablePrefixBloomProbes); - - /** - * The number of hash probes per key used in the mem-table. - * - * @return the number of hash probes per key. - */ - int memtablePrefixBloomProbes(); + double memtablePrefixBloomSizeRatio(); /** * Control locality of bloom filter probes to improve cache miss rate. @@ -1074,7 +1059,7 @@ public interface ColumnFamilyOptionsInterface { * Default: 0 * * @return the level of locality of bloom-filter probes. - * @see #setMemtablePrefixBloomProbes(int) + * @see #setBloomLocality(int) */ int bloomLocality(); diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index 56fd5ab18..e8f898610 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -1005,26 +1005,13 @@ public class Options extends RocksObject } @Override - public int memtablePrefixBloomBits() { - return memtablePrefixBloomBits(nativeHandle_); + public double memtablePrefixBloomSizeRatio() { + return memtablePrefixBloomSizeRatio(nativeHandle_); } @Override - public Options setMemtablePrefixBloomBits( - final int memtablePrefixBloomBits) { - setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits); - return this; - } - - @Override - public int memtablePrefixBloomProbes() { - return memtablePrefixBloomProbes(nativeHandle_); - } - - @Override - public Options setMemtablePrefixBloomProbes( - final int memtablePrefixBloomProbes) { - setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes); + public Options setMemtablePrefixBloomSizeRatio(final double memtablePrefixBloomSizeRatio) { + setMemtablePrefixBloomSizeRatio(nativeHandle_, memtablePrefixBloomSizeRatio); return this; } @@ -1293,12 +1280,9 @@ public class Options extends RocksObject long handle, long inplaceUpdateNumLocks) throws IllegalArgumentException; private native long inplaceUpdateNumLocks(long handle); - private native void setMemtablePrefixBloomBits( - long handle, int memtablePrefixBloomBits); - private native int memtablePrefixBloomBits(long handle); - private native void setMemtablePrefixBloomProbes( - long handle, int memtablePrefixBloomProbes); - private native int memtablePrefixBloomProbes(long handle); + private native void setMemtablePrefixBloomSizeRatio( + long handle, double memtablePrefixBloomSizeRatio); + private native double memtablePrefixBloomSizeRatio(long handle); private native void setBloomLocality( long handle, int bloomLocality); private native int bloomLocality(long handle); diff --git a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java index e2ebf13d8..1cb2ed1bd 100644 --- a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java @@ -305,20 +305,11 @@ public class ColumnFamilyOptionsTest { } @Test - public void memtablePrefixBloomBits() { + public void memtablePrefixBloomSizeRatio() { try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { - final int intValue = rand.nextInt(); - opt.setMemtablePrefixBloomBits(intValue); - assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue); - } - } - - @Test - public void memtablePrefixBloomProbes() { - try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { - final int intValue = rand.nextInt(); - opt.setMemtablePrefixBloomProbes(intValue); - assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue); + final double doubleValue = rand.nextDouble(); + opt.setMemtablePrefixBloomSizeRatio(doubleValue); + assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue); } } diff --git a/java/src/test/java/org/rocksdb/OptionsTest.java b/java/src/test/java/org/rocksdb/OptionsTest.java index a588b5de7..fba9a5dde 100644 --- a/java/src/test/java/org/rocksdb/OptionsTest.java +++ b/java/src/test/java/org/rocksdb/OptionsTest.java @@ -268,20 +268,11 @@ public class OptionsTest { } @Test - public void memtablePrefixBloomBits() { + public void memtablePrefixBloomSizeRatio() { try (final Options opt = new Options()) { - final int intValue = rand.nextInt(); - opt.setMemtablePrefixBloomBits(intValue); - assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue); - } - } - - @Test - public void memtablePrefixBloomProbes() { - try (final Options opt = new Options()) { - final int intValue = rand.nextInt(); - opt.setMemtablePrefixBloomProbes(intValue); - assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue); + final double doubleValue = rand.nextDouble(); + opt.setMemtablePrefixBloomSizeRatio(doubleValue); + assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue); } } diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 17bb38dfa..5e5bcf681 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -396,8 +396,9 @@ DEFINE_int32(skip_table_builder_flush, false, "Skip flushing block in " DEFINE_int32(bloom_bits, -1, "Bloom filter bits per key. Negative means" " use default settings."); -DEFINE_int32(memtable_bloom_bits, 0, "Bloom filter bits per key for memtable. " - "Negative means no bloom filter."); +DEFINE_double(memtable_bloom_size_ratio, 0, + "Ratio of memtable size used for bloom filter. 0 means no bloom " + "filter."); DEFINE_bool(use_existing_db, false, "If true, do not destroy the existing" " database. If you set this flag and also specify a benchmark that" @@ -2461,7 +2462,7 @@ class Benchmark { exit(1); } } - options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits; + options.memtable_prefix_bloom_size_ratio = FLAGS_memtable_bloom_size_ratio; options.bloom_locality = FLAGS_bloom_locality; options.max_file_opening_threads = FLAGS_file_opening_threads; options.new_table_reader_for_compaction_inputs = diff --git a/util/mutable_cf_options.cc b/util/mutable_cf_options.cc index 83eb2fafb..74d5603ff 100644 --- a/util/mutable_cf_options.cc +++ b/util/mutable_cf_options.cc @@ -70,10 +70,8 @@ void MutableCFOptions::Dump(Logger* log) const { max_write_buffer_number); Log(log, " arena_block_size: %" ROCKSDB_PRIszt, arena_block_size); - Log(log, " memtable_prefix_bloom_bits: %" PRIu32, - memtable_prefix_bloom_bits); - Log(log, " memtable_prefix_bloom_probes: %" PRIu32, - memtable_prefix_bloom_probes); + Log(log, " memtable_prefix_bloom_ratio: %f", + memtable_prefix_bloom_size_ratio); Log(log, " memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt, memtable_prefix_bloom_huge_page_tlb_size); Log(log, " max_successive_merges: %" ROCKSDB_PRIszt, diff --git a/util/mutable_cf_options.h b/util/mutable_cf_options.h index a0978070e..e198771c8 100644 --- a/util/mutable_cf_options.h +++ b/util/mutable_cf_options.h @@ -17,8 +17,8 @@ struct MutableCFOptions { : write_buffer_size(options.write_buffer_size), max_write_buffer_number(options.max_write_buffer_number), arena_block_size(options.arena_block_size), - memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits), - memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes), + memtable_prefix_bloom_size_ratio( + options.memtable_prefix_bloom_size_ratio), memtable_prefix_bloom_huge_page_tlb_size( options.memtable_prefix_bloom_huge_page_tlb_size), max_successive_merges(options.max_successive_merges), @@ -58,8 +58,7 @@ struct MutableCFOptions { : write_buffer_size(0), max_write_buffer_number(0), arena_block_size(0), - memtable_prefix_bloom_bits(0), - memtable_prefix_bloom_probes(0), + memtable_prefix_bloom_size_ratio(0), memtable_prefix_bloom_huge_page_tlb_size(0), max_successive_merges(0), filter_deletes(false), @@ -110,8 +109,7 @@ struct MutableCFOptions { size_t write_buffer_size; int max_write_buffer_number; size_t arena_block_size; - uint32_t memtable_prefix_bloom_bits; - uint32_t memtable_prefix_bloom_probes; + double memtable_prefix_bloom_size_ratio; size_t memtable_prefix_bloom_huge_page_tlb_size; size_t max_successive_merges; bool filter_deletes; diff --git a/util/options.cc b/util/options.cc index f73eabca5..7c3867087 100644 --- a/util/options.cc +++ b/util/options.cc @@ -121,8 +121,7 @@ ColumnFamilyOptions::ColumnFamilyOptions() inplace_update_support(false), inplace_update_num_locks(10000), inplace_callback(nullptr), - memtable_prefix_bloom_bits(0), - memtable_prefix_bloom_probes(6), + memtable_prefix_bloom_size_ratio(0.0), memtable_prefix_bloom_huge_page_tlb_size(0), bloom_locality(0), max_successive_merges(0), @@ -190,8 +189,8 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options) inplace_update_support(options.inplace_update_support), inplace_update_num_locks(options.inplace_update_num_locks), inplace_callback(options.inplace_callback), - memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits), - memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes), + memtable_prefix_bloom_size_ratio( + options.memtable_prefix_bloom_size_ratio), memtable_prefix_bloom_huge_page_tlb_size( options.memtable_prefix_bloom_huge_page_tlb_size), bloom_locality(options.bloom_locality), @@ -596,10 +595,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const { Header(log, " Options.min_partial_merge_operands: %u", min_partial_merge_operands); // TODO: easier config for bloom (maybe based on avg key/value size) - Header(log, " Options.memtable_prefix_bloom_bits: %d", - memtable_prefix_bloom_bits); - Header(log, " Options.memtable_prefix_bloom_probes: %d", - memtable_prefix_bloom_probes); + Header(log, " Options.memtable_prefix_bloom_size_ratio: %f", + memtable_prefix_bloom_size_ratio); Header(log, " Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt, diff --git a/util/options_helper.cc b/util/options_helper.cc index fbf6a5766..a9f50cb26 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -537,9 +537,11 @@ bool ParseMemtableOptions(const std::string& name, const std::string& value, } else if (name == "arena_block_size") { new_options->arena_block_size = ParseSizeT(value); } else if (name == "memtable_prefix_bloom_bits") { - new_options->memtable_prefix_bloom_bits = ParseUint32(value); + // deprecated + } else if (name == "memtable_prefix_bloom_size_ratio") { + new_options->memtable_prefix_bloom_size_ratio = ParseDouble(value); } else if (name == "memtable_prefix_bloom_probes") { - new_options->memtable_prefix_bloom_probes = ParseUint32(value); + // Deprecated } else if (name == "memtable_prefix_bloom_huge_page_tlb_size") { new_options->memtable_prefix_bloom_huge_page_tlb_size = ParseSizeT(value); @@ -1437,10 +1439,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions( cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size; cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number; cf_opts.arena_block_size = mutable_cf_options.arena_block_size; - cf_opts.memtable_prefix_bloom_bits = - mutable_cf_options.memtable_prefix_bloom_bits; - cf_opts.memtable_prefix_bloom_probes = - mutable_cf_options.memtable_prefix_bloom_probes; + cf_opts.memtable_prefix_bloom_size_ratio = + mutable_cf_options.memtable_prefix_bloom_size_ratio; cf_opts.memtable_prefix_bloom_huge_page_tlb_size = mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size; cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges; diff --git a/util/options_helper.h b/util/options_helper.h index 2a72ac071..423e6b784 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -437,11 +437,12 @@ static std::unordered_map cf_options_type_info = { {offsetof(struct ColumnFamilyOptions, bloom_locality), OptionType::kUInt32T, OptionVerificationType::kNormal}}, {"memtable_prefix_bloom_bits", - {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_bits), - OptionType::kUInt32T, OptionVerificationType::kNormal}}, + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}}, + {"memtable_prefix_bloom_size_ratio", + {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_size_ratio), + OptionType::kDouble, OptionVerificationType::kNormal}}, {"memtable_prefix_bloom_probes", - {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_probes), - OptionType::kUInt32T, OptionVerificationType::kNormal}}, + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}}, {"min_partial_merge_operands", {offsetof(struct ColumnFamilyOptions, min_partial_merge_operands), OptionType::kUInt32T, OptionVerificationType::kNormal}}, diff --git a/util/options_settable_test.cc b/util/options_settable_test.cc index f83ed9886..338a4017f 100644 --- a/util/options_settable_test.cc +++ b/util/options_settable_test.cc @@ -413,14 +413,13 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { "max_write_buffer_number_to_maintain=84;" "verify_checksums_in_compaction=false;" "merge_operator=aabcxehazrMergeOperator;" - "memtable_prefix_bloom_bits=4642;" + "memtable_prefix_bloom_size_ratio=0.4642;" "paranoid_file_checks=true;" "inplace_update_num_locks=7429;" "optimize_filters_for_hits=false;" "level_compaction_dynamic_level_bytes=false;" "inplace_update_support=false;" "compaction_style=kCompactionStyleFIFO;" - "memtable_prefix_bloom_probes=2511;" "purge_redundant_kvs_while_flush=true;" "filter_deletes=false;" "hard_pending_compaction_bytes_limit=0;" diff --git a/util/options_test.cc b/util/options_test.cc index a4c259de2..cc4176557 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -86,8 +86,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"report_bg_io_stats", "true"}, {"compaction_measure_io_stats", "false"}, {"inplace_update_num_locks", "25"}, - {"memtable_prefix_bloom_bits", "26"}, - {"memtable_prefix_bloom_probes", "27"}, + {"memtable_prefix_bloom_size_ratio", "0.26"}, {"memtable_prefix_bloom_huge_page_tlb_size", "28"}, {"bloom_locality", "29"}, {"max_successive_merges", "30"}, @@ -187,8 +186,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { static_cast(24)); ASSERT_EQ(new_cf_opt.inplace_update_support, true); ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U); - ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 26U); - ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_probes, 27U); + ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26); ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_huge_page_tlb_size, 28U); ASSERT_EQ(new_cf_opt.bloom_locality, 29U); ASSERT_EQ(new_cf_opt.max_successive_merges, 30U); @@ -300,10 +298,8 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { const int64_t tera = 1024 * giga; // Units (k) - ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, - "memtable_prefix_bloom_bits=14k;max_write_buffer_number=-15K", - &new_cf_opt)); - ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 14UL * kilo); + ASSERT_OK(GetColumnFamilyOptionsFromString( + base_cf_opt, "max_write_buffer_number=-15K", &new_cf_opt)); ASSERT_EQ(new_cf_opt.max_write_buffer_number, -15 * kilo); // Units (m) ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, diff --git a/util/testutil.cc b/util/testutil.cc index fbaa75e64..e0de8b1c2 100644 --- a/util/testutil.cc +++ b/util/testutil.cc @@ -307,6 +307,8 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) { // double options cf_opt->hard_rate_limit = static_cast(rnd->Uniform(10000)) / 13; cf_opt->soft_rate_limit = static_cast(rnd->Uniform(10000)) / 13; + cf_opt->memtable_prefix_bloom_size_ratio = + static_cast(rnd->Uniform(10000)) / 20000.0; // int options cf_opt->expanded_compaction_factor = rnd->Uniform(100); @@ -332,8 +334,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) { // uint32_t options cf_opt->bloom_locality = rnd->Uniform(10000); - cf_opt->memtable_prefix_bloom_bits = rnd->Uniform(10000); - cf_opt->memtable_prefix_bloom_probes = rnd->Uniform(10000); cf_opt->min_partial_merge_operands = rnd->Uniform(10000); cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);