Make FilterPolicy Customizable (#9590)
Summary: Make FilterPolicy into a Customizable class. Allow new FilterPolicy to be discovered through the ObjectRegistry Pull Request resolved: https://github.com/facebook/rocksdb/pull/9590 Reviewed By: pdillinger Differential Revision: D34327367 Pulled By: mrambacher fbshipit-source-id: 37e7edac90ec9457422b72f359ab8ef48829c190
This commit is contained in:
parent
f066b5cecb
commit
30b08878d8
@ -43,8 +43,7 @@
|
|||||||
string. Use something like "filter_policy=ribbonfilter:10" instead.
|
string. Use something like "filter_policy=ribbonfilter:10" instead.
|
||||||
* Allow configuration string like "filter_policy=bloomfilter:10" without
|
* Allow configuration string like "filter_policy=bloomfilter:10" without
|
||||||
bool, to minimize acknowledgement of obsolete block-based filter.
|
bool, to minimize acknowledgement of obsolete block-based filter.
|
||||||
* A `filter_policy` loaded from an OPTIONS file can read existing filters
|
* Made FilterPolicy Customizable. Configuration of filter_policy is now accurately saved in OPTIONS file and can be loaded with LoadOptionsFromFile. (Loading an OPTIONS file generated by a previous version only enables reading and using existing filters, not generating new filters. Previously, no filter_policy would be configured from a saved OPTIONS file.)
|
||||||
but still does not support writing new filters.
|
|
||||||
* Change meaning of nullptr return from GetBuilderWithContext() from "use
|
* Change meaning of nullptr return from GetBuilderWithContext() from "use
|
||||||
block-based filter" to "generate no filter in this case."
|
block-based filter" to "generate no filter in this case."
|
||||||
* Also, when user specifies bits_per_key < 0.5, we now round this down
|
* Also, when user specifies bits_per_key < 0.5, we now round this down
|
||||||
|
@ -33,14 +33,15 @@ std::shared_ptr<const FilterPolicy> Create(double bits_per_key,
|
|||||||
const std::string& name) {
|
const std::string& name) {
|
||||||
return BloomLikeFilterPolicy::Create(name, bits_per_key);
|
return BloomLikeFilterPolicy::Create(name, bits_per_key);
|
||||||
}
|
}
|
||||||
const std::string kLegacyBloom = test::LegacyBloomFilterPolicy::kName();
|
const std::string kLegacyBloom = test::LegacyBloomFilterPolicy::kClassName();
|
||||||
const std::string kDeprecatedBlock =
|
const std::string kDeprecatedBlock =
|
||||||
DeprecatedBlockBasedBloomFilterPolicy::kName();
|
DeprecatedBlockBasedBloomFilterPolicy::kClassName();
|
||||||
const std::string kFastLocalBloom = test::FastLocalBloomFilterPolicy::kName();
|
const std::string kFastLocalBloom =
|
||||||
|
test::FastLocalBloomFilterPolicy::kClassName();
|
||||||
const std::string kStandard128Ribbon =
|
const std::string kStandard128Ribbon =
|
||||||
test::Standard128RibbonFilterPolicy::kName();
|
test::Standard128RibbonFilterPolicy::kClassName();
|
||||||
const std::string kAutoBloom = BloomFilterPolicy::kName();
|
const std::string kAutoBloom = BloomFilterPolicy::kClassName();
|
||||||
const std::string kAutoRibbon = RibbonFilterPolicy::kName();
|
const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
// DB tests related to bloom filter.
|
// DB tests related to bloom filter.
|
||||||
@ -593,10 +594,9 @@ class AlwaysTrueBitsBuilder : public FilterBitsBuilder {
|
|||||||
size_t ApproximateNumEntries(size_t) override { return SIZE_MAX; }
|
size_t ApproximateNumEntries(size_t) override { return SIZE_MAX; }
|
||||||
};
|
};
|
||||||
|
|
||||||
class AlwaysTrueFilterPolicy : public BloomLikeFilterPolicy {
|
class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
|
||||||
public:
|
public:
|
||||||
explicit AlwaysTrueFilterPolicy(bool skip)
|
explicit AlwaysTrueFilterPolicy(bool skip) : skip_(skip) {}
|
||||||
: BloomLikeFilterPolicy(/* ignored */ 10), skip_(skip) {}
|
|
||||||
|
|
||||||
FilterBitsBuilder* GetBuilderWithContext(
|
FilterBitsBuilder* GetBuilderWithContext(
|
||||||
const FilterBuildingContext&) const override {
|
const FilterBuildingContext&) const override {
|
||||||
@ -607,10 +607,6 @@ class AlwaysTrueFilterPolicy : public BloomLikeFilterPolicy {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string GetId() const override {
|
|
||||||
return "rocksdb.test.AlwaysTrueFilterPolicy";
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
bool skip_;
|
bool skip_;
|
||||||
};
|
};
|
||||||
@ -1606,11 +1602,11 @@ class TestingContextCustomFilterPolicy
|
|||||||
test_report_ +=
|
test_report_ +=
|
||||||
OptionsHelper::compaction_style_to_string[context.compaction_style];
|
OptionsHelper::compaction_style_to_string[context.compaction_style];
|
||||||
test_report_ += ",n=";
|
test_report_ += ",n=";
|
||||||
test_report_ += ToString(context.num_levels);
|
test_report_ += ROCKSDB_NAMESPACE::ToString(context.num_levels);
|
||||||
test_report_ += ",l=";
|
test_report_ += ",l=";
|
||||||
test_report_ += ToString(context.level_at_creation);
|
test_report_ += ROCKSDB_NAMESPACE::ToString(context.level_at_creation);
|
||||||
test_report_ += ",b=";
|
test_report_ += ",b=";
|
||||||
test_report_ += ToString(int{context.is_bottommost});
|
test_report_ += ROCKSDB_NAMESPACE::ToString(int{context.is_bottommost});
|
||||||
test_report_ += ",r=";
|
test_report_ += ",r=";
|
||||||
test_report_ += table_file_creation_reason_to_string[context.reason];
|
test_report_ += table_file_creation_reason_to_string[context.reason];
|
||||||
test_report_ += "\n";
|
test_report_ += "\n";
|
||||||
|
@ -28,7 +28,7 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "rocksdb/advanced_options.h"
|
#include "rocksdb/advanced_options.h"
|
||||||
#include "rocksdb/memory_allocator.h"
|
#include "rocksdb/customizable.h"
|
||||||
#include "rocksdb/status.h"
|
#include "rocksdb/status.h"
|
||||||
#include "rocksdb/types.h"
|
#include "rocksdb/types.h"
|
||||||
|
|
||||||
@ -85,9 +85,10 @@ struct FilterBuildingContext {
|
|||||||
// defer to other built-in policies (see NewBloomFilterPolicy and
|
// defer to other built-in policies (see NewBloomFilterPolicy and
|
||||||
// NewRibbonFilterPolicy) based on the context provided to
|
// NewRibbonFilterPolicy) based on the context provided to
|
||||||
// GetBuilderWithContext.
|
// GetBuilderWithContext.
|
||||||
class FilterPolicy {
|
class FilterPolicy : public Customizable {
|
||||||
public:
|
public:
|
||||||
virtual ~FilterPolicy();
|
virtual ~FilterPolicy();
|
||||||
|
static const char* Type() { return "FilterPolicy"; }
|
||||||
|
|
||||||
// Creates a new FilterPolicy based on the input value string and returns the
|
// Creates a new FilterPolicy based on the input value string and returns the
|
||||||
// result The value might be an ID, and ID with properties, or an old-style
|
// result The value might be an ID, and ID with properties, or an old-style
|
||||||
@ -101,12 +102,6 @@ class FilterPolicy {
|
|||||||
const std::string& value,
|
const std::string& value,
|
||||||
std::shared_ptr<const FilterPolicy>* result);
|
std::shared_ptr<const FilterPolicy>* result);
|
||||||
|
|
||||||
// Return the name of this policy. Note that if the filter encoding
|
|
||||||
// changes in an incompatible way, the name returned by this method
|
|
||||||
// must be changed. Otherwise, old incompatible filters may be
|
|
||||||
// passed to methods of this type.
|
|
||||||
virtual const char* Name() const = 0;
|
|
||||||
|
|
||||||
// Return a new FilterBitsBuilder for constructing full or partitioned
|
// Return a new FilterBitsBuilder for constructing full or partitioned
|
||||||
// filter blocks, or return nullptr to indicate "no filter". Custom
|
// filter blocks, or return nullptr to indicate "no filter". Custom
|
||||||
// implementations should defer to a built-in FilterPolicy to get a
|
// implementations should defer to a built-in FilterPolicy to get a
|
||||||
|
@ -35,7 +35,6 @@ enum class OptionType {
|
|||||||
kCompactionPri,
|
kCompactionPri,
|
||||||
kCompressionType,
|
kCompressionType,
|
||||||
kCompactionStopStyle,
|
kCompactionStopStyle,
|
||||||
kFilterPolicy,
|
|
||||||
kChecksumType,
|
kChecksumType,
|
||||||
kEncodingType,
|
kEncodingType,
|
||||||
kEnv,
|
kEnv,
|
||||||
|
@ -21,6 +21,7 @@
|
|||||||
#include "rocksdb/convenience.h"
|
#include "rocksdb/convenience.h"
|
||||||
#include "rocksdb/env_encryption.h"
|
#include "rocksdb/env_encryption.h"
|
||||||
#include "rocksdb/file_checksum.h"
|
#include "rocksdb/file_checksum.h"
|
||||||
|
#include "rocksdb/filter_policy.h"
|
||||||
#include "rocksdb/flush_block_policy.h"
|
#include "rocksdb/flush_block_policy.h"
|
||||||
#include "rocksdb/memory_allocator.h"
|
#include "rocksdb/memory_allocator.h"
|
||||||
#include "rocksdb/rate_limiter.h"
|
#include "rocksdb/rate_limiter.h"
|
||||||
@ -31,6 +32,7 @@
|
|||||||
#include "rocksdb/utilities/customizable_util.h"
|
#include "rocksdb/utilities/customizable_util.h"
|
||||||
#include "rocksdb/utilities/object_registry.h"
|
#include "rocksdb/utilities/object_registry.h"
|
||||||
#include "rocksdb/utilities/options_type.h"
|
#include "rocksdb/utilities/options_type.h"
|
||||||
|
#include "table/block_based/filter_policy_internal.h"
|
||||||
#include "table/block_based/flush_block_policy.h"
|
#include "table/block_based/flush_block_policy.h"
|
||||||
#include "table/mock_table.h"
|
#include "table/mock_table.h"
|
||||||
#include "test_util/mock_time_env.h"
|
#include "test_util/mock_time_env.h"
|
||||||
@ -1481,6 +1483,20 @@ class MockRateLimiter : public RateLimiter {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class MockFilterPolicy : public FilterPolicy {
|
||||||
|
public:
|
||||||
|
static const char* kClassName() { return "MockFilterPolicy"; }
|
||||||
|
const char* Name() const override { return kClassName(); }
|
||||||
|
FilterBitsBuilder* GetBuilderWithContext(
|
||||||
|
const FilterBuildingContext&) const override {
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
FilterBitsReader* GetFilterBitsReader(
|
||||||
|
const Slice& /*contents*/) const override {
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
static int RegisterLocalObjects(ObjectLibrary& library,
|
static int RegisterLocalObjects(ObjectLibrary& library,
|
||||||
const std::string& /*arg*/) {
|
const std::string& /*arg*/) {
|
||||||
@ -1605,6 +1621,14 @@ static int RegisterLocalObjects(ObjectLibrary& library,
|
|||||||
return guard->get();
|
return guard->get();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
MockFilterPolicy::kClassName(),
|
||||||
|
[](const std::string& /*uri*/, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(new MockFilterPolicy());
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
|
||||||
return static_cast<int>(library.GetFactoryCount(&num_types));
|
return static_cast<int>(library.GetFactoryCount(&num_types));
|
||||||
}
|
}
|
||||||
#endif // !ROCKSDB_LITE
|
#endif // !ROCKSDB_LITE
|
||||||
@ -2114,6 +2138,55 @@ TEST_F(LoadCustomizableTest, LoadRateLimiterTest) {
|
|||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(LoadCustomizableTest, LoadFilterPolicyTest) {
|
||||||
|
std::shared_ptr<TableFactory> table;
|
||||||
|
std::shared_ptr<const FilterPolicy> result;
|
||||||
|
ASSERT_NOK(FilterPolicy::CreateFromString(
|
||||||
|
config_options_, MockFilterPolicy::kClassName(), &result));
|
||||||
|
|
||||||
|
ASSERT_OK(FilterPolicy::CreateFromString(config_options_, "", &result));
|
||||||
|
ASSERT_EQ(result, nullptr);
|
||||||
|
ASSERT_OK(FilterPolicy::CreateFromString(
|
||||||
|
config_options_, ReadOnlyBuiltinFilterPolicy::kClassName(), &result));
|
||||||
|
ASSERT_NE(result, nullptr);
|
||||||
|
ASSERT_STREQ(result->Name(), ReadOnlyBuiltinFilterPolicy::kClassName());
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
std::string table_opts = "id=BlockBasedTable; filter_policy=";
|
||||||
|
ASSERT_OK(TableFactory::CreateFromString(config_options_,
|
||||||
|
table_opts + "nullptr", &table));
|
||||||
|
ASSERT_NE(table.get(), nullptr);
|
||||||
|
auto bbto = table->GetOptions<BlockBasedTableOptions>();
|
||||||
|
ASSERT_NE(bbto, nullptr);
|
||||||
|
ASSERT_EQ(bbto->filter_policy.get(), nullptr);
|
||||||
|
ASSERT_OK(TableFactory::CreateFromString(
|
||||||
|
config_options_, table_opts + ReadOnlyBuiltinFilterPolicy::kClassName(),
|
||||||
|
&table));
|
||||||
|
bbto = table->GetOptions<BlockBasedTableOptions>();
|
||||||
|
ASSERT_NE(bbto, nullptr);
|
||||||
|
ASSERT_NE(bbto->filter_policy.get(), nullptr);
|
||||||
|
ASSERT_STREQ(bbto->filter_policy->Name(),
|
||||||
|
ReadOnlyBuiltinFilterPolicy::kClassName());
|
||||||
|
ASSERT_OK(TableFactory::CreateFromString(
|
||||||
|
config_options_, table_opts + MockFilterPolicy::kClassName(), &table));
|
||||||
|
bbto = table->GetOptions<BlockBasedTableOptions>();
|
||||||
|
ASSERT_NE(bbto, nullptr);
|
||||||
|
ASSERT_EQ(bbto->filter_policy.get(), nullptr);
|
||||||
|
if (RegisterTests("Test")) {
|
||||||
|
ASSERT_OK(FilterPolicy::CreateFromString(
|
||||||
|
config_options_, MockFilterPolicy::kClassName(), &result));
|
||||||
|
ASSERT_NE(result, nullptr);
|
||||||
|
ASSERT_STREQ(result->Name(), MockFilterPolicy::kClassName());
|
||||||
|
ASSERT_OK(TableFactory::CreateFromString(
|
||||||
|
config_options_, table_opts + MockFilterPolicy::kClassName(), &table));
|
||||||
|
bbto = table->GetOptions<BlockBasedTableOptions>();
|
||||||
|
ASSERT_NE(bbto, nullptr);
|
||||||
|
ASSERT_NE(bbto->filter_policy.get(), nullptr);
|
||||||
|
ASSERT_STREQ(bbto->filter_policy->Name(), MockFilterPolicy::kClassName());
|
||||||
|
}
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
}
|
||||||
|
|
||||||
TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) {
|
TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) {
|
||||||
std::shared_ptr<TableFactory> table;
|
std::shared_ptr<TableFactory> table;
|
||||||
std::shared_ptr<FlushBlockPolicyFactory> result;
|
std::shared_ptr<FlushBlockPolicyFactory> result;
|
||||||
|
@ -512,12 +512,6 @@ bool SerializeSingleOptionHelper(const void* opt_address,
|
|||||||
compression_type_string_map,
|
compression_type_string_map,
|
||||||
*(static_cast<const CompressionType*>(opt_address)), value);
|
*(static_cast<const CompressionType*>(opt_address)), value);
|
||||||
break;
|
break;
|
||||||
case OptionType::kFilterPolicy: {
|
|
||||||
const auto* ptr =
|
|
||||||
static_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
|
|
||||||
*value = ptr->get() ? ptr->get()->Name() : kNullptrString;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case OptionType::kChecksumType:
|
case OptionType::kChecksumType:
|
||||||
return SerializeEnum<ChecksumType>(
|
return SerializeEnum<ChecksumType>(
|
||||||
checksum_type_string_map,
|
checksum_type_string_map,
|
||||||
|
@ -844,6 +844,7 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
|
|||||||
ConfigOptions config_options;
|
ConfigOptions config_options;
|
||||||
config_options.input_strings_escaped = false;
|
config_options.input_strings_escaped = false;
|
||||||
config_options.ignore_unknown_options = false;
|
config_options.ignore_unknown_options = false;
|
||||||
|
config_options.ignore_unsupported_options = false;
|
||||||
|
|
||||||
// make sure default values are overwritten by something else
|
// make sure default values are overwritten by something else
|
||||||
ASSERT_OK(GetBlockBasedTableOptionsFromString(
|
ASSERT_OK(GetBlockBasedTableOptionsFromString(
|
||||||
@ -878,8 +879,8 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
|
|||||||
ASSERT_EQ(new_opt.detect_filter_construct_corruption, true);
|
ASSERT_EQ(new_opt.detect_filter_construct_corruption, true);
|
||||||
ASSERT_EQ(new_opt.reserve_table_builder_memory, true);
|
ASSERT_EQ(new_opt.reserve_table_builder_memory, true);
|
||||||
ASSERT_TRUE(new_opt.filter_policy != nullptr);
|
ASSERT_TRUE(new_opt.filter_policy != nullptr);
|
||||||
const BloomFilterPolicy* bfp =
|
auto bfp = new_opt.filter_policy->CheckedCast<BloomFilterPolicy>();
|
||||||
dynamic_cast<const BloomFilterPolicy*>(new_opt.filter_policy.get());
|
ASSERT_NE(bfp, nullptr);
|
||||||
EXPECT_EQ(bfp->GetMillibitsPerKey(), 4567);
|
EXPECT_EQ(bfp->GetMillibitsPerKey(), 4567);
|
||||||
EXPECT_EQ(bfp->GetWholeBitsPerKey(), 5);
|
EXPECT_EQ(bfp->GetWholeBitsPerKey(), 5);
|
||||||
// Verify that only the lower 32bits are stored in
|
// Verify that only the lower 32bits are stored in
|
||||||
@ -1104,6 +1105,25 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
|
|||||||
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
|
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
|
||||||
->GetHighPriPoolRatio(),
|
->GetHighPriPoolRatio(),
|
||||||
0.5);
|
0.5);
|
||||||
|
|
||||||
|
ASSERT_OK(GetBlockBasedTableOptionsFromString(
|
||||||
|
config_options, table_opt, "filter_policy=rocksdb.BloomFilter:1.234",
|
||||||
|
&new_opt));
|
||||||
|
ASSERT_TRUE(new_opt.filter_policy != nullptr);
|
||||||
|
ASSERT_TRUE(
|
||||||
|
new_opt.filter_policy->IsInstanceOf(BloomFilterPolicy::kClassName()));
|
||||||
|
ASSERT_TRUE(
|
||||||
|
new_opt.filter_policy->IsInstanceOf(BloomFilterPolicy::kNickName()));
|
||||||
|
|
||||||
|
// Ribbon filter policy alternative name
|
||||||
|
ASSERT_OK(GetBlockBasedTableOptionsFromString(
|
||||||
|
config_options, table_opt, "filter_policy=rocksdb.RibbonFilter:6.789:5;",
|
||||||
|
&new_opt));
|
||||||
|
ASSERT_TRUE(new_opt.filter_policy != nullptr);
|
||||||
|
ASSERT_TRUE(
|
||||||
|
new_opt.filter_policy->IsInstanceOf(RibbonFilterPolicy::kClassName()));
|
||||||
|
ASSERT_TRUE(
|
||||||
|
new_opt.filter_policy->IsInstanceOf(RibbonFilterPolicy::kNickName()));
|
||||||
}
|
}
|
||||||
#endif // !ROCKSDB_LITE
|
#endif // !ROCKSDB_LITE
|
||||||
|
|
||||||
|
@ -314,44 +314,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
|
|||||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||||
OptionTypeFlags::kNone}},
|
OptionTypeFlags::kNone}},
|
||||||
{"filter_policy",
|
{"filter_policy",
|
||||||
{offsetof(struct BlockBasedTableOptions, filter_policy),
|
OptionTypeInfo::AsCustomSharedPtr<const FilterPolicy>(
|
||||||
OptionType::kUnknown, OptionVerificationType::kByNameAllowFromNull,
|
offsetof(struct BlockBasedTableOptions, filter_policy),
|
||||||
OptionTypeFlags::kNone,
|
OptionVerificationType::kByNameAllowFromNull,
|
||||||
// Parses the Filter policy
|
OptionTypeFlags::kNone)},
|
||||||
[](const ConfigOptions& opts, const std::string&,
|
|
||||||
const std::string& value, void* addr) {
|
|
||||||
auto* policy =
|
|
||||||
static_cast<std::shared_ptr<const FilterPolicy>*>(addr);
|
|
||||||
return FilterPolicy::CreateFromString(opts, value, policy);
|
|
||||||
},
|
|
||||||
// Converts the FilterPolicy to its string representation
|
|
||||||
[](const ConfigOptions&, const std::string&, const void* addr,
|
|
||||||
std::string* value) {
|
|
||||||
const auto* policy =
|
|
||||||
static_cast<const std::shared_ptr<const FilterPolicy>*>(addr);
|
|
||||||
if (policy->get()) {
|
|
||||||
*value = (*policy)->Name();
|
|
||||||
} else {
|
|
||||||
*value = kNullptrString;
|
|
||||||
}
|
|
||||||
return Status::OK();
|
|
||||||
},
|
|
||||||
// Compares two FilterPolicy objects for equality
|
|
||||||
[](const ConfigOptions&, const std::string&, const void* addr1,
|
|
||||||
const void* addr2, std::string*) {
|
|
||||||
const auto* policy1 =
|
|
||||||
static_cast<const std::shared_ptr<const FilterPolicy>*>(addr1)
|
|
||||||
->get();
|
|
||||||
const auto* policy2 =
|
|
||||||
static_cast<const std::shared_ptr<FilterPolicy>*>(addr2)->get();
|
|
||||||
if (policy1 == policy2) {
|
|
||||||
return true;
|
|
||||||
} else if (policy1 != nullptr && policy2 != nullptr) {
|
|
||||||
return (strcmp(policy1->Name(), policy2->Name()) == 0);
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}}},
|
|
||||||
{"whole_key_filtering",
|
{"whole_key_filtering",
|
||||||
{offsetof(struct BlockBasedTableOptions, whole_key_filtering),
|
{offsetof(struct BlockBasedTableOptions, whole_key_filtering),
|
||||||
OptionType::kBoolean, OptionVerificationType::kNormal,
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
||||||
@ -913,6 +879,8 @@ Status GetBlockBasedTableOptionsFromString(
|
|||||||
config_options.input_strings_escaped = false;
|
config_options.input_strings_escaped = false;
|
||||||
config_options.ignore_unknown_options = false;
|
config_options.ignore_unknown_options = false;
|
||||||
config_options.invoke_prepare_options = false;
|
config_options.invoke_prepare_options = false;
|
||||||
|
config_options.ignore_unsupported_options = false;
|
||||||
|
|
||||||
return GetBlockBasedTableOptionsFromString(config_options, table_options,
|
return GetBlockBasedTableOptionsFromString(config_options, table_options,
|
||||||
opts_str, new_table_options);
|
opts_str, new_table_options);
|
||||||
}
|
}
|
||||||
|
@ -20,8 +20,10 @@
|
|||||||
#include "cache/cache_reservation_manager.h"
|
#include "cache/cache_reservation_manager.h"
|
||||||
#include "logging/logging.h"
|
#include "logging/logging.h"
|
||||||
#include "port/lang.h"
|
#include "port/lang.h"
|
||||||
|
#include "rocksdb/convenience.h"
|
||||||
#include "rocksdb/rocksdb_namespace.h"
|
#include "rocksdb/rocksdb_namespace.h"
|
||||||
#include "rocksdb/slice.h"
|
#include "rocksdb/slice.h"
|
||||||
|
#include "rocksdb/utilities/object_registry.h"
|
||||||
#include "table/block_based/block_based_filter_block.h"
|
#include "table/block_based/block_based_filter_block.h"
|
||||||
#include "table/block_based/block_based_table_reader.h"
|
#include "table/block_based/block_based_table_reader.h"
|
||||||
#include "table/block_based/filter_policy_internal.h"
|
#include "table/block_based/filter_policy_internal.h"
|
||||||
@ -1311,6 +1313,18 @@ Status XXPH3FilterBitsBuilder::MaybePostVerify(const Slice& filter_content) {
|
|||||||
}
|
}
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
|
const char* BuiltinFilterPolicy::kClassName() {
|
||||||
|
return "rocksdb.internal.BuiltinFilter";
|
||||||
|
}
|
||||||
|
|
||||||
|
bool BuiltinFilterPolicy::IsInstanceOf(const std::string& name) const {
|
||||||
|
if (name == kClassName()) {
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
return FilterPolicy::IsInstanceOf(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
BloomLikeFilterPolicy::BloomLikeFilterPolicy(double bits_per_key)
|
BloomLikeFilterPolicy::BloomLikeFilterPolicy(double bits_per_key)
|
||||||
: warned_(false), aggregate_rounding_balance_(0) {
|
: warned_(false), aggregate_rounding_balance_(0) {
|
||||||
// Sanitize bits_per_key
|
// Sanitize bits_per_key
|
||||||
@ -1345,17 +1359,28 @@ BloomLikeFilterPolicy::BloomLikeFilterPolicy(double bits_per_key)
|
|||||||
}
|
}
|
||||||
|
|
||||||
BloomLikeFilterPolicy::~BloomLikeFilterPolicy() {}
|
BloomLikeFilterPolicy::~BloomLikeFilterPolicy() {}
|
||||||
|
const char* BloomLikeFilterPolicy::kClassName() {
|
||||||
|
return "rocksdb.internal.BloomLikeFilter";
|
||||||
|
}
|
||||||
|
|
||||||
const char* BuiltinFilterPolicy::Name() const {
|
bool BloomLikeFilterPolicy::IsInstanceOf(const std::string& name) const {
|
||||||
|
if (name == kClassName()) {
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
return BuiltinFilterPolicy::IsInstanceOf(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* ReadOnlyBuiltinFilterPolicy::kClassName() {
|
||||||
return "rocksdb.BuiltinBloomFilter";
|
return "rocksdb.BuiltinBloomFilter";
|
||||||
}
|
}
|
||||||
|
|
||||||
const char* DeprecatedBlockBasedBloomFilterPolicy::kName() {
|
const char* DeprecatedBlockBasedBloomFilterPolicy::kClassName() {
|
||||||
return "rocksdb.internal.DeprecatedBlockBasedBloomFilter";
|
return "rocksdb.internal.DeprecatedBlockBasedBloomFilter";
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string DeprecatedBlockBasedBloomFilterPolicy::GetId() const {
|
std::string BloomLikeFilterPolicy::GetId() const {
|
||||||
return kName() + GetBitsPerKeySuffix();
|
return Name() + GetBitsPerKeySuffix();
|
||||||
}
|
}
|
||||||
|
|
||||||
DeprecatedBlockBasedBloomFilterPolicy::DeprecatedBlockBasedBloomFilterPolicy(
|
DeprecatedBlockBasedBloomFilterPolicy::DeprecatedBlockBasedBloomFilterPolicy(
|
||||||
@ -1446,12 +1471,13 @@ FilterBitsBuilder* BloomFilterPolicy::GetBuilderWithContext(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const char* BloomFilterPolicy::kName() { return "bloomfilter"; }
|
const char* BloomFilterPolicy::kClassName() { return "bloomfilter"; }
|
||||||
|
const char* BloomFilterPolicy::kNickName() { return "rocksdb.BloomFilter"; }
|
||||||
|
|
||||||
std::string BloomFilterPolicy::GetId() const {
|
std::string BloomFilterPolicy::GetId() const {
|
||||||
// Including ":false" for better forward-compatibility with 6.29 and earlier
|
// Including ":false" for better forward-compatibility with 6.29 and earlier
|
||||||
// which required a boolean `use_block_based_builder` parameter
|
// which required a boolean `use_block_based_builder` parameter
|
||||||
return kName() + GetBitsPerKeySuffix() + ":false";
|
return BloomLikeFilterPolicy::GetId() + ":false";
|
||||||
}
|
}
|
||||||
|
|
||||||
FilterBitsBuilder* BloomLikeFilterPolicy::GetFastLocalBloomBuilderWithContext(
|
FilterBitsBuilder* BloomLikeFilterPolicy::GetFastLocalBloomBuilderWithContext(
|
||||||
@ -1543,14 +1569,10 @@ FilterBitsBuilder* BuiltinFilterPolicy::GetBuilderFromContext(
|
|||||||
// For testing only, but always constructable with internal names
|
// For testing only, but always constructable with internal names
|
||||||
namespace test {
|
namespace test {
|
||||||
|
|
||||||
const char* LegacyBloomFilterPolicy::kName() {
|
const char* LegacyBloomFilterPolicy::kClassName() {
|
||||||
return "rocksdb.internal.LegacyBloomFilter";
|
return "rocksdb.internal.LegacyBloomFilter";
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string LegacyBloomFilterPolicy::GetId() const {
|
|
||||||
return kName() + GetBitsPerKeySuffix();
|
|
||||||
}
|
|
||||||
|
|
||||||
FilterBitsBuilder* LegacyBloomFilterPolicy::GetBuilderWithContext(
|
FilterBitsBuilder* LegacyBloomFilterPolicy::GetBuilderWithContext(
|
||||||
const FilterBuildingContext& context) const {
|
const FilterBuildingContext& context) const {
|
||||||
if (GetMillibitsPerKey() == 0) {
|
if (GetMillibitsPerKey() == 0) {
|
||||||
@ -1560,14 +1582,10 @@ FilterBitsBuilder* LegacyBloomFilterPolicy::GetBuilderWithContext(
|
|||||||
return GetLegacyBloomBuilderWithContext(context);
|
return GetLegacyBloomBuilderWithContext(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
const char* FastLocalBloomFilterPolicy::kName() {
|
const char* FastLocalBloomFilterPolicy::kClassName() {
|
||||||
return "rocksdb.internal.FastLocalBloomFilter";
|
return "rocksdb.internal.FastLocalBloomFilter";
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string FastLocalBloomFilterPolicy::GetId() const {
|
|
||||||
return kName() + GetBitsPerKeySuffix();
|
|
||||||
}
|
|
||||||
|
|
||||||
FilterBitsBuilder* FastLocalBloomFilterPolicy::GetBuilderWithContext(
|
FilterBitsBuilder* FastLocalBloomFilterPolicy::GetBuilderWithContext(
|
||||||
const FilterBuildingContext& context) const {
|
const FilterBuildingContext& context) const {
|
||||||
if (GetMillibitsPerKey() == 0) {
|
if (GetMillibitsPerKey() == 0) {
|
||||||
@ -1577,14 +1595,10 @@ FilterBitsBuilder* FastLocalBloomFilterPolicy::GetBuilderWithContext(
|
|||||||
return GetFastLocalBloomBuilderWithContext(context);
|
return GetFastLocalBloomBuilderWithContext(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
const char* Standard128RibbonFilterPolicy::kName() {
|
const char* Standard128RibbonFilterPolicy::kClassName() {
|
||||||
return "rocksdb.internal.Standard128RibbonFilter";
|
return "rocksdb.internal.Standard128RibbonFilter";
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string Standard128RibbonFilterPolicy::GetId() const {
|
|
||||||
return kName() + GetBitsPerKeySuffix();
|
|
||||||
}
|
|
||||||
|
|
||||||
FilterBitsBuilder* Standard128RibbonFilterPolicy::GetBuilderWithContext(
|
FilterBitsBuilder* Standard128RibbonFilterPolicy::GetBuilderWithContext(
|
||||||
const FilterBuildingContext& context) const {
|
const FilterBuildingContext& context) const {
|
||||||
if (GetMillibitsPerKey() == 0) {
|
if (GetMillibitsPerKey() == 0) {
|
||||||
@ -1816,10 +1830,11 @@ FilterBitsBuilder* RibbonFilterPolicy::GetBuilderWithContext(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const char* RibbonFilterPolicy::kName() { return "ribbonfilter"; }
|
const char* RibbonFilterPolicy::kClassName() { return "ribbonfilter"; }
|
||||||
|
const char* RibbonFilterPolicy::kNickName() { return "rocksdb.RibbonFilter"; }
|
||||||
|
|
||||||
std::string RibbonFilterPolicy::GetId() const {
|
std::string RibbonFilterPolicy::GetId() const {
|
||||||
return kName() + GetBitsPerKeySuffix() + ":" +
|
return BloomLikeFilterPolicy::GetId() + ":" +
|
||||||
ROCKSDB_NAMESPACE::ToString(bloom_before_level_);
|
ROCKSDB_NAMESPACE::ToString(bloom_before_level_);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1837,19 +1852,19 @@ FilterPolicy::~FilterPolicy() { }
|
|||||||
|
|
||||||
std::shared_ptr<const FilterPolicy> BloomLikeFilterPolicy::Create(
|
std::shared_ptr<const FilterPolicy> BloomLikeFilterPolicy::Create(
|
||||||
const std::string& name, double bits_per_key) {
|
const std::string& name, double bits_per_key) {
|
||||||
if (name == test::LegacyBloomFilterPolicy::kName()) {
|
if (name == test::LegacyBloomFilterPolicy::kClassName()) {
|
||||||
return std::make_shared<test::LegacyBloomFilterPolicy>(bits_per_key);
|
return std::make_shared<test::LegacyBloomFilterPolicy>(bits_per_key);
|
||||||
} else if (name == test::FastLocalBloomFilterPolicy::kName()) {
|
} else if (name == test::FastLocalBloomFilterPolicy::kClassName()) {
|
||||||
return std::make_shared<test::FastLocalBloomFilterPolicy>(bits_per_key);
|
return std::make_shared<test::FastLocalBloomFilterPolicy>(bits_per_key);
|
||||||
} else if (name == test::Standard128RibbonFilterPolicy::kName()) {
|
} else if (name == test::Standard128RibbonFilterPolicy::kClassName()) {
|
||||||
return std::make_shared<test::Standard128RibbonFilterPolicy>(bits_per_key);
|
return std::make_shared<test::Standard128RibbonFilterPolicy>(bits_per_key);
|
||||||
} else if (name == DeprecatedBlockBasedBloomFilterPolicy::kName()) {
|
} else if (name == DeprecatedBlockBasedBloomFilterPolicy::kClassName()) {
|
||||||
return std::make_shared<DeprecatedBlockBasedBloomFilterPolicy>(
|
return std::make_shared<DeprecatedBlockBasedBloomFilterPolicy>(
|
||||||
bits_per_key);
|
bits_per_key);
|
||||||
} else if (name == BloomFilterPolicy::kName()) {
|
} else if (name == BloomFilterPolicy::kClassName()) {
|
||||||
// For testing
|
// For testing
|
||||||
return std::make_shared<BloomFilterPolicy>(bits_per_key);
|
return std::make_shared<BloomFilterPolicy>(bits_per_key);
|
||||||
} else if (name == RibbonFilterPolicy::kName()) {
|
} else if (name == RibbonFilterPolicy::kClassName()) {
|
||||||
// For testing
|
// For testing
|
||||||
return std::make_shared<RibbonFilterPolicy>(bits_per_key,
|
return std::make_shared<RibbonFilterPolicy>(bits_per_key,
|
||||||
/*bloom_before_level*/ 0);
|
/*bloom_before_level*/ 0);
|
||||||
@ -1858,59 +1873,173 @@ std::shared_ptr<const FilterPolicy> BloomLikeFilterPolicy::Create(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
namespace {
|
||||||
|
static ObjectLibrary::PatternEntry FilterPatternEntryWithBits(
|
||||||
|
const char* name) {
|
||||||
|
return ObjectLibrary::PatternEntry(name, false).AddNumber(":", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
T* NewBuiltinFilterPolicyWithBits(const std::string& uri) {
|
||||||
|
const std::vector<std::string> vals = StringSplit(uri, ':');
|
||||||
|
double bits_per_key = ParseDouble(vals[1]);
|
||||||
|
return new T(bits_per_key);
|
||||||
|
}
|
||||||
|
static int RegisterBuiltinFilterPolicies(ObjectLibrary& library,
|
||||||
|
const std::string& /*arg*/) {
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
ReadOnlyBuiltinFilterPolicy::kClassName(),
|
||||||
|
[](const std::string& /*uri*/, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(new ReadOnlyBuiltinFilterPolicy());
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(BloomFilterPolicy::kClassName())
|
||||||
|
.AnotherName(BloomFilterPolicy::kNickName()),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(NewBuiltinFilterPolicyWithBits<BloomFilterPolicy>(uri));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(BloomFilterPolicy::kClassName())
|
||||||
|
.AnotherName(BloomFilterPolicy::kNickName())
|
||||||
|
.AddSuffix(":false"),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(NewBuiltinFilterPolicyWithBits<BloomFilterPolicy>(uri));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(BloomFilterPolicy::kClassName())
|
||||||
|
.AnotherName(BloomFilterPolicy::kNickName())
|
||||||
|
.AddSuffix(":true"),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
const std::vector<std::string> vals = StringSplit(uri, ':');
|
||||||
|
double bits_per_key = ParseDouble(vals[1]);
|
||||||
|
// NOTE: This case previously configured the deprecated block-based
|
||||||
|
// filter, but old ways of configuring that now map to full filter. We
|
||||||
|
// defer to the corresponding API to ensure consistency in case that
|
||||||
|
// change is reverted.
|
||||||
|
guard->reset(NewBloomFilterPolicy(bits_per_key, true));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(RibbonFilterPolicy::kClassName())
|
||||||
|
.AnotherName(RibbonFilterPolicy::kNickName()),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
const std::vector<std::string> vals = StringSplit(uri, ':');
|
||||||
|
double bits_per_key = ParseDouble(vals[1]);
|
||||||
|
guard->reset(NewRibbonFilterPolicy(bits_per_key));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(RibbonFilterPolicy::kClassName())
|
||||||
|
.AnotherName(RibbonFilterPolicy::kNickName())
|
||||||
|
.AddNumber(":", true),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
const std::vector<std::string> vals = StringSplit(uri, ':');
|
||||||
|
double bits_per_key = ParseDouble(vals[1]);
|
||||||
|
int bloom_before_level = ParseInt(vals[2]);
|
||||||
|
guard->reset(NewRibbonFilterPolicy(bits_per_key, bloom_before_level));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(test::LegacyBloomFilterPolicy::kClassName()),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(
|
||||||
|
NewBuiltinFilterPolicyWithBits<test::LegacyBloomFilterPolicy>(uri));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(
|
||||||
|
test::FastLocalBloomFilterPolicy::kClassName()),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(
|
||||||
|
NewBuiltinFilterPolicyWithBits<test::FastLocalBloomFilterPolicy>(
|
||||||
|
uri));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(
|
||||||
|
test::Standard128RibbonFilterPolicy::kClassName()),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(
|
||||||
|
NewBuiltinFilterPolicyWithBits<test::Standard128RibbonFilterPolicy>(
|
||||||
|
uri));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.AddFactory<const FilterPolicy>(
|
||||||
|
FilterPatternEntryWithBits(
|
||||||
|
DeprecatedBlockBasedBloomFilterPolicy::kClassName()),
|
||||||
|
[](const std::string& uri, std::unique_ptr<const FilterPolicy>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(NewBuiltinFilterPolicyWithBits<
|
||||||
|
DeprecatedBlockBasedBloomFilterPolicy>(uri));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
size_t num_types;
|
||||||
|
return static_cast<int>(library.GetFactoryCount(&num_types));
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
Status FilterPolicy::CreateFromString(
|
Status FilterPolicy::CreateFromString(
|
||||||
const ConfigOptions& /*options*/, const std::string& value,
|
const ConfigOptions& options, const std::string& value,
|
||||||
std::shared_ptr<const FilterPolicy>* policy) {
|
std::shared_ptr<const FilterPolicy>* policy) {
|
||||||
if (value == kNullptrString) {
|
if (value == kNullptrString || value.empty()) {
|
||||||
policy->reset();
|
policy->reset();
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
} else if (value == "rocksdb.BuiltinBloomFilter") {
|
} else if (value == ReadOnlyBuiltinFilterPolicy::kClassName()) {
|
||||||
*policy = std::make_shared<ReadOnlyBuiltinFilterPolicy>();
|
*policy = std::make_shared<ReadOnlyBuiltinFilterPolicy>();
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string id;
|
||||||
|
std::unordered_map<std::string, std::string> opt_map;
|
||||||
|
Status status =
|
||||||
|
Customizable::GetOptionsMap(options, policy->get(), value, &id, &opt_map);
|
||||||
|
if (!status.ok()) { // GetOptionsMap failed
|
||||||
|
return status;
|
||||||
|
} else if (id.empty()) { // We have no Id but have options. Not good
|
||||||
|
return Status::NotSupported("Cannot reset object ", id);
|
||||||
|
} else {
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
const std::vector<std::string> vals = StringSplit(value, ':');
|
static std::once_flag loaded;
|
||||||
if (vals.size() < 2) {
|
std::call_once(loaded, [&]() {
|
||||||
return Status::NotFound("Invalid filter policy name ", value);
|
RegisterBuiltinFilterPolicies(*(ObjectLibrary::Default().get()), "");
|
||||||
}
|
});
|
||||||
const std::string& name = vals[0];
|
status = options.registry->NewSharedObject(id, policy);
|
||||||
double bits_per_key = ParseDouble(trim(vals[1]));
|
|
||||||
if (name == BloomFilterPolicy::kName()) {
|
|
||||||
bool use_block_based_builder = false;
|
|
||||||
if (vals.size() > 2) {
|
|
||||||
use_block_based_builder =
|
|
||||||
ParseBoolean("use_block_based_builder", trim(vals[2]));
|
|
||||||
}
|
|
||||||
policy->reset(NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
|
|
||||||
} else if (name == RibbonFilterPolicy::kName()) {
|
|
||||||
int bloom_before_level;
|
|
||||||
if (vals.size() < 3) {
|
|
||||||
bloom_before_level = 0;
|
|
||||||
} else {
|
|
||||||
bloom_before_level = ParseInt(trim(vals[2]));
|
|
||||||
}
|
|
||||||
policy->reset(NewRibbonFilterPolicy(/*bloom_equivalent*/ bits_per_key,
|
|
||||||
bloom_before_level));
|
|
||||||
} else {
|
|
||||||
*policy = BloomLikeFilterPolicy::Create(name, bits_per_key);
|
|
||||||
}
|
|
||||||
if (*policy) {
|
|
||||||
return Status::OK();
|
|
||||||
} else {
|
|
||||||
return Status::NotFound("Invalid filter policy name ", value);
|
|
||||||
}
|
|
||||||
#else
|
#else
|
||||||
return Status::NotSupported("Cannot load filter policy in LITE mode ", value);
|
status =
|
||||||
|
Status::NotSupported("Cannot load filter policy in LITE mode ", value);
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
}
|
||||||
|
if (options.ignore_unsupported_options && status.IsNotSupported()) {
|
||||||
|
return Status::OK();
|
||||||
|
} else if (status.ok()) {
|
||||||
|
status = Customizable::ConfigureNewObject(
|
||||||
|
options, const_cast<FilterPolicy*>(policy->get()), opt_map);
|
||||||
|
}
|
||||||
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::vector<std::string>& BloomLikeFilterPolicy::GetAllFixedImpls() {
|
const std::vector<std::string>& BloomLikeFilterPolicy::GetAllFixedImpls() {
|
||||||
STATIC_AVOID_DESTRUCTION(std::vector<std::string>, impls){
|
STATIC_AVOID_DESTRUCTION(std::vector<std::string>, impls){
|
||||||
// Match filter_bench -impl=x ordering
|
// Match filter_bench -impl=x ordering
|
||||||
test::LegacyBloomFilterPolicy::kName(),
|
test::LegacyBloomFilterPolicy::kClassName(),
|
||||||
DeprecatedBlockBasedBloomFilterPolicy::kName(),
|
DeprecatedBlockBasedBloomFilterPolicy::kClassName(),
|
||||||
test::FastLocalBloomFilterPolicy::kName(),
|
test::FastLocalBloomFilterPolicy::kClassName(),
|
||||||
test::Standard128RibbonFilterPolicy::kName(),
|
test::Standard128RibbonFilterPolicy::kClassName(),
|
||||||
};
|
};
|
||||||
return impls;
|
return impls;
|
||||||
}
|
}
|
||||||
|
@ -128,20 +128,13 @@ class BuiltinFilterBitsReader : public FilterBitsReader {
|
|||||||
// be used even when you change between built-in policies).
|
// be used even when you change between built-in policies).
|
||||||
class BuiltinFilterPolicy : public FilterPolicy {
|
class BuiltinFilterPolicy : public FilterPolicy {
|
||||||
public: // overrides
|
public: // overrides
|
||||||
// Shared name because any built-in policy can read filters from
|
|
||||||
// any other
|
|
||||||
// FIXME when making filter policies Configurable. For now, this
|
|
||||||
// is still rocksdb.BuiltinBloomFilter
|
|
||||||
const char* Name() const override;
|
|
||||||
|
|
||||||
// Convert to a string understood by FilterPolicy::CreateFromString
|
|
||||||
virtual std::string GetId() const = 0;
|
|
||||||
|
|
||||||
// Read metadata to determine what kind of FilterBitsReader is needed
|
// Read metadata to determine what kind of FilterBitsReader is needed
|
||||||
// and return a new one. This must successfully process any filter data
|
// and return a new one. This must successfully process any filter data
|
||||||
// generated by a built-in FilterBitsBuilder, regardless of the impl
|
// generated by a built-in FilterBitsBuilder, regardless of the impl
|
||||||
// chosen for this BloomFilterPolicy. Not compatible with CreateFilter.
|
// chosen for this BloomFilterPolicy. Not compatible with CreateFilter.
|
||||||
FilterBitsReader* GetFilterBitsReader(const Slice& contents) const override;
|
FilterBitsReader* GetFilterBitsReader(const Slice& contents) const override;
|
||||||
|
static const char* kClassName();
|
||||||
|
bool IsInstanceOf(const std::string& id) const override;
|
||||||
|
|
||||||
public: // new
|
public: // new
|
||||||
// An internal function for the implementation of
|
// An internal function for the implementation of
|
||||||
@ -175,8 +168,8 @@ class BuiltinFilterPolicy : public FilterPolicy {
|
|||||||
// This class is considered internal API and subject to change.
|
// This class is considered internal API and subject to change.
|
||||||
class ReadOnlyBuiltinFilterPolicy : public BuiltinFilterPolicy {
|
class ReadOnlyBuiltinFilterPolicy : public BuiltinFilterPolicy {
|
||||||
public:
|
public:
|
||||||
// Convert to a string understood by FilterPolicy::CreateFromString
|
const char* Name() const override { return kClassName(); }
|
||||||
virtual std::string GetId() const override { return Name(); }
|
static const char* kClassName();
|
||||||
|
|
||||||
// Does not write filters.
|
// Does not write filters.
|
||||||
FilterBitsBuilder* GetBuilderWithContext(
|
FilterBitsBuilder* GetBuilderWithContext(
|
||||||
@ -194,6 +187,10 @@ class BloomLikeFilterPolicy : public BuiltinFilterPolicy {
|
|||||||
explicit BloomLikeFilterPolicy(double bits_per_key);
|
explicit BloomLikeFilterPolicy(double bits_per_key);
|
||||||
|
|
||||||
~BloomLikeFilterPolicy() override;
|
~BloomLikeFilterPolicy() override;
|
||||||
|
static const char* kClassName();
|
||||||
|
bool IsInstanceOf(const std::string& id) const override;
|
||||||
|
|
||||||
|
std::string GetId() const override;
|
||||||
|
|
||||||
// Essentially for testing only: configured millibits/key
|
// Essentially for testing only: configured millibits/key
|
||||||
int GetMillibitsPerKey() const { return millibits_per_key_; }
|
int GetMillibitsPerKey() const { return millibits_per_key_; }
|
||||||
@ -268,7 +265,10 @@ class BloomFilterPolicy : public BloomLikeFilterPolicy {
|
|||||||
FilterBitsBuilder* GetBuilderWithContext(
|
FilterBitsBuilder* GetBuilderWithContext(
|
||||||
const FilterBuildingContext&) const override;
|
const FilterBuildingContext&) const override;
|
||||||
|
|
||||||
static const char* kName();
|
static const char* kClassName();
|
||||||
|
const char* Name() const override { return kClassName(); }
|
||||||
|
static const char* kNickName();
|
||||||
|
const char* NickName() const override { return kNickName(); }
|
||||||
std::string GetId() const override;
|
std::string GetId() const override;
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -287,7 +287,10 @@ class RibbonFilterPolicy : public BloomLikeFilterPolicy {
|
|||||||
|
|
||||||
int GetBloomBeforeLevel() const { return bloom_before_level_; }
|
int GetBloomBeforeLevel() const { return bloom_before_level_; }
|
||||||
|
|
||||||
static const char* kName();
|
static const char* kClassName();
|
||||||
|
const char* Name() const override { return kClassName(); }
|
||||||
|
static const char* kNickName();
|
||||||
|
const char* NickName() const override { return kNickName(); }
|
||||||
std::string GetId() const override;
|
std::string GetId() const override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@ -310,8 +313,8 @@ class DeprecatedBlockBasedBloomFilterPolicy : public BloomLikeFilterPolicy {
|
|||||||
const FilterBuildingContext&) const override;
|
const FilterBuildingContext&) const override;
|
||||||
static constexpr size_t kSecretBitsPerKeyStart = 1234567890U;
|
static constexpr size_t kSecretBitsPerKeyStart = 1234567890U;
|
||||||
|
|
||||||
static const char* kName();
|
static const char* kClassName();
|
||||||
std::string GetId() const override;
|
const char* Name() const override { return kClassName(); }
|
||||||
|
|
||||||
static void CreateFilter(const Slice* keys, int n, int bits_per_key,
|
static void CreateFilter(const Slice* keys, int n, int bits_per_key,
|
||||||
std::string* dst);
|
std::string* dst);
|
||||||
@ -329,8 +332,8 @@ class LegacyBloomFilterPolicy : public BloomLikeFilterPolicy {
|
|||||||
FilterBitsBuilder* GetBuilderWithContext(
|
FilterBitsBuilder* GetBuilderWithContext(
|
||||||
const FilterBuildingContext& context) const override;
|
const FilterBuildingContext& context) const override;
|
||||||
|
|
||||||
static const char* kName();
|
static const char* kClassName();
|
||||||
std::string GetId() const override;
|
const char* Name() const override { return kClassName(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
class FastLocalBloomFilterPolicy : public BloomLikeFilterPolicy {
|
class FastLocalBloomFilterPolicy : public BloomLikeFilterPolicy {
|
||||||
@ -341,8 +344,8 @@ class FastLocalBloomFilterPolicy : public BloomLikeFilterPolicy {
|
|||||||
FilterBitsBuilder* GetBuilderWithContext(
|
FilterBitsBuilder* GetBuilderWithContext(
|
||||||
const FilterBuildingContext& context) const override;
|
const FilterBuildingContext& context) const override;
|
||||||
|
|
||||||
static const char* kName();
|
static const char* kClassName();
|
||||||
std::string GetId() const override;
|
const char* Name() const override { return kClassName(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
class Standard128RibbonFilterPolicy : public BloomLikeFilterPolicy {
|
class Standard128RibbonFilterPolicy : public BloomLikeFilterPolicy {
|
||||||
@ -353,8 +356,8 @@ class Standard128RibbonFilterPolicy : public BloomLikeFilterPolicy {
|
|||||||
FilterBitsBuilder* GetBuilderWithContext(
|
FilterBitsBuilder* GetBuilderWithContext(
|
||||||
const FilterBuildingContext& context) const override;
|
const FilterBuildingContext& context) const override;
|
||||||
|
|
||||||
static const char* kName();
|
static const char* kClassName();
|
||||||
std::string GetId() const override;
|
const char* Name() const override { return kClassName(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace test
|
} // namespace test
|
||||||
|
@ -1872,7 +1872,7 @@ TEST_P(BlockBasedTableTest, FilterPolicyNameProperties) {
|
|||||||
c.Finish(options, ioptions, moptions, table_options,
|
c.Finish(options, ioptions, moptions, table_options,
|
||||||
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
|
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
|
||||||
auto& props = *c.GetTableReader()->GetTableProperties();
|
auto& props = *c.GetTableReader()->GetTableProperties();
|
||||||
ASSERT_EQ("rocksdb.BuiltinBloomFilter", props.filter_policy_name);
|
ASSERT_EQ(table_options.filter_policy->Name(), props.filter_policy_name);
|
||||||
c.ResetTableReader();
|
c.ResetTableReader();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -40,10 +40,11 @@ DEFINE_int32(bits_per_key, 10, "");
|
|||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
const std::string kLegacyBloom = test::LegacyBloomFilterPolicy::kName();
|
const std::string kLegacyBloom = test::LegacyBloomFilterPolicy::kClassName();
|
||||||
const std::string kFastLocalBloom = test::FastLocalBloomFilterPolicy::kName();
|
const std::string kFastLocalBloom =
|
||||||
|
test::FastLocalBloomFilterPolicy::kClassName();
|
||||||
const std::string kStandard128Ribbon =
|
const std::string kStandard128Ribbon =
|
||||||
test::Standard128RibbonFilterPolicy::kName();
|
test::Standard128RibbonFilterPolicy::kClassName();
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
static const int kVerbose = 1;
|
static const int kVerbose = 1;
|
||||||
|
@ -19,6 +19,9 @@ namespace {
|
|||||||
bool MatchesInteger(const std::string &target, size_t start, size_t pos) {
|
bool MatchesInteger(const std::string &target, size_t start, size_t pos) {
|
||||||
// If it is numeric, everything up to the match must be a number
|
// If it is numeric, everything up to the match must be a number
|
||||||
int digits = 0;
|
int digits = 0;
|
||||||
|
if (target[start] == '-') {
|
||||||
|
start++; // Allow negative numbers
|
||||||
|
}
|
||||||
while (start < pos) {
|
while (start < pos) {
|
||||||
if (!isdigit(target[start++])) {
|
if (!isdigit(target[start++])) {
|
||||||
return false;
|
return false;
|
||||||
@ -31,6 +34,9 @@ bool MatchesInteger(const std::string &target, size_t start, size_t pos) {
|
|||||||
|
|
||||||
bool MatchesDecimal(const std::string &target, size_t start, size_t pos) {
|
bool MatchesDecimal(const std::string &target, size_t start, size_t pos) {
|
||||||
int digits = 0;
|
int digits = 0;
|
||||||
|
if (target[start] == '-') {
|
||||||
|
start++; // Allow negative numbers
|
||||||
|
}
|
||||||
for (bool point = false; start < pos; start++) {
|
for (bool point = false; start < pos; start++) {
|
||||||
if (target[start] == '.') {
|
if (target[start] == '.') {
|
||||||
if (point) {
|
if (point) {
|
||||||
|
Loading…
Reference in New Issue
Block a user