Remember whole key/prefix filtering on/off in SST file

Summary: Remember whole key or prefix filtering on/off in SST files. If user opens the DB with a different setting that cannot be satisfied while reading the SST file, ignore the bloom filter.

Test Plan: Add a unit test for it

Reviewers: yhchiang, igor, rven

Reviewed By: rven

Subscribers: leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D32889
This commit is contained in:
sdong 2015-02-04 17:03:57 -08:00
parent fd5970b454
commit 68af7811ea
13 changed files with 266 additions and 50 deletions

View File

@ -14,6 +14,7 @@
* MemEnv (env that stores data in memory) is now available in default library build. You can create it by calling NewMemEnv().
* Add SliceTransform.SameResultWhenAppended() to help users determine it is safe to apply prefix bloom/hash.
* Block based table now makes use of prefix bloom filter if it is a full fulter.
* Block based table remembers whether a whole key or prefix based bloom filter is supported in SST files. Do a sanity check when reading the file with users' configuration.
### Public API changes
* Deprecated skip_log_error_on_recovery option

View File

@ -2018,6 +2018,155 @@ TEST(DBTest, GetFilterByPrefixBloom) {
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
}
TEST(DBTest, WholeKeyFilterProp) {
Options options = last_options_;
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
options.statistics = rocksdb::CreateDBStatistics();
BlockBasedTableOptions bbto;
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
bbto.whole_key_filtering = false;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
DestroyAndReopen(options);
WriteOptions wo;
ReadOptions ro;
FlushOptions fo;
fo.wait = true;
std::string value;
ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
// Needs insert some keys to make sure files are not filtered out by key
// ranges.
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
dbfull()->Flush(fo);
Reopen(options);
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
ASSERT_EQ("NOT_FOUND", Get("bar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("foo", Get("foobar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
// Reopen with whole key filtering enabled and prefix extractor
// NULL. Bloom filter should be off for both of whole key and
// prefix bloom.
bbto.whole_key_filtering = true;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
options.prefix_extractor.reset();
Reopen(options);
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("NOT_FOUND", Get("bar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("foo", Get("foobar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
// Write DB with only full key filtering.
ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
// Needs insert some keys to make sure files are not filtered out by key
// ranges.
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
db_->CompactRange(nullptr, nullptr);
// Reopen with both of whole key off and prefix extractor enabled.
// Still no bloom filter should be used.
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
bbto.whole_key_filtering = false;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
Reopen(options);
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("NOT_FOUND", Get("bar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("foo", Get("foobar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
// Try to create a DB with mixed files:
ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
// Needs insert some keys to make sure files are not filtered out by key
// ranges.
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
db_->CompactRange(nullptr, nullptr);
options.prefix_extractor.reset();
bbto.whole_key_filtering = true;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
Reopen(options);
// Try to create a DB with mixed files.
ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
// In this case needs insert some keys to make sure files are
// not filtered out by key ranges.
ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
Flush();
// Now we have two files:
// File 1: An older file with prefix bloom.
// File 2: A newer file with whole bloom filter.
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
ASSERT_EQ("NOT_FOUND", Get("bar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 3);
ASSERT_EQ("foo", Get("foobar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
ASSERT_EQ("bar", Get("barfoo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
// Reopen with the same setting: only whole key is used
Reopen(options);
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 5);
ASSERT_EQ("NOT_FOUND", Get("bar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 6);
ASSERT_EQ("foo", Get("foobar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
ASSERT_EQ("bar", Get("barfoo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
// Restart with both filters are allowed
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
bbto.whole_key_filtering = true;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
Reopen(options);
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
// File 1 will has it filtered out.
// File 2 will not, as prefix `foo` exists in the file.
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 8);
ASSERT_EQ("NOT_FOUND", Get("bar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 10);
ASSERT_EQ("foo", Get("foobar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
ASSERT_EQ("bar", Get("barfoo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
// Restart with only prefix bloom is allowed.
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
bbto.whole_key_filtering = false;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
Reopen(options);
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
ASSERT_EQ("NOT_FOUND", Get("bar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
ASSERT_EQ("foo", Get("foobar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
ASSERT_EQ("bar", Get("barfoo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
}
TEST(DBTest, IterSeekBeforePrev) {
ASSERT_OK(Put("a", "b"));
ASSERT_OK(Put("c", "d"));

View File

@ -147,6 +147,10 @@ struct BlockBasedTableOptions {
struct BlockBasedTablePropertyNames {
// value of this propertis is a fixed int32 number.
static const std::string kIndexType;
// value is "1" for true and "0" for false.
static const std::string kWholeKeyFiltering;
// value is "1" for true and "0" for false.
static const std::string kPrefixFiltering;
};
// Create default block based table factory.

View File

@ -171,10 +171,11 @@ void BlockBasedFilterBlockBuilder::GenerateFilter() {
BlockBasedFilterBlockReader::BlockBasedFilterBlockReader(
const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt, BlockContents&& contents)
const BlockBasedTableOptions& table_opt, bool whole_key_filtering,
BlockContents&& contents)
: policy_(table_opt.filter_policy.get()),
prefix_extractor_(prefix_extractor),
whole_key_filtering_(table_opt.whole_key_filtering),
whole_key_filtering_(whole_key_filtering),
data_(nullptr),
offset_(nullptr),
num_(0),

View File

@ -74,6 +74,7 @@ class BlockBasedFilterBlockReader : public FilterBlockReader {
// REQUIRES: "contents" and *policy must stay live while *this is live.
BlockBasedFilterBlockReader(const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt,
bool whole_key_filtering,
BlockContents&& contents);
virtual bool IsBlockBased() override { return true; }
virtual bool KeyMayMatch(const Slice& key,

View File

@ -57,7 +57,8 @@ TEST(FilterBlockTest, EmptyBuilder) {
BlockBasedFilterBlockBuilder builder(nullptr, table_options_);
BlockContents block(builder.Finish(), false, kNoCompression);
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block.data));
BlockBasedFilterBlockReader reader(nullptr, table_options_, std::move(block));
BlockBasedFilterBlockReader reader(nullptr, table_options_, true,
std::move(block));
ASSERT_TRUE(reader.KeyMayMatch("foo", 0));
ASSERT_TRUE(reader.KeyMayMatch("foo", 100000));
}
@ -73,7 +74,8 @@ TEST(FilterBlockTest, SingleChunk) {
builder.StartBlock(300);
builder.Add("hello");
BlockContents block(builder.Finish(), false, kNoCompression);
BlockBasedFilterBlockReader reader(nullptr, table_options_, std::move(block));
BlockBasedFilterBlockReader reader(nullptr, table_options_, true,
std::move(block));
ASSERT_TRUE(reader.KeyMayMatch("foo", 100));
ASSERT_TRUE(reader.KeyMayMatch("bar", 100));
ASSERT_TRUE(reader.KeyMayMatch("box", 100));
@ -104,7 +106,8 @@ TEST(FilterBlockTest, MultiChunk) {
builder.Add("hello");
BlockContents block(builder.Finish(), false, kNoCompression);
BlockBasedFilterBlockReader reader(nullptr, table_options_, std::move(block));
BlockBasedFilterBlockReader reader(nullptr, table_options_, true,
std::move(block));
// Check first filter
ASSERT_TRUE(reader.KeyMayMatch("foo", 0));
@ -150,7 +153,7 @@ TEST(BlockBasedFilterBlockTest, BlockBasedEmptyBuilder) {
BlockContents block(builder->Finish(), false, kNoCompression);
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block.data));
FilterBlockReader* reader = new BlockBasedFilterBlockReader(
nullptr, table_options_, std::move(block));
nullptr, table_options_, true, std::move(block));
ASSERT_TRUE(reader->KeyMayMatch("foo", 0));
ASSERT_TRUE(reader->KeyMayMatch("foo", 100000));
@ -171,7 +174,7 @@ TEST(BlockBasedFilterBlockTest, BlockBasedSingleChunk) {
builder->Add("hello");
BlockContents block(builder->Finish(), false, kNoCompression);
FilterBlockReader* reader = new BlockBasedFilterBlockReader(
nullptr, table_options_, std::move(block));
nullptr, table_options_, true, std::move(block));
ASSERT_TRUE(reader->KeyMayMatch("foo", 100));
ASSERT_TRUE(reader->KeyMayMatch("bar", 100));
ASSERT_TRUE(reader->KeyMayMatch("box", 100));
@ -207,7 +210,7 @@ TEST(BlockBasedFilterBlockTest, BlockBasedMultiChunk) {
BlockContents block(builder->Finish(), false, kNoCompression);
FilterBlockReader* reader = new BlockBasedFilterBlockReader(
nullptr, table_options_, std::move(block));
nullptr, table_options_, true, std::move(block));
// Check first filter
ASSERT_TRUE(reader->KeyMayMatch("foo", 0));

View File

@ -33,6 +33,7 @@
#include "table/block_builder.h"
#include "table/filter_block.h"
#include "table/block_based_filter_block.h"
#include "table/block_based_table_factory.h"
#include "table/full_filter_block.h"
#include "table/format.h"
#include "table/meta_blocks.h"
@ -292,7 +293,8 @@ FilterBlockBuilder* CreateFilterBlockBuilder(const ImmutableCFOptions& opt,
if (filter_bits_builder == nullptr) {
return new BlockBasedFilterBlockBuilder(opt.prefix_extractor, table_opt);
} else {
return new FullFilterBlockBuilder(opt.prefix_extractor, table_opt,
return new FullFilterBlockBuilder(opt.prefix_extractor,
table_opt.whole_key_filtering,
filter_bits_builder);
}
}
@ -387,8 +389,11 @@ class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector
: public TablePropertiesCollector {
public:
explicit BlockBasedTablePropertiesCollector(
BlockBasedTableOptions::IndexType index_type)
: index_type_(index_type) {}
BlockBasedTableOptions::IndexType index_type, bool whole_key_filtering,
bool prefix_filtering)
: index_type_(index_type),
whole_key_filtering_(whole_key_filtering),
prefix_filtering_(prefix_filtering) {}
virtual Status Add(const Slice& key, const Slice& value) {
// Intentionally left blank. Have no interest in collecting stats for
@ -400,6 +405,10 @@ class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector
std::string val;
PutFixed32(&val, static_cast<uint32_t>(index_type_));
properties->insert({BlockBasedTablePropertyNames::kIndexType, val});
properties->insert({BlockBasedTablePropertyNames::kWholeKeyFiltering,
whole_key_filtering_ ? kPropTrue : kPropFalse});
properties->insert({BlockBasedTablePropertyNames::kPrefixFiltering,
prefix_filtering_ ? kPropTrue : kPropFalse});
return Status::OK();
}
@ -415,6 +424,8 @@ class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector
private:
BlockBasedTableOptions::IndexType index_type_;
bool whole_key_filtering_;
bool prefix_filtering_;
};
struct BlockBasedTableBuilder::Rep {
@ -473,7 +484,9 @@ struct BlockBasedTableBuilder::Rep {
collector_factories->CreateTablePropertiesCollector());
}
table_properties_collectors.emplace_back(
new BlockBasedTablePropertiesCollector(table_options.index_type));
new BlockBasedTablePropertiesCollector(
table_options.index_type, table_options.whole_key_filtering,
_ioptions.prefix_extractor != nullptr));
}
};
@ -851,5 +864,4 @@ uint64_t BlockBasedTableBuilder::FileSize() const {
const std::string BlockBasedTable::kFilterBlockPrefix = "filter.";
const std::string BlockBasedTable::kFullFilterBlockPrefix = "fullfilter.";
} // namespace rocksdb

View File

@ -158,8 +158,14 @@ TableFactory* NewBlockBasedTableFactory(
const std::string BlockBasedTablePropertyNames::kIndexType =
"rocksdb.block.based.table.index.type";
const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering =
"rocksdb.block.based.table.whole.key.filtering";
const std::string BlockBasedTablePropertyNames::kPrefixFiltering =
"rocksdb.block.based.table.prefix.filtering";
const std::string kHashIndexPrefixesBlock = "rocksdb.hashindex.prefixes";
const std::string kHashIndexPrefixesMetadataBlock =
"rocksdb.hashindex.metadata";
const std::string kPropTrue = "1";
const std::string kPropFalse = "0";
} // namespace rocksdb

View File

@ -59,5 +59,7 @@ class BlockBasedTableFactory : public TableFactory {
extern const std::string kHashIndexPrefixesBlock;
extern const std::string kHashIndexPrefixesMetadataBlock;
extern const std::string kPropTrue;
extern const std::string kPropFalse;
} // namespace rocksdb

View File

@ -27,6 +27,7 @@
#include "table/block.h"
#include "table/filter_block.h"
#include "table/block_based_filter_block.h"
#include "table/block_based_table_factory.h"
#include "table/full_filter_block.h"
#include "table/block_hash_index.h"
#include "table/block_prefix_index.h"
@ -324,7 +325,9 @@ struct BlockBasedTable::Rep {
env_options(_env_options),
table_options(_table_opt),
filter_policy(_table_opt.filter_policy.get()),
internal_comparator(_internal_comparator) {}
internal_comparator(_internal_comparator),
whole_key_filtering(_table_opt.whole_key_filtering),
prefix_filtering(true) {}
const ImmutableCFOptions& ioptions;
const EnvOptions& env_options;
@ -349,6 +352,8 @@ struct BlockBasedTable::Rep {
std::shared_ptr<const TableProperties> table_properties;
BlockBasedTableOptions::IndexType index_type;
bool hash_index_allow_collision;
bool whole_key_filtering;
bool prefix_filtering;
// TODO(kailiu) It is very ugly to use internal key in table, since table
// module should not be relying on db module. However to make things easier
// and compatible with existing code, we introduce a wrapper that allows
@ -427,6 +432,27 @@ void BlockBasedTable::GenerateCachePrefix(Cache* cc,
}
}
namespace {
// Return True if table_properties has `user_prop_name` has a `true` value
// or it doesn't contain this property (for backward compatible).
bool IsFeatureSupported(const TableProperties& table_properties,
const std::string& user_prop_name, Logger* info_log) {
auto& props = table_properties.user_collected_properties;
auto pos = props.find(user_prop_name);
// Older version doesn't have this value set. Skip this check.
if (pos != props.end()) {
if (pos->second == kPropFalse) {
return false;
} else if (pos->second != kPropTrue) {
Log(InfoLogLevel::WARN_LEVEL, info_log,
"Property %s has invalidate value %s", user_prop_name.c_str(),
pos->second.c_str());
}
}
return true;
}
} // namespace
Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
const EnvOptions& env_options,
const BlockBasedTableOptions& table_options,
@ -496,6 +522,17 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
"Cannot find Properties block from file.");
}
// Determine whether whole key filtering is supported.
if (rep->table_properties) {
rep->whole_key_filtering &=
IsFeatureSupported(*(rep->table_properties),
BlockBasedTablePropertyNames::kWholeKeyFiltering,
rep->ioptions.info_log);
rep->prefix_filtering &= IsFeatureSupported(
*(rep->table_properties),
BlockBasedTablePropertyNames::kPrefixFiltering, rep->ioptions.info_log);
}
// Will use block cache for index/filter blocks access?
if (table_options.cache_index_and_filter_blocks) {
assert(table_options.block_cache != nullptr);
@ -748,16 +785,15 @@ FilterBlockReader* BlockBasedTable::ReadFilter(
assert(rep->filter_policy);
if (kFilterBlockPrefix == prefix) {
return new BlockBasedFilterBlockReader(
rep->ioptions.prefix_extractor, rep->table_options,
std::move(block));
rep->prefix_filtering ? rep->ioptions.prefix_extractor : nullptr,
rep->table_options, rep->whole_key_filtering, std::move(block));
} else if (kFullFilterBlockPrefix == prefix) {
auto filter_bits_reader = rep->filter_policy->
GetFilterBitsReader(block.data);
if (filter_bits_reader != nullptr) {
return new FullFilterBlockReader(rep->ioptions.prefix_extractor,
rep->table_options,
std::move(block),
filter_bits_reader);
return new FullFilterBlockReader(
rep->prefix_filtering ? rep->ioptions.prefix_extractor : nullptr,
rep->whole_key_filtering, std::move(block), filter_bits_reader);
}
} else {
assert(false);
@ -1403,9 +1439,9 @@ Status BlockBasedTable::DumpTable(WritableFile* out_file) {
BlockContents block;
if (ReadBlockContents(rep_->file.get(), rep_->footer, ReadOptions(),
handle, &block, rep_->ioptions.env, false).ok()) {
rep_->filter.reset(
new BlockBasedFilterBlockReader(rep_->ioptions.prefix_extractor,
table_options, std::move(block)));
rep_->filter.reset(new BlockBasedFilterBlockReader(
rep_->ioptions.prefix_extractor, table_options,
table_options.whole_key_filtering, std::move(block)));
}
}
}

View File

@ -12,11 +12,10 @@
namespace rocksdb {
FullFilterBlockBuilder::FullFilterBlockBuilder(
const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt,
const SliceTransform* prefix_extractor, bool whole_key_filtering,
FilterBitsBuilder* filter_bits_builder)
: prefix_extractor_(prefix_extractor),
whole_key_filtering_(table_opt.whole_key_filtering),
whole_key_filtering_(whole_key_filtering),
num_added_(0) {
assert(filter_bits_builder != nullptr);
filter_bits_builder_.reset(filter_bits_builder);
@ -53,22 +52,20 @@ Slice FullFilterBlockBuilder::Finish() {
}
FullFilterBlockReader::FullFilterBlockReader(
const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt, const Slice& contents,
FilterBitsReader* filter_bits_reader)
const SliceTransform* prefix_extractor, bool whole_key_filtering,
const Slice& contents, FilterBitsReader* filter_bits_reader)
: prefix_extractor_(prefix_extractor),
whole_key_filtering_(table_opt.whole_key_filtering),
whole_key_filtering_(whole_key_filtering),
contents_(contents) {
assert(filter_bits_reader != nullptr);
filter_bits_reader_.reset(filter_bits_reader);
}
FullFilterBlockReader::FullFilterBlockReader(
const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt, BlockContents&& contents,
FilterBitsReader* filter_bits_reader)
: FullFilterBlockReader(prefix_extractor, table_opt, contents.data,
filter_bits_reader) {
const SliceTransform* prefix_extractor, bool whole_key_filtering,
BlockContents&& contents, FilterBitsReader* filter_bits_reader)
: FullFilterBlockReader(prefix_extractor, whole_key_filtering,
contents.data, filter_bits_reader) {
block_contents_ = std::move(contents);
}

View File

@ -36,7 +36,7 @@ class FilterBitsReader;
class FullFilterBlockBuilder : public FilterBlockBuilder {
public:
explicit FullFilterBlockBuilder(const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt,
bool whole_key_filtering,
FilterBitsBuilder* filter_bits_builder);
// bits_builder is created in filter_policy, it should be passed in here
// directly. and be deleted here
@ -73,11 +73,11 @@ class FullFilterBlockReader : public FilterBlockReader {
// REQUIRES: "contents" and filter_bits_reader must stay live
// while *this is live.
explicit FullFilterBlockReader(const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt,
bool whole_key_filtering,
const Slice& contents,
FilterBitsReader* filter_bits_reader);
explicit FullFilterBlockReader(const SliceTransform* prefix_extractor,
const BlockBasedTableOptions& table_opt,
bool whole_key_filtering,
BlockContents&& contents,
FilterBitsReader* filter_bits_reader);

View File

@ -103,27 +103,29 @@ class PluginFullFilterBlockTest {
};
TEST(PluginFullFilterBlockTest, PluginEmptyBuilder) {
FullFilterBlockBuilder builder(nullptr, table_options_,
table_options_.filter_policy->GetFilterBitsBuilder());
FullFilterBlockBuilder builder(
nullptr, true, table_options_.filter_policy->GetFilterBitsBuilder());
Slice block = builder.Finish();
ASSERT_EQ("", EscapeString(block));
FullFilterBlockReader reader(nullptr, table_options_, block,
FullFilterBlockReader reader(
nullptr, true, block,
table_options_.filter_policy->GetFilterBitsReader(block));
// Remain same symantic with blockbased filter
ASSERT_TRUE(reader.KeyMayMatch("foo"));
}
TEST(PluginFullFilterBlockTest, PluginSingleChunk) {
FullFilterBlockBuilder builder(nullptr, table_options_,
table_options_.filter_policy->GetFilterBitsBuilder());
FullFilterBlockBuilder builder(
nullptr, true, table_options_.filter_policy->GetFilterBitsBuilder());
builder.Add("foo");
builder.Add("bar");
builder.Add("box");
builder.Add("box");
builder.Add("hello");
Slice block = builder.Finish();
FullFilterBlockReader reader(nullptr, table_options_, block,
FullFilterBlockReader reader(
nullptr, true, block,
table_options_.filter_policy->GetFilterBitsReader(block));
ASSERT_TRUE(reader.KeyMayMatch("foo"));
ASSERT_TRUE(reader.KeyMayMatch("bar"));
@ -146,27 +148,29 @@ class FullFilterBlockTest {
};
TEST(FullFilterBlockTest, EmptyBuilder) {
FullFilterBlockBuilder builder(nullptr, table_options_,
table_options_.filter_policy->GetFilterBitsBuilder());
FullFilterBlockBuilder builder(
nullptr, true, table_options_.filter_policy->GetFilterBitsBuilder());
Slice block = builder.Finish();
ASSERT_EQ("", EscapeString(block));
FullFilterBlockReader reader(nullptr, table_options_, block,
FullFilterBlockReader reader(
nullptr, true, block,
table_options_.filter_policy->GetFilterBitsReader(block));
// Remain same symantic with blockbased filter
ASSERT_TRUE(reader.KeyMayMatch("foo"));
}
TEST(FullFilterBlockTest, SingleChunk) {
FullFilterBlockBuilder builder(nullptr, table_options_,
table_options_.filter_policy->GetFilterBitsBuilder());
FullFilterBlockBuilder builder(
nullptr, true, table_options_.filter_policy->GetFilterBitsBuilder());
builder.Add("foo");
builder.Add("bar");
builder.Add("box");
builder.Add("box");
builder.Add("hello");
Slice block = builder.Finish();
FullFilterBlockReader reader(nullptr, table_options_, block,
FullFilterBlockReader reader(
nullptr, true, block,
table_options_.filter_policy->GetFilterBitsReader(block));
ASSERT_TRUE(reader.KeyMayMatch("foo"));
ASSERT_TRUE(reader.KeyMayMatch("bar"));