PrefixMayMatch: remove unnecessary check for prefix_extractor_ (#4067)

Summary:
with https://github.com/facebook/rocksdb/pull/3601 and https://github.com/facebook/rocksdb/pull/3899, `prefix_extractor_` is not really being used in block based filter and full filter's version of `PrefixMayMatch` because now `prefix_extractor` is passed as an argument. Also it is now possible that prefix_extractor_ may be initialized to nullptr when a non-standard prefix_extractor is used and also for ROCKSDB_LITE. Removing these checks should not break any existing tests.
Closes https://github.com/facebook/rocksdb/pull/4067

Differential Revision: D8669002

Pulled By: miasantreble

fbshipit-source-id: 0e701ba912b8a26734fadb72d15bb1b266b6176a
This commit is contained in:
Zhongyi Xie 2018-06-27 20:38:53 -07:00 committed by Facebook Github Bot
parent 1f6efabe23
commit 14f409c0f1
5 changed files with 143 additions and 7 deletions

View File

@ -39,6 +39,26 @@ class DBBloomFilterTestWithParam
}
};
class SliceTransformLimitedDomainGeneric : public SliceTransform {
const char* Name() const override {
return "SliceTransformLimitedDomainGeneric";
}
Slice Transform(const Slice& src) const override {
return Slice(src.data(), 5);
}
bool InDomain(const Slice& src) const override {
// prefix will be x????
return src.size() >= 5;
}
bool InRange(const Slice& dst) const override {
// prefix will be x????
return dst.size() == 5;
}
};
// KeyMayExist can lead to a few false positives, but not false negatives.
// To make test deterministic, use a much larger number of bits per key-20 than
// bits in the key, so that false positives are eliminated
@ -117,6 +137,53 @@ TEST_P(DBBloomFilterTestWithParam, KeyMayExist) {
ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction));
}
TEST_F(DBBloomFilterTest, GetFilterByPrefixBloomCustomPrefixExtractor) {
for (bool partition_filters : {true, false}) {
Options options = last_options_;
options.prefix_extractor =
std::make_shared<SliceTransformLimitedDomainGeneric>();
options.statistics = rocksdb::CreateDBStatistics();
BlockBasedTableOptions bbto;
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
if (partition_filters) {
bbto.partition_filters = true;
bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
}
bbto.whole_key_filtering = false;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
DestroyAndReopen(options);
WriteOptions wo;
ReadOptions ro;
FlushOptions fo;
fo.wait = true;
std::string value;
ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
dbfull()->Flush(fo);
ASSERT_EQ("foo", Get("barbarbar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
ASSERT_EQ("foo2", Get("barbarbar2"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
ro.total_order_seek = true;
ASSERT_TRUE(db_->Get(ro, "foobarbar", &value).IsNotFound());
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
}
}
TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
for (bool partition_filters : {true, false}) {
Options options = last_options_;

View File

@ -1537,6 +1537,26 @@ TEST_P(DBIteratorTest, PinnedDataIteratorReadAfterUpdate) {
delete iter;
}
class SliceTransformLimitedDomainGeneric : public SliceTransform {
const char* Name() const override {
return "SliceTransformLimitedDomainGeneric";
}
Slice Transform(const Slice& src) const override {
return Slice(src.data(), 1);
}
bool InDomain(const Slice& src) const override {
// prefix will be x????
return src.size() >= 1;
}
bool InRange(const Slice& dst) const override {
// prefix will be x????
return dst.size() == 1;
}
};
TEST_P(DBIteratorTest, IterSeekForPrevCrossingFiles) {
Options options = CurrentOptions();
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
@ -1591,6 +1611,61 @@ TEST_P(DBIteratorTest, IterSeekForPrevCrossingFiles) {
}
}
TEST_P(DBIteratorTest, IterSeekForPrevCrossingFilesCustomPrefixExtractor) {
Options options = CurrentOptions();
options.prefix_extractor =
std::make_shared<SliceTransformLimitedDomainGeneric>();
options.disable_auto_compactions = true;
// Enable prefix bloom for SST files
BlockBasedTableOptions table_options;
table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options);
ASSERT_OK(Put("a1", "va1"));
ASSERT_OK(Put("a2", "va2"));
ASSERT_OK(Put("a3", "va3"));
ASSERT_OK(Flush());
ASSERT_OK(Put("b1", "vb1"));
ASSERT_OK(Put("b2", "vb2"));
ASSERT_OK(Put("b3", "vb3"));
ASSERT_OK(Flush());
ASSERT_OK(Put("b4", "vb4"));
ASSERT_OK(Put("d1", "vd1"));
ASSERT_OK(Put("d2", "vd2"));
ASSERT_OK(Put("d4", "vd4"));
ASSERT_OK(Flush());
MoveFilesToLevel(1);
{
ReadOptions ro;
Iterator* iter = NewIterator(ro);
iter->SeekForPrev("a4");
ASSERT_EQ(iter->key().ToString(), "a3");
ASSERT_EQ(iter->value().ToString(), "va3");
iter->SeekForPrev("c2");
ASSERT_EQ(iter->key().ToString(), "b3");
iter->SeekForPrev("d3");
ASSERT_EQ(iter->key().ToString(), "d2");
iter->SeekForPrev("b5");
ASSERT_EQ(iter->key().ToString(), "b4");
delete iter;
}
{
ReadOptions ro;
ro.prefix_same_as_start = true;
Iterator* iter = NewIterator(ro);
iter->SeekForPrev("c2");
ASSERT_TRUE(!iter->Valid());
delete iter;
}
}
TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocks) {
Options options = CurrentOptions();
BlockBasedTableOptions table_options;

View File

@ -201,9 +201,6 @@ bool BlockBasedFilterBlockReader::PrefixMayMatch(
uint64_t block_offset, const bool /*no_io*/,
const Slice* const /*const_ikey_ptr*/) {
assert(block_offset != kNotValid);
if (!prefix_extractor_) {
return true;
}
return MayMatch(prefix, block_offset);
}

View File

@ -135,9 +135,6 @@ bool FullFilterBlockReader::PrefixMayMatch(
(void)block_offset;
#endif
assert(block_offset == kNotValid);
if (!prefix_extractor_) {
return true;
}
return MayMatch(prefix);
}

View File

@ -186,7 +186,7 @@ bool PartitionedFilterBlockReader::PrefixMayMatch(
#endif
assert(const_ikey_ptr != nullptr);
assert(block_offset == kNotValid);
if (!prefix_extractor_) {
if (!prefix_extractor_ && !prefix_extractor) {
return true;
}
if (UNLIKELY(idx_on_fltr_blk_->size() == 0)) {