2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2013-10-16 23:59:46 +02:00
|
|
|
//
|
2012-04-17 17:36:46 +02:00
|
|
|
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2014-05-09 17:34:18 +02:00
|
|
|
#ifndef GFLAGS
|
|
|
|
#include <cstdio>
|
|
|
|
int main() {
|
2015-09-16 03:10:36 +02:00
|
|
|
fprintf(stderr, "Please install gflags to run this test... Skipping...\n");
|
|
|
|
return 0;
|
2014-05-09 17:34:18 +02:00
|
|
|
}
|
|
|
|
#else
|
|
|
|
|
2014-09-08 19:37:05 +02:00
|
|
|
#include <vector>
|
2013-11-27 23:27:02 +01:00
|
|
|
|
2019-06-01 02:19:43 +02:00
|
|
|
#include "logging/logging.h"
|
2019-05-31 02:39:43 +02:00
|
|
|
#include "memory/arena.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/filter_policy.h"
|
2017-07-02 19:36:10 +02:00
|
|
|
#include "table/full_filter_bits_builder.h"
|
2019-05-30 20:21:38 +02:00
|
|
|
#include "test_util/testharness.h"
|
|
|
|
#include "test_util/testutil.h"
|
2019-05-31 02:39:43 +02:00
|
|
|
#include "util/gflags_compat.h"
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2017-12-01 19:40:45 +01:00
|
|
|
using GFLAGS_NAMESPACE::ParseCommandLineFlags;
|
2014-05-09 17:34:18 +02:00
|
|
|
|
2013-11-27 23:27:02 +01:00
|
|
|
DEFINE_int32(bits_per_key, 10, "");
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
static const int kVerbose = 1;
|
|
|
|
|
|
|
|
static Slice Key(int i, char* buffer) {
|
2017-04-22 05:41:37 +02:00
|
|
|
std::string s;
|
|
|
|
PutFixed32(&s, static_cast<uint32_t>(i));
|
|
|
|
memcpy(buffer, s.c_str(), sizeof(i));
|
2012-04-17 17:36:46 +02:00
|
|
|
return Slice(buffer, sizeof(i));
|
|
|
|
}
|
|
|
|
|
2014-09-08 19:37:05 +02:00
|
|
|
static int NextLength(int length) {
|
|
|
|
if (length < 10) {
|
|
|
|
length += 1;
|
|
|
|
} else if (length < 100) {
|
|
|
|
length += 10;
|
|
|
|
} else if (length < 1000) {
|
|
|
|
length += 100;
|
|
|
|
} else {
|
|
|
|
length += 1000;
|
|
|
|
}
|
|
|
|
return length;
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
class BloomTest : public testing::Test {
|
2012-04-17 17:36:46 +02:00
|
|
|
private:
|
|
|
|
const FilterPolicy* policy_;
|
|
|
|
std::string filter_;
|
|
|
|
std::vector<std::string> keys_;
|
|
|
|
|
|
|
|
public:
|
2014-09-08 19:37:05 +02:00
|
|
|
BloomTest() : policy_(
|
|
|
|
NewBloomFilterPolicy(FLAGS_bits_per_key)) {}
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2019-02-14 22:52:47 +01:00
|
|
|
~BloomTest() override { delete policy_; }
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
void Reset() {
|
|
|
|
keys_.clear();
|
|
|
|
filter_.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
void Add(const Slice& s) {
|
|
|
|
keys_.push_back(s.ToString());
|
|
|
|
}
|
|
|
|
|
|
|
|
void Build() {
|
|
|
|
std::vector<Slice> key_slices;
|
|
|
|
for (size_t i = 0; i < keys_.size(); i++) {
|
|
|
|
key_slices.push_back(Slice(keys_[i]));
|
|
|
|
}
|
|
|
|
filter_.clear();
|
2014-11-11 22:47:22 +01:00
|
|
|
policy_->CreateFilter(&key_slices[0], static_cast<int>(key_slices.size()),
|
|
|
|
&filter_);
|
2012-04-17 17:36:46 +02:00
|
|
|
keys_.clear();
|
|
|
|
if (kVerbose >= 2) DumpFilter();
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t FilterSize() const {
|
|
|
|
return filter_.size();
|
|
|
|
}
|
|
|
|
|
|
|
|
void DumpFilter() {
|
|
|
|
fprintf(stderr, "F(");
|
|
|
|
for (size_t i = 0; i+1 < filter_.size(); i++) {
|
|
|
|
const unsigned int c = static_cast<unsigned int>(filter_[i]);
|
|
|
|
for (int j = 0; j < 8; j++) {
|
|
|
|
fprintf(stderr, "%c", (c & (1 <<j)) ? '1' : '.');
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fprintf(stderr, ")\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Matches(const Slice& s) {
|
|
|
|
if (!keys_.empty()) {
|
|
|
|
Build();
|
|
|
|
}
|
|
|
|
return policy_->KeyMayMatch(s, filter_);
|
|
|
|
}
|
|
|
|
|
|
|
|
double FalsePositiveRate() {
|
|
|
|
char buffer[sizeof(int)];
|
|
|
|
int result = 0;
|
|
|
|
for (int i = 0; i < 10000; i++) {
|
|
|
|
if (Matches(Key(i + 1000000000, buffer))) {
|
|
|
|
result++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result / 10000.0;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(BloomTest, EmptyFilter) {
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_TRUE(! Matches("hello"));
|
|
|
|
ASSERT_TRUE(! Matches("world"));
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(BloomTest, Small) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Add("hello");
|
|
|
|
Add("world");
|
|
|
|
ASSERT_TRUE(Matches("hello"));
|
|
|
|
ASSERT_TRUE(Matches("world"));
|
|
|
|
ASSERT_TRUE(! Matches("x"));
|
|
|
|
ASSERT_TRUE(! Matches("foo"));
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(BloomTest, VaryingLengths) {
|
2012-04-17 17:36:46 +02:00
|
|
|
char buffer[sizeof(int)];
|
|
|
|
|
|
|
|
// Count number of filters that significantly exceed the false positive rate
|
|
|
|
int mediocre_filters = 0;
|
|
|
|
int good_filters = 0;
|
|
|
|
|
|
|
|
for (int length = 1; length <= 10000; length = NextLength(length)) {
|
|
|
|
Reset();
|
|
|
|
for (int i = 0; i < length; i++) {
|
|
|
|
Add(Key(i, buffer));
|
|
|
|
}
|
|
|
|
Build();
|
|
|
|
|
2012-11-06 21:02:18 +01:00
|
|
|
ASSERT_LE(FilterSize(), (size_t)((length * 10 / 8) + 40)) << length;
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
// All added keys must match
|
|
|
|
for (int i = 0; i < length; i++) {
|
|
|
|
ASSERT_TRUE(Matches(Key(i, buffer)))
|
|
|
|
<< "Length " << length << "; key " << i;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check false positive rate
|
|
|
|
double rate = FalsePositiveRate();
|
|
|
|
if (kVerbose >= 1) {
|
|
|
|
fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n",
|
|
|
|
rate*100.0, length, static_cast<int>(FilterSize()));
|
|
|
|
}
|
|
|
|
ASSERT_LE(rate, 0.02); // Must not be over 2%
|
|
|
|
if (rate > 0.0125) mediocre_filters++; // Allowed, but not too often
|
|
|
|
else good_filters++;
|
|
|
|
}
|
|
|
|
if (kVerbose >= 1) {
|
|
|
|
fprintf(stderr, "Filters: %d good, %d mediocre\n",
|
|
|
|
good_filters, mediocre_filters);
|
|
|
|
}
|
|
|
|
ASSERT_LE(mediocre_filters, good_filters/5);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Different bits-per-byte
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
class FullBloomTest : public testing::Test {
|
2014-09-08 19:37:05 +02:00
|
|
|
private:
|
|
|
|
const FilterPolicy* policy_;
|
|
|
|
std::unique_ptr<FilterBitsBuilder> bits_builder_;
|
|
|
|
std::unique_ptr<FilterBitsReader> bits_reader_;
|
|
|
|
std::unique_ptr<const char[]> buf_;
|
|
|
|
size_t filter_size_;
|
|
|
|
|
|
|
|
public:
|
|
|
|
FullBloomTest() :
|
|
|
|
policy_(NewBloomFilterPolicy(FLAGS_bits_per_key, false)),
|
|
|
|
filter_size_(0) {
|
|
|
|
Reset();
|
|
|
|
}
|
|
|
|
|
2019-02-14 22:52:47 +01:00
|
|
|
~FullBloomTest() override { delete policy_; }
|
2014-09-08 19:37:05 +02:00
|
|
|
|
2017-07-02 19:36:10 +02:00
|
|
|
FullFilterBitsBuilder* GetFullFilterBitsBuilder() {
|
|
|
|
return dynamic_cast<FullFilterBitsBuilder*>(bits_builder_.get());
|
|
|
|
}
|
|
|
|
|
2014-09-08 19:37:05 +02:00
|
|
|
void Reset() {
|
|
|
|
bits_builder_.reset(policy_->GetFilterBitsBuilder());
|
|
|
|
bits_reader_.reset(nullptr);
|
|
|
|
buf_.reset(nullptr);
|
|
|
|
filter_size_ = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void Add(const Slice& s) {
|
|
|
|
bits_builder_->AddKey(s);
|
|
|
|
}
|
|
|
|
|
|
|
|
void Build() {
|
|
|
|
Slice filter = bits_builder_->Finish(&buf_);
|
|
|
|
bits_reader_.reset(policy_->GetFilterBitsReader(filter));
|
|
|
|
filter_size_ = filter.size();
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t FilterSize() const {
|
|
|
|
return filter_size_;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Matches(const Slice& s) {
|
|
|
|
if (bits_reader_ == nullptr) {
|
|
|
|
Build();
|
|
|
|
}
|
|
|
|
return bits_reader_->MayMatch(s);
|
|
|
|
}
|
|
|
|
|
|
|
|
double FalsePositiveRate() {
|
|
|
|
char buffer[sizeof(int)];
|
|
|
|
int result = 0;
|
|
|
|
for (int i = 0; i < 10000; i++) {
|
|
|
|
if (Matches(Key(i + 1000000000, buffer))) {
|
|
|
|
result++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result / 10000.0;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2017-07-02 19:36:10 +02:00
|
|
|
TEST_F(FullBloomTest, FilterSize) {
|
|
|
|
uint32_t dont_care1, dont_care2;
|
|
|
|
auto full_bits_builder = GetFullFilterBitsBuilder();
|
|
|
|
for (int n = 1; n < 100; n++) {
|
|
|
|
auto space = full_bits_builder->CalculateSpace(n, &dont_care1, &dont_care2);
|
|
|
|
auto n2 = full_bits_builder->CalculateNumEntry(space);
|
|
|
|
ASSERT_GE(n2, n);
|
|
|
|
auto space2 =
|
|
|
|
full_bits_builder->CalculateSpace(n2, &dont_care1, &dont_care2);
|
|
|
|
ASSERT_EQ(space, space2);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(FullBloomTest, FullEmptyFilter) {
|
2014-09-08 19:37:05 +02:00
|
|
|
// Empty filter is not match, at this level
|
|
|
|
ASSERT_TRUE(!Matches("hello"));
|
|
|
|
ASSERT_TRUE(!Matches("world"));
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(FullBloomTest, FullSmall) {
|
2014-09-08 19:37:05 +02:00
|
|
|
Add("hello");
|
|
|
|
Add("world");
|
|
|
|
ASSERT_TRUE(Matches("hello"));
|
|
|
|
ASSERT_TRUE(Matches("world"));
|
|
|
|
ASSERT_TRUE(!Matches("x"));
|
|
|
|
ASSERT_TRUE(!Matches("foo"));
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(FullBloomTest, FullVaryingLengths) {
|
2014-09-08 19:37:05 +02:00
|
|
|
char buffer[sizeof(int)];
|
|
|
|
|
|
|
|
// Count number of filters that significantly exceed the false positive rate
|
|
|
|
int mediocre_filters = 0;
|
|
|
|
int good_filters = 0;
|
|
|
|
|
|
|
|
for (int length = 1; length <= 10000; length = NextLength(length)) {
|
|
|
|
Reset();
|
|
|
|
for (int i = 0; i < length; i++) {
|
|
|
|
Add(Key(i, buffer));
|
|
|
|
}
|
|
|
|
Build();
|
|
|
|
|
2019-09-06 02:01:58 +02:00
|
|
|
ASSERT_LE(FilterSize(), (size_t)((length * 10 / 8) + CACHE_LINE_SIZE * 2 + 5)) << length;
|
2014-09-08 19:37:05 +02:00
|
|
|
|
|
|
|
// All added keys must match
|
|
|
|
for (int i = 0; i < length; i++) {
|
|
|
|
ASSERT_TRUE(Matches(Key(i, buffer)))
|
|
|
|
<< "Length " << length << "; key " << i;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check false positive rate
|
|
|
|
double rate = FalsePositiveRate();
|
|
|
|
if (kVerbose >= 1) {
|
|
|
|
fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n",
|
|
|
|
rate*100.0, length, static_cast<int>(FilterSize()));
|
|
|
|
}
|
|
|
|
ASSERT_LE(rate, 0.02); // Must not be over 2%
|
|
|
|
if (rate > 0.0125)
|
|
|
|
mediocre_filters++; // Allowed, but not too often
|
|
|
|
else
|
|
|
|
good_filters++;
|
|
|
|
}
|
|
|
|
if (kVerbose >= 1) {
|
|
|
|
fprintf(stderr, "Filters: %d good, %d mediocre\n",
|
|
|
|
good_filters, mediocre_filters);
|
|
|
|
}
|
|
|
|
ASSERT_LE(mediocre_filters, good_filters/5);
|
|
|
|
}
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
2015-03-17 22:08:00 +01:00
|
|
|
::testing::InitGoogleTest(&argc, argv);
|
2014-05-09 17:34:18 +02:00
|
|
|
ParseCommandLineFlags(&argc, &argv, true);
|
2013-11-27 23:27:02 +01:00
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
return RUN_ALL_TESTS();
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2014-05-09 17:34:18 +02:00
|
|
|
|
|
|
|
#endif // GFLAGS
|