rocksdb/tools/db_sanity_test.cc

298 lines
8.4 KiB
C++
Raw Normal View History

// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include <cstdio>
2014-05-15 06:31:03 +02:00
#include <cstdlib>
#include <vector>
#include <memory>
#include "rocksdb/db.h"
#include "rocksdb/options.h"
#include "rocksdb/env.h"
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "rocksdb/comparator.h"
#include "rocksdb/table.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/filter_policy.h"
#include "port/port.h"
#include "util/string_util.h"
namespace rocksdb {
class SanityTest {
public:
explicit SanityTest(const std::string& path)
: env_(Env::Default()), path_(path) {
env_->CreateDirIfMissing(path);
}
virtual ~SanityTest() {}
virtual std::string Name() const = 0;
virtual Options GetOptions() const = 0;
Status Create() {
Options options = GetOptions();
options.create_if_missing = true;
std::string dbname = path_ + Name();
DestroyDB(dbname, options);
DB* db = nullptr;
Status s = DB::Open(options, dbname, &db);
std::unique_ptr<DB> db_guard(db);
if (!s.ok()) {
return s;
}
for (int i = 0; i < 1000000; ++i) {
std::string k = "key" + ToString(i);
std::string v = "value" + ToString(i);
s = db->Put(WriteOptions(), Slice(k), Slice(v));
if (!s.ok()) {
return s;
}
}
return db->Flush(FlushOptions());
}
Status Verify() {
DB* db = nullptr;
std::string dbname = path_ + Name();
Status s = DB::Open(GetOptions(), dbname, &db);
std::unique_ptr<DB> db_guard(db);
if (!s.ok()) {
return s;
}
for (int i = 0; i < 1000000; ++i) {
std::string k = "key" + ToString(i);
std::string v = "value" + ToString(i);
std::string result;
s = db->Get(ReadOptions(), Slice(k), &result);
if (!s.ok()) {
return s;
}
if (result != v) {
return Status::Corruption("Unexpected value for key " + k);
}
}
return Status::OK();
}
private:
Env* env_;
std::string const path_;
};
class SanityTestBasic : public SanityTest {
public:
explicit SanityTestBasic(const std::string& path) : SanityTest(path) {}
virtual Options GetOptions() const override {
Options options;
options.create_if_missing = true;
return options;
}
virtual std::string Name() const override { return "Basic"; }
};
class SanityTestSpecialComparator : public SanityTest {
public:
explicit SanityTestSpecialComparator(const std::string& path)
: SanityTest(path) {
options_.comparator = new NewComparator();
}
~SanityTestSpecialComparator() { delete options_.comparator; }
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "SpecialComparator"; }
private:
class NewComparator : public Comparator {
public:
virtual const char* Name() const override {
return "rocksdb.NewComparator";
}
virtual int Compare(const Slice& a, const Slice& b) const override {
return BytewiseComparator()->Compare(a, b);
}
virtual void FindShortestSeparator(std::string* s,
const Slice& l) const override {
BytewiseComparator()->FindShortestSeparator(s, l);
}
virtual void FindShortSuccessor(std::string* key) const override {
BytewiseComparator()->FindShortSuccessor(key);
}
};
Options options_;
};
class SanityTestZlibCompression : public SanityTest {
public:
explicit SanityTestZlibCompression(const std::string& path)
: SanityTest(path) {
options_.compression = kZlibCompression;
}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "ZlibCompression"; }
private:
Options options_;
};
class SanityTestZlibCompressionVersion2 : public SanityTest {
public:
explicit SanityTestZlibCompressionVersion2(const std::string& path)
: SanityTest(path) {
options_.compression = kZlibCompression;
BlockBasedTableOptions table_options;
#if ROCKSDB_MAJOR > 3 || (ROCKSDB_MAJOR == 3 && ROCKSDB_MINOR >= 10)
table_options.format_version = 2;
#endif
options_.table_factory.reset(NewBlockBasedTableFactory(table_options));
}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override {
return "ZlibCompressionVersion2";
}
private:
Options options_;
};
class SanityTestLZ4Compression : public SanityTest {
public:
explicit SanityTestLZ4Compression(const std::string& path)
: SanityTest(path) {
options_.compression = kLZ4Compression;
}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "LZ4Compression"; }
private:
Options options_;
};
class SanityTestLZ4HCCompression : public SanityTest {
public:
explicit SanityTestLZ4HCCompression(const std::string& path)
: SanityTest(path) {
options_.compression = kLZ4HCCompression;
}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "LZ4HCCompression"; }
private:
Options options_;
};
class SanityTestZSTDCompression : public SanityTest {
public:
explicit SanityTestZSTDCompression(const std::string& path)
: SanityTest(path) {
options_.compression = kZSTDNotFinalCompression;
}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "ZSTDCompression"; }
private:
Options options_;
};
#ifndef ROCKSDB_LITE
class SanityTestPlainTableFactory : public SanityTest {
public:
explicit SanityTestPlainTableFactory(const std::string& path)
: SanityTest(path) {
options_.table_factory.reset(NewPlainTableFactory());
2014-04-01 02:06:53 +02:00
options_.prefix_extractor.reset(NewFixedPrefixTransform(2));
options_.allow_mmap_reads = true;
}
2014-04-01 02:06:53 +02:00
~SanityTestPlainTableFactory() {}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "PlainTable"; }
private:
Options options_;
};
#endif // ROCKSDB_LITE
Implement full filter for block based table. Summary: 1. Make filter_block.h a base class. Derive block_based_filter_block and full_filter_block. The previous one is the traditional filter block. The full_filter_block is newly added. It would generate a filter block that contain all the keys in SST file. 2. When querying a key, table would first check if full_filter is available. If not, it would go to the exact data block and check using block_based filter. 3. User could choose to use full_filter or tradional(block_based_filter). They would be stored in SST file with different meta index name. "filter.filter_policy" or "full_filter.filter_policy". Then, Table reader is able to know the fllter block type. 4. Some optimizations have been done for full_filter_block, thus it requires a different interface compared to the original one in filter_policy.h. 5. Actual implementation of filter bits coding/decoding is placed in util/bloom_impl.cc Benchmark: base commit 1d23b5c470844c1208301311f0889eca750431c0 Command: db_bench --db=/dev/shm/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --write_buffer_size=134217728 --max_write_buffer_number=2 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --verify_checksum=false --max_background_compactions=4 --use_plain_table=0 --memtablerep=prefix_hash --open_files=-1 --mmap_read=1 --mmap_write=0 --bloom_bits=10 --bloom_locality=1 --memtable_bloom_bits=500000 --compression_type=lz4 --num=393216000 --use_hash_search=1 --block_size=1024 --block_restart_interval=16 --use_existing_db=1 --threads=1 --benchmarks=readrandom —disable_auto_compactions=1 Read QPS increase for about 30% from 2230002 to 2991411. Test Plan: make all check valgrind db_test db_stress --use_block_based_filter = 0 ./auto_sanity_test.sh Reviewers: igor, yhchiang, ljin, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D20979
2014-09-08 19:37:05 +02:00
class SanityTestBloomFilter : public SanityTest {
public:
explicit SanityTestBloomFilter(const std::string& path) : SanityTest(path) {
BlockBasedTableOptions table_options;
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
options_.table_factory.reset(NewBlockBasedTableFactory(table_options));
Implement full filter for block based table. Summary: 1. Make filter_block.h a base class. Derive block_based_filter_block and full_filter_block. The previous one is the traditional filter block. The full_filter_block is newly added. It would generate a filter block that contain all the keys in SST file. 2. When querying a key, table would first check if full_filter is available. If not, it would go to the exact data block and check using block_based filter. 3. User could choose to use full_filter or tradional(block_based_filter). They would be stored in SST file with different meta index name. "filter.filter_policy" or "full_filter.filter_policy". Then, Table reader is able to know the fllter block type. 4. Some optimizations have been done for full_filter_block, thus it requires a different interface compared to the original one in filter_policy.h. 5. Actual implementation of filter bits coding/decoding is placed in util/bloom_impl.cc Benchmark: base commit 1d23b5c470844c1208301311f0889eca750431c0 Command: db_bench --db=/dev/shm/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --write_buffer_size=134217728 --max_write_buffer_number=2 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --verify_checksum=false --max_background_compactions=4 --use_plain_table=0 --memtablerep=prefix_hash --open_files=-1 --mmap_read=1 --mmap_write=0 --bloom_bits=10 --bloom_locality=1 --memtable_bloom_bits=500000 --compression_type=lz4 --num=393216000 --use_hash_search=1 --block_size=1024 --block_restart_interval=16 --use_existing_db=1 --threads=1 --benchmarks=readrandom —disable_auto_compactions=1 Read QPS increase for about 30% from 2230002 to 2991411. Test Plan: make all check valgrind db_test db_stress --use_block_based_filter = 0 ./auto_sanity_test.sh Reviewers: igor, yhchiang, ljin, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D20979
2014-09-08 19:37:05 +02:00
}
~SanityTestBloomFilter() {}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "BloomFilter"; }
Implement full filter for block based table. Summary: 1. Make filter_block.h a base class. Derive block_based_filter_block and full_filter_block. The previous one is the traditional filter block. The full_filter_block is newly added. It would generate a filter block that contain all the keys in SST file. 2. When querying a key, table would first check if full_filter is available. If not, it would go to the exact data block and check using block_based filter. 3. User could choose to use full_filter or tradional(block_based_filter). They would be stored in SST file with different meta index name. "filter.filter_policy" or "full_filter.filter_policy". Then, Table reader is able to know the fllter block type. 4. Some optimizations have been done for full_filter_block, thus it requires a different interface compared to the original one in filter_policy.h. 5. Actual implementation of filter bits coding/decoding is placed in util/bloom_impl.cc Benchmark: base commit 1d23b5c470844c1208301311f0889eca750431c0 Command: db_bench --db=/dev/shm/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --write_buffer_size=134217728 --max_write_buffer_number=2 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --verify_checksum=false --max_background_compactions=4 --use_plain_table=0 --memtablerep=prefix_hash --open_files=-1 --mmap_read=1 --mmap_write=0 --bloom_bits=10 --bloom_locality=1 --memtable_bloom_bits=500000 --compression_type=lz4 --num=393216000 --use_hash_search=1 --block_size=1024 --block_restart_interval=16 --use_existing_db=1 --threads=1 --benchmarks=readrandom —disable_auto_compactions=1 Read QPS increase for about 30% from 2230002 to 2991411. Test Plan: make all check valgrind db_test db_stress --use_block_based_filter = 0 ./auto_sanity_test.sh Reviewers: igor, yhchiang, ljin, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D20979
2014-09-08 19:37:05 +02:00
private:
Options options_;
};
namespace {
bool RunSanityTests(const std::string& command, const std::string& path) {
bool result = true;
// Suppress false positive clang static anaylzer warnings.
#ifndef __clang_analyzer__
std::vector<SanityTest*> sanity_tests = {
new SanityTestBasic(path),
new SanityTestSpecialComparator(path),
new SanityTestZlibCompression(path),
new SanityTestZlibCompressionVersion2(path),
new SanityTestLZ4Compression(path),
new SanityTestLZ4HCCompression(path),
new SanityTestZSTDCompression(path),
#ifndef ROCKSDB_LITE
Implement full filter for block based table. Summary: 1. Make filter_block.h a base class. Derive block_based_filter_block and full_filter_block. The previous one is the traditional filter block. The full_filter_block is newly added. It would generate a filter block that contain all the keys in SST file. 2. When querying a key, table would first check if full_filter is available. If not, it would go to the exact data block and check using block_based filter. 3. User could choose to use full_filter or tradional(block_based_filter). They would be stored in SST file with different meta index name. "filter.filter_policy" or "full_filter.filter_policy". Then, Table reader is able to know the fllter block type. 4. Some optimizations have been done for full_filter_block, thus it requires a different interface compared to the original one in filter_policy.h. 5. Actual implementation of filter bits coding/decoding is placed in util/bloom_impl.cc Benchmark: base commit 1d23b5c470844c1208301311f0889eca750431c0 Command: db_bench --db=/dev/shm/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --write_buffer_size=134217728 --max_write_buffer_number=2 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --verify_checksum=false --max_background_compactions=4 --use_plain_table=0 --memtablerep=prefix_hash --open_files=-1 --mmap_read=1 --mmap_write=0 --bloom_bits=10 --bloom_locality=1 --memtable_bloom_bits=500000 --compression_type=lz4 --num=393216000 --use_hash_search=1 --block_size=1024 --block_restart_interval=16 --use_existing_db=1 --threads=1 --benchmarks=readrandom —disable_auto_compactions=1 Read QPS increase for about 30% from 2230002 to 2991411. Test Plan: make all check valgrind db_test db_stress --use_block_based_filter = 0 ./auto_sanity_test.sh Reviewers: igor, yhchiang, ljin, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D20979
2014-09-08 19:37:05 +02:00
new SanityTestPlainTableFactory(path),
#endif // ROCKSDB_LITE
Implement full filter for block based table. Summary: 1. Make filter_block.h a base class. Derive block_based_filter_block and full_filter_block. The previous one is the traditional filter block. The full_filter_block is newly added. It would generate a filter block that contain all the keys in SST file. 2. When querying a key, table would first check if full_filter is available. If not, it would go to the exact data block and check using block_based filter. 3. User could choose to use full_filter or tradional(block_based_filter). They would be stored in SST file with different meta index name. "filter.filter_policy" or "full_filter.filter_policy". Then, Table reader is able to know the fllter block type. 4. Some optimizations have been done for full_filter_block, thus it requires a different interface compared to the original one in filter_policy.h. 5. Actual implementation of filter bits coding/decoding is placed in util/bloom_impl.cc Benchmark: base commit 1d23b5c470844c1208301311f0889eca750431c0 Command: db_bench --db=/dev/shm/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --write_buffer_size=134217728 --max_write_buffer_number=2 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --verify_checksum=false --max_background_compactions=4 --use_plain_table=0 --memtablerep=prefix_hash --open_files=-1 --mmap_read=1 --mmap_write=0 --bloom_bits=10 --bloom_locality=1 --memtable_bloom_bits=500000 --compression_type=lz4 --num=393216000 --use_hash_search=1 --block_size=1024 --block_restart_interval=16 --use_existing_db=1 --threads=1 --benchmarks=readrandom —disable_auto_compactions=1 Read QPS increase for about 30% from 2230002 to 2991411. Test Plan: make all check valgrind db_test db_stress --use_block_based_filter = 0 ./auto_sanity_test.sh Reviewers: igor, yhchiang, ljin, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D20979
2014-09-08 19:37:05 +02:00
new SanityTestBloomFilter(path)};
if (command == "create") {
fprintf(stderr, "Creating...\n");
} else {
fprintf(stderr, "Verifying...\n");
}
for (auto sanity_test : sanity_tests) {
Status s;
fprintf(stderr, "%s -- ", sanity_test->Name().c_str());
if (command == "create") {
s = sanity_test->Create();
} else {
assert(command == "verify");
s = sanity_test->Verify();
}
fprintf(stderr, "%s\n", s.ToString().c_str());
if (!s.ok()) {
fprintf(stderr, "FAIL\n");
result = false;
}
delete sanity_test;
}
#endif // __clang_analyzer__
return result;
}
} // namespace
} // namespace rocksdb
int main(int argc, char** argv) {
std::string path, command;
bool ok = (argc == 3);
if (ok) {
path = std::string(argv[1]);
command = std::string(argv[2]);
ok = (command == "create" || command == "verify");
}
if (!ok) {
fprintf(stderr, "Usage: %s <path> [create|verify] \n", argv[0]);
exit(1);
}
if (path.back() != '/') {
path += "/";
}
bool sanity_ok = rocksdb::RunSanityTests(command, path);
return sanity_ok ? 0 : 1;
}