andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix db_bench_tool_test. Fixes 7341 (#7344)

Browse Source 
Summary:
1.  Failed to compile because of use of FileSystem* instead of Env* to some methods;

2.  Failed to compile with addition of ConfigOptions to some methods

3.  Failed to run successfully because the database and/or db_bench would change some of the options, invalidating the comparison

4.  Failed to run successfully if Snappy was not available.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7344

Reviewed By: siying

Differential Revision: D23501093

Pulled By: jay-zhuang

fbshipit-source-id: 81fd947e95fff9db8a4c5ff419d69d4c36bef23f
This commit is contained in:
mrambacher 2020-09-09 09:03:39 -07:00 committed by Facebook GitHub Bot
parent f1e99b36f5
commit a6ac51b99a
1 changed files with 51 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
tools/db_bench_tool_test.cc
View File

@ -8,6 +8,8 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/db_bench_tool.h" #include "rocksdb/db_bench_tool.h"
#include "db/db_impl/db_impl.h"
#include "options/options_parser.h" #include "options/options_parser.h"
#include "rocksdb/utilities/options_util.h" #include "rocksdb/utilities/options_util.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
@ -30,6 +32,7 @@ class DBBenchTest : public testing::Test {
Env::Default()->CreateDir(test_path_); Env::Default()->CreateDir(test_path_);
db_path_ = test_path_ + "/db"; db_path_ = test_path_ + "/db";
wal_path_ = test_path_ + "/wal"; wal_path_ = test_path_ + "/wal";
fs_.reset(new LegacyFileSystemWrapper(Env::Default()));
} }
~DBBenchTest() { ~DBBenchTest() {
@ -53,9 +56,29 @@ class DBBenchTest : public testing::Test {
} }
} }
// Gets the default options for this test/db_bench.
// Note that db_bench may change some of the default option values and that
// the database might as well. The options changed by db_bench are
// specified here; the ones by the DB are set via SanitizeOptions
Options GetDefaultOptions(CompactionStyle style = kCompactionStyleLevel,
int levels = 7) const {
Options opt;
opt.create_if_missing = true;
opt.max_open_files = 256;
opt.max_background_compactions = 10;
opt.dump_malloc_stats = true; // db_bench uses a different default
opt.compaction_style = style;
opt.num_levels = levels;
opt.compression = kNoCompression;
opt.arena_block_size = 8388608;
return SanitizeOptions(db_path_, opt);
}
void RunDbBench(const std::string& options_file_name) { void RunDbBench(const std::string& options_file_name) {
AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0", AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0",
"--num=1000", "--num=1000", "--compression_type=none",
std::string(std::string("--db=") + db_path_).c_str(), std::string(std::string("--db=") + db_path_).c_str(),
std::string(std::string("--wal_dir=") + wal_path_).c_str(), std::string(std::string("--wal_dir=") + wal_path_).c_str(),
std::string(std::string("--options_file=") + options_file_name) std::string(std::string("--options_file=") + options_file_name)
@ -66,19 +89,22 @@ class DBBenchTest : public testing::Test {
void VerifyOptions(const Options& opt) { void VerifyOptions(const Options& opt) {
DBOptions loaded_db_opts; DBOptions loaded_db_opts;
std::vector<ColumnFamilyDescriptor> cf_descs; std::vector<ColumnFamilyDescriptor> cf_descs;
ASSERT_OK(LoadLatestOptions(db_path_, FileSystem::Default(), ASSERT_OK(LoadLatestOptions(db_path_, Env::Default(), &loaded_db_opts,
&loaded_db_opts, &cf_descs)); &cf_descs));
ASSERT_OK( ConfigOptions exact;
RocksDBOptionsParser::VerifyDBOptions(DBOptions(opt), loaded_db_opts)); exact.input_strings_escaped = false;
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(opt), exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
cf_descs[0].options)); ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, DBOptions(opt),
loaded_db_opts));
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
exact, ColumnFamilyOptions(opt), cf_descs[0].options));
// check with the default rocksdb options and expect failure // check with the default rocksdb options and expect failure
ASSERT_NOK( ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(exact, DBOptions(),
RocksDBOptionsParser::VerifyDBOptions(DBOptions(), loaded_db_opts)); loaded_db_opts));
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(), ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
cf_descs[0].options)); exact, ColumnFamilyOptions(), cf_descs[0].options));
} }
char** argv() { return argv_; } char** argv() { return argv_; }
@ -88,6 +114,7 @@ class DBBenchTest : public testing::Test {
std::string db_path_; std::string db_path_;
std::string test_path_; std::string test_path_;
std::string wal_path_; std::string wal_path_;
std::unique_ptr<LegacyFileSystemWrapper> fs_;
char arg_buffer_[kArgBufferSize]; char arg_buffer_[kArgBufferSize];
char* argv_[kMaxArgCount]; char* argv_[kMaxArgCount];
@ -100,21 +127,17 @@ namespace {} // namespace
TEST_F(DBBenchTest, OptionsFile) { TEST_F(DBBenchTest, OptionsFile) {
const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
Options opt = GetDefaultOptions();
Options opt;
opt.create_if_missing = true;
opt.max_open_files = 256;
opt.max_background_compactions = 10;
opt.arena_block_size = 8388608;
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions(opt)}, kOptionsFileName, {ColumnFamilyOptions()}, kOptionsFileName,
Env::Default())); fs_.get()));
// override the following options as db_bench will not take these // override the following options as db_bench will not take these
// options from the options file // options from the options file
opt.wal_dir = wal_path_; opt.wal_dir = wal_path_;
RunDbBench(kOptionsFileName); RunDbBench(kOptionsFileName);
opt.delayed_write_rate = 16 * 1024 * 1024; // Set by SanitizeOptions
VerifyOptions(opt); VerifyOptions(opt);
} }
@ -122,21 +145,15 @@ TEST_F(DBBenchTest, OptionsFile) {
TEST_F(DBBenchTest, OptionsFileUniversal) { TEST_F(DBBenchTest, OptionsFileUniversal) {
const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
Options opt; Options opt = GetDefaultOptions(kCompactionStyleUniversal, 1);
opt.compaction_style = kCompactionStyleUniversal;
opt.num_levels = 1;
opt.create_if_missing = true;
opt.max_open_files = 256;
opt.max_background_compactions = 10;
opt.arena_block_size = 8388608;
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions(opt)}, kOptionsFileName, {ColumnFamilyOptions(opt)}, kOptionsFileName,
Env::Default())); fs_.get()));
// override the following options as db_bench will not take these // override the following options as db_bench will not take these
// options from the options file // options from the options file
opt.wal_dir = wal_path_; opt.wal_dir = wal_path_;
RunDbBench(kOptionsFileName); RunDbBench(kOptionsFileName);
VerifyOptions(opt); VerifyOptions(opt);
@ -145,23 +162,17 @@ TEST_F(DBBenchTest, OptionsFileUniversal) {
TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) { TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) {
const std::string kOptionsFileName = test_path_ + "/OPTIONS_test"; const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
Options opt; Options opt = GetDefaultOptions(kCompactionStyleUniversal, 12);
opt.compaction_style = kCompactionStyleUniversal;
opt.num_levels = 12;
opt.create_if_missing = true;
opt.max_open_files = 256;
opt.max_background_compactions = 10;
opt.arena_block_size = 8388608;
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions(opt)}, kOptionsFileName, {ColumnFamilyOptions(opt)}, kOptionsFileName,
Env::Default())); fs_.get()));
// override the following options as db_bench will not take these // override the following options as db_bench will not take these
// options from the options file // options from the options file
opt.wal_dir = wal_path_; opt.wal_dir = wal_path_;
RunDbBench(kOptionsFileName); RunDbBench(kOptionsFileName);
VerifyOptions(opt); VerifyOptions(opt);
} }
@ -213,7 +224,7 @@ const std::string options_file_content = R"OPTIONS_FILE(
max_log_file_size=83886080 max_log_file_size=83886080
random_access_max_buffer_size=1048576 random_access_max_buffer_size=1048576
advise_random_on_open=true advise_random_on_open=true
dump_malloc_stats=true
[CFOptions "default"] [CFOptions "default"]
compaction_filter_factory=nullptr compaction_filter_factory=nullptr
@ -236,7 +247,7 @@ const std::string options_file_content = R"OPTIONS_FILE(
max_grandparent_overlap_factor=10 max_grandparent_overlap_factor=10
max_bytes_for_level_multiplier=10 max_bytes_for_level_multiplier=10
memtable_factory=SkipListFactory memtable_factory=SkipListFactory
compression=kSnappyCompression compression=kNoCompression
min_partial_merge_operands=2 min_partial_merge_operands=2
level0_stop_writes_trigger=100 level0_stop_writes_trigger=100
num_levels=1 num_levels=1
@ -292,7 +303,6 @@ TEST_F(DBBenchTest, OptionsFileFromFile) {
ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt, ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt,
&cf_descs)); &cf_descs));
Options opt(db_opt, cf_descs[0].options); Options opt(db_opt, cf_descs[0].options);
opt.create_if_missing = true; opt.create_if_missing = true;
// override the following options as db_bench will not take these // override the following options as db_bench will not take these
@ -301,7 +311,7 @@ TEST_F(DBBenchTest, OptionsFileFromFile) {
RunDbBench(kOptionsFileName); RunDbBench(kOptionsFileName);
VerifyOptions(opt); VerifyOptions(SanitizeOptions(db_path_, opt));
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE