rocksdb/tools/db_bench_tool_test.cc
Aaron Gao 44fa8ece9b change use_direct_writes to use_direct_io_for_flush_and_compaction
Summary:
Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction
Now if Options::use_direct_io_for_flush_and_compaction = true, we will enable direct io for both reads and writes for flush and compaction job. Whereas Options::use_direct_reads controls user reads like iterator and Get().
Closes https://github.com/facebook/rocksdb/pull/2117

Differential Revision: D4860912

Pulled By: lightmark

fbshipit-source-id: d93575a8a5e780cf7e40797287edc425ee648c19
2017-04-13 16:12:04 -07:00

323 lines
9.7 KiB
C++

// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/db_bench_tool.h"
#include "options/options_parser.h"
#include "rocksdb/utilities/options_util.h"
#include "util/random.h"
#include "util/testharness.h"
#include "util/testutil.h"
#ifdef GFLAGS
#include <gflags/gflags.h>
namespace rocksdb {
namespace {
static const int kMaxArgCount = 100;
static const size_t kArgBufferSize = 100000;
} // namespace
class DBBenchTest : public testing::Test {
public:
DBBenchTest() : rnd_(0xFB) {
test_path_ = test::TmpDir() + "/db_bench_test";
Env::Default()->CreateDir(test_path_);
db_path_ = test_path_ + "/db";
wal_path_ = test_path_ + "/wal";
}
~DBBenchTest() {
// DestroyDB(db_path_, Options());
}
void ResetArgs() {
argc_ = 0;
cursor_ = 0;
memset(arg_buffer_, 0, kArgBufferSize);
}
void AppendArgs(const std::vector<std::string>& args) {
for (const auto& arg : args) {
ASSERT_LE(cursor_ + arg.size() + 1, kArgBufferSize);
ASSERT_LE(argc_ + 1, kMaxArgCount);
snprintf(arg_buffer_ + cursor_, arg.size() + 1, "%s", arg.c_str());
argv_[argc_++] = arg_buffer_ + cursor_;
cursor_ += arg.size() + 1;
}
}
void RunDbBench(const std::string& options_file_name) {
AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0",
"--num=1000",
std::string(std::string("--db=") + db_path_).c_str(),
std::string(std::string("--wal_dir=") + wal_path_).c_str(),
std::string(std::string("--options_file=") + options_file_name)
.c_str()});
ASSERT_EQ(0, db_bench_tool(argc(), argv()));
}
void VerifyOptions(const Options& opt) {
DBOptions loaded_db_opts;
std::vector<ColumnFamilyDescriptor> cf_descs;
ASSERT_OK(LoadLatestOptions(db_path_, Env::Default(), &loaded_db_opts,
&cf_descs));
ASSERT_OK(
RocksDBOptionsParser::VerifyDBOptions(DBOptions(opt), loaded_db_opts));
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(opt),
cf_descs[0].options));
// check with the default rocksdb options and expect failure
ASSERT_NOK(
RocksDBOptionsParser::VerifyDBOptions(DBOptions(), loaded_db_opts));
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(),
cf_descs[0].options));
}
char** argv() { return argv_; }
int argc() { return argc_; }
std::string db_path_;
std::string test_path_;
std::string wal_path_;
char arg_buffer_[kArgBufferSize];
char* argv_[kMaxArgCount];
int argc_ = 0;
int cursor_ = 0;
Random rnd_;
};
namespace {} // namespace
TEST_F(DBBenchTest, OptionsFile) {
const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
Options opt;
opt.create_if_missing = true;
opt.max_open_files = 256;
opt.base_background_compactions = 5;
opt.max_background_compactions = 10;
opt.arena_block_size = 8388608;
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions(opt)}, kOptionsFileName,
Env::Default()));
// override the following options as db_bench will not take these
// options from the options file
opt.wal_dir = wal_path_;
RunDbBench(kOptionsFileName);
VerifyOptions(opt);
}
TEST_F(DBBenchTest, OptionsFileUniversal) {
const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
Options opt;
opt.compaction_style = kCompactionStyleUniversal;
opt.num_levels = 1;
opt.create_if_missing = true;
opt.max_open_files = 256;
opt.base_background_compactions = 5;
opt.max_background_compactions = 10;
opt.arena_block_size = 8388608;
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions(opt)}, kOptionsFileName,
Env::Default()));
// override the following options as db_bench will not take these
// options from the options file
opt.wal_dir = wal_path_;
RunDbBench(kOptionsFileName);
VerifyOptions(opt);
}
TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) {
const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
Options opt;
opt.compaction_style = kCompactionStyleUniversal;
opt.num_levels = 12;
opt.create_if_missing = true;
opt.max_open_files = 256;
opt.base_background_compactions = 5;
opt.max_background_compactions = 10;
opt.arena_block_size = 8388608;
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions(opt)}, kOptionsFileName,
Env::Default()));
// override the following options as db_bench will not take these
// options from the options file
opt.wal_dir = wal_path_;
RunDbBench(kOptionsFileName);
VerifyOptions(opt);
}
const std::string options_file_content = R"OPTIONS_FILE(
[Version]
rocksdb_version=4.3.1
options_file_version=1.1
[DBOptions]
wal_bytes_per_sync=1048576
delete_obsolete_files_period_micros=0
WAL_ttl_seconds=0
WAL_size_limit_MB=0
db_write_buffer_size=0
max_subcompactions=1
table_cache_numshardbits=4
max_open_files=-1
max_file_opening_threads=10
base_background_compactions=3
max_background_compactions=5
use_fsync=false
use_adaptive_mutex=false
max_total_wal_size=18446744073709551615
compaction_readahead_size=0
new_table_reader_for_compaction_inputs=false
keep_log_file_num=10
skip_stats_update_on_db_open=false
max_manifest_file_size=18446744073709551615
db_log_dir=
skip_log_error_on_recovery=false
writable_file_max_buffer_size=1048576
paranoid_checks=true
is_fd_close_on_exec=true
bytes_per_sync=1048576
enable_thread_tracking=true
recycle_log_file_num=0
create_missing_column_families=false
log_file_time_to_roll=0
max_background_flushes=1
create_if_missing=true
error_if_exists=false
delayed_write_rate=1048576
manifest_preallocation_size=4194304
allow_mmap_reads=false
allow_mmap_writes=false
use_direct_reads=false
use_direct_io_for_flush_and_compaction=false
stats_dump_period_sec=600
allow_fallocate=true
max_log_file_size=83886080
random_access_max_buffer_size=1048576
advise_random_on_open=true
[CFOptions "default"]
compaction_filter_factory=nullptr
table_factory=BlockBasedTable
prefix_extractor=nullptr
comparator=leveldb.BytewiseComparator
compression_per_level=
max_bytes_for_level_base=104857600
bloom_locality=0
target_file_size_base=10485760
memtable_huge_page_size=0
max_successive_merges=1000
max_sequential_skip_in_iterations=8
arena_block_size=52428800
target_file_size_multiplier=1
source_compaction_factor=1
min_write_buffer_number_to_merge=1
max_write_buffer_number=2
write_buffer_size=419430400
max_grandparent_overlap_factor=10
max_bytes_for_level_multiplier=10
memtable_factory=SkipListFactory
compression=kSnappyCompression
min_partial_merge_operands=2
level0_stop_writes_trigger=100
num_levels=1
level0_slowdown_writes_trigger=50
level0_file_num_compaction_trigger=10
expanded_compaction_factor=25
soft_rate_limit=0.000000
max_write_buffer_number_to_maintain=0
verify_checksums_in_compaction=true
merge_operator=nullptr
memtable_prefix_bloom_bits=0
paranoid_file_checks=false
inplace_update_num_locks=10000
optimize_filters_for_hits=false
level_compaction_dynamic_level_bytes=false
inplace_update_support=false
compaction_style=kCompactionStyleUniversal
memtable_prefix_bloom_probes=6
purge_redundant_kvs_while_flush=true
filter_deletes=false
hard_pending_compaction_bytes_limit=0
disable_auto_compactions=false
compaction_measure_io_stats=false
[TableOptions/BlockBasedTable "default"]
format_version=0
skip_table_builder_flush=false
cache_index_and_filter_blocks=false
flush_block_policy_factory=FlushBlockBySizePolicyFactory
hash_index_allow_collision=true
index_type=kBinarySearch
whole_key_filtering=true
checksum=kCRC32c
no_block_cache=false
block_size=32768
block_size_deviation=10
block_restart_interval=16
filter_policy=rocksdb.BuiltinBloomFilter
)OPTIONS_FILE";
TEST_F(DBBenchTest, OptionsFileFromFile) {
const std::string kOptionsFileName = test_path_ + "/OPTIONS_flash";
unique_ptr<WritableFile> writable;
ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName, &writable,
EnvOptions()));
ASSERT_OK(writable->Append(options_file_content));
ASSERT_OK(writable->Close());
DBOptions db_opt;
std::vector<ColumnFamilyDescriptor> cf_descs;
ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt,
&cf_descs));
Options opt(db_opt, cf_descs[0].options);
opt.create_if_missing = true;
// override the following options as db_bench will not take these
// options from the options file
opt.wal_dir = wal_path_;
RunDbBench(kOptionsFileName);
VerifyOptions(opt);
}
} // namespace rocksdb
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
google::ParseCommandLineFlags(&argc, &argv, true);
return RUN_ALL_TESTS();
}
#else
int main(int argc, char** argv) {
printf("Skip db_bench_tool_test as the required library GFLAG is missing.");
}
#endif // #ifdef GFLAGS