Allows db_bench to take an options file

Summary:
This patch allows db_bench to initialize it's RocksDB Options via a
options file, specified by the --options_file flag.  Note that if
--options_file flag is set, then it has higher priority than the
command-line argument.

Test Plan: db_bench_tool_test

Reviewers: sdong, IslamAbdelRahman, kradhakrishnan, yiwu, andrewkr

Reviewed By: andrewkr

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D58533

Makefile

@@ -1117,6 +1117,9 @@ options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
 options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+db_bench_tool_test: tools/db_bench_tool_test.o $(BENCHTOOLOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 event_logger_test: util/event_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 

src.mk

@@ -249,6 +249,7 @@ TEST_BENCH_SOURCES =                                                    \
   table/merger_test.cc                                                  \
   table/table_reader_bench.cc                                           \
   table/table_test.cc                                                   \
+  tools/db_bench_tool_test.cc                                           \
   tools/db_sanity_test.cc                                               \
   tools/ldb_cmd_test.cc                                                 \
   tools/reduce_levels_test.cc                                           \

tools/db_bench_tool.cc

@@ -51,6 +51,7 @@
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/utilities/flashcache.h"
 #include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "rocksdb/utilities/options_util.h"
 #include "rocksdb/utilities/sim_cache.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
@@ -530,6 +531,23 @@ DEFINE_int32(transaction_sleep, 0,
 DEFINE_uint64(transaction_lock_timeout, 100,
               "If using a transaction_db, specifies the lock wait timeout in"
               " milliseconds before failing a transaction waiting on a lock");
+DEFINE_string(
+    options_file, "",
+    "The path to a RocksDB options file.  If specified, then db_bench will "
+    "run with the RocksDB options in the default column family of the "
+    "specified options file. "
+    "Note that with this setting, db_bench will ONLY accept the following "
+    "RocksDB options related command-line arguments, all other arguments "
+    "that are related to RocksDB options will be ignored:\n"
+    "\t--use_existing_db\n"
+    "\t--statistics\n"
+    "\t--row_cache_size\n"
+    "\t--row_cache_numshardbits\n"
+    "\t--enable_io_prio\n"
+    "\t--disable_flashcache_for_background_threads\n"
+    "\t--flashcache_dev\n"
+    "\t--dump_malloc_stats\n"
+    "\t--num_multi_db\n");
 #endif  // ROCKSDB_LITE
 
 DEFINE_bool(report_bg_io_stats, false,
@@ -2385,13 +2403,36 @@ class Benchmark {
     }
   }
 
-  void Open(Options* opts) {
+  // Returns true if the options is initialized from the specified
+  // options file.
+  bool InitializeOptionsFromFile(Options* opts) {
+#ifndef ROCKSDB_LITE
+    printf("Initializing RocksDB Options from the specified file\n");
+    DBOptions db_opts;
+    std::vector<ColumnFamilyDescriptor> cf_descs;
+    if (FLAGS_options_file != "") {
+      auto s = LoadOptionsFromFile(FLAGS_options_file, Env::Default(), &db_opts,
+                                   &cf_descs);
+      if (s.ok()) {
+        *opts = Options(db_opts, cf_descs[0].options);
+        return true;
+      }
+      fprintf(stderr, "Unable to load options file %s --- %s\n",
+              FLAGS_options_file.c_str(), s.ToString().c_str());
+      exit(1);
+    }
+#endif
+    return false;
+  }
+
+  void InitializeOptionsFromFlags(Options* opts) {
+    printf("Initializing RocksDB Options from command-line flags\n");
     Options& options = *opts;
 
     assert(db_.db == nullptr);
 
-    options.create_if_missing = !FLAGS_use_existing_db;
     options.create_missing_column_families = FLAGS_num_column_families > 1;
+    options.max_open_files = FLAGS_open_files;
     options.db_write_buffer_size = FLAGS_db_write_buffer_size;
     options.write_buffer_size = FLAGS_write_buffer_size;
     options.max_write_buffer_number = FLAGS_max_write_buffer_number;
@@ -2417,39 +2458,14 @@ class Benchmark {
     }
     options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits;
     options.bloom_locality = FLAGS_bloom_locality;
-    options.max_open_files = FLAGS_open_files;
     options.max_file_opening_threads = FLAGS_file_opening_threads;
     options.new_table_reader_for_compaction_inputs =
         FLAGS_new_table_reader_for_compaction_inputs;
     options.compaction_readahead_size = FLAGS_compaction_readahead_size;
     options.random_access_max_buffer_size = FLAGS_random_access_max_buffer_size;
     options.writable_file_max_buffer_size = FLAGS_writable_file_max_buffer_size;
-    options.statistics = dbstats;
-    if (FLAGS_enable_io_prio) {
-      FLAGS_env->LowerThreadPoolIOPriority(Env::LOW);
-      FLAGS_env->LowerThreadPoolIOPriority(Env::HIGH);
-    }
-    if (FLAGS_disable_flashcache_for_background_threads &&
-        cachedev_fd_ == -1) {
-      // Avoid creating the env twice when an use_existing_db is true
-      cachedev_fd_ = open(FLAGS_flashcache_dev.c_str(), O_RDONLY);
-      if (cachedev_fd_ < 0) {
-        fprintf(stderr, "Open flash device failed\n");
-        exit(1);
-      }
-      flashcache_aware_env_ = NewFlashcacheAwareEnv(FLAGS_env, cachedev_fd_);
-      if (flashcache_aware_env_.get() == nullptr) {
-        fprintf(stderr, "Failed to open flashcache device at %s\n",
-                FLAGS_flashcache_dev.c_str());
-        std::abort();
-      }
-      options.env = flashcache_aware_env_.get();
-    } else {
-      options.env = FLAGS_env;
-    }
     options.disableDataSync = FLAGS_disable_data_sync;
     options.use_fsync = FLAGS_use_fsync;
-    options.wal_dir = FLAGS_wal_dir;
     options.num_levels = FLAGS_num_levels;
     options.target_file_size_base = FLAGS_target_file_size_base;
     options.target_file_size_multiplier = FLAGS_target_file_size_multiplier;
@@ -2459,14 +2475,6 @@ class Benchmark {
     options.max_bytes_for_level_multiplier =
         FLAGS_max_bytes_for_level_multiplier;
     options.filter_deletes = FLAGS_filter_deletes;
-    if (FLAGS_row_cache_size) {
-      if (FLAGS_cache_numshardbits >= 1) {
-        options.row_cache =
-            NewLRUCache(FLAGS_row_cache_size, FLAGS_cache_numshardbits);
-      } else {
-        options.row_cache = NewLRUCache(FLAGS_row_cache_size);
-      }
-    }
     if ((FLAGS_prefix_size == 0) && (FLAGS_rep_factory == kPrefixHash ||
                                      FLAGS_rep_factory == kHashLinkedList)) {
       fprintf(stderr, "prefix_size should be non-zero if PrefixHash or "
@@ -2689,6 +2697,48 @@ class Benchmark {
     }
 #endif  // ROCKSDB_LITE
 
+    if (FLAGS_min_level_to_compress >= 0) {
+      options.compression_per_level.clear();
+    }
+  }
+
+  void InitializeOptionsGeneral(Options* opts) {
+    Options& options = *opts;
+
+    options.statistics = dbstats;
+    options.wal_dir = FLAGS_wal_dir;
+    options.create_if_missing = !FLAGS_use_existing_db;
+
+    if (FLAGS_row_cache_size) {
+      if (FLAGS_cache_numshardbits >= 1) {
+        options.row_cache =
+            NewLRUCache(FLAGS_row_cache_size, FLAGS_cache_numshardbits);
+      } else {
+        options.row_cache = NewLRUCache(FLAGS_row_cache_size);
+      }
+    }
+    if (FLAGS_enable_io_prio) {
+      FLAGS_env->LowerThreadPoolIOPriority(Env::LOW);
+      FLAGS_env->LowerThreadPoolIOPriority(Env::HIGH);
+    }
+    if (FLAGS_disable_flashcache_for_background_threads && cachedev_fd_ == -1) {
+      // Avoid creating the env twice when an use_existing_db is true
+      cachedev_fd_ = open(FLAGS_flashcache_dev.c_str(), O_RDONLY);
+      if (cachedev_fd_ < 0) {
+        fprintf(stderr, "Open flash device failed\n");
+        exit(1);
+      }
+      flashcache_aware_env_ = NewFlashcacheAwareEnv(FLAGS_env, cachedev_fd_);
+      if (flashcache_aware_env_.get() == nullptr) {
+        fprintf(stderr, "Failed to open flashcache device at %s\n",
+                FLAGS_flashcache_dev.c_str());
+        std::abort();
+      }
+      options.env = flashcache_aware_env_.get();
+    } else {
+      options.env = FLAGS_env;
+    }
+
     if (FLAGS_num_multi_db <= 1) {
       OpenDb(options, FLAGS_db, &db_);
     } else {
@@ -2698,11 +2748,15 @@ class Benchmark {
         OpenDb(options, GetDbNameForMultiple(FLAGS_db, i), &multi_dbs_[i]);
       }
     }
-    if (FLAGS_min_level_to_compress >= 0) {
+    options.dump_malloc_stats = FLAGS_dump_malloc_stats;
-      options.compression_per_level.clear();
+  }
+
+  void Open(Options* opts) {
+    if (!InitializeOptionsFromFile(opts)) {
+      InitializeOptionsFromFlags(opts);
     }
 
-    options.dump_malloc_stats = FLAGS_dump_malloc_stats;
+    InitializeOptionsGeneral(opts);
   }
 
   void OpenDb(const Options& options, const std::string& db_name,
@@ -3992,8 +4046,12 @@ class Benchmark {
 
 int db_bench_tool(int argc, char** argv) {
   rocksdb::port::InstallStackTraceHandler();
-  SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
+  static bool initialized = false;
-                  " [OPTIONS]...");
+  if (!initialized) {
+    SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
+                    " [OPTIONS]...");
+    initialized = true;
+  }
   ParseCommandLineFlags(&argc, &argv, true);
 
   FLAGS_compaction_style_e = (rocksdb::CompactionStyle) FLAGS_compaction_style;

tools/db_bench_tool_test.cc

@@ -0,0 +1,323 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "rocksdb/db_bench_tool.h"
+#include "rocksdb/utilities/options_util.h"
+#include "util/options_parser.h"
+#include "util/random.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+#ifdef GFLAGS
+#include <gflags/gflags.h>
+
+namespace rocksdb {
+namespace {
+static const int kMaxArgCount = 100;
+static const size_t kArgBufferSize = 100000;
+}  // namespace
+
+class DBBenchTest : public testing::Test {
+ public:
+  DBBenchTest() : rnd_(0xFB) {
+    test_path_ = test::TmpDir() + "/db_bench_test";
+    Env::Default()->CreateDir(test_path_);
+    db_path_ = test_path_ + "/db";
+    wal_path_ = test_path_ + "/wal";
+  }
+
+  ~DBBenchTest() {
+    //  DestroyDB(db_path_, Options());
+  }
+
+  void ResetArgs() {
+    argc_ = 0;
+    cursor_ = 0;
+    memset(arg_buffer_, 0, kArgBufferSize);
+  }
+
+  void AppendArgs(const std::vector<std::string>& args) {
+    for (const auto& arg : args) {
+      ASSERT_LE(cursor_ + arg.size() + 1, kArgBufferSize);
+      ASSERT_LE(argc_ + 1, kMaxArgCount);
+      snprintf(arg_buffer_ + cursor_, arg.size() + 1, "%s", arg.c_str());
+
+      argv_[argc_++] = arg_buffer_ + cursor_;
+      cursor_ += arg.size() + 1;
+    }
+  }
+
+  void RunDbBench(const std::string& options_file_name) {
+    AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0",
+                "--num=1000",
+                std::string(std::string("--db=") + db_path_).c_str(),
+                std::string(std::string("--wal_dir=") + wal_path_).c_str(),
+                std::string(std::string("--options_file=") + options_file_name)
+                    .c_str()});
+    ASSERT_EQ(0, db_bench_tool(argc(), argv()));
+  }
+
+  void VerifyOptions(const Options& opt) {
+    DBOptions loaded_db_opts;
+    std::vector<ColumnFamilyDescriptor> cf_descs;
+    ASSERT_OK(LoadLatestOptions(db_path_, Env::Default(), &loaded_db_opts,
+                                &cf_descs));
+
+    ASSERT_OK(
+        RocksDBOptionsParser::VerifyDBOptions(DBOptions(opt), loaded_db_opts));
+    ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(opt),
+                                                    cf_descs[0].options));
+
+    // check with the default rocksdb options and expect failure
+    ASSERT_NOK(
+        RocksDBOptionsParser::VerifyDBOptions(DBOptions(), loaded_db_opts));
+    ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(),
+                                                     cf_descs[0].options));
+  }
+
+  char** argv() { return argv_; }
+
+  int argc() { return argc_; }
+
+  std::string db_path_;
+  std::string test_path_;
+  std::string wal_path_;
+
+  char arg_buffer_[kArgBufferSize];
+  char* argv_[kMaxArgCount];
+  int argc_ = 0;
+  int cursor_ = 0;
+  Random rnd_;
+};
+
+namespace {}  // namespace
+
+TEST_F(DBBenchTest, OptionsFile) {
+  const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
+
+  Options opt;
+  opt.create_if_missing = true;
+  opt.max_open_files = 256;
+  opt.base_background_compactions = 5;
+  opt.max_background_compactions = 10;
+  opt.arena_block_size = 8388608;
+  ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
+                                  {ColumnFamilyOptions(opt)}, kOptionsFileName,
+                                  Env::Default()));
+
+  // override the following options as db_bench will not take these
+  // options from the options file
+  opt.wal_dir = wal_path_;
+
+  RunDbBench(kOptionsFileName);
+
+  VerifyOptions(opt);
+}
+
+TEST_F(DBBenchTest, OptionsFileUniversal) {
+  const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
+
+  Options opt;
+  opt.compaction_style = kCompactionStyleUniversal;
+  opt.num_levels = 1;
+  opt.create_if_missing = true;
+  opt.max_open_files = 256;
+  opt.base_background_compactions = 5;
+  opt.max_background_compactions = 10;
+  opt.arena_block_size = 8388608;
+  ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
+                                  {ColumnFamilyOptions(opt)}, kOptionsFileName,
+                                  Env::Default()));
+
+  // override the following options as db_bench will not take these
+  // options from the options file
+  opt.wal_dir = wal_path_;
+
+  RunDbBench(kOptionsFileName);
+
+  VerifyOptions(opt);
+}
+
+TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) {
+  const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
+
+  Options opt;
+  opt.compaction_style = kCompactionStyleUniversal;
+  opt.num_levels = 12;
+  opt.create_if_missing = true;
+  opt.max_open_files = 256;
+  opt.base_background_compactions = 5;
+  opt.max_background_compactions = 10;
+  opt.arena_block_size = 8388608;
+  ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
+                                  {ColumnFamilyOptions(opt)}, kOptionsFileName,
+                                  Env::Default()));
+
+  // override the following options as db_bench will not take these
+  // options from the options file
+  opt.wal_dir = wal_path_;
+
+  RunDbBench(kOptionsFileName);
+
+  VerifyOptions(opt);
+}
+
+const std::string options_file_content = R"OPTIONS_FILE(
+[Version]
+  rocksdb_version=4.3.1
+  options_file_version=1.1
+
+[DBOptions]
+  wal_bytes_per_sync=1048576
+  delete_obsolete_files_period_micros=0
+  WAL_ttl_seconds=0
+  WAL_size_limit_MB=0
+  db_write_buffer_size=0
+  max_subcompactions=1
+  table_cache_numshardbits=4
+  max_open_files=-1
+  max_file_opening_threads=10
+  base_background_compactions=3
+  max_background_compactions=5
+  use_fsync=false
+  use_adaptive_mutex=false
+  max_total_wal_size=18446744073709551615
+  compaction_readahead_size=0
+  new_table_reader_for_compaction_inputs=false
+  keep_log_file_num=10
+  skip_stats_update_on_db_open=false
+  max_manifest_file_size=18446744073709551615
+  db_log_dir=
+  skip_log_error_on_recovery=false
+  writable_file_max_buffer_size=1048576
+  paranoid_checks=true
+  is_fd_close_on_exec=true
+  bytes_per_sync=1048576
+  enable_thread_tracking=true
+  disable_data_sync=false
+  recycle_log_file_num=0
+  disableDataSync=false
+  create_missing_column_families=false
+  log_file_time_to_roll=0
+  max_background_flushes=1
+  create_if_missing=true
+  error_if_exists=false
+  allow_os_buffer=true
+  delayed_write_rate=1048576
+  manifest_preallocation_size=4194304
+  allow_mmap_writes=false
+  stats_dump_period_sec=600
+  allow_fallocate=true
+  allow_mmap_reads=false
+  max_log_file_size=83886080
+  random_access_max_buffer_size=1048576
+  advise_random_on_open=true
+
+
+[CFOptions "default"]
+  compaction_filter_factory=nullptr
+  table_factory=BlockBasedTable
+  prefix_extractor=nullptr
+  comparator=leveldb.BytewiseComparator
+  compression_per_level=
+  max_bytes_for_level_base=104857600
+  bloom_locality=0
+  target_file_size_base=10485760
+  memtable_prefix_bloom_huge_page_tlb_size=0
+  max_successive_merges=1000
+  max_sequential_skip_in_iterations=8
+  arena_block_size=52428800
+  target_file_size_multiplier=1
+  source_compaction_factor=1
+  min_write_buffer_number_to_merge=1
+  max_write_buffer_number=2
+  write_buffer_size=419430400
+  max_grandparent_overlap_factor=10
+  max_bytes_for_level_multiplier=10
+  memtable_factory=SkipListFactory
+  compression=kSnappyCompression
+  min_partial_merge_operands=2
+  level0_stop_writes_trigger=100
+  num_levels=1
+  level0_slowdown_writes_trigger=50
+  level0_file_num_compaction_trigger=10
+  expanded_compaction_factor=25
+  soft_rate_limit=0.000000
+  max_write_buffer_number_to_maintain=0
+  verify_checksums_in_compaction=true
+  merge_operator=nullptr
+  memtable_prefix_bloom_bits=0
+  paranoid_file_checks=false
+  inplace_update_num_locks=10000
+  optimize_filters_for_hits=false
+  level_compaction_dynamic_level_bytes=false
+  inplace_update_support=false
+  compaction_style=kCompactionStyleUniversal
+  memtable_prefix_bloom_probes=6
+  purge_redundant_kvs_while_flush=true
+  filter_deletes=false
+  hard_pending_compaction_bytes_limit=0
+  disable_auto_compactions=false
+  compaction_measure_io_stats=false
+
+[TableOptions/BlockBasedTable "default"]
+  format_version=0
+  skip_table_builder_flush=false
+  cache_index_and_filter_blocks=false
+  flush_block_policy_factory=FlushBlockBySizePolicyFactory
+  hash_index_allow_collision=true
+  index_type=kBinarySearch
+  whole_key_filtering=true
+  checksum=kCRC32c
+  no_block_cache=false
+  block_size=32768
+  block_size_deviation=10
+  block_restart_interval=16
+  filter_policy=rocksdb.BuiltinBloomFilter
+)OPTIONS_FILE";
+
+TEST_F(DBBenchTest, OptionsFileFromFile) {
+  const std::string kOptionsFileName = test_path_ + "/OPTIONS_flash";
+  unique_ptr<WritableFile> writable;
+  ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName, &writable,
+                                            EnvOptions()));
+  ASSERT_OK(writable->Append(options_file_content));
+  ASSERT_OK(writable->Close());
+
+  DBOptions db_opt;
+  std::vector<ColumnFamilyDescriptor> cf_descs;
+  ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt,
+                                &cf_descs));
+  Options opt(db_opt, cf_descs[0].options);
+
+  opt.create_if_missing = true;
+
+  // override the following options as db_bench will not take these
+  // options from the options file
+  opt.wal_dir = wal_path_;
+
+  RunDbBench(kOptionsFileName);
+
+  VerifyOptions(opt);
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}
+
+#else
+
+int main(int argc, char** argv) {
+  printf("Skip db_bench_tool_test as the required library GFLAG is missing.");
+}
+#endif  // #ifdef GFLAGS