From a4396f9218d47dc828bef4cff0d20a94c29df4ef Mon Sep 17 00:00:00 2001 From: Rashmi Sharma Date: Thu, 21 Mar 2019 16:16:40 -0700 Subject: [PATCH] Make it easier for users to load options from option file and set shared block cache. (#5063) Summary: [RocksDB] Make it easier for users to load options from option file and set shared block cache. Right now, it requires several dynamic casting for users to set the shared block cache to their option struct cast from the option file. If people don't do that, every CF of every DB will generate its own 8MB block cache. It's not a usable setting. So we are dragging every user who loads options from the file into such a mess. Instead, we should allow them to pass their cache object to LoadLatestOptions() and LoadOptionsFromFile(), so that those loaded option structs will have the shared block cache. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5063 Differential Revision: D14518584 Pulled By: rashmishrm fbshipit-source-id: c91430ff9425a0e67d76fc67931d755f491ca5aa --- env/io_posix.cc | 2 +- include/rocksdb/utilities/options_util.h | 6 ++-- utilities/options/options_util.cc | 20 +++++++---- utilities/options/options_util_test.cc | 46 ++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/env/io_posix.cc b/env/io_posix.cc index 967a6ace8..de796c653 100644 --- a/env/io_posix.cc +++ b/env/io_posix.cc @@ -826,7 +826,7 @@ Status PosixWritableFile::Close() { int dummy __attribute__((__unused__)); dummy = ftruncate(fd_, filesize_); #if defined(ROCKSDB_FALLOCATE_PRESENT) && defined(FALLOC_FL_PUNCH_HOLE) && \ - !defined(TRAVIS) + !defined(TRAVIS) // in some file systems, ftruncate only trims trailing space if the // new file size is smaller than the current size. Calling fallocate // with FALLOC_FL_PUNCH_HOLE flag to explicitly release these unused diff --git a/include/rocksdb/utilities/options_util.h b/include/rocksdb/utilities/options_util.h index d02c57410..aa5116e0a 100644 --- a/include/rocksdb/utilities/options_util.h +++ b/include/rocksdb/utilities/options_util.h @@ -58,7 +58,8 @@ namespace rocksdb { Status LoadLatestOptions(const std::string& dbpath, Env* env, DBOptions* db_options, std::vector* cf_descs, - bool ignore_unknown_options = false); + bool ignore_unknown_options = false, + std::shared_ptr* cache = {}); // Similar to LoadLatestOptions, this function constructs the DBOptions // and ColumnFamilyDescriptors based on the specified RocksDB Options file. @@ -67,7 +68,8 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env, Status LoadOptionsFromFile(const std::string& options_file_name, Env* env, DBOptions* db_options, std::vector* cf_descs, - bool ignore_unknown_options = false); + bool ignore_unknown_options = false, + std::shared_ptr* cache = {}); // Returns the latest options file name under the specified db path. Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, diff --git a/utilities/options/options_util.cc b/utilities/options/options_util.cc index 21734923f..3975eadd7 100644 --- a/utilities/options/options_util.cc +++ b/utilities/options/options_util.cc @@ -15,20 +15,28 @@ namespace rocksdb { Status LoadOptionsFromFile(const std::string& file_name, Env* env, DBOptions* db_options, std::vector* cf_descs, - bool ignore_unknown_options) { + bool ignore_unknown_options, + std::shared_ptr* cache) { RocksDBOptionsParser parser; Status s = parser.Parse(file_name, env, ignore_unknown_options); if (!s.ok()) { return s; } - *db_options = *parser.db_opt(); - const std::vector& cf_names = *parser.cf_names(); const std::vector& cf_opts = *parser.cf_opts(); cf_descs->clear(); for (size_t i = 0; i < cf_opts.size(); ++i) { cf_descs->push_back({cf_names[i], cf_opts[i]}); + if (cache != nullptr) { + TableFactory* tf = cf_opts[i].table_factory.get(); + if (tf != nullptr && tf->GetOptions() != nullptr && + tf->Name() == BlockBasedTableFactory().Name()) { + auto* loaded_bbt_opt = + reinterpret_cast(tf->GetOptions()); + loaded_bbt_opt->block_cache = *cache; + } + } } return Status::OK(); } @@ -63,15 +71,15 @@ Status GetLatestOptionsFileName(const std::string& dbpath, Status LoadLatestOptions(const std::string& dbpath, Env* env, DBOptions* db_options, std::vector* cf_descs, - bool ignore_unknown_options) { + bool ignore_unknown_options, + std::shared_ptr* cache) { std::string options_file_name; Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); if (!s.ok()) { return s; } - return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, db_options, - cf_descs, ignore_unknown_options); + cf_descs, ignore_unknown_options, cache); } Status CheckOptionsCompatibility( diff --git a/utilities/options/options_util_test.cc b/utilities/options/options_util_test.cc index b1c668221..ed7bfdfd6 100644 --- a/utilities/options/options_util_test.cc +++ b/utilities/options/options_util_test.cc @@ -94,6 +94,52 @@ TEST_F(OptionsUtilTest, SaveAndLoad) { } } +TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) { + // creating db + DBOptions db_opt; + db_opt.create_if_missing = true; + // initialize BlockBasedTableOptions + std::shared_ptr cache = NewLRUCache(1 * 1024); + BlockBasedTableOptions bbt_opts; + bbt_opts.block_size = 32 * 1024; + // saving cf options + std::vector cf_opts; + ColumnFamilyOptions default_column_family_opt = ColumnFamilyOptions(); + default_column_family_opt.table_factory.reset( + NewBlockBasedTableFactory(bbt_opts)); + cf_opts.push_back(default_column_family_opt); + + ColumnFamilyOptions cf_opt_sample = ColumnFamilyOptions(); + cf_opt_sample.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); + cf_opts.push_back(cf_opt_sample); + + ColumnFamilyOptions cf_opt_plain_table_opt = ColumnFamilyOptions(); + cf_opt_plain_table_opt.table_factory.reset(NewPlainTableFactory()); + cf_opts.push_back(cf_opt_plain_table_opt); + + std::vector cf_names; + cf_names.push_back(kDefaultColumnFamilyName); + cf_names.push_back("cf_sample"); + cf_names.push_back("cf_plain_table_sample"); + // Saving DB in file + const std::string kFileName = "OPTIONS-LOAD_CACHE_123456"; + PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, env_.get()); + DBOptions loaded_db_opt; + std::vector loaded_cf_descs; + ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, + &loaded_cf_descs, false, &cache)); + for (size_t i = 0; i < loaded_cf_descs.size(); i++) { + if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { + auto* loaded_bbt_opt = reinterpret_cast( + loaded_cf_descs[i].options.table_factory->GetOptions()); + // Expect the same cache will be loaded + if (loaded_bbt_opt != nullptr) { + ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); + } + } + } +} + namespace { class DummyTableFactory : public TableFactory { public: