andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make it easier for users to load options from option file and set shared block cache. (#5063)

Browse Source 
Summary:
[RocksDB] Make it easier for users to load options from option file and set shared block cache.
Right now, it requires several dynamic casting for users to set the shared block cache to their option struct cast from the option file.
If people don't do that, every CF of every DB will generate its own 8MB block cache. It's not a usable setting. So we are dragging every user who loads options from the file into such a mess.
Instead, we should allow them to pass their cache object to LoadLatestOptions() and LoadOptionsFromFile(), so that those loaded option structs will have the shared block cache.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5063

Differential Revision: D14518584

Pulled By: rashmishrm

fbshipit-source-id: c91430ff9425a0e67d76fc67931d755f491ca5aa
This commit is contained in:
Rashmi Sharma 2019-03-21 16:16:40 -07:00 committed by Facebook Github Bot
parent 88d85b6820
commit a4396f9218
4 changed files with 65 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/rocksdb/utilities/options_util.h
View File

@ -58,7 +58,8 @@ namespace rocksdb {
Status LoadLatestOptions(const std::string& dbpath, Env* env,
DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options = false);
bool ignore_unknown_options = false,
std::shared_ptr<Cache>* cache = {});
// Similar to LoadLatestOptions, this function constructs the DBOptions
// and ColumnFamilyDescriptors based on the specified RocksDB Options file.
@ -67,7 +68,8 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env,
Status LoadOptionsFromFile(const std::string& options_file_name, Env* env,
DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options = false);
bool ignore_unknown_options = false,
std::shared_ptr<Cache>* cache = {});
// Returns the latest options file name under the specified db path.
Status GetLatestOptionsFileName(const std::string& dbpath, Env* env,

20
utilities/options/options_util.cc
View File

@ -15,20 +15,28 @@ namespace rocksdb {
Status LoadOptionsFromFile(const std::string& file_name, Env* env,
DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options) {
bool ignore_unknown_options,
std::shared_ptr<Cache>* cache) {
RocksDBOptionsParser parser;
Status s = parser.Parse(file_name, env, ignore_unknown_options);
if (!s.ok()) {
return s;
}
*db_options = *parser.db_opt();
const std::vector<std::string>& cf_names = *parser.cf_names();
const std::vector<ColumnFamilyOptions>& cf_opts = *parser.cf_opts();
cf_descs->clear();
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs->push_back({cf_names[i], cf_opts[i]});
if (cache != nullptr) {
TableFactory* tf = cf_opts[i].table_factory.get();
if (tf != nullptr && tf->GetOptions() != nullptr &&
tf->Name() == BlockBasedTableFactory().Name()) {
auto* loaded_bbt_opt =
reinterpret_cast<BlockBasedTableOptions*>(tf->GetOptions());
loaded_bbt_opt->block_cache = *cache;
}
}
}
return Status::OK();
}
@ -63,15 +71,15 @@ Status GetLatestOptionsFileName(const std::string& dbpath,
Status LoadLatestOptions(const std::string& dbpath, Env* env,
DBOptions* db_options,
std::vector<ColumnFamilyDescriptor>* cf_descs,
bool ignore_unknown_options) {
bool ignore_unknown_options,
std::shared_ptr<Cache>* cache) {
std::string options_file_name;
Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name);
if (!s.ok()) {
return s;
}
return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, db_options,
cf_descs, ignore_unknown_options);
cf_descs, ignore_unknown_options, cache);
}
Status CheckOptionsCompatibility(

46
utilities/options/options_util_test.cc
View File

@ -94,6 +94,52 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
}
}
TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) {
// creating db
DBOptions db_opt;
db_opt.create_if_missing = true;
// initialize BlockBasedTableOptions
std::shared_ptr<Cache> cache = NewLRUCache(1 * 1024);
BlockBasedTableOptions bbt_opts;
bbt_opts.block_size = 32 * 1024;
// saving cf options
std::vector<ColumnFamilyOptions> cf_opts;
ColumnFamilyOptions default_column_family_opt = ColumnFamilyOptions();
default_column_family_opt.table_factory.reset(
NewBlockBasedTableFactory(bbt_opts));
cf_opts.push_back(default_column_family_opt);
ColumnFamilyOptions cf_opt_sample = ColumnFamilyOptions();
cf_opt_sample.table_factory.reset(NewBlockBasedTableFactory(bbt_opts));
cf_opts.push_back(cf_opt_sample);
ColumnFamilyOptions cf_opt_plain_table_opt = ColumnFamilyOptions();
cf_opt_plain_table_opt.table_factory.reset(NewPlainTableFactory());
cf_opts.push_back(cf_opt_plain_table_opt);
std::vector<std::string> cf_names;
cf_names.push_back(kDefaultColumnFamilyName);
cf_names.push_back("cf_sample");
cf_names.push_back("cf_plain_table_sample");
// Saving DB in file
const std::string kFileName = "OPTIONS-LOAD_CACHE_123456";
PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, env_.get());
DBOptions loaded_db_opt;
std::vector<ColumnFamilyDescriptor> loaded_cf_descs;
ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt,
&loaded_cf_descs, false, &cache));
for (size_t i = 0; i < loaded_cf_descs.size(); i++) {
if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) {
auto* loaded_bbt_opt = reinterpret_cast<BlockBasedTableOptions*>(
loaded_cf_descs[i].options.table_factory->GetOptions());
// Expect the same cache will be loaded
if (loaded_bbt_opt != nullptr) {
ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get());
}
}
}
}
namespace {
class DummyTableFactory : public TableFactory {
public: