andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

improve ldb CLI option support

Browse Source 
Summary:
- Made CLI arguments take precedence over options file when both are provided. Note some of the CLI args are not settable via options file, like `--compression_max_dict_bytes`, so it's necessary to allow both ways of providing options simultaneously.
- Changed `PrepareOptionsForOpenDB` to update the proper `ColumnFamilyOptions` if one exists for the user's `--column_family_name` argument. I supported this only in the base class, `LDBCommand`, so it works for the general arguments. Will defer adding support for subcommand-specific arguments.
- Made the command fail if `--try_load_options` is provided and loading options file returns NotFound. I found the previous behavior of silently continuing confusing.
Closes https://github.com/facebook/rocksdb/pull/3144

Differential Revision: D6270544

Pulled By: ajkr

fbshipit-source-id: 7c2eac9f9b38720523d74466fb9e78db53561367
This commit is contained in:
Andrew Kryczka 2017-11-28 17:20:47 -08:00 committed by Facebook Github Bot
parent c85f8ccca3
commit ed3af9ef99
1 changed files with 46 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
tools/ldb_cmd.cc
View File

@ -312,14 +312,10 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
}
void LDBCommand::OpenDB() {
Options opt;
bool opt_set = false;
if (!create_if_missing_ && try_load_options_) {
Status s = LoadLatestOptions(db_path_, Env::Default(), &opt,
Status s = LoadLatestOptions(db_path_, Env::Default(), &options_,
&column_families_, ignore_unknown_options_);
if (s.ok()) {
opt_set = true;
} else if (!s.IsNotFound()) {
if (!s.ok()) {
// Option file exists but load option file error.
std::string msg = s.ToString();
exec_state_ = LDBCommandExecuteResult::Failed(msg);
@ -327,9 +323,7 @@ void LDBCommand::OpenDB() {
return;
}
}
if (!opt_set) {
opt = PrepareOptionsForOpenDB();
}
options_ = PrepareOptionsForOpenDB();
if (!exec_state_.IsNotStarted()) {
return;
}
@ -343,13 +337,13 @@ void LDBCommand::OpenDB() {
"ldb doesn't support TTL DB with multiple column families");
}
if (is_read_only_) {
st = DBWithTTL::Open(opt, db_path_, &db_ttl_, 0, true);
st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true);
} else {
st = DBWithTTL::Open(opt, db_path_, &db_ttl_);
st = DBWithTTL::Open(options_, db_path_, &db_ttl_);
}
db_ = db_ttl_;
} else {
if (!opt_set && column_families_.empty()) {
if (column_families_.empty()) {
// Try to figure out column family lists
std::vector<std::string> cf_list;
st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list);
@ -360,22 +354,22 @@ void LDBCommand::OpenDB() {
if (st.ok() && cf_list.size() > 1) {
// Ignore single column family DB.
for (auto cf_name : cf_list) {
column_families_.emplace_back(cf_name, opt);
column_families_.emplace_back(cf_name, options_);
}
}
}
if (is_read_only_) {
if (column_families_.empty()) {
st = DB::OpenForReadOnly(opt, db_path_, &db_);
st = DB::OpenForReadOnly(options_, db_path_, &db_);
} else {
st = DB::OpenForReadOnly(opt, db_path_, column_families_,
st = DB::OpenForReadOnly(options_, db_path_, column_families_,
&handles_opened, &db_);
}
} else {
if (column_families_.empty()) {
st = DB::Open(opt, db_path_, &db_);
st = DB::Open(options_, db_path_, &db_);
} else {
st = DB::Open(opt, db_path_, column_families_, &handles_opened, &db_);
st = DB::Open(options_, db_path_, column_families_, &handles_opened, &db_);
}
}
}
@ -405,8 +399,6 @@ void LDBCommand::OpenDB() {
CloseDB();
}
}
options_ = opt;
}
void LDBCommand::CloseDB() {
@ -499,9 +491,19 @@ bool LDBCommand::ParseStringOption(
}
Options LDBCommand::PrepareOptionsForOpenDB() {
Options opt = options_;
opt.create_if_missing = false;
ColumnFamilyOptions* cf_opts;
auto column_families_iter =
std::find_if(column_families_.begin(), column_families_.end(),
[this](const ColumnFamilyDescriptor& cf_desc) {
return cf_desc.name == column_family_name_;
});
if (column_families_iter != column_families_.end()) {
cf_opts = &column_families_iter->options;
} else {
cf_opts = static_cast<ColumnFamilyOptions*>(&options_);
}
DBOptions* db_opts = static_cast<DBOptions*>(&options_);
db_opts->create_if_missing = false;
std::map<std::string, std::string>::const_iterator itr;
@ -530,33 +532,33 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
}
if (use_table_options) {
opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
cf_opts->table_factory.reset(NewBlockBasedTableFactory(table_options));
}
itr = option_map_.find(ARG_AUTO_COMPACTION);
if (itr != option_map_.end()) {
opt.disable_auto_compactions = ! StringToBool(itr->second);
cf_opts->disable_auto_compactions = !StringToBool(itr->second);
}
itr = option_map_.find(ARG_COMPRESSION_TYPE);
if (itr != option_map_.end()) {
std::string comp = itr->second;
if (comp == "no") {
opt.compression = kNoCompression;
cf_opts->compression = kNoCompression;
} else if (comp == "snappy") {
opt.compression = kSnappyCompression;
cf_opts->compression = kSnappyCompression;
} else if (comp == "zlib") {
opt.compression = kZlibCompression;
cf_opts->compression = kZlibCompression;
} else if (comp == "bzip2") {
opt.compression = kBZip2Compression;
cf_opts->compression = kBZip2Compression;
} else if (comp == "lz4") {
opt.compression = kLZ4Compression;
cf_opts->compression = kLZ4Compression;
} else if (comp == "lz4hc") {
opt.compression = kLZ4HCCompression;
cf_opts->compression = kLZ4HCCompression;
} else if (comp == "xpress") {
opt.compression = kXpressCompression;
cf_opts->compression = kXpressCompression;
} else if (comp == "zstd") {
opt.compression = kZSTD;
cf_opts->compression = kZSTD;
} else {
// Unknown compression.
exec_state_ =
@ -568,7 +570,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES,
compression_max_dict_bytes, exec_state_)) {
if (compression_max_dict_bytes >= 0) {
opt.compression_opts.max_dict_bytes = compression_max_dict_bytes;
cf_opts->compression_opts.max_dict_bytes = compression_max_dict_bytes;
} else {
exec_state_ = LDBCommandExecuteResult::Failed(
ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0.");
@ -579,7 +581,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE,
db_write_buffer_size, exec_state_)) {
if (db_write_buffer_size >= 0) {
opt.db_write_buffer_size = db_write_buffer_size;
db_opts->db_write_buffer_size = db_write_buffer_size;
} else {
exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE +
" must be >= 0.");
@ -590,7 +592,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size,
exec_state_)) {
if (write_buffer_size > 0) {
opt.write_buffer_size = write_buffer_size;
cf_opts->write_buffer_size = write_buffer_size;
} else {
exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE +
" must be > 0.");
@ -600,30 +602,33 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
int file_size;
if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) {
if (file_size > 0) {
opt.target_file_size_base = file_size;
cf_opts->target_file_size_base = file_size;
} else {
exec_state_ =
LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0.");
}
}
if (opt.db_paths.size() == 0) {
opt.db_paths.emplace_back(db_path_, std::numeric_limits<uint64_t>::max());
if (db_opts->db_paths.size() == 0) {
db_opts->db_paths.emplace_back(db_path_,
std::numeric_limits<uint64_t>::max());
}
int fix_prefix_len;
if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len,
exec_state_)) {
if (fix_prefix_len > 0) {
opt.prefix_extractor.reset(
cf_opts->prefix_extractor.reset(
NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len)));
} else {
exec_state_ =
LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0.");
}
}
return opt;
// TODO(ajkr): this return value doesn't reflect the CF options changed, so
// subcommands that rely on this won't see the effect of CF-related CLI args.
// Such subcommands need to be changed to properly support CFs.
return options_;
}
bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key,