Options conversion function for convenience

Summary: as title

Test Plan: options_test

Reviewers: sdong, yhchiang, igor

Reviewed By: igor

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D23283
This commit is contained in:
Lei Jin 2014-09-17 12:46:32 -07:00
parent a7c2094526
commit e4eca6a1e5
3 changed files with 471 additions and 0 deletions

View File

@ -14,6 +14,7 @@
#include <memory>
#include <vector>
#include <stdint.h>
#include <unordered_map>
#include "rocksdb/version.h"
#include "rocksdb/universal_compaction.h"
@ -1012,6 +1013,12 @@ extern Options GetOptions(size_t total_write_buffer_limit,
int read_amplification_threshold = 8,
int write_amplification_threshold = 32,
uint64_t target_db_size = 68719476736 /* 64GB */);
bool GetOptionsFromStrings(
const Options& base_options,
const std::unordered_map<std::string, std::string>& options_map,
Options* new_options);
} // namespace rocksdb
#endif // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_

292
util/options_helper.cc Normal file
View File

@ -0,0 +1,292 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <cassert>
#include "rocksdb/options.h"
namespace rocksdb {
namespace {
CompressionType ParseCompressionType(const std::string& type) {
if (type == "kNoCompression") {
return kNoCompression;
} else if (type == "kSnappyCompression") {
return kSnappyCompression;
} else if (type == "kZlibCompression") {
return kZlibCompression;
} else if (type == "kBZip2Compression") {
return kBZip2Compression;
} else if (type == "kLZ4Compression") {
return kLZ4Compression;
} else if (type == "kLZ4HCCompression") {
return kLZ4HCCompression;
} else {
throw "unknown compression type: " + type;
}
return kNoCompression;
}
bool ParseBoolean(const std::string& type, const std::string& value) {
if (value == "true" || value == "1") {
return true;
} else if (value == "false" || value == "0") {
return false;
} else {
throw type;
}
}
uint32_t ParseInt(const std::string& value) {
return std::stoi(value);
}
uint32_t ParseUint32(const std::string& value) {
return std::stoul(value);
}
uint64_t ParseUint64(const std::string& value) {
return std::stoull(value);
}
int64_t ParseInt64(const std::string& value) {
return std::stol(value);
}
double ParseDouble(const std::string& value) {
return std::stod(value);
}
CompactionStyle ParseCompactionStyle(const std::string& type) {
if (type == "kCompactionStyleLevel") {
return kCompactionStyleLevel;
} else if (type == "kCompactionStyleUniversal") {
return kCompactionStyleUniversal;
} else if (type == "kCompactionStyleFIFO") {
return kCompactionStyleFIFO;
} else {
throw "unknown compaction style: " + type;
}
return kCompactionStyleLevel;
}
} // anonymouse namespace
bool GetOptionsFromStrings(
const Options& base_options,
const std::unordered_map<std::string, std::string>& options_map,
Options* new_options) {
assert(new_options);
*new_options = base_options;
for (const auto& o : options_map) {
try {
if (o.first == "write_buffer_size") {
new_options->write_buffer_size = ParseInt64(o.second);
} else if (o.first == "max_write_buffer_number") {
new_options->max_write_buffer_number = ParseInt(o.second);
} else if (o.first == "min_write_buffer_number_to_merge") {
new_options->min_write_buffer_number_to_merge = ParseInt(o.second);
} else if (o.first == "compression") {
new_options->compression = ParseCompressionType(o.second);
} else if (o.first == "compression_per_level") {
new_options->compression_per_level.clear();
size_t start = 0;
while (true) {
size_t end = o.second.find_first_of(':', start);
if (end == std::string::npos) {
new_options->compression_per_level.push_back(
ParseCompressionType(o.second.substr(start)));
break;
} else {
new_options->compression_per_level.push_back(
ParseCompressionType(o.second.substr(start, end - start)));
start = end + 1;
}
}
} else if (o.first == "compression_opts") {
size_t start = 0;
size_t end = o.second.find_first_of(':');
if (end == std::string::npos) {
throw o.first;
}
new_options->compression_opts.window_bits =
ParseInt(o.second.substr(start, end - start));
start = end + 1;
end = o.second.find_first_of(':', start);
if (end == std::string::npos) {
throw o.first;
}
new_options->compression_opts.level =
ParseInt(o.second.substr(start, end - start));
start = end + 1;
if (start >= o.second.size()) {
throw o.first;
}
new_options->compression_opts.strategy =
ParseInt(o.second.substr(start, o.second.size() - start));
} else if (o.first == "num_levels") {
new_options->num_levels = ParseInt(o.second);
} else if (o.first == "level0_file_num_compaction_trigger") {
new_options->level0_file_num_compaction_trigger = ParseInt(o.second);
} else if (o.first == "level0_slowdown_writes_trigger") {
new_options->level0_slowdown_writes_trigger = ParseInt(o.second);
} else if (o.first == "level0_stop_writes_trigger") {
new_options->level0_stop_writes_trigger = ParseInt(o.second);
} else if (o.first == "max_mem_compaction_level") {
new_options->max_mem_compaction_level = ParseInt(o.second);
} else if (o.first == "target_file_size_base") {
new_options->target_file_size_base = ParseInt(o.second);
} else if (o.first == "target_file_size_multiplier") {
new_options->target_file_size_multiplier = ParseInt(o.second);
} else if (o.first == "max_bytes_for_level_base") {
new_options->max_bytes_for_level_base = ParseUint64(o.second);
} else if (o.first == "max_bytes_for_level_multiplier") {
new_options->max_bytes_for_level_multiplier = ParseInt(o.second);
} else if (o.first == "max_bytes_for_level_multiplier_additional") {
new_options->max_bytes_for_level_multiplier_additional.clear();
size_t start = 0;
while (true) {
size_t end = o.second.find_first_of(':', start);
if (end == std::string::npos) {
new_options->max_bytes_for_level_multiplier_additional.push_back(
ParseInt(o.second.substr(start)));
break;
} else {
new_options->max_bytes_for_level_multiplier_additional.push_back(
ParseInt(o.second.substr(start, end - start)));
start = end + 1;
}
}
} else if (o.first == "expanded_compaction_factor") {
new_options->expanded_compaction_factor = ParseInt(o.second);
} else if (o.first == "source_compaction_factor") {
new_options->source_compaction_factor = ParseInt(o.second);
} else if (o.first == "max_grandparent_overlap_factor") {
new_options->max_grandparent_overlap_factor = ParseInt(o.second);
} else if (o.first == "soft_rate_limit") {
new_options->soft_rate_limit = ParseDouble(o.second);
} else if (o.first == "hard_rate_limit") {
new_options->hard_rate_limit = ParseDouble(o.second);
} else if (o.first == "arena_block_size") {
new_options->arena_block_size = ParseInt64(o.second);
} else if (o.first == "disable_auto_compactions") {
new_options->disable_auto_compactions = ParseBoolean(o.first, o.second);
} else if (o.first == "purge_redundant_kvs_while_flush") {
new_options->purge_redundant_kvs_while_flush =
ParseBoolean(o.first, o.second);
} else if (o.first == "compaction_style") {
new_options->compaction_style = ParseCompactionStyle(o.second);
} else if (o.first == "verify_checksums_in_compaction") {
new_options->verify_checksums_in_compaction =
ParseBoolean(o.first, o.second);
} else if (o.first == "compaction_options_universal") {
// TODO(ljin): add support
throw o.first;
} else if (o.first == "compaction_options_fifo") {
new_options->compaction_options_fifo.max_table_files_size
= ParseUint64(o.second);
} else if (o.first == "filter_deletes") {
new_options->filter_deletes = ParseBoolean(o.first, o.second);
} else if (o.first == "max_sequential_skip_in_iterations") {
new_options->max_sequential_skip_in_iterations = ParseUint64(o.second);
} else if (o.first == "inplace_update_support") {
new_options->inplace_update_support = ParseBoolean(o.first, o.second);
} else if (o.first == "inplace_update_num_locks") {
new_options->inplace_update_num_locks = ParseInt64(o.second);
} else if (o.first == "memtable_prefix_bloom_bits") {
new_options->memtable_prefix_bloom_bits = stoul(o.second);
} else if (o.first == "memtable_prefix_bloom_probes") {
new_options->memtable_prefix_bloom_probes = stoul(o.second);
} else if (o.first == "memtable_prefix_bloom_huge_page_tlb_size") {
new_options->memtable_prefix_bloom_huge_page_tlb_size =
ParseInt64(o.second);
} else if (o.first == "bloom_locality") {
new_options->bloom_locality = ParseUint32(o.second);
} else if (o.first == "max_successive_merges") {
new_options->max_successive_merges = ParseInt64(o.second);
} else if (o.first == "min_partial_merge_operands") {
new_options->min_partial_merge_operands = ParseUint32(o.second);
} else if (o.first == "create_if_missing") {
new_options->create_if_missing = ParseBoolean(o.first, o.second);
} else if (o.first == "create_missing_column_families") {
new_options->create_missing_column_families =
ParseBoolean(o.first, o.second);
} else if (o.first == "error_if_exists") {
new_options->error_if_exists = ParseBoolean(o.first, o.second);
} else if (o.first == "paranoid_checks") {
new_options->paranoid_checks = ParseBoolean(o.first, o.second);
} else if (o.first == "max_open_files") {
new_options->max_open_files = ParseInt(o.second);
} else if (o.first == "max_total_wal_size") {
new_options->max_total_wal_size = ParseUint64(o.second);
} else if (o.first == "disable_data_sync") {
new_options->disableDataSync = ParseBoolean(o.first, o.second);
} else if (o.first == "use_fsync") {
new_options->use_fsync = ParseBoolean(o.first, o.second);
} else if (o.first == "db_paths") {
// TODO(ljin): add support
throw o.first;
} else if (o.first == "db_log_dir") {
new_options->db_log_dir = o.second;
} else if (o.first == "wal_dir") {
new_options->wal_dir = o.second;
} else if (o.first == "delete_obsolete_files_period_micros") {
new_options->delete_obsolete_files_period_micros =
ParseUint64(o.second);
} else if (o.first == "max_background_compactions") {
new_options->max_background_compactions = ParseInt(o.second);
} else if (o.first == "max_background_flushes") {
new_options->max_background_flushes = ParseInt(o.second);
} else if (o.first == "max_log_file_size") {
new_options->max_log_file_size = ParseInt64(o.second);
} else if (o.first == "log_file_time_to_roll") {
new_options->log_file_time_to_roll = ParseInt64(o.second);
} else if (o.first == "keep_log_file_num") {
new_options->keep_log_file_num = ParseInt64(o.second);
} else if (o.first == "max_manifest_file_size") {
new_options->max_manifest_file_size = ParseUint64(o.second);
} else if (o.first == "table_cache_numshardbits") {
new_options->table_cache_numshardbits = ParseInt(o.second);
} else if (o.first == "table_cache_remove_scan_count_limit") {
new_options->table_cache_remove_scan_count_limit = ParseInt(o.second);
} else if (o.first == "WAL_ttl_seconds") {
new_options->WAL_ttl_seconds = ParseUint64(o.second);
} else if (o.first == "WAL_size_limit_MB") {
new_options->WAL_size_limit_MB = ParseUint64(o.second);
} else if (o.first == "manifest_preallocation_size") {
new_options->manifest_preallocation_size = ParseInt64(o.second);
} else if (o.first == "allow_os_buffer") {
new_options->allow_os_buffer = ParseBoolean(o.first, o.second);
} else if (o.first == "allow_mmap_reads") {
new_options->allow_mmap_reads = ParseBoolean(o.first, o.second);
} else if (o.first == "allow_mmap_writes") {
new_options->allow_mmap_writes = ParseBoolean(o.first, o.second);
} else if (o.first == "is_fd_close_on_exec") {
new_options->is_fd_close_on_exec = ParseBoolean(o.first, o.second);
} else if (o.first == "skip_log_error_on_recovery") {
new_options->skip_log_error_on_recovery =
ParseBoolean(o.first, o.second);
} else if (o.first == "stats_dump_period_sec") {
new_options->stats_dump_period_sec = ParseUint32(o.second);
} else if (o.first == "advise_random_on_open") {
new_options->advise_random_on_open = ParseBoolean(o.first, o.second);
} else if (o.first == "use_adaptive_mutex") {
new_options->use_adaptive_mutex = ParseBoolean(o.first, o.second);
} else if (o.first == "allow_thread_local") {
new_options->allow_thread_local = ParseBoolean(o.first, o.second);
} else if (o.first == "bytes_per_sync") {
new_options->bytes_per_sync = ParseUint64(o.second);
} else {
return false;
}
} catch (std::exception) {
return false;
}
}
return true;
}
} // namespace rocksdb

View File

@ -11,6 +11,7 @@
#define __STDC_FORMAT_MACROS
#endif
#include <unordered_map>
#include <inttypes.h>
#include <gflags/gflags.h>
@ -75,6 +76,177 @@ TEST(OptionsTest, LooseCondition) {
// Both tight amplifications
PrintAndGetOptions(128 * 1024 * 1024, 4, 8);
}
TEST(OptionsTest, GetOptionsFromStringsTest) {
std::unordered_map<std::string, std::string> options_map = {
{"write_buffer_size", "1"},
{"max_write_buffer_number", "2"},
{"min_write_buffer_number_to_merge", "3"},
{"compression", "kSnappyCompression"},
{"compression_per_level", "kNoCompression:"
"kSnappyCompression:"
"kZlibCompression:"
"kBZip2Compression:"
"kLZ4Compression:"
"kLZ4HCCompression"},
{"compression_opts", "4:5:6"},
{"num_levels", "7"},
{"level0_file_num_compaction_trigger", "8"},
{"level0_slowdown_writes_trigger", "9"},
{"level0_stop_writes_trigger", "10"},
{"max_mem_compaction_level", "11"},
{"target_file_size_base", "12"},
{"target_file_size_multiplier", "13"},
{"max_bytes_for_level_base", "14"},
{"max_bytes_for_level_multiplier", "15"},
{"max_bytes_for_level_multiplier_additional", "16:17:18"},
{"expanded_compaction_factor", "19"},
{"source_compaction_factor", "20"},
{"max_grandparent_overlap_factor", "21"},
{"soft_rate_limit", "1.1"},
{"hard_rate_limit", "2.1"},
{"arena_block_size", "22"},
{"disable_auto_compactions", "true"},
{"purge_redundant_kvs_while_flush", "1"},
{"compaction_style", "kCompactionStyleLevel"},
{"verify_checksums_in_compaction", "false"},
{"compaction_options_fifo", "23"},
{"filter_deletes", "0"},
{"max_sequential_skip_in_iterations", "24"},
{"inplace_update_support", "true"},
{"inplace_update_num_locks", "25"},
{"memtable_prefix_bloom_bits", "26"},
{"memtable_prefix_bloom_probes", "27"},
{"memtable_prefix_bloom_huge_page_tlb_size", "28"},
{"bloom_locality", "29"},
{"max_successive_merges", "30"},
{"min_partial_merge_operands", "31"},
{"create_if_missing", "false"},
{"create_missing_column_families", "true"},
{"error_if_exists", "false"},
{"paranoid_checks", "true"},
{"max_open_files", "32"},
{"max_total_wal_size", "33"},
{"disable_data_sync", "false"},
{"use_fsync", "true"},
{"db_log_dir", "/db_log_dir"},
{"wal_dir", "/wal_dir"},
{"delete_obsolete_files_period_micros", "34"},
{"max_background_compactions", "35"},
{"max_background_flushes", "36"},
{"max_log_file_size", "37"},
{"log_file_time_to_roll", "38"},
{"keep_log_file_num", "39"},
{"max_manifest_file_size", "40"},
{"table_cache_numshardbits", "41"},
{"table_cache_remove_scan_count_limit", "42"},
{"WAL_ttl_seconds", "43"},
{"WAL_size_limit_MB", "44"},
{"manifest_preallocation_size", "45"},
{"allow_os_buffer", "false"},
{"allow_mmap_reads", "true"},
{"allow_mmap_writes", "false"},
{"is_fd_close_on_exec", "true"},
{"skip_log_error_on_recovery", "false"},
{"stats_dump_period_sec", "46"},
{"advise_random_on_open", "true"},
{"use_adaptive_mutex", "false"},
{"allow_thread_local", "true"},
{"bytes_per_sync", "47"},
};
Options base_opt;
Options new_opt;
ASSERT_TRUE(GetOptionsFromStrings(base_opt, options_map, &new_opt));
ASSERT_EQ(new_opt.write_buffer_size, 1);
ASSERT_EQ(new_opt.max_write_buffer_number, 2);
ASSERT_EQ(new_opt.min_write_buffer_number_to_merge, 3);
ASSERT_EQ(new_opt.compression, kSnappyCompression);
ASSERT_EQ(new_opt.compression_per_level.size(), 6);
ASSERT_EQ(new_opt.compression_per_level[0], kNoCompression);
ASSERT_EQ(new_opt.compression_per_level[1], kSnappyCompression);
ASSERT_EQ(new_opt.compression_per_level[2], kZlibCompression);
ASSERT_EQ(new_opt.compression_per_level[3], kBZip2Compression);
ASSERT_EQ(new_opt.compression_per_level[4], kLZ4Compression);
ASSERT_EQ(new_opt.compression_per_level[5], kLZ4HCCompression);
ASSERT_EQ(new_opt.compression_opts.window_bits, 4);
ASSERT_EQ(new_opt.compression_opts.level, 5);
ASSERT_EQ(new_opt.compression_opts.strategy, 6);
ASSERT_EQ(new_opt.num_levels, 7);
ASSERT_EQ(new_opt.level0_file_num_compaction_trigger, 8);
ASSERT_EQ(new_opt.level0_slowdown_writes_trigger, 9);
ASSERT_EQ(new_opt.level0_stop_writes_trigger, 10);
ASSERT_EQ(new_opt.max_mem_compaction_level, 11);
ASSERT_EQ(new_opt.target_file_size_base, 12);
ASSERT_EQ(new_opt.target_file_size_multiplier, 13);
ASSERT_EQ(new_opt.max_bytes_for_level_base, 14);
ASSERT_EQ(new_opt.max_bytes_for_level_multiplier, 15);
ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional.size(), 3);
ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional[0], 16);
ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional[1], 17);
ASSERT_EQ(new_opt.max_bytes_for_level_multiplier_additional[2], 18);
ASSERT_EQ(new_opt.expanded_compaction_factor, 19);
ASSERT_EQ(new_opt.source_compaction_factor, 20);
ASSERT_EQ(new_opt.max_grandparent_overlap_factor, 21);
ASSERT_EQ(new_opt.soft_rate_limit, 1.1);
ASSERT_EQ(new_opt.hard_rate_limit, 2.1);
ASSERT_EQ(new_opt.arena_block_size, 22);
ASSERT_EQ(new_opt.disable_auto_compactions, true);
ASSERT_EQ(new_opt.purge_redundant_kvs_while_flush, true);
ASSERT_EQ(new_opt.compaction_style, kCompactionStyleLevel);
ASSERT_EQ(new_opt.verify_checksums_in_compaction, false);
ASSERT_EQ(new_opt.compaction_options_fifo.max_table_files_size, 23);
ASSERT_EQ(new_opt.filter_deletes, false);
ASSERT_EQ(new_opt.max_sequential_skip_in_iterations, 24);
ASSERT_EQ(new_opt.inplace_update_support, true);
ASSERT_EQ(new_opt.inplace_update_num_locks, 25);
ASSERT_EQ(new_opt.memtable_prefix_bloom_bits, 26);
ASSERT_EQ(new_opt.memtable_prefix_bloom_probes, 27);
ASSERT_EQ(new_opt.memtable_prefix_bloom_huge_page_tlb_size, 28);
ASSERT_EQ(new_opt.bloom_locality, 29);
ASSERT_EQ(new_opt.max_successive_merges, 30);
ASSERT_EQ(new_opt.min_partial_merge_operands, 31);
ASSERT_EQ(new_opt.create_if_missing, false);
ASSERT_EQ(new_opt.create_missing_column_families, true);
ASSERT_EQ(new_opt.error_if_exists, false);
ASSERT_EQ(new_opt.paranoid_checks, true);
ASSERT_EQ(new_opt.max_open_files, 32);
ASSERT_EQ(new_opt.max_total_wal_size, 33);
ASSERT_EQ(new_opt.disableDataSync, false);
ASSERT_EQ(new_opt.use_fsync, true);
ASSERT_EQ(new_opt.db_log_dir, "/db_log_dir");
ASSERT_EQ(new_opt.wal_dir, "/wal_dir");
ASSERT_EQ(new_opt.delete_obsolete_files_period_micros, 34);
ASSERT_EQ(new_opt.max_background_compactions, 35);
ASSERT_EQ(new_opt.max_background_flushes, 36);
ASSERT_EQ(new_opt.max_log_file_size, 37);
ASSERT_EQ(new_opt.log_file_time_to_roll, 38);
ASSERT_EQ(new_opt.keep_log_file_num, 39);
ASSERT_EQ(new_opt.max_manifest_file_size, 40);
ASSERT_EQ(new_opt.table_cache_numshardbits, 41);
ASSERT_EQ(new_opt.table_cache_remove_scan_count_limit, 42);
ASSERT_EQ(new_opt.WAL_ttl_seconds, 43);
ASSERT_EQ(new_opt.WAL_size_limit_MB, 44);
ASSERT_EQ(new_opt.manifest_preallocation_size, 45);
ASSERT_EQ(new_opt.allow_os_buffer, false);
ASSERT_EQ(new_opt.allow_mmap_reads, true);
ASSERT_EQ(new_opt.allow_mmap_writes, false);
ASSERT_EQ(new_opt.is_fd_close_on_exec, true);
ASSERT_EQ(new_opt.skip_log_error_on_recovery, false);
ASSERT_EQ(new_opt.stats_dump_period_sec, 46);
ASSERT_EQ(new_opt.advise_random_on_open, true);
ASSERT_EQ(new_opt.use_adaptive_mutex, false);
ASSERT_EQ(new_opt.allow_thread_local, true);
ASSERT_EQ(new_opt.bytes_per_sync, 47);
options_map["write_buffer_size"] = "hello";
ASSERT_TRUE(!GetOptionsFromStrings(base_opt, options_map, &new_opt));
options_map["write_buffer_size"] = "1";
ASSERT_TRUE(GetOptionsFromStrings(base_opt, options_map, &new_opt));
options_map["unknown_option"] = "1";
ASSERT_TRUE(!GetOptionsFromStrings(base_opt, options_map, &new_opt));
}
} // namespace rocksdb
int main(int argc, char** argv) {