c815351038
Summary: For transactions, we are using the memtables to validate that there are no write conflicts. But after flushing, we don't have any memtables, and transactions could fail to commit. So we want to someone keep around some extra history to use for conflict checking. In addition, we want to provide a way to increase the size of this history if too many transactions fail to commit. After chatting with people, it seems like everyone prefers just using Memtables to store this history (instead of a separate history structure). It seems like the best place for this is abstracted inside the memtable_list. I decide to create a separate list in MemtableListVersion as using the same list complicated the flush/installalflushresults logic too much. This diff adds a new parameter to control how much memtable history to keep around after flushing. However, it sounds like people aren't too fond of adding new parameters. So I am making the default size of flushed+not-flushed memtables be set to max_write_buffers. This should not change the maximum amount of memory used, but make it more likely we're using closer the the limit. (We are now postponing deleting flushed memtables until the max_write_buffer limit is reached). So while we might use more memory on average, we are still obeying the limit set (and you could argue it's better to go ahead and use up memory now instead of waiting for a write stall to happen to test this limit). However, if people are opposed to this default behavior, we can easily set it to 0 and require this parameter be set in order to use transactions. Test Plan: Added a xfunc test to play around with setting different values of this parameter in all tests. Added testing in memtablelist_test and planning on adding more testing here. Reviewers: sdong, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D37443
725 lines
26 KiB
C++
725 lines
26 KiB
C++
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under the BSD-style license found in the
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
#include <cassert>
|
|
#include <cctype>
|
|
#include <cstdlib>
|
|
#include <unordered_set>
|
|
#include "rocksdb/cache.h"
|
|
#include "rocksdb/filter_policy.h"
|
|
#include "rocksdb/options.h"
|
|
#include "rocksdb/rate_limiter.h"
|
|
#include "rocksdb/slice_transform.h"
|
|
#include "rocksdb/table.h"
|
|
#include "rocksdb/utilities/convenience.h"
|
|
#include "table/block_based_table_factory.h"
|
|
#include "util/logging.h"
|
|
#include "util/options_helper.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
namespace {
|
|
CompressionType ParseCompressionType(const std::string& type) {
|
|
if (type == "kNoCompression") {
|
|
return kNoCompression;
|
|
} else if (type == "kSnappyCompression") {
|
|
return kSnappyCompression;
|
|
} else if (type == "kZlibCompression") {
|
|
return kZlibCompression;
|
|
} else if (type == "kBZip2Compression") {
|
|
return kBZip2Compression;
|
|
} else if (type == "kLZ4Compression") {
|
|
return kLZ4Compression;
|
|
} else if (type == "kLZ4HCCompression") {
|
|
return kLZ4HCCompression;
|
|
} else {
|
|
throw std::invalid_argument("Unknown compression type: " + type);
|
|
}
|
|
return kNoCompression;
|
|
}
|
|
|
|
BlockBasedTableOptions::IndexType ParseBlockBasedTableIndexType(
|
|
const std::string& type) {
|
|
if (type == "kBinarySearch") {
|
|
return BlockBasedTableOptions::kBinarySearch;
|
|
} else if (type == "kHashSearch") {
|
|
return BlockBasedTableOptions::kHashSearch;
|
|
}
|
|
throw std::invalid_argument("Unknown index type: " + type);
|
|
}
|
|
|
|
ChecksumType ParseBlockBasedTableChecksumType(
|
|
const std::string& type) {
|
|
if (type == "kNoChecksum") {
|
|
return kNoChecksum;
|
|
} else if (type == "kCRC32c") {
|
|
return kCRC32c;
|
|
} else if (type == "kxxHash") {
|
|
return kxxHash;
|
|
}
|
|
throw std::invalid_argument("Unknown checksum type: " + type);
|
|
}
|
|
|
|
bool ParseBoolean(const std::string& type, const std::string& value) {
|
|
if (value == "true" || value == "1") {
|
|
return true;
|
|
} else if (value == "false" || value == "0") {
|
|
return false;
|
|
}
|
|
throw std::invalid_argument(type);
|
|
}
|
|
|
|
uint64_t ParseUint64(const std::string& value) {
|
|
size_t endchar;
|
|
#ifndef CYGWIN
|
|
uint64_t num = std::stoull(value.c_str(), &endchar);
|
|
#else
|
|
char* endptr;
|
|
uint64_t num = std::strtoul(value.c_str(), &endptr, 0);
|
|
endchar = endptr - value.c_str();
|
|
#endif
|
|
|
|
if (endchar < value.length()) {
|
|
char c = value[endchar];
|
|
if (c == 'k' || c == 'K')
|
|
num <<= 10LL;
|
|
else if (c == 'm' || c == 'M')
|
|
num <<= 20LL;
|
|
else if (c == 'g' || c == 'G')
|
|
num <<= 30LL;
|
|
else if (c == 't' || c == 'T')
|
|
num <<= 40LL;
|
|
}
|
|
|
|
return num;
|
|
}
|
|
|
|
size_t ParseSizeT(const std::string& value) {
|
|
return static_cast<size_t>(ParseUint64(value));
|
|
}
|
|
|
|
uint32_t ParseUint32(const std::string& value) {
|
|
uint64_t num = ParseUint64(value);
|
|
if ((num >> 32LL) == 0) {
|
|
return static_cast<uint32_t>(num);
|
|
} else {
|
|
throw std::out_of_range(value);
|
|
}
|
|
}
|
|
|
|
int ParseInt(const std::string& value) {
|
|
size_t endchar;
|
|
#ifndef CYGWIN
|
|
int num = std::stoi(value.c_str(), &endchar);
|
|
#else
|
|
char* endptr;
|
|
int num = std::strtoul(value.c_str(), &endptr, 0);
|
|
endchar = endptr - value.c_str();
|
|
#endif
|
|
|
|
if (endchar < value.length()) {
|
|
char c = value[endchar];
|
|
if (c == 'k' || c == 'K')
|
|
num <<= 10;
|
|
else if (c == 'm' || c == 'M')
|
|
num <<= 20;
|
|
else if (c == 'g' || c == 'G')
|
|
num <<= 30;
|
|
}
|
|
|
|
return num;
|
|
}
|
|
|
|
double ParseDouble(const std::string& value) {
|
|
#ifndef CYGWIN
|
|
return std::stod(value);
|
|
#else
|
|
return std::strtod(value.c_str(), 0);
|
|
#endif
|
|
}
|
|
|
|
CompactionStyle ParseCompactionStyle(const std::string& type) {
|
|
if (type == "kCompactionStyleLevel") {
|
|
return kCompactionStyleLevel;
|
|
} else if (type == "kCompactionStyleUniversal") {
|
|
return kCompactionStyleUniversal;
|
|
} else if (type == "kCompactionStyleFIFO") {
|
|
return kCompactionStyleFIFO;
|
|
} else {
|
|
throw std::invalid_argument("unknown compaction style: " + type);
|
|
}
|
|
return kCompactionStyleLevel;
|
|
}
|
|
} // anonymouse namespace
|
|
|
|
template<typename OptionsType>
|
|
bool ParseMemtableOptions(const std::string& name, const std::string& value,
|
|
OptionsType* new_options) {
|
|
if (name == "write_buffer_size") {
|
|
new_options->write_buffer_size = ParseSizeT(value);
|
|
} else if (name == "arena_block_size") {
|
|
new_options->arena_block_size = ParseSizeT(value);
|
|
} else if (name == "memtable_prefix_bloom_bits") {
|
|
new_options->memtable_prefix_bloom_bits = ParseUint32(value);
|
|
} else if (name == "memtable_prefix_bloom_probes") {
|
|
new_options->memtable_prefix_bloom_probes = ParseUint32(value);
|
|
} else if (name == "memtable_prefix_bloom_huge_page_tlb_size") {
|
|
new_options->memtable_prefix_bloom_huge_page_tlb_size =
|
|
ParseSizeT(value);
|
|
} else if (name == "max_successive_merges") {
|
|
new_options->max_successive_merges = ParseSizeT(value);
|
|
} else if (name == "filter_deletes") {
|
|
new_options->filter_deletes = ParseBoolean(name, value);
|
|
} else if (name == "max_write_buffer_number") {
|
|
new_options->max_write_buffer_number = ParseInt(value);
|
|
} else if (name == "inplace_update_num_locks") {
|
|
new_options->inplace_update_num_locks = ParseSizeT(value);
|
|
} else {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
template<typename OptionsType>
|
|
bool ParseCompactionOptions(const std::string& name, const std::string& value,
|
|
OptionsType* new_options) {
|
|
if (name == "disable_auto_compactions") {
|
|
new_options->disable_auto_compactions = ParseBoolean(name, value);
|
|
} else if (name == "soft_rate_limit") {
|
|
new_options->soft_rate_limit = ParseDouble(value);
|
|
} else if (name == "hard_rate_limit") {
|
|
new_options->hard_rate_limit = ParseDouble(value);
|
|
} else if (name == "level0_file_num_compaction_trigger") {
|
|
new_options->level0_file_num_compaction_trigger = ParseInt(value);
|
|
} else if (name == "level0_slowdown_writes_trigger") {
|
|
new_options->level0_slowdown_writes_trigger = ParseInt(value);
|
|
} else if (name == "level0_stop_writes_trigger") {
|
|
new_options->level0_stop_writes_trigger = ParseInt(value);
|
|
} else if (name == "max_grandparent_overlap_factor") {
|
|
new_options->max_grandparent_overlap_factor = ParseInt(value);
|
|
} else if (name == "expanded_compaction_factor") {
|
|
new_options->expanded_compaction_factor = ParseInt(value);
|
|
} else if (name == "source_compaction_factor") {
|
|
new_options->source_compaction_factor = ParseInt(value);
|
|
} else if (name == "target_file_size_base") {
|
|
new_options->target_file_size_base = ParseInt(value);
|
|
} else if (name == "target_file_size_multiplier") {
|
|
new_options->target_file_size_multiplier = ParseInt(value);
|
|
} else if (name == "max_bytes_for_level_base") {
|
|
new_options->max_bytes_for_level_base = ParseUint64(value);
|
|
} else if (name == "max_bytes_for_level_multiplier") {
|
|
new_options->max_bytes_for_level_multiplier = ParseInt(value);
|
|
} else if (name == "max_bytes_for_level_multiplier_additional") {
|
|
new_options->max_bytes_for_level_multiplier_additional.clear();
|
|
size_t start = 0;
|
|
while (true) {
|
|
size_t end = value.find(':', start);
|
|
if (end == std::string::npos) {
|
|
new_options->max_bytes_for_level_multiplier_additional.push_back(
|
|
ParseInt(value.substr(start)));
|
|
break;
|
|
} else {
|
|
new_options->max_bytes_for_level_multiplier_additional.push_back(
|
|
ParseInt(value.substr(start, end - start)));
|
|
start = end + 1;
|
|
}
|
|
}
|
|
} else if (name == "max_mem_compaction_level") {
|
|
new_options->max_mem_compaction_level = ParseInt(value);
|
|
} else if (name == "verify_checksums_in_compaction") {
|
|
new_options->verify_checksums_in_compaction = ParseBoolean(name, value);
|
|
} else {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
template<typename OptionsType>
|
|
bool ParseMiscOptions(const std::string& name, const std::string& value,
|
|
OptionsType* new_options) {
|
|
if (name == "max_sequential_skip_in_iterations") {
|
|
new_options->max_sequential_skip_in_iterations = ParseUint64(value);
|
|
} else if (name == "paranoid_file_checks") {
|
|
new_options->paranoid_file_checks = ParseBoolean(name, value);
|
|
} else {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
Status GetMutableOptionsFromStrings(
|
|
const MutableCFOptions& base_options,
|
|
const std::unordered_map<std::string, std::string>& options_map,
|
|
MutableCFOptions* new_options) {
|
|
assert(new_options);
|
|
*new_options = base_options;
|
|
for (const auto& o : options_map) {
|
|
try {
|
|
if (ParseMemtableOptions(o.first, o.second, new_options)) {
|
|
} else if (ParseCompactionOptions(o.first, o.second, new_options)) {
|
|
} else if (ParseMiscOptions(o.first, o.second, new_options)) {
|
|
} else {
|
|
return Status::InvalidArgument(
|
|
"unsupported dynamic option: " + o.first);
|
|
}
|
|
} catch (std::exception& e) {
|
|
return Status::InvalidArgument("error parsing " + o.first + ":" +
|
|
std::string(e.what()));
|
|
}
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
namespace {
|
|
|
|
std::string trim(const std::string& str) {
|
|
size_t start = 0;
|
|
size_t end = str.size() - 1;
|
|
while (isspace(str[start]) != 0 && start <= end) {
|
|
++start;
|
|
}
|
|
while (isspace(str[end]) != 0 && start <= end) {
|
|
--end;
|
|
}
|
|
if (start <= end) {
|
|
return str.substr(start, end - start + 1);
|
|
}
|
|
return std::string();
|
|
}
|
|
|
|
} // anonymous namespace
|
|
|
|
Status StringToMap(const std::string& opts_str,
|
|
std::unordered_map<std::string, std::string>* opts_map) {
|
|
assert(opts_map);
|
|
// Example:
|
|
// opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
|
|
// "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
|
|
size_t pos = 0;
|
|
std::string opts = trim(opts_str);
|
|
while (pos < opts.size()) {
|
|
size_t eq_pos = opts.find('=', pos);
|
|
if (eq_pos == std::string::npos) {
|
|
return Status::InvalidArgument("Mismatched key value pair, '=' expected");
|
|
}
|
|
std::string key = trim(opts.substr(pos, eq_pos - pos));
|
|
if (key.empty()) {
|
|
return Status::InvalidArgument("Empty key found");
|
|
}
|
|
|
|
// skip space after '=' and look for '{' for possible nested options
|
|
pos = eq_pos + 1;
|
|
while (pos < opts.size() && isspace(opts[pos])) {
|
|
++pos;
|
|
}
|
|
// Empty value at the end
|
|
if (pos >= opts.size()) {
|
|
(*opts_map)[key] = "";
|
|
break;
|
|
}
|
|
if (opts[pos] == '{') {
|
|
int count = 1;
|
|
size_t brace_pos = pos + 1;
|
|
while (brace_pos < opts.size()) {
|
|
if (opts[brace_pos] == '{') {
|
|
++count;
|
|
} else if (opts[brace_pos] == '}') {
|
|
--count;
|
|
if (count == 0) {
|
|
break;
|
|
}
|
|
}
|
|
++brace_pos;
|
|
}
|
|
// found the matching closing brace
|
|
if (count == 0) {
|
|
(*opts_map)[key] = trim(opts.substr(pos + 1, brace_pos - pos - 1));
|
|
// skip all whitespace and move to the next ';'
|
|
// brace_pos points to the next position after the matching '}'
|
|
pos = brace_pos + 1;
|
|
while (pos < opts.size() && isspace(opts[pos])) {
|
|
++pos;
|
|
}
|
|
if (pos < opts.size() && opts[pos] != ';') {
|
|
return Status::InvalidArgument(
|
|
"Unexpected chars after nested options");
|
|
}
|
|
++pos;
|
|
} else {
|
|
return Status::InvalidArgument(
|
|
"Mismatched curly braces for nested options");
|
|
}
|
|
} else {
|
|
size_t sc_pos = opts.find(';', pos);
|
|
if (sc_pos == std::string::npos) {
|
|
(*opts_map)[key] = trim(opts.substr(pos));
|
|
// It either ends with a trailing semi-colon or the last key-value pair
|
|
break;
|
|
} else {
|
|
(*opts_map)[key] = trim(opts.substr(pos, sc_pos - pos));
|
|
}
|
|
pos = sc_pos + 1;
|
|
}
|
|
}
|
|
|
|
return Status::OK();
|
|
}
|
|
|
|
bool ParseColumnFamilyOption(const std::string& name, const std::string& value,
|
|
ColumnFamilyOptions* new_options) {
|
|
try {
|
|
if (ParseMemtableOptions(name, value, new_options)) {
|
|
} else if (ParseCompactionOptions(name, value, new_options)) {
|
|
} else if (ParseMiscOptions(name, value, new_options)) {
|
|
} else if (name == "block_based_table_factory") {
|
|
// Nested options
|
|
BlockBasedTableOptions table_opt, base_table_options;
|
|
auto block_based_table_factory = dynamic_cast<BlockBasedTableFactory*>(
|
|
new_options->table_factory.get());
|
|
if (block_based_table_factory != nullptr) {
|
|
base_table_options = block_based_table_factory->GetTableOptions();
|
|
}
|
|
Status table_opt_s = GetBlockBasedTableOptionsFromString(
|
|
base_table_options, value, &table_opt);
|
|
if (!table_opt_s.ok()) {
|
|
return false;
|
|
}
|
|
new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
|
|
} else if (name == "min_write_buffer_number_to_merge") {
|
|
new_options->min_write_buffer_number_to_merge = ParseInt(value);
|
|
} else if (name == "max_write_buffer_number_to_maintain") {
|
|
new_options->max_write_buffer_number_to_maintain = ParseInt(value);
|
|
} else if (name == "compression") {
|
|
new_options->compression = ParseCompressionType(value);
|
|
} else if (name == "compression_per_level") {
|
|
new_options->compression_per_level.clear();
|
|
size_t start = 0;
|
|
while (true) {
|
|
size_t end = value.find(':', start);
|
|
if (end == std::string::npos) {
|
|
new_options->compression_per_level.push_back(
|
|
ParseCompressionType(value.substr(start)));
|
|
break;
|
|
} else {
|
|
new_options->compression_per_level.push_back(
|
|
ParseCompressionType(value.substr(start, end - start)));
|
|
start = end + 1;
|
|
}
|
|
}
|
|
} else if (name == "compression_opts") {
|
|
size_t start = 0;
|
|
size_t end = value.find(':');
|
|
if (end == std::string::npos) {
|
|
return false;
|
|
}
|
|
new_options->compression_opts.window_bits =
|
|
ParseInt(value.substr(start, end - start));
|
|
start = end + 1;
|
|
end = value.find(':', start);
|
|
if (end == std::string::npos) {
|
|
return false;
|
|
}
|
|
new_options->compression_opts.level =
|
|
ParseInt(value.substr(start, end - start));
|
|
start = end + 1;
|
|
if (start >= value.size()) {
|
|
return false;
|
|
}
|
|
new_options->compression_opts.strategy =
|
|
ParseInt(value.substr(start, value.size() - start));
|
|
} else if (name == "num_levels") {
|
|
new_options->num_levels = ParseInt(value);
|
|
} else if (name == "level_compaction_dynamic_level_bytes") {
|
|
new_options->level_compaction_dynamic_level_bytes =
|
|
ParseBoolean(name, value);
|
|
} else if (name == "purge_redundant_kvs_while_flush") {
|
|
new_options->purge_redundant_kvs_while_flush =
|
|
ParseBoolean(name, value);
|
|
} else if (name == "compaction_style") {
|
|
new_options->compaction_style = ParseCompactionStyle(value);
|
|
} else if (name == "compaction_options_universal") {
|
|
// TODO(ljin): add support
|
|
return false;
|
|
} else if (name == "compaction_options_fifo") {
|
|
new_options->compaction_options_fifo.max_table_files_size =
|
|
ParseUint64(value);
|
|
} else if (name == "bloom_locality") {
|
|
new_options->bloom_locality = ParseUint32(value);
|
|
} else if (name == "min_partial_merge_operands") {
|
|
new_options->min_partial_merge_operands = ParseUint32(value);
|
|
} else if (name == "inplace_update_support") {
|
|
new_options->inplace_update_support = ParseBoolean(name, value);
|
|
} else if (name == "prefix_extractor") {
|
|
const std::string kFixedPrefixName = "fixed:";
|
|
const std::string kCappedPrefixName = "capped:";
|
|
auto& pe_value = value;
|
|
if (pe_value.size() > kFixedPrefixName.size() &&
|
|
pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
|
|
int prefix_length =
|
|
ParseInt(trim(value.substr(kFixedPrefixName.size())));
|
|
new_options->prefix_extractor.reset(
|
|
NewFixedPrefixTransform(prefix_length));
|
|
} else if (pe_value.size() > kCappedPrefixName.size() &&
|
|
pe_value.compare(0, kCappedPrefixName.size(),
|
|
kCappedPrefixName) == 0) {
|
|
int prefix_length =
|
|
ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
|
|
new_options->prefix_extractor.reset(
|
|
NewCappedPrefixTransform(prefix_length));
|
|
} else {
|
|
return false;
|
|
}
|
|
} else if (name == "optimize_filters_for_hits") {
|
|
new_options->optimize_filters_for_hits = ParseBoolean(name, value);
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
catch (std::exception& e) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool ParseDBOption(const std::string& name, const std::string& value,
|
|
DBOptions* new_options) {
|
|
try {
|
|
if (name == "create_if_missing") {
|
|
new_options->create_if_missing = ParseBoolean(name, value);
|
|
} else if (name == "create_missing_column_families") {
|
|
new_options->create_missing_column_families =
|
|
ParseBoolean(name, value);
|
|
} else if (name == "error_if_exists") {
|
|
new_options->error_if_exists = ParseBoolean(name, value);
|
|
} else if (name == "paranoid_checks") {
|
|
new_options->paranoid_checks = ParseBoolean(name, value);
|
|
} else if (name == "rate_limiter_bytes_per_sec") {
|
|
new_options->rate_limiter.reset(
|
|
NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value))));
|
|
} else if (name == "max_open_files") {
|
|
new_options->max_open_files = ParseInt(value);
|
|
} else if (name == "max_total_wal_size") {
|
|
new_options->max_total_wal_size = ParseUint64(value);
|
|
} else if (name == "disable_data_sync") {
|
|
new_options->disableDataSync = ParseBoolean(name, value);
|
|
} else if (name == "use_fsync") {
|
|
new_options->use_fsync = ParseBoolean(name, value);
|
|
} else if (name == "db_paths") {
|
|
// TODO(ljin): add support
|
|
return false;
|
|
} else if (name == "db_log_dir") {
|
|
new_options->db_log_dir = value;
|
|
} else if (name == "wal_dir") {
|
|
new_options->wal_dir = value;
|
|
} else if (name == "delete_obsolete_files_period_micros") {
|
|
new_options->delete_obsolete_files_period_micros = ParseUint64(value);
|
|
} else if (name == "max_background_compactions") {
|
|
new_options->max_background_compactions = ParseInt(value);
|
|
} else if (name == "max_background_flushes") {
|
|
new_options->max_background_flushes = ParseInt(value);
|
|
} else if (name == "max_log_file_size") {
|
|
new_options->max_log_file_size = ParseSizeT(value);
|
|
} else if (name == "log_file_time_to_roll") {
|
|
new_options->log_file_time_to_roll = ParseSizeT(value);
|
|
} else if (name == "keep_log_file_num") {
|
|
new_options->keep_log_file_num = ParseSizeT(value);
|
|
} else if (name == "max_manifest_file_size") {
|
|
new_options->max_manifest_file_size = ParseUint64(value);
|
|
} else if (name == "table_cache_numshardbits") {
|
|
new_options->table_cache_numshardbits = ParseInt(value);
|
|
} else if (name == "WAL_ttl_seconds") {
|
|
new_options->WAL_ttl_seconds = ParseUint64(value);
|
|
} else if (name == "WAL_size_limit_MB") {
|
|
new_options->WAL_size_limit_MB = ParseUint64(value);
|
|
} else if (name == "manifest_preallocation_size") {
|
|
new_options->manifest_preallocation_size = ParseSizeT(value);
|
|
} else if (name == "allow_os_buffer") {
|
|
new_options->allow_os_buffer = ParseBoolean(name, value);
|
|
} else if (name == "allow_mmap_reads") {
|
|
new_options->allow_mmap_reads = ParseBoolean(name, value);
|
|
} else if (name == "allow_mmap_writes") {
|
|
new_options->allow_mmap_writes = ParseBoolean(name, value);
|
|
} else if (name == "is_fd_close_on_exec") {
|
|
new_options->is_fd_close_on_exec = ParseBoolean(name, value);
|
|
} else if (name == "skip_log_error_on_recovery") {
|
|
new_options->skip_log_error_on_recovery = ParseBoolean(name, value);
|
|
} else if (name == "stats_dump_period_sec") {
|
|
new_options->stats_dump_period_sec = ParseUint32(value);
|
|
} else if (name == "advise_random_on_open") {
|
|
new_options->advise_random_on_open = ParseBoolean(name, value);
|
|
} else if (name == "db_write_buffer_size") {
|
|
new_options->db_write_buffer_size = ParseUint64(value);
|
|
} else if (name == "use_adaptive_mutex") {
|
|
new_options->use_adaptive_mutex = ParseBoolean(name, value);
|
|
} else if (name == "bytes_per_sync") {
|
|
new_options->bytes_per_sync = ParseUint64(value);
|
|
} else if (name == "wal_bytes_per_sync") {
|
|
new_options->wal_bytes_per_sync = ParseUint64(value);
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
catch (std::exception& e) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
Status GetBlockBasedTableOptionsFromMap(
|
|
const BlockBasedTableOptions& table_options,
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
BlockBasedTableOptions* new_table_options) {
|
|
|
|
assert(new_table_options);
|
|
*new_table_options = table_options;
|
|
for (const auto& o : opts_map) {
|
|
try {
|
|
if (o.first == "cache_index_and_filter_blocks") {
|
|
new_table_options->cache_index_and_filter_blocks =
|
|
ParseBoolean(o.first, o.second);
|
|
} else if (o.first == "index_type") {
|
|
new_table_options->index_type = ParseBlockBasedTableIndexType(o.second);
|
|
} else if (o.first == "hash_index_allow_collision") {
|
|
new_table_options->hash_index_allow_collision =
|
|
ParseBoolean(o.first, o.second);
|
|
} else if (o.first == "checksum") {
|
|
new_table_options->checksum =
|
|
ParseBlockBasedTableChecksumType(o.second);
|
|
} else if (o.first == "no_block_cache") {
|
|
new_table_options->no_block_cache = ParseBoolean(o.first, o.second);
|
|
} else if (o.first == "block_cache") {
|
|
new_table_options->block_cache = NewLRUCache(ParseSizeT(o.second));
|
|
} else if (o.first == "block_cache_compressed") {
|
|
new_table_options->block_cache_compressed =
|
|
NewLRUCache(ParseSizeT(o.second));
|
|
} else if (o.first == "block_size") {
|
|
new_table_options->block_size = ParseSizeT(o.second);
|
|
} else if (o.first == "block_size_deviation") {
|
|
new_table_options->block_size_deviation = ParseInt(o.second);
|
|
} else if (o.first == "block_restart_interval") {
|
|
new_table_options->block_restart_interval = ParseInt(o.second);
|
|
} else if (o.first == "filter_policy") {
|
|
// Expect the following format
|
|
// bloomfilter:int:bool
|
|
const std::string kName = "bloomfilter:";
|
|
if (o.second.compare(0, kName.size(), kName) != 0) {
|
|
return Status::InvalidArgument("Invalid filter policy name");
|
|
}
|
|
size_t pos = o.second.find(':', kName.size());
|
|
if (pos == std::string::npos) {
|
|
return Status::InvalidArgument("Invalid filter policy config, "
|
|
"missing bits_per_key");
|
|
}
|
|
int bits_per_key = ParseInt(
|
|
trim(o.second.substr(kName.size(), pos - kName.size())));
|
|
bool use_block_based_builder =
|
|
ParseBoolean("use_block_based_builder",
|
|
trim(o.second.substr(pos + 1)));
|
|
new_table_options->filter_policy.reset(
|
|
NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
|
|
} else if (o.first == "whole_key_filtering") {
|
|
new_table_options->whole_key_filtering =
|
|
ParseBoolean(o.first, o.second);
|
|
} else {
|
|
return Status::InvalidArgument("Unrecognized option: " + o.first);
|
|
}
|
|
} catch (std::exception& e) {
|
|
return Status::InvalidArgument("error parsing " + o.first + ":" +
|
|
std::string(e.what()));
|
|
}
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
Status GetBlockBasedTableOptionsFromString(
|
|
const BlockBasedTableOptions& table_options,
|
|
const std::string& opts_str,
|
|
BlockBasedTableOptions* new_table_options) {
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
return GetBlockBasedTableOptionsFromMap(table_options, opts_map,
|
|
new_table_options);
|
|
}
|
|
|
|
Status GetColumnFamilyOptionsFromMap(
|
|
const ColumnFamilyOptions& base_options,
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
ColumnFamilyOptions* new_options) {
|
|
assert(new_options);
|
|
*new_options = base_options;
|
|
for (const auto& o : opts_map) {
|
|
if (!ParseColumnFamilyOption(o.first, o.second, new_options)) {
|
|
return Status::InvalidArgument("Can't parse option " + o.first);
|
|
}
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
Status GetColumnFamilyOptionsFromString(
|
|
const ColumnFamilyOptions& base_options,
|
|
const std::string& opts_str,
|
|
ColumnFamilyOptions* new_options) {
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
return GetColumnFamilyOptionsFromMap(base_options, opts_map, new_options);
|
|
}
|
|
|
|
Status GetDBOptionsFromMap(
|
|
const DBOptions& base_options,
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
DBOptions* new_options) {
|
|
assert(new_options);
|
|
*new_options = base_options;
|
|
for (const auto& o : opts_map) {
|
|
if (!ParseDBOption(o.first, o.second, new_options)) {
|
|
return Status::InvalidArgument("Can't parse option " + o.first);
|
|
}
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
Status GetDBOptionsFromString(
|
|
const DBOptions& base_options,
|
|
const std::string& opts_str,
|
|
DBOptions* new_options) {
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
return GetDBOptionsFromMap(base_options, opts_map, new_options);
|
|
}
|
|
|
|
Status GetOptionsFromString(const Options& base_options,
|
|
const std::string& opts_str, Options* new_options) {
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
DBOptions new_db_options(base_options);
|
|
ColumnFamilyOptions new_cf_options(base_options);
|
|
for (const auto& o : opts_map) {
|
|
if (ParseDBOption(o.first, o.second, &new_db_options)) {
|
|
} else if (ParseColumnFamilyOption(o.first, o.second, &new_cf_options)) {
|
|
} else {
|
|
return Status::InvalidArgument("Can't parse option " + o.first);
|
|
}
|
|
}
|
|
*new_options = Options(new_db_options, new_cf_options);
|
|
return Status::OK();
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
} // namespace rocksdb
|