andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[RocksDB Options] Support more options in RocksDBOptionParser for sanity check.

Browse Source 
Summary:
RocksDBOptionsParser now supports CompressionType and the following
pointer-typed options in RocksDBOptionParser
for sanity check:
  prefix_extractor
  table_factory
  comparator
  compaction_filter
  compaction_filter_factory
  merge_operator
  memtable_factory

In the RocksDB Options file, only high level information about pointer-typed
options are serialized, and those information is only used for verification
/ sanity check purpose.

Test Plan: added more tests in options_test

Reviewers: igor, IslamAbdelRahman, sdong, anthony

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D47925
This commit is contained in:
Yueh-Hsuan Chiang 2015-10-02 15:35:32 -07:00
parent 115427ef63
commit 5c7bf56d35
6 changed files with 624 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

327
util/options_helper.cc
View File

@ -8,9 +8,13 @@
#include <cctype>
#include <cstdlib>
#include <unordered_set>
#include <vector>
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/convenience.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/options.h"
#include "rocksdb/rate_limiter.h"
#include "rocksdb/slice_transform.h"
@ -85,25 +89,89 @@ std::string UnescapeOptionString(const std::string& escaped_string) {
}
namespace {
CompressionType ParseCompressionType(const std::string& type) {
if (type == "kNoCompression") {
return kNoCompression;
} else if (type == "kSnappyCompression") {
return kSnappyCompression;
} else if (type == "kZlibCompression") {
return kZlibCompression;
} else if (type == "kBZip2Compression") {
return kBZip2Compression;
} else if (type == "kLZ4Compression") {
return kLZ4Compression;
} else if (type == "kLZ4HCCompression") {
return kLZ4HCCompression;
} else if (type == "kZSTDNotFinalCompression") {
return kZSTDNotFinalCompression;
} else {
throw std::invalid_argument("Unknown compression type: " + type);
std::string trim(const std::string& str) {
if (str.empty()) return std::string();
size_t start = 0;
size_t end = str.size() - 1;
while (isspace(str[start]) != 0 && start <= end) {
++start;
}
return kNoCompression;
while (isspace(str[end]) != 0 && start <= end) {
--end;
}
if (start <= end) {
return str.substr(start, end - start + 1);
}
return std::string();
}
bool SerializeCompressionType(const CompressionType& type, std::string* value) {
switch (type) {
case kNoCompression:
*value = "kNoCompression";
return true;
case kSnappyCompression:
*value = "kSnappyCompression";
return true;
case kZlibCompression:
*value = "kZlibCompression";
return true;
case kBZip2Compression:
*value = "kBZip2Compression";
return true;
case kLZ4Compression:
*value = "kLZ4Compression";
return true;
case kLZ4HCCompression:
*value = "kLZ4HCCompression";
return true;
case kZSTDNotFinalCompression:
*value = "kZSTDNotFinalCompression";
return true;
default:
return false;
}
}
bool SerializeVectorCompressionType(const std::vector<CompressionType>& types,
std::string* value) {
std::stringstream ss;
bool result;
for (size_t i = 0; i < types.size(); ++i) {
if (i > 0) {
ss << ':';
}
std::string string_type;
result = SerializeCompressionType(types[i], &string_type);
if (result == false) {
return result;
}
ss << string_type;
}
*value = ss.str();
return true;
}
bool ParseCompressionType(const std::string& string_value,
CompressionType* type) {
if (string_value == "kNoCompression") {
*type = kNoCompression;
} else if (string_value == "kSnappyCompression") {
*type = kSnappyCompression;
} else if (string_value == "kZlibCompression") {
*type = kZlibCompression;
} else if (string_value == "kBZip2Compression") {
*type = kBZip2Compression;
} else if (string_value == "kLZ4Compression") {
*type = kLZ4Compression;
} else if (string_value == "kLZ4HCCompression") {
*type = kLZ4HCCompression;
} else if (string_value == "kZSTDNotFinalCompression") {
*type = kZSTDNotFinalCompression;
} else {
return false;
}
return true;
}
BlockBasedTableOptions::IndexType ParseBlockBasedTableIndexType(
@ -205,7 +273,6 @@ double ParseDouble(const std::string& value) {
return std::strtod(value.c_str(), 0);
#endif
}
static const std::unordered_map<char, std::string>
compaction_style_to_string_map = {
{kCompactionStyleLevel, "kCompactionStyleLevel"},
@ -229,6 +296,83 @@ std::string CompactionStyleToString(const CompactionStyle style) {
return iter->second;
}
bool ParseVectorCompressionType(
const std::string& value,
std::vector<CompressionType>* compression_per_level) {
compression_per_level->clear();
size_t start = 0;
while (start < value.size()) {
size_t end = value.find(':', start);
bool is_ok;
CompressionType type;
if (end == std::string::npos) {
is_ok = ParseCompressionType(value.substr(start), &type);
if (!is_ok) {
return false;
}
compression_per_level->emplace_back(type);
break;
} else {
is_ok = ParseCompressionType(value.substr(start, end - start), &type);
if (!is_ok) {
return false;
}
compression_per_level->emplace_back(type);
start = end + 1;
}
}
return true;
}
bool ParseSliceTransformHelper(
const std::string& kFixedPrefixName, const std::string& kCappedPrefixName,
const std::string& value,
std::shared_ptr<const SliceTransform>* slice_transform) {
auto& pe_value = value;
if (pe_value.size() > kFixedPrefixName.size() &&
pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
int prefix_length = ParseInt(trim(value.substr(kFixedPrefixName.size())));
slice_transform->reset(NewFixedPrefixTransform(prefix_length));
} else if (pe_value.size() > kCappedPrefixName.size() &&
pe_value.compare(0, kCappedPrefixName.size(), kCappedPrefixName) ==
0) {
int prefix_length =
ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
slice_transform->reset(NewCappedPrefixTransform(prefix_length));
} else if (value == "nullptr") {
slice_transform->reset();
} else {
return false;
}
return true;
}
bool ParseSliceTransform(
const std::string& value,
std::shared_ptr<const SliceTransform>* slice_transform) {
// While we normally don't convert the string representation of a
// pointer-typed option into its instance, here we do so for backward
// compatibility as we allow this action in SetOption().
// TODO(yhchiang): A possible better place for these serialization /
// deserialization is inside the class definition of pointer-typed
// option itself, but this requires a bigger change of public API.
bool result =
ParseSliceTransformHelper("fixed:", "capped:", value, slice_transform);
if (result) {
return result;
}
result = ParseSliceTransformHelper(
"rocksdb.FixedPrefix.", "rocksdb.CappedPrefix.", value, slice_transform);
if (result) {
return result;
}
// TODO(yhchiang): we can further support other default
// SliceTransforms here.
return false;
}
bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
const std::string& value) {
switch (opt_type) {
@ -260,12 +404,24 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
*reinterpret_cast<CompactionStyle*>(opt_address) =
ParseCompactionStyle(value);
break;
case OptionType::kCompressionType:
return ParseCompressionType(
value, reinterpret_cast<CompressionType*>(opt_address));
case OptionType::kVectorCompressionType:
return ParseVectorCompressionType(
value, reinterpret_cast<std::vector<CompressionType>*>(opt_address));
case OptionType::kSliceTransform:
return ParseSliceTransform(
value, reinterpret_cast<std::shared_ptr<const SliceTransform>*>(
opt_address));
default:
return false;
}
return true;
}
} // anonymouse namespace
bool SerializeSingleOptionHelper(const char* opt_address,
const OptionType opt_type,
std::string* value) {
@ -300,13 +456,69 @@ bool SerializeSingleOptionHelper(const char* opt_address,
*value = CompactionStyleToString(
*(reinterpret_cast<const CompactionStyle*>(opt_address)));
break;
case OptionType::kCompressionType:
return SerializeCompressionType(
*(reinterpret_cast<const CompressionType*>(opt_address)), value);
case OptionType::kVectorCompressionType:
return SerializeVectorCompressionType(
*(reinterpret_cast<const std::vector<CompressionType>*>(opt_address)),
value);
break;
case OptionType::kSliceTransform: {
const auto* slice_transform_ptr =
reinterpret_cast<const std::shared_ptr<const SliceTransform>*>(
opt_address);
*value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name()
: "nullptr";
break;
}
case OptionType::kTableFactory: {
const auto* table_factory_ptr =
reinterpret_cast<const std::shared_ptr<const TableFactory>*>(
opt_address);
*value = table_factory_ptr->get() ? table_factory_ptr->get()->Name()
: "nullptr";
break;
}
case OptionType::kComparator: {
// it's a const pointer of const Comparator*
const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address);
*value = *ptr ? (*ptr)->Name() : "nullptr";
break;
}
case OptionType::kCompactionFilter: {
// it's a const pointer of const CompactionFilter*
const auto* ptr =
reinterpret_cast<const CompactionFilter* const*>(opt_address);
*value = *ptr ? (*ptr)->Name() : "nullptr";
break;
}
case OptionType::kCompactionFilterFactory: {
const auto* ptr =
reinterpret_cast<const std::shared_ptr<CompactionFilterFactory>*>(
opt_address);
*value = ptr->get() ? ptr->get()->Name() : "nullptr";
break;
}
case OptionType::kMemTableRepFactory: {
const auto* ptr =
reinterpret_cast<const std::shared_ptr<MemTableRepFactory>*>(
opt_address);
*value = ptr->get() ? ptr->get()->Name() : "nullptr";
break;
}
case OptionType::kMergeOperator: {
const auto* ptr =
reinterpret_cast<const std::shared_ptr<MergeOperator>*>(opt_address);
*value = ptr->get() ? ptr->get()->Name() : "nullptr";
break;
}
default:
return false;
}
return true;
}
} // anonymouse namespace
template<typename OptionsType>
bool ParseMemtableOptions(const std::string& name, const std::string& value,
@ -427,26 +639,6 @@ Status GetMutableOptionsFromStrings(
return Status::OK();
}
namespace {
std::string trim(const std::string& str) {
if (str.empty()) return std::string();
size_t start = 0;
size_t end = str.size() - 1;
while (isspace(str[start]) != 0 && start <= end) {
++start;
}
while (isspace(str[end]) != 0 && start <= end) {
--end;
}
if (start <= end) {
return str.substr(start, end - start + 1);
}
return std::string();
}
} // anonymous namespace
Status StringToMap(const std::string& opts_str,
std::unordered_map<std::string, std::string>* opts_map) {
assert(opts_map);
@ -559,23 +751,6 @@ bool ParseColumnFamilyOption(const std::string& name,
return false;
}
new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
} else if (name == "compression") {
new_options->compression = ParseCompressionType(value);
} else if (name == "compression_per_level") {
new_options->compression_per_level.clear();
size_t start = 0;
while (true) {
size_t end = value.find(':', start);
if (end == std::string::npos) {
new_options->compression_per_level.push_back(
ParseCompressionType(value.substr(start)));
break;
} else {
new_options->compression_per_level.push_back(
ParseCompressionType(value.substr(start, end - start)));
start = end + 1;
}
}
} else if (name == "compression_opts") {
size_t start = 0;
size_t end = value.find(':');
@ -603,26 +778,6 @@ bool ParseColumnFamilyOption(const std::string& name,
} else if (name == "compaction_options_fifo") {
new_options->compaction_options_fifo.max_table_files_size =
ParseUint64(value);
} else if (name == "prefix_extractor") {
const std::string kFixedPrefixName = "fixed:";
const std::string kCappedPrefixName = "capped:";
auto& pe_value = value;
if (pe_value.size() > kFixedPrefixName.size() &&
pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
int prefix_length =
ParseInt(trim(value.substr(kFixedPrefixName.size())));
new_options->prefix_extractor.reset(
NewFixedPrefixTransform(prefix_length));
} else if (pe_value.size() > kCappedPrefixName.size() &&
pe_value.compare(0, kCappedPrefixName.size(),
kCappedPrefixName) == 0) {
int prefix_length =
ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
new_options->prefix_extractor.reset(
NewCappedPrefixTransform(prefix_length));
} else {
return false;
}
} else {
auto iter = cf_options_type_info.find(name);
if (iter == cf_options_type_info.end()) {
@ -740,9 +895,12 @@ bool ParseDBOption(const std::string& name, const std::string& org_value,
return false;
}
const auto& opt_info = iter->second;
return ParseOptionHelper(
reinterpret_cast<char*>(new_options) + opt_info.offset, opt_info.type,
value);
if (opt_info.verification != OptionVerificationType::kByName &&
opt_info.verification != OptionVerificationType::kDeprecated) {
return ParseOptionHelper(
reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value);
}
}
} catch (const std::exception& e) {
return false;
@ -881,7 +1039,12 @@ Status GetColumnFamilyOptionsFromMap(
for (const auto& o : opts_map) {
if (!ParseColumnFamilyOption(o.first, o.second, new_options,
input_strings_escaped)) {
return Status::InvalidArgument("Can't parse option " + o.first);
auto iter = cf_options_type_info.find(o.first);
if (iter == cf_options_type_info.end() ||
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification != OptionVerificationType::kDeprecated)) {
return Status::InvalidArgument("Can't parse option " + o.first);
}
}
}
return Status::OK();
@ -907,6 +1070,8 @@ Status GetDBOptionsFromMap(
*new_options = base_options;
for (const auto& o : opts_map) {
if (!ParseDBOption(o.first, o.second, new_options, input_strings_escaped)) {
// Note that options with kDeprecated validation will pass ParseDBOption
// and will not hit the below statement.
return Status::InvalidArgument("Can't parse option " + o.first);
}
}

52
util/options_helper.h
View File

@ -65,11 +65,22 @@ enum class OptionType {
kString,
kDouble,
kCompactionStyle,
kSliceTransform,
kCompressionType,
kVectorCompressionType,
kTableFactory,
kComparator,
kCompactionFilter,
kCompactionFilterFactory,
kMergeOperator,
kMemTableRepFactory,
kUnknown
};
enum class OptionVerificationType {
kNormal,
kByName, // The option is pointer typed so we can only verify
// based on it's name.
kDeprecated // The option is no longer used in rocksdb. The RocksDB
// OptionsParser will still accept this option if it
// happen to exists in some Options file. However, the
@ -85,6 +96,11 @@ struct OptionTypeInfo {
OptionVerificationType verification;
};
// A helper function that converts "opt_address" to a std::string
// based on the specified OptionType.
bool SerializeSingleOptionHelper(const char* opt_address,
const OptionType opt_type, std::string* value);
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
/*
// not yet supported
@ -226,7 +242,6 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
CompactionOptionsFIFO compaction_options_fifo;
CompactionOptionsUniversal compaction_options_universal;
CompressionOptions compression_opts;
CompressionType compression;
TablePropertiesCollectorFactories table_properties_collector_factories;
typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
TablePropertiesCollectorFactories;
@ -234,14 +249,6 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
uint34_t* existing_value_size,
Slice delta_value,
std::string* merged_value);
const CompactionFilter* compaction_filter;
const Comparator* comparator;
std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
std::shared_ptr<MemTableRepFactory> memtable_factory;
std::shared_ptr<MergeOperator> merge_operator;
std::shared_ptr<TableFactory> table_factory;
std::shared_ptr<const SliceTransform> prefix_extractor;
std::vector<CompressionType> compression_per_level;
std::vector<int> max_bytes_for_level_multiplier_additional;
*/
{"compaction_measure_io_stats",
@ -360,6 +367,33 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{"rate_limit_delay_max_milliseconds",
{offsetof(struct ColumnFamilyOptions, rate_limit_delay_max_milliseconds),
OptionType::kUInt, OptionVerificationType::kDeprecated}},
{"compression",
{offsetof(struct ColumnFamilyOptions, compression),
OptionType::kCompressionType, OptionVerificationType::kNormal}},
{"compression_per_level",
{offsetof(struct ColumnFamilyOptions, compression_per_level),
OptionType::kVectorCompressionType, OptionVerificationType::kNormal}},
{"comparator",
{offsetof(struct ColumnFamilyOptions, comparator), OptionType::kComparator,
OptionVerificationType::kByName}},
{"prefix_extractor",
{offsetof(struct ColumnFamilyOptions, prefix_extractor),
OptionType::kSliceTransform, OptionVerificationType::kByName}},
{"memtable_factory",
{offsetof(struct ColumnFamilyOptions, memtable_factory),
OptionType::kMemTableRepFactory, OptionVerificationType::kByName}},
{"table_factory",
{offsetof(struct ColumnFamilyOptions, table_factory),
OptionType::kTableFactory, OptionVerificationType::kByName}},
{"compaction_filter",
{offsetof(struct ColumnFamilyOptions, compaction_filter),
OptionType::kCompactionFilter, OptionVerificationType::kByName}},
{"compaction_filter_factory",
{offsetof(struct ColumnFamilyOptions, compaction_filter_factory),
OptionType::kCompactionFilterFactory, OptionVerificationType::kByName}},
{"merge_operator",
{offsetof(struct ColumnFamilyOptions, merge_operator),
OptionType::kMergeOperator, OptionVerificationType::kByName}},
{"compaction_style",
{offsetof(struct ColumnFamilyOptions, compaction_style),
OptionType::kCompactionStyle, OptionVerificationType::kNormal}}};

59
util/options_parser.cc
View File

@ -88,8 +88,10 @@ RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }
void RocksDBOptionsParser::Reset() {
db_opt_ = DBOptions();
db_opt_map_.clear();
cf_names_.clear();
cf_opts_.clear();
cf_opt_maps_.clear();
has_version_section_ = false;
has_db_options_ = false;
has_default_cf_options_ = false;
@ -356,6 +358,7 @@ Status RocksDBOptionsParser::EndSection(
if (!s.ok()) {
return s;
}
db_opt_map_ = opt_map;
} else if (section == kOptionSectionCFOptions) {
// This condition should be ensured earlier in ParseSection
// so we make an assertion here.
@ -367,6 +370,8 @@ Status RocksDBOptionsParser::EndSection(
if (!s.ok()) {
return s;
}
// keep the parsed string.
cf_opt_maps_.emplace_back(opt_map);
} else if (section == kOptionSectionVersion) {
for (const auto pair : opt_map) {
if (pair.first == "rocksdb_version") {
@ -444,8 +449,10 @@ bool AreEqualDoubles(const double a, const double b) {
return (fabs(a - b) < 0.00001);
}
bool AreEqualOptions(const char* opt1, const char* opt2,
const OptionTypeInfo& type_info) {
bool AreEqualOptions(
const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
const std::string& opt_name,
const std::unordered_map<std::string, std::string>* opt_map) {
const char* offset1 = opt1 + type_info.offset;
const char* offset2 = opt2 + type_info.offset;
switch (type_info.type) {
@ -476,7 +483,34 @@ bool AreEqualOptions(const char* opt1, const char* opt2,
case OptionType::kCompactionStyle:
return (*reinterpret_cast<const CompactionStyle*>(offset1) ==
*reinterpret_cast<const CompactionStyle*>(offset2));
case OptionType::kCompressionType:
return (*reinterpret_cast<const CompressionType*>(offset1) ==
*reinterpret_cast<const CompressionType*>(offset2));
case OptionType::kVectorCompressionType: {
const auto* vec1 =
reinterpret_cast<const std::vector<CompressionType>*>(offset1);
const auto* vec2 =
reinterpret_cast<const std::vector<CompressionType>*>(offset2);
return (*vec1 == *vec2);
}
default:
if (type_info.verification == OptionVerificationType::kByName) {
std::string value1;
bool result =
SerializeSingleOptionHelper(offset1, type_info.type, &value1);
if (result == false) {
return false;
}
if (opt_map == nullptr) {
return true;
}
auto iter = opt_map->find(opt_name);
if (iter == opt_map->end()) {
return true;
} else {
return (value1 == iter->second);
}
}
return false;
}
}
@ -495,7 +529,7 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
}
// Verify DBOptions
s = VerifyDBOptions(db_opt, *parser.db_opt());
s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map());
if (!s.ok()) {
return s;
}
@ -522,7 +556,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
"the same number of column families as the db instance.");
}
for (size_t i = 0; i < cf_opts.size(); ++i) {
s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i));
s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i),
&(parser.cf_opt_maps()->at(i)));
if (!s.ok()) {
return s;
}
@ -531,8 +566,9 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
return Status::OK();
}
Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
const DBOptions& new_opt) {
Status RocksDBOptionsParser::VerifyDBOptions(
const DBOptions& base_opt, const DBOptions& new_opt,
const std::unordered_map<std::string, std::string>* opt_map) {
for (auto pair : db_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
@ -540,8 +576,8 @@ Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
continue;
}
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&new_opt),
pair.second)) {
reinterpret_cast<const char*>(&new_opt), pair.second,
pair.first, nullptr)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on DBOptions::",
@ -552,7 +588,8 @@ Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
}
Status RocksDBOptionsParser::VerifyCFOptions(
const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt) {
const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
const std::unordered_map<std::string, std::string>* new_opt_map) {
for (auto& pair : cf_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
@ -560,8 +597,8 @@ Status RocksDBOptionsParser::VerifyCFOptions(
continue;
}
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&new_opt),
pair.second)) {
reinterpret_cast<const char*>(&new_opt), pair.second,
pair.first, new_opt_map)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on ColumnFamilyOptions::",

21
util/options_parser.h
View File

@ -45,8 +45,15 @@ class RocksDBOptionsParser {
const bool trim_only = false);
const DBOptions* db_opt() const { return &db_opt_; }
const std::unordered_map<std::string, std::string>* db_opt_map() const {
return &db_opt_map_;
}
const std::vector<ColumnFamilyOptions>* cf_opts() const { return &cf_opts_; }
const std::vector<std::string>* cf_names() const { return &cf_names_; }
const std::vector<std::unordered_map<std::string, std::string>>* cf_opt_maps()
const {
return &cf_opt_maps_;
}
const ColumnFamilyOptions* GetCFOptions(const std::string& name) const {
assert(cf_names_.size() == cf_opts_.size());
@ -64,11 +71,15 @@ class RocksDBOptionsParser {
const std::vector<ColumnFamilyOptions>& cf_opts,
const std::string& file_name, Env* env);
static Status VerifyDBOptions(const DBOptions& base_opt,
const DBOptions& new_opt);
static Status VerifyDBOptions(
const DBOptions& base_opt, const DBOptions& new_opt,
const std::unordered_map<std::string, std::string>* new_opt_map =
nullptr);
static Status VerifyCFOptions(const ColumnFamilyOptions& base_opt,
const ColumnFamilyOptions& new_opt);
static Status VerifyCFOptions(
const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
const std::unordered_map<std::string, std::string>* new_opt_map =
nullptr);
static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
@ -97,8 +108,10 @@ class RocksDBOptionsParser {
private:
DBOptions db_opt_;
std::unordered_map<std::string, std::string> db_opt_map_;
std::vector<std::string> cf_names_;
std::vector<ColumnFamilyOptions> cf_opts_;
std::vector<std::unordered_map<std::string, std::string>> cf_opt_maps_;
bool has_version_section_;
bool has_db_options_;
bool has_default_cf_options_;

264
util/options_test.cc
View File

@ -16,7 +16,9 @@
#include <inttypes.h>
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/convenience.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/options.h"
#include "rocksdb/table.h"
#include "rocksdb/utilities/leveldb_options.h"
@ -748,7 +750,132 @@ TEST_F(OptionsTest, DBOptionsSerialization) {
}
namespace {
CompressionType RandomCompressionType(Random* rnd) {
return static_cast<CompressionType>(rnd->Uniform(6));
}
void RandomCompressionTypeVector(const size_t count,
std::vector<CompressionType>* types,
Random* rnd) {
types->clear();
for (size_t i = 0; i < count; ++i) {
types->emplace_back(RandomCompressionType(rnd));
}
}
const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined = -1) {
int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
switch (random_num) {
case 0:
return NewFixedPrefixTransform(rnd->Uniform(20) + 1);
case 1:
return NewCappedPrefixTransform(rnd->Uniform(20) + 1);
case 2:
return NewNoopTransform();
default:
return nullptr;
}
}
TableFactory* RandomTableFactory(Random* rnd, int pre_defined = -1) {
int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(3);
switch (random_num) {
case 0:
return NewPlainTableFactory();
case 1:
return NewCuckooTableFactory();
default:
return NewBlockBasedTableFactory();
}
}
std::string RandomString(Random* rnd, const size_t len) {
std::stringstream ss;
for (size_t i = 0; i < len; ++i) {
ss << static_cast<char>(rnd->Uniform(26) + 'a');
}
return ss.str();
}
class ChanglingMergeOperator : public MergeOperator {
public:
explicit ChanglingMergeOperator(const std::string& name)
: name_(name + "MergeOperator") {}
~ChanglingMergeOperator() {}
void SetName(const std::string& name) { name_ = name; }
virtual bool FullMerge(const Slice& key, const Slice* existing_value,
const std::deque<std::string>& operand_list,
std::string* new_value,
Logger* logger) const override {
return false;
}
virtual bool PartialMergeMulti(const Slice& key,
const std::deque<Slice>& operand_list,
std::string* new_value,
Logger* logger) const override {
return false;
}
virtual const char* Name() const override { return name_.c_str(); }
protected:
std::string name_;
};
MergeOperator* RandomMergeOperator(Random* rnd) {
return new ChanglingMergeOperator(RandomString(rnd, 10));
}
class ChanglingCompactionFilter : public CompactionFilter {
public:
explicit ChanglingCompactionFilter(const std::string& name)
: name_(name + "CompactionFilter") {}
~ChanglingCompactionFilter() {}
void SetName(const std::string& name) { name_ = name; }
bool Filter(int level, const Slice& key, const Slice& existing_value,
std::string* new_value, bool* value_changed) const override {
return false;
}
const char* Name() const override { return name_.c_str(); }
private:
std::string name_;
};
CompactionFilter* RandomCompactionFilter(Random* rnd) {
return new ChanglingCompactionFilter(RandomString(rnd, 10));
}
class ChanglingCompactionFilterFactory : public CompactionFilterFactory {
public:
explicit ChanglingCompactionFilterFactory(const std::string& name)
: name_(name + "CompactionFilterFactory") {}
~ChanglingCompactionFilterFactory() {}
void SetName(const std::string& name) { name_ = name; }
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
const CompactionFilter::Context& context) override {
return std::unique_ptr<CompactionFilter>();
}
// Returns a name that identifies this compaction filter factory.
const char* Name() const override { return name_.c_str(); }
protected:
std::string name_;
};
CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) {
return new ChanglingCompactionFilterFactory(RandomString(rnd, 10));
}
// Note that the caller is responsible for releasing non-null
// cf_opt->compaction_filter.
void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4));
@ -803,6 +930,21 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// unsigned int options
cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000);
// pointer typed options
cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd));
cf_opt->table_factory.reset(RandomTableFactory(rnd));
cf_opt->merge_operator.reset(RandomMergeOperator(rnd));
if (cf_opt->compaction_filter) {
delete cf_opt->compaction_filter;
}
cf_opt->compaction_filter = RandomCompactionFilter(rnd);
cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd));
// custom typed options
cf_opt->compression = RandomCompressionType(rnd);
RandomCompressionTypeVector(cf_opt->num_levels,
&cf_opt->compression_per_level, rnd);
}
} // namespace
@ -824,6 +966,9 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
ASSERT_OK(GetColumnFamilyOptionsFromString(
ColumnFamilyOptions(), base_options_file_content, &new_opt));
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt));
if (base_opt.compaction_filter) {
delete base_opt.compaction_filter;
}
}
#endif // !ROCKSDB_LITE
@ -1268,12 +1413,101 @@ TEST_F(OptionsParserTest, ParseVersion) {
}
}
void VerifyCFPointerTypedOptions(
ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
const std::unordered_map<std::string, std::string>* new_cf_opt_map) {
std::string name_buffer;
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
new_cf_opt_map));
// change the name of merge operator back-and-forth
{
auto* merge_operator = dynamic_cast<ChanglingMergeOperator*>(
base_cf_opt->merge_operator.get());
if (merge_operator != nullptr) {
name_buffer = merge_operator->Name();
// change the name and expect non-ok status
merge_operator->SetName("some-other-name");
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
// change the name back and expect ok status
merge_operator->SetName(name_buffer);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
new_cf_opt_map));
}
}
// change the name of the compaction filter factory back-and-forth
{
auto* compaction_filter_factory =
dynamic_cast<ChanglingCompactionFilterFactory*>(
base_cf_opt->compaction_filter_factory.get());
if (compaction_filter_factory != nullptr) {
name_buffer = compaction_filter_factory->Name();
// change the name and expect non-ok status
compaction_filter_factory->SetName("some-other-name");
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
// change the name back and expect ok status
compaction_filter_factory->SetName(name_buffer);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
new_cf_opt_map));
}
}
// test by setting compaction_filter to nullptr
{
auto* tmp_compaction_filter = base_cf_opt->compaction_filter;
if (tmp_compaction_filter != nullptr) {
base_cf_opt->compaction_filter = nullptr;
// set compaction_filter to nullptr and expect non-ok status
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
// set the value back and expect ok status
base_cf_opt->compaction_filter = tmp_compaction_filter;
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
new_cf_opt_map));
}
}
// test by setting table_factory to nullptr
{
auto tmp_table_factory = base_cf_opt->table_factory;
if (tmp_table_factory != nullptr) {
base_cf_opt->table_factory.reset();
// set table_factory to nullptr and expect non-ok status
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
// set the value back and expect ok status
base_cf_opt->table_factory = tmp_table_factory;
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
new_cf_opt_map));
}
}
// test by setting memtable_factory to nullptr
{
auto tmp_memtable_factory = base_cf_opt->memtable_factory;
if (tmp_memtable_factory != nullptr) {
base_cf_opt->memtable_factory.reset();
// set memtable_factory to nullptr and expect non-ok status
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
// set the value back and expect ok status
base_cf_opt->memtable_factory = tmp_memtable_factory;
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
new_cf_opt_map));
}
}
}
TEST_F(OptionsParserTest, DumpAndParse) {
DBOptions base_db_opt;
std::vector<ColumnFamilyOptions> base_cf_opts;
std::vector<std::string> cf_names = {
// special characters are also included.
"default", "p\\i\\k\\a\\chu\\\\\\", "###rocksdb#1-testcf#2###"};
std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3",
"c:f:4:4:4"
"p\\i\\k\\a\\chu\\\\\\",
"###rocksdb#1-testcf#2###"};
const int num_cf = static_cast<int>(cf_names.size());
Random rnd(302);
RandomInitDBOptions(&base_db_opt, &rnd);
@ -1282,6 +1516,12 @@ TEST_F(OptionsParserTest, DumpAndParse) {
ColumnFamilyOptions cf_opt;
Random cf_rnd(0xFB + c);
RandomInitCFOptions(&cf_opt, &cf_rnd);
if (c < 4) {
cf_opt.prefix_extractor.reset(RandomSliceTransform(&rnd, c));
}
if (c < 3) {
cf_opt.table_factory.reset(RandomTableFactory(&rnd, c));
}
base_cf_opts.emplace_back(cf_opt);
}
@ -1300,13 +1540,29 @@ TEST_F(OptionsParserTest, DumpAndParse) {
for (int c = 0; c < num_cf; ++c) {
const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
ASSERT_NE(cf_opt, nullptr);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*cf_opt, base_cf_opts[c]));
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c))));
}
// Further verify pointer-typed options
for (int c = 0; c < num_cf; ++c) {
const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
ASSERT_NE(cf_opt, nullptr);
VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt,
&(parser.cf_opt_maps()->at(c)));
}
ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr);
base_db_opt.max_open_files++;
ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get()));
for (int c = 0; c < num_cf; ++c) {
if (base_cf_opts[c].compaction_filter) {
delete base_cf_opts[c].compaction_filter;
}
}
}
namespace {

10
util/slice.cc
View File

@ -25,6 +25,11 @@ class FixedPrefixTransform : public SliceTransform {
public:
explicit FixedPrefixTransform(size_t prefix_len)
: prefix_len_(prefix_len),
// Note that if any part of the name format changes, it will require
// changes on options_helper in order to make RocksDBOptionsParser work
// for the new change.
// TODO(yhchiang): move serialization / deserializaion code inside
// the class implementation itself.
name_("rocksdb.FixedPrefix." + ToString(prefix_len_)) {}
virtual const char* Name() const override { return name_.c_str(); }
@ -55,6 +60,11 @@ class CappedPrefixTransform : public SliceTransform {
public:
explicit CappedPrefixTransform(size_t cap_len)
: cap_len_(cap_len),
// Note that if any part of the name format changes, it will require
// changes on options_helper in order to make RocksDBOptionsParser work
// for the new change.
// TODO(yhchiang): move serialization / deserializaion code inside
// the class implementation itself.
name_("rocksdb.CappedPrefix." + ToString(cap_len_)) {}
virtual const char* Name() const override { return name_.c_str(); }