diff --git a/cache/cache.cc b/cache/cache.cc index f4a480533..f4e49c481 100644 --- a/cache/cache.cc +++ b/cache/cache.cc @@ -14,7 +14,30 @@ #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { -Status Cache::CreateFromString(const ConfigOptions& /*opts*/, +#ifndef ROCKSDB_LITE +static std::unordered_map + lru_cache_options_type_info = { + {"capacity", + {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct LRUCacheOptions, capacity)}}, + {"num_shard_bits", + {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct LRUCacheOptions, num_shard_bits)}}, + {"strict_capacity_limit", + {offsetof(struct LRUCacheOptions, strict_capacity_limit), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, + {"high_pri_pool_ratio", + {offsetof(struct LRUCacheOptions, high_pri_pool_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; +#endif // ROCKSDB_LITE + +Status Cache::CreateFromString(const ConfigOptions& config_options, const std::string& value, std::shared_ptr* result) { Status status; @@ -24,12 +47,14 @@ Status Cache::CreateFromString(const ConfigOptions& /*opts*/, } else { #ifndef ROCKSDB_LITE LRUCacheOptions cache_opts; - if (!ParseOptionHelper(reinterpret_cast(&cache_opts), - OptionType::kLRUCacheOptions, value)) { - status = Status::InvalidArgument("Invalid cache options"); + status = OptionTypeInfo::ParseStruct( + config_options, "", &lru_cache_options_type_info, "", value, + reinterpret_cast(&cache_opts)); + if (status.ok()) { + cache = NewLRUCache(cache_opts); } - cache = NewLRUCache(cache_opts); #else + (void)config_options; status = Status::NotSupported("Cannot load cache in LITE mode ", value); #endif //! ROCKSDB_LITE } diff --git a/options/cf_options.cc b/options/cf_options.cc index a3352c07b..c27c3f960 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -114,6 +114,63 @@ static Status ParseCompressionOptions(const std::string& value, const std::string kOptNameBMCompOpts = "bottommost_compression_opts"; const std::string kOptNameCompOpts = "compression_opts"; +static std::unordered_map + fifo_compaction_options_type_info = { + {"max_table_files_size", + {offsetof(struct CompactionOptionsFIFO, max_table_files_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, + {"ttl", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"allow_compaction", + {offsetof(struct CompactionOptionsFIFO, allow_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; + +static std::unordered_map + universal_compaction_options_type_info = { + {"size_ratio", + {offsetof(class CompactionOptionsUniversal, size_ratio), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, size_ratio)}}, + {"min_merge_width", + {offsetof(class CompactionOptionsUniversal, min_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, min_merge_width)}}, + {"max_merge_width", + {offsetof(class CompactionOptionsUniversal, max_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, max_merge_width)}}, + {"max_size_amplification_percent", + {offsetof(class CompactionOptionsUniversal, + max_size_amplification_percent), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, + max_size_amplification_percent)}}, + {"compression_size_percent", + {offsetof(class CompactionOptionsUniversal, compression_size_percent), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, + compression_size_percent)}}, + {"stop_style", + {offsetof(class CompactionOptionsUniversal, stop_style), + OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, stop_style)}}, + {"allow_trivial_move", + {offsetof(class CompactionOptionsUniversal, allow_trivial_move), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}}; + std::unordered_map OptionsHelper::cf_options_type_info = { /* not yet supported @@ -473,15 +530,36 @@ std::unordered_map OptionType::kCompactionPri, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, {"compaction_options_fifo", - {offset_of(&ColumnFamilyOptions::compaction_options_fifo), - OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, compaction_options_fifo)}}, + OptionTypeInfo::Struct( + "compaction_options_fifo", &fifo_compaction_options_type_info, + offset_of(&ColumnFamilyOptions::compaction_options_fifo), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compaction_options_fifo), + [](const ConfigOptions& opts, const std::string& name, + const std::string& value, char* addr) { + // This is to handle backward compatibility, where + // compaction_options_fifo could be assigned a single scalar + // value, say, like "23", which would be assigned to + // max_table_files_size. + if (name == "compaction_options_fifo" && + value.find("=") == std::string::npos) { + // Old format. Parse just a single uint64_t value. + auto options = reinterpret_cast(addr); + options->max_table_files_size = ParseUint64(value); + return Status::OK(); + } else { + return OptionTypeInfo::ParseStruct( + opts, "compaction_options_fifo", + &fifo_compaction_options_type_info, name, value, addr); + } + })}, {"compaction_options_universal", - {offset_of(&ColumnFamilyOptions::compaction_options_universal), - OptionType::kCompactionOptionsUniversal, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, compaction_options_universal)}}, + OptionTypeInfo::Struct( + "compaction_options_universal", + &universal_compaction_options_type_info, + offset_of(&ColumnFamilyOptions::compaction_options_universal), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compaction_options_universal))}, {"ttl", {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T, OptionVerificationType::kNormal, OptionTypeFlags::kMutable, @@ -534,14 +612,16 @@ Status ParseColumnFamilyOption(const ConfigOptions& config_options, ? UnescapeOptionString(org_value) : org_value; try { - auto iter = cf_options_type_info.find(name); - if (iter == cf_options_type_info.end()) { + std::string elem; + const auto opt_info = + OptionTypeInfo::Find(name, cf_options_type_info, &elem); + if (opt_info != nullptr) { + return opt_info->Parse( + config_options, elem, value, + reinterpret_cast(new_options) + opt_info->offset_); + } else { return Status::InvalidArgument( "Unable to parse the specified CF option " + name); - } else { - return iter->second.ParseOption( - config_options, name, value, - reinterpret_cast(new_options) + iter->second.offset); } } catch (const std::exception&) { return Status::InvalidArgument("unable to parse the specified option " + diff --git a/options/options_helper.cc b/options/options_helper.cc index cecfd4d2b..2f8ce4c55 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -322,80 +322,6 @@ bool ParseVectorCompressionType( } return true; } - -// This is to handle backward compatibility, where compaction_options_fifo -// could be assigned a single scalar value, say, like "23", which would be -// assigned to max_table_files_size. -bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str, - CompactionOptionsFIFO* options) { - if (opt_str.find("=") != std::string::npos) { - // New format. Go do your new parsing using ParseStructOptions. - return false; - } - - // Old format. Parse just a single uint64_t value. - options->max_table_files_size = ParseUint64(opt_str); - return true; -} - -static bool SerializeStruct( - const void* const opt_ptr, std::string* value, - const std::unordered_map& type_info_map) { - ConfigOptions config_options; - config_options.delimiter = ";"; - - std::string opt_str; - Status s = - GetStringFromStruct(config_options, opt_ptr, type_info_map, &opt_str); - if (!s.ok()) { - return false; - } - *value = "{" + opt_str + "}"; - return true; -} - -static bool ParseSingleStructOption( - const ConfigOptions& config_options, const std::string& opt_val_str, - void* options, - const std::unordered_map& type_info_map) { - size_t end = opt_val_str.find('='); - std::string key = opt_val_str.substr(0, end); - std::string value = opt_val_str.substr(end + 1); - auto iter = type_info_map.find(key); - if (iter == type_info_map.end()) { - return false; - } - const auto& opt_info = iter->second; - Status s = opt_info.ParseOption( - config_options, key, value, - reinterpret_cast(options) + opt_info.mutable_offset); - return s.ok(); -} - -static bool ParseStructOptions( - const std::string& opt_str, void* options, - const std::unordered_map& type_info_map) { - assert(!opt_str.empty()); - ConfigOptions config_options; - - size_t start = 0; - if (opt_str[0] == '{') { - start++; - } - while ((start != std::string::npos) && (start < opt_str.size())) { - if (opt_str[start] == '}') { - break; - } - size_t end = opt_str.find(';', start); - size_t len = (end == std::string::npos) ? end : end - start; - if (!ParseSingleStructOption(config_options, opt_str.substr(start, len), - options, type_info_map)) { - return false; - } - start = (end == std::string::npos) ? end : end + 1; - } - return true; -} } // anonymouse namespace bool ParseSliceTransformHelper( @@ -516,21 +442,6 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, return ParseEnum( encoding_type_string_map, value, reinterpret_cast(opt_address)); - case OptionType::kCompactionOptionsFIFO: { - if (!FIFOCompactionOptionsSpecialCase( - value, reinterpret_cast(opt_address))) { - return ParseStructOptions(value, opt_address, - fifo_compaction_options_type_info); - } - return true; - } - case OptionType::kLRUCacheOptions: { - return ParseStructOptions(value, opt_address, - lru_cache_options_type_info); - } - case OptionType::kCompactionOptionsUniversal: - return ParseStructOptions(value, opt_address, - universal_compaction_options_type_info); case OptionType::kCompactionStopStyle: return ParseEnum( compaction_stop_style_string_map, value, @@ -691,12 +602,6 @@ bool SerializeSingleOptionHelper(const char* opt_address, return SerializeEnum( encoding_type_string_map, *reinterpret_cast(opt_address), value); - case OptionType::kCompactionOptionsFIFO: - return SerializeStruct(opt_address, value, - fifo_compaction_options_type_info); - case OptionType::kCompactionOptionsUniversal: - return SerializeStruct(opt_address, value, - universal_compaction_options_type_info); case OptionType::kCompactionStopStyle: return SerializeEnum( compaction_stop_style_string_map, @@ -715,26 +620,25 @@ Status GetMutableOptionsFromStrings( *new_options = base_options; ConfigOptions config_options; for (const auto& o : options_map) { - auto iter = cf_options_type_info.find(o.first); - if (iter == cf_options_type_info.end()) { + std::string elem; + const auto opt_info = + OptionTypeInfo::Find(o.first, cf_options_type_info, &elem); + if (opt_info == nullptr) { return Status::InvalidArgument("Unrecognized option: " + o.first); - } - const auto& opt_info = iter->second; - if (!opt_info.IsMutable()) { + } else if (!opt_info->IsMutable()) { return Status::InvalidArgument("Option not changeable: " + o.first); - } - if (opt_info.IsDeprecated()) { + } else if (opt_info->IsDeprecated()) { // log warning when user tries to set a deprecated option but don't fail // the call for compatibility. ROCKS_LOG_WARN(info_log, "%s is a deprecated option and cannot be set", o.first.c_str()); - continue; - } - Status s = opt_info.ParseOption( - config_options, o.first, o.second, - reinterpret_cast(new_options) + opt_info.mutable_offset); - if (!s.ok()) { - return s; + } else { + Status s = opt_info->Parse( + config_options, elem, o.second, + reinterpret_cast(new_options) + opt_info->mutable_offset_); + if (!s.ok()) { + return s; + } } } return Status::OK(); @@ -750,19 +654,20 @@ Status GetMutableDBOptionsFromStrings( for (const auto& o : options_map) { try { - auto iter = db_options_type_info.find(o.first); - if (iter == db_options_type_info.end()) { + std::string elem; + const auto opt_info = + OptionTypeInfo::Find(o.first, db_options_type_info, &elem); + if (opt_info == nullptr) { return Status::InvalidArgument("Unrecognized option: " + o.first); - } - const auto& opt_info = iter->second; - if (!opt_info.IsMutable()) { + } else if (!opt_info->IsMutable()) { return Status::InvalidArgument("Option not changeable: " + o.first); - } - Status s = opt_info.ParseOption( - config_options, o.first, o.second, - reinterpret_cast(new_options) + opt_info.mutable_offset); - if (!s.ok()) { - return s; + } else { + Status s = opt_info->Parse( + config_options, elem, o.second, + reinterpret_cast(new_options) + opt_info->mutable_offset_); + if (!s.ok()) { + return s; + } } } catch (std::exception& e) { return Status::InvalidArgument("Error parsing " + o.first + ":" + @@ -780,6 +685,11 @@ Status StringToMap(const std::string& opts_str, // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100" size_t pos = 0; std::string opts = trim(opts_str); + // If the input string starts and ends with "{...}", strip off the brackets + while (opts.size() > 2 && opts[0] == '{' && opts[opts.size() - 1] == '}') { + opts = trim(opts.substr(1, opts.size() - 2)); + } + while (pos < opts.size()) { size_t eq_pos = opts.find('=', pos); if (eq_pos == std::string::npos) { @@ -860,10 +770,10 @@ Status GetStringFromStruct( // we skip it in the serialization. if (opt_info.ShouldSerialize()) { const char* opt_addr = - reinterpret_cast(opt_ptr) + opt_info.offset; + reinterpret_cast(opt_ptr) + opt_info.offset_; std::string value; - Status s = opt_info.SerializeOption(config_options, iter.first, opt_addr, - &value); + Status s = + opt_info.Serialize(config_options, iter.first, opt_addr, &value); if (s.ok()) { opt_string->append(iter.first + "=" + value + config_options.delimiter); } else { @@ -923,13 +833,14 @@ static Status ParseDBOption(const ConfigOptions& config_options, const std::string& value = config_options.input_strings_escaped ? UnescapeOptionString(org_value) : org_value; - auto iter = db_options_type_info.find(name); - if (iter == db_options_type_info.end()) { + std::string elem; + const auto opt_info = OptionTypeInfo::Find(name, db_options_type_info, &elem); + if (opt_info == nullptr) { return Status::InvalidArgument("Unrecognized option DBOptions:", name); } else { - return iter->second.ParseOption( - config_options, name, value, - reinterpret_cast(new_options) + iter->second.offset); + return opt_info->Parse( + config_options, elem, value, + reinterpret_cast(new_options) + opt_info->offset_); } } @@ -1170,121 +1081,24 @@ std::unordered_map {"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst}, {"kMinOverlappingRatio", kMinOverlappingRatio}}; -LRUCacheOptions OptionsHelper::dummy_lru_cache_options; -CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal; -CompactionOptionsFIFO OptionsHelper::dummy_comp_options; - -template -int offset_of(T1 LRUCacheOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - - size_t(&OptionsHelper::dummy_lru_cache_options)); -} - -template -int offset_of(T1 CompactionOptionsFIFO::*member) { - return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) - - size_t(&OptionsHelper::dummy_comp_options)); -} -template -int offset_of(T1 CompactionOptionsUniversal::*member) { - return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) - - size_t(&OptionsHelper::dummy_comp_options_universal)); -} - -std::unordered_map - OptionsHelper::fifo_compaction_options_type_info = { - {"max_table_files_size", - {offset_of(&CompactionOptionsFIFO::max_table_files_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, - {"ttl", - {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, - {"allow_compaction", - {offset_of(&CompactionOptionsFIFO::allow_compaction), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; - -std::unordered_map - OptionsHelper::universal_compaction_options_type_info = { - {"size_ratio", - {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, size_ratio)}}, - {"min_merge_width", - {offset_of(&CompactionOptionsUniversal::min_merge_width), - OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, min_merge_width)}}, - {"max_merge_width", - {offset_of(&CompactionOptionsUniversal::max_merge_width), - OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, max_merge_width)}}, - {"max_size_amplification_percent", - {offset_of( - &CompactionOptionsUniversal::max_size_amplification_percent), - OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, - max_size_amplification_percent)}}, - {"compression_size_percent", - {offset_of(&CompactionOptionsUniversal::compression_size_percent), - OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, - compression_size_percent)}}, - {"stop_style", - {offset_of(&CompactionOptionsUniversal::stop_style), - OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, stop_style)}}, - {"allow_trivial_move", - {offset_of(&CompactionOptionsUniversal::allow_trivial_move), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}}; - std::unordered_map OptionsHelper::compaction_stop_style_string_map = { {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize}, {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}}; -std::unordered_map - OptionsHelper::lru_cache_options_type_info = { - {"capacity", - {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, capacity)}}, - {"num_shard_bits", - {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, num_shard_bits)}}, - {"strict_capacity_limit", - {offset_of(&LRUCacheOptions::strict_capacity_limit), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, - {"high_pri_pool_ratio", - {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; - -Status OptionTypeInfo::ParseOption(const ConfigOptions& config_options, - const std::string& opt_name, - const std::string& opt_value, - char* opt_addr) const { +Status OptionTypeInfo::Parse(const ConfigOptions& config_options, + const std::string& opt_name, + const std::string& opt_value, + char* opt_addr) const { if (IsDeprecated()) { return Status::OK(); } try { if (opt_addr == nullptr) { return Status::NotFound("Could not find option: ", opt_name); - } else if (parser_func != nullptr) { - return parser_func(config_options, opt_name, opt_value, opt_addr); - } else if (ParseOptionHelper(opt_addr, type, opt_value)) { + } else if (parse_func_ != nullptr) { + return parse_func_(config_options, opt_name, opt_value, opt_addr); + } else if (ParseOptionHelper(opt_addr, type_, opt_value)) { return Status::OK(); } else if (IsByName()) { return Status::NotSupported("Deserializing the option " + opt_name + @@ -1298,23 +1112,122 @@ Status OptionTypeInfo::ParseOption(const ConfigOptions& config_options, } } -Status OptionTypeInfo::SerializeOption(const ConfigOptions& config_options, - const std::string& opt_name, - const char* opt_addr, - std::string* opt_value) const { +Status OptionTypeInfo::ParseStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* struct_map, + const std::string& opt_name, const std::string& opt_value, char* opt_addr) { + assert(struct_map); + Status status; + if (opt_name == struct_name || EndsWith(opt_name, "." + struct_name)) { + // This option represents the entire struct + std::unordered_map opt_map; + status = StringToMap(opt_value, &opt_map); + if (status.ok()) { + for (const auto& map_iter : opt_map) { + const auto iter = struct_map->find(map_iter.first); + if (iter != struct_map->end()) { + status = iter->second.Parse(config_options, map_iter.first, + map_iter.second, + opt_addr + iter->second.offset_); + } else { + return Status::InvalidArgument("Unrecognized option: ", + struct_name + "." + map_iter.first); + } + } + } + } else if (StartsWith(opt_name, struct_name + ".")) { + // This option represents a nested field in the struct (e.g, struct.field) + std::string elem_name; + const auto opt_info = + Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); + if (opt_info != nullptr) { + status = opt_info->Parse(config_options, elem_name, opt_value, + opt_addr + opt_info->offset_); + } else { + status = Status::InvalidArgument("Unrecognized option: ", opt_name); + } + } else { + // This option represents a field in the struct (e.g. field) + std::string elem_name; + const auto opt_info = Find(opt_name, *struct_map, &elem_name); + if (opt_info != nullptr) { + status = opt_info->Parse(config_options, elem_name, opt_value, + opt_addr + opt_info->offset_); + } else { + status = Status::InvalidArgument("Unrecognized option: ", + struct_name + "." + opt_name); + } + } + return status; +} + +Status OptionTypeInfo::Serialize(const ConfigOptions& config_options, + const std::string& opt_name, + const char* opt_addr, + std::string* opt_value) const { // If the option is no longer used in rocksdb and marked as deprecated, // we skip it in the serialization. - Status s; - if (opt_addr == nullptr || IsDeprecated()) { - s = Status::OK(); - } else if (string_func != nullptr) { - s = string_func(config_options, opt_name, opt_addr, opt_value); - } else if (SerializeSingleOptionHelper(opt_addr, type, opt_value)) { - s = Status::OK(); - } else { - s = Status::InvalidArgument("Cannot serialize option: ", opt_name); + if (opt_addr != nullptr && ShouldSerialize()) { + if (serialize_func_ != nullptr) { + return serialize_func_(config_options, opt_name, opt_addr, opt_value); + } else if (!SerializeSingleOptionHelper(opt_addr, type_, opt_value)) { + return Status::InvalidArgument("Cannot serialize option: ", opt_name); + } } - return s; + return Status::OK(); +} + +Status OptionTypeInfo::SerializeStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* struct_map, + const std::string& opt_name, const char* opt_addr, std::string* value) { + assert(struct_map); + Status status; + if (EndsWith(opt_name, struct_name)) { + // We are going to write the struct as "{ prop1=value1; prop2=value2;}. + // Set the delimiter to ";" so that the everything will be on one line. + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + + // This option represents the entire struct + std::string result; + for (const auto& iter : *struct_map) { + std::string single; + const auto& opt_info = iter.second; + if (opt_info.ShouldSerialize()) { + status = opt_info.Serialize(embedded, iter.first, + opt_addr + opt_info.offset_, &single); + if (!status.ok()) { + return status; + } else { + result.append(iter.first + "=" + single + embedded.delimiter); + } + } + } + *value = "{" + result + "}"; + } else if (StartsWith(opt_name, struct_name + ".")) { + // This option represents a nested field in the struct (e.g, struct.field) + std::string elem_name; + const auto opt_info = + Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); + if (opt_info != nullptr) { + status = opt_info->Serialize(config_options, elem_name, + opt_addr + opt_info->offset_, value); + } else { + status = Status::InvalidArgument("Unrecognized option: ", opt_name); + } + } else { + // This option represents a field in the struct (e.g. field) + std::string elem_name; + const auto opt_info = Find(opt_name, *struct_map, &elem_name); + if (opt_info == nullptr) { + return Status::InvalidArgument("Unrecognized option: ", opt_name); + } else if (opt_info->ShouldSerialize()) { + return opt_info->Serialize(config_options, opt_name + "." + elem_name, + opt_addr + opt_info->offset_, value); + } + } + return Status::OK(); } template @@ -1380,44 +1293,15 @@ static bool AreOptionsEqual(OptionType type, const char* this_offset, return IsOptionEqual(this_offset, that_offset); case OptionType::kEncodingType: return IsOptionEqual(this_offset, that_offset); - case OptionType::kCompactionOptionsFIFO: { - CompactionOptionsFIFO lhs = - *reinterpret_cast(this_offset); - CompactionOptionsFIFO rhs = - *reinterpret_cast(that_offset); - if (lhs.max_table_files_size == rhs.max_table_files_size && - lhs.allow_compaction == rhs.allow_compaction) { - return true; - } - return false; - } - case OptionType::kCompactionOptionsUniversal: { - CompactionOptionsUniversal lhs = - *reinterpret_cast(this_offset); - CompactionOptionsUniversal rhs = - *reinterpret_cast(that_offset); - if (lhs.size_ratio == rhs.size_ratio && - lhs.min_merge_width == rhs.min_merge_width && - lhs.max_merge_width == rhs.max_merge_width && - lhs.max_size_amplification_percent == - rhs.max_size_amplification_percent && - lhs.compression_size_percent == rhs.compression_size_percent && - lhs.stop_style == rhs.stop_style && - lhs.allow_trivial_move == rhs.allow_trivial_move) { - return true; - } - return false; - } default: return false; } // End switch } -bool OptionTypeInfo::MatchesOption(const ConfigOptions& config_options, - const std::string& opt_name, - const char* this_addr, const char* that_addr, - - std::string* mismatch) const { +bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options, + const std::string& opt_name, + const char* this_addr, const char* that_addr, + std::string* mismatch) const { if (!config_options.IsCheckEnabled(GetSanityLevel())) { return true; // If the sanity level is not being checked, skip it } @@ -1425,11 +1309,12 @@ bool OptionTypeInfo::MatchesOption(const ConfigOptions& config_options, if (this_addr == that_addr) { return true; } - } else if (equals_func != nullptr) { - if (equals_func(config_options, opt_name, this_addr, that_addr, mismatch)) { + } else if (equals_func_ != nullptr) { + if (equals_func_(config_options, opt_name, this_addr, that_addr, + mismatch)) { return true; } - } else if (AreOptionsEqual(type, this_addr, that_addr)) { + } else if (AreOptionsEqual(type_, this_addr, that_addr)) { return true; } if (mismatch->empty()) { @@ -1438,29 +1323,82 @@ bool OptionTypeInfo::MatchesOption(const ConfigOptions& config_options, return false; } -bool OptionTypeInfo::MatchesByName(const ConfigOptions& config_options, - const std::string& opt_name, - const char* this_addr, - const char* that_addr) const { +bool OptionTypeInfo::StructsAreEqual( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* struct_map, + const std::string& opt_name, const char* this_addr, const char* that_addr, + std::string* mismatch) { + assert(struct_map); + Status status; + bool matches = true; + std::string result; + if (EndsWith(opt_name, struct_name)) { + // This option represents the entire struct + for (const auto& iter : *struct_map) { + const auto& opt_info = iter.second; + + matches = opt_info.AreEqual(config_options, iter.first, + this_addr + opt_info.offset_, + that_addr + opt_info.offset_, &result); + if (!matches) { + *mismatch = struct_name + "." + result; + return false; + } + } + } else if (StartsWith(opt_name, struct_name + ".")) { + // This option represents a nested field in the struct (e.g, struct.field) + std::string elem_name; + const auto opt_info = + Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); + assert(opt_info); + if (opt_info == nullptr) { + *mismatch = opt_name; + matches = false; + } else if (!opt_info->AreEqual(config_options, elem_name, + this_addr + opt_info->offset_, + that_addr + opt_info->offset_, &result)) { + matches = false; + *mismatch = struct_name + "." + result; + } + } else { + // This option represents a field in the struct (e.g. field) + std::string elem_name; + const auto opt_info = Find(opt_name, *struct_map, &elem_name); + assert(opt_info); + if (opt_info == nullptr) { + *mismatch = struct_name + "." + opt_name; + matches = false; + } else if (!opt_info->AreEqual(config_options, elem_name, + this_addr + opt_info->offset_, + that_addr + opt_info->offset_, &result)) { + matches = false; + *mismatch = struct_name + "." + result; + } + } + return matches; +} + +bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, + const char* this_addr, + const char* that_addr) const { if (IsByName()) { std::string that_value; - if (SerializeOption(config_options, opt_name, that_addr, &that_value) - .ok()) { - return MatchesByName(config_options, opt_name, this_addr, that_value); + if (Serialize(config_options, opt_name, that_addr, &that_value).ok()) { + return AreEqualByName(config_options, opt_name, this_addr, that_value); } } return false; } -bool OptionTypeInfo::MatchesByName(const ConfigOptions& config_options, - const std::string& opt_name, - const char* opt_addr, - const std::string& that_value) const { +bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, + const char* opt_addr, + const std::string& that_value) const { std::string this_value; if (!IsByName()) { return false; - } else if (!SerializeOption(config_options, opt_name, opt_addr, &this_value) - .ok()) { + } else if (!Serialize(config_options, opt_name, opt_addr, &this_value).ok()) { return false; } else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) { if (that_value == kNullptrString) { @@ -1473,6 +1411,30 @@ bool OptionTypeInfo::MatchesByName(const ConfigOptions& config_options, } return (this_value == that_value); } + +const OptionTypeInfo* OptionTypeInfo::Find( + const std::string& opt_name, + const std::unordered_map& opt_map, + std::string* elem_name) { + const auto iter = opt_map.find(opt_name); // Look up the value in the map + if (iter != opt_map.end()) { // Found the option in the map + *elem_name = opt_name; // Return the name + return &(iter->second); // Return the contents of the iterator + } else { + auto idx = opt_name.find("."); // Look for a separator + if (idx > 0 && idx != std::string::npos) { // We found a separator + auto siter = + opt_map.find(opt_name.substr(0, idx)); // Look for the short name + if (siter != opt_map.end()) { // We found the short name + if (siter->second.IsStruct()) { // If the object is a struct + *elem_name = opt_name.substr(idx + 1); // Return the rest + return &(siter->second); // Return the contents of the iterator + } + } + } + } + return nullptr; +} #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/options/options_helper.h b/options/options_helper.h index 08780e3a6..7fbc6095e 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -98,24 +98,15 @@ struct OptionsHelper { compression_type_string_map; #ifndef ROCKSDB_LITE static std::unordered_map cf_options_type_info; - static std::unordered_map - fifo_compaction_options_type_info; - static std::unordered_map - universal_compaction_options_type_info; static std::unordered_map compaction_stop_style_string_map; static std::unordered_map db_options_type_info; - static std::unordered_map - lru_cache_options_type_info; static std::unordered_map encoding_type_string_map; static std::unordered_map compaction_style_string_map; static std::unordered_map compaction_pri_string_map; static ColumnFamilyOptions dummy_cf_options; - static CompactionOptionsFIFO dummy_comp_options; - static LRUCacheOptions dummy_lru_cache_options; - static CompactionOptionsUniversal dummy_comp_options_universal; #endif // !ROCKSDB_LITE }; @@ -128,15 +119,9 @@ static auto& compaction_stop_style_to_string = static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map; #ifndef ROCKSDB_LITE static auto& cf_options_type_info = OptionsHelper::cf_options_type_info; -static auto& fifo_compaction_options_type_info = - OptionsHelper::fifo_compaction_options_type_info; -static auto& universal_compaction_options_type_info = - OptionsHelper::universal_compaction_options_type_info; static auto& compaction_stop_style_string_map = OptionsHelper::compaction_stop_style_string_map; static auto& db_options_type_info = OptionsHelper::db_options_type_info; -static auto& lru_cache_options_type_info = - OptionsHelper::lru_cache_options_type_info; static auto& compression_type_string_map = OptionsHelper::compression_type_string_map; static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map; diff --git a/options/options_parser.cc b/options/options_parser.cc index 82dedda79..36db390ed 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -610,14 +610,14 @@ Status RocksDBOptionsParser::VerifyDBOptions( const auto& opt_info = pair.second; if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) { const char* base_addr = - reinterpret_cast(&base_opt) + opt_info.offset; + reinterpret_cast(&base_opt) + opt_info.offset_; const char* file_addr = - reinterpret_cast(&file_opt) + opt_info.offset; + reinterpret_cast(&file_opt) + opt_info.offset_; std::string mismatch; - if (!opt_info.MatchesOption(config_options, pair.first, base_addr, - file_addr, &mismatch) && - !opt_info.MatchesByName(config_options, pair.first, base_addr, - file_addr)) { + if (!opt_info.AreEqual(config_options, pair.first, base_addr, file_addr, + &mismatch) && + !opt_info.AreEqualByName(config_options, pair.first, base_addr, + file_addr)) { const size_t kBufferSize = 2048; char buffer[kBufferSize]; std::string base_value; @@ -627,11 +627,11 @@ Status RocksDBOptionsParser::VerifyDBOptions( "[RocksDBOptionsParser]: " "failed the verification on ColumnFamilyOptions::%s", pair.first.c_str()); - Status s = opt_info.SerializeOption(config_options, pair.first, - base_addr, &base_value); + Status s = opt_info.Serialize(config_options, pair.first, base_addr, + &base_value); if (s.ok()) { - s = opt_info.SerializeOption(config_options, pair.first, file_addr, - &file_value); + s = opt_info.Serialize(config_options, pair.first, file_addr, + &file_value); } snprintf(buffer, sizeof(buffer), "[RocksDBOptionsParser]: " @@ -668,11 +668,11 @@ Status RocksDBOptionsParser::VerifyCFOptions( if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) { std::string mismatch; const char* base_addr = - reinterpret_cast(&base_opt) + opt_info.offset; + reinterpret_cast(&base_opt) + opt_info.offset_; const char* file_addr = - reinterpret_cast(&file_opt) + opt_info.offset; - bool matches = opt_info.MatchesOption(config_options, pair.first, - base_addr, file_addr, &mismatch); + reinterpret_cast(&file_opt) + opt_info.offset_; + bool matches = opt_info.AreEqual(config_options, pair.first, base_addr, + file_addr, &mismatch); if (!matches && opt_info.IsByName()) { if (opt_map == nullptr) { matches = true; @@ -681,8 +681,8 @@ Status RocksDBOptionsParser::VerifyCFOptions( if (iter == opt_map->end()) { matches = true; } else { - matches = opt_info.MatchesByName(config_options, pair.first, - base_addr, iter->second); + matches = opt_info.AreEqualByName(config_options, pair.first, + base_addr, iter->second); } } } @@ -692,11 +692,11 @@ Status RocksDBOptionsParser::VerifyCFOptions( char buffer[kBufferSize]; std::string base_value; std::string file_value; - Status s = opt_info.SerializeOption(config_options, pair.first, - base_addr, &base_value); + Status s = opt_info.Serialize(config_options, pair.first, base_addr, + &base_value); if (s.ok()) { - s = opt_info.SerializeOption(config_options, pair.first, file_addr, - &file_value); + s = opt_info.Serialize(config_options, pair.first, file_addr, + &file_value); } int offset = snprintf(buffer, sizeof(buffer), diff --git a/options/options_test.cc b/options/options_test.cc index c34fb8666..26f2153b6 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -2999,13 +2999,12 @@ static void TestAndCompareOption(const ConfigOptions& config_options, const std::string& opt_name, void* base_ptr, void* comp_ptr) { std::string result, mismatch; - char* base_addr = reinterpret_cast(base_ptr) + opt_info.offset; - char* comp_addr = reinterpret_cast(comp_ptr) + opt_info.offset; - ASSERT_OK( - opt_info.SerializeOption(config_options, opt_name, base_addr, &result)); - ASSERT_OK(opt_info.ParseOption(config_options, opt_name, result, comp_addr)); - ASSERT_TRUE(opt_info.MatchesOption(config_options, opt_name, base_addr, - comp_addr, &mismatch)); + char* base_addr = reinterpret_cast(base_ptr) + opt_info.offset_; + char* comp_addr = reinterpret_cast(comp_ptr) + opt_info.offset_; + ASSERT_OK(opt_info.Serialize(config_options, opt_name, base_addr, &result)); + ASSERT_OK(opt_info.Parse(config_options, opt_name, result, comp_addr)); + ASSERT_TRUE(opt_info.AreEqual(config_options, opt_name, base_addr, comp_addr, + &mismatch)); } static void TestAndCompareOption(const ConfigOptions& config_options, @@ -3013,9 +3012,8 @@ static void TestAndCompareOption(const ConfigOptions& config_options, const std::string& opt_name, const std::string& opt_value, void* base_ptr, void* comp_ptr) { - char* base_addr = reinterpret_cast(base_ptr) + opt_info.offset; - ASSERT_OK( - opt_info.ParseOption(config_options, opt_name, opt_value, base_addr)); + char* base_addr = reinterpret_cast(base_ptr) + opt_info.offset_; + ASSERT_OK(opt_info.Parse(config_options, opt_name, opt_value, base_addr)); TestAndCompareOption(config_options, opt_info, opt_name, base_ptr, comp_ptr); } @@ -3026,8 +3024,8 @@ void TestOptInfo(const ConfigOptions& config_options, OptionType opt_type, OptionTypeInfo opt_info(0, opt_type); char* base_addr = reinterpret_cast(base); char* comp_addr = reinterpret_cast(comp); - ASSERT_FALSE(opt_info.MatchesOption(config_options, "base", base_addr, - comp_addr, &result)); + ASSERT_FALSE( + opt_info.AreEqual(config_options, "base", base_addr, comp_addr, &result)); ASSERT_EQ(result, "base"); ASSERT_NE(*base, *comp); TestAndCompareOption(config_options, opt_info, "base", base_addr, comp_addr); @@ -3092,38 +3090,33 @@ TEST_F(OptionTypeInfoTest, TestInvalidArgs) { size_t sz; double d; + ASSERT_NOK(OptionTypeInfo(0, OptionType::kBoolean) + .Parse(config_options, "b", "x", reinterpret_cast(&b))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt) + .Parse(config_options, "b", "x", reinterpret_cast(&i))); ASSERT_NOK( - OptionTypeInfo(0, OptionType::kBoolean) - .ParseOption(config_options, "b", "x", reinterpret_cast(&b))); + OptionTypeInfo(0, OptionType::kInt32T) + .Parse(config_options, "b", "x", reinterpret_cast(&i32))); ASSERT_NOK( - OptionTypeInfo(0, OptionType::kInt) - .ParseOption(config_options, "b", "x", reinterpret_cast(&i))); - ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt32T) - .ParseOption(config_options, "b", "x", - reinterpret_cast(&i32))); - ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt64T) - .ParseOption(config_options, "b", "x", - reinterpret_cast(&i64))); + OptionTypeInfo(0, OptionType::kInt64T) + .Parse(config_options, "b", "x", reinterpret_cast(&i64))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt) + .Parse(config_options, "b", "x", reinterpret_cast(&u))); ASSERT_NOK( - OptionTypeInfo(0, OptionType::kUInt) - .ParseOption(config_options, "b", "x", reinterpret_cast(&u))); - ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt32T) - .ParseOption(config_options, "b", "x", - reinterpret_cast(&u32))); - ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt64T) - .ParseOption(config_options, "b", "x", - reinterpret_cast(&u64))); + OptionTypeInfo(0, OptionType::kUInt32T) + .Parse(config_options, "b", "x", reinterpret_cast(&u32))); + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kUInt64T) + .Parse(config_options, "b", "x", reinterpret_cast(&u64))); ASSERT_NOK( OptionTypeInfo(0, OptionType::kSizeT) - .ParseOption(config_options, "b", "x", reinterpret_cast(&sz))); - ASSERT_NOK( - OptionTypeInfo(0, OptionType::kDouble) - .ParseOption(config_options, "b", "x", reinterpret_cast(&d))); + .Parse(config_options, "b", "x", reinterpret_cast(&sz))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kDouble) + .Parse(config_options, "b", "x", reinterpret_cast(&d))); // Don't know how to convert Unknowns to anything else - ASSERT_NOK( - OptionTypeInfo(0, OptionType::kUnknown) - .ParseOption(config_options, "b", "x", reinterpret_cast(&d))); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUnknown) + .Parse(config_options, "b", "x", reinterpret_cast(&d))); // Verify that if the parse function throws an exception, it is also trapped OptionTypeInfo func_info(0, OptionType::kUnknown, @@ -3135,10 +3128,10 @@ TEST_F(OptionTypeInfoTest, TestInvalidArgs) { *ptr = ParseInt(value); return Status::OK(); }); - ASSERT_OK(func_info.ParseOption(config_options, "b", "1", - reinterpret_cast(&i))); - ASSERT_NOK(func_info.ParseOption(config_options, "b", "x", - reinterpret_cast(&i))); + ASSERT_OK( + func_info.Parse(config_options, "b", "1", reinterpret_cast(&i))); + ASSERT_NOK( + func_info.Parse(config_options, "b", "x", reinterpret_cast(&i))); } TEST_F(OptionTypeInfoTest, TestParseFunc) { @@ -3157,11 +3150,11 @@ TEST_F(OptionTypeInfoTest, TestParseFunc) { }); ConfigOptions config_options; std::string base; - ASSERT_OK(opt_info.ParseOption(config_options, "World", "Hello", - reinterpret_cast(&base))); + ASSERT_OK(opt_info.Parse(config_options, "World", "Hello", + reinterpret_cast(&base))); ASSERT_EQ(base, "Hello World"); - ASSERT_NOK(opt_info.ParseOption(config_options, "Oops", "Hello", - reinterpret_cast(&base))); + ASSERT_NOK(opt_info.Parse(config_options, "Oops", "Hello", + reinterpret_cast(&base))); } TEST_F(OptionTypeInfoTest, TestSerializeFunc) { @@ -3181,11 +3174,11 @@ TEST_F(OptionTypeInfoTest, TestSerializeFunc) { ConfigOptions config_options; std::string base; std::string value; - ASSERT_OK(opt_info.SerializeOption(config_options, "Hello", - reinterpret_cast(&base), &value)); + ASSERT_OK(opt_info.Serialize(config_options, "Hello", + reinterpret_cast(&base), &value)); ASSERT_EQ(value, "Hello"); - ASSERT_NOK(opt_info.SerializeOption(config_options, "Oops", - reinterpret_cast(&base), &value)); + ASSERT_NOK(opt_info.Serialize(config_options, "Oops", + reinterpret_cast(&base), &value)); } TEST_F(OptionTypeInfoTest, TestEqualsFunc) { @@ -3212,17 +3205,17 @@ TEST_F(OptionTypeInfoTest, TestEqualsFunc) { int int1 = 100; int int2 = 200; std::string mismatch; - ASSERT_TRUE(opt_info.MatchesOption( + ASSERT_TRUE(opt_info.AreEqual( config_options, "LT", reinterpret_cast(&int1), reinterpret_cast(&int2), &mismatch)); ASSERT_EQ(mismatch, ""); - ASSERT_FALSE(opt_info.MatchesOption( - config_options, "GT", reinterpret_cast(&int1), - reinterpret_cast(&int2), &mismatch)); + ASSERT_FALSE(opt_info.AreEqual(config_options, "GT", + reinterpret_cast(&int1), + reinterpret_cast(&int2), &mismatch)); ASSERT_EQ(mismatch, "GT"); - ASSERT_FALSE(opt_info.MatchesOption( - config_options, "NO", reinterpret_cast(&int1), - reinterpret_cast(&int2), &mismatch)); + ASSERT_FALSE(opt_info.AreEqual(config_options, "NO", + reinterpret_cast(&int1), + reinterpret_cast(&int2), &mismatch)); ASSERT_EQ(mismatch, "NO???"); } @@ -3244,37 +3237,37 @@ TEST_F(OptionTypeInfoTest, TestOptionFlags) { std::string comp = "comp"; // If marked string none, the serialization returns okay but does nothing - ASSERT_OK(opt_none.SerializeOption(config_options, "None", - reinterpret_cast(&base), &base)); + ASSERT_OK(opt_none.Serialize(config_options, "None", + reinterpret_cast(&base), &base)); // If marked never compare, they match even when they do not - ASSERT_TRUE(opt_never.MatchesOption(config_options, "Never", - reinterpret_cast(&base), - reinterpret_cast(&comp), &base)); - ASSERT_FALSE(opt_none.MatchesOption(config_options, "Never", - reinterpret_cast(&base), - reinterpret_cast(&comp), &base)); + ASSERT_TRUE(opt_never.AreEqual(config_options, "Never", + reinterpret_cast(&base), + reinterpret_cast(&comp), &base)); + ASSERT_FALSE(opt_none.AreEqual(config_options, "Never", + reinterpret_cast(&base), + reinterpret_cast(&comp), &base)); // An alias can change the value via parse, but does nothing on serialize on // match std::string result; - ASSERT_OK(opt_alias.ParseOption(config_options, "Alias", "Alias", - reinterpret_cast(&base))); - ASSERT_OK(opt_alias.SerializeOption(config_options, "Alias", - reinterpret_cast(&base), &result)); - ASSERT_TRUE(opt_alias.MatchesOption(config_options, "Alias", - reinterpret_cast(&base), - reinterpret_cast(&comp), &result)); + ASSERT_OK(opt_alias.Parse(config_options, "Alias", "Alias", + reinterpret_cast(&base))); + ASSERT_OK(opt_alias.Serialize(config_options, "Alias", + reinterpret_cast(&base), &result)); + ASSERT_TRUE(opt_alias.AreEqual(config_options, "Alias", + reinterpret_cast(&base), + reinterpret_cast(&comp), &result)); ASSERT_EQ(base, "Alias"); ASSERT_NE(base, comp); // Deprecated options do nothing on any of the commands - ASSERT_OK(opt_deprecated.ParseOption(config_options, "Alias", "Deprecated", - reinterpret_cast(&base))); - ASSERT_OK(opt_deprecated.SerializeOption( - config_options, "Alias", reinterpret_cast(&base), &result)); - ASSERT_TRUE(opt_deprecated.MatchesOption( - config_options, "Alias", reinterpret_cast(&base), - reinterpret_cast(&comp), &result)); + ASSERT_OK(opt_deprecated.Parse(config_options, "Alias", "Deprecated", + reinterpret_cast(&base))); + ASSERT_OK(opt_deprecated.Serialize(config_options, "Alias", + reinterpret_cast(&base), &result)); + ASSERT_TRUE(opt_deprecated.AreEqual(config_options, "Alias", + reinterpret_cast(&base), + reinterpret_cast(&comp), &result)); ASSERT_EQ(base, "Alias"); ASSERT_NE(base, comp); } @@ -3293,16 +3286,16 @@ TEST_F(OptionTypeInfoTest, TestCustomEnum) { e2 = TestEnum::kA; - ASSERT_OK(opt_info.ParseOption(config_options, "", "B", - reinterpret_cast(&e1))); - ASSERT_OK(opt_info.SerializeOption(config_options, "", - reinterpret_cast(&e1), &result)); + ASSERT_OK( + opt_info.Parse(config_options, "", "B", reinterpret_cast(&e1))); + ASSERT_OK(opt_info.Serialize(config_options, "", reinterpret_cast(&e1), + &result)); ASSERT_EQ(e1, TestEnum::kB); ASSERT_EQ(result, "B"); - ASSERT_FALSE(opt_info.MatchesOption(config_options, "Enum", - reinterpret_cast(&e1), - reinterpret_cast(&e2), &mismatch)); + ASSERT_FALSE(opt_info.AreEqual(config_options, "Enum", + reinterpret_cast(&e1), + reinterpret_cast(&e2), &mismatch)); ASSERT_EQ(mismatch, "Enum"); TestAndCompareOption(config_options, opt_info, "", "C", @@ -3310,8 +3303,8 @@ TEST_F(OptionTypeInfoTest, TestCustomEnum) { reinterpret_cast(&e2)); ASSERT_EQ(e2, TestEnum::kC); - ASSERT_NOK(opt_info.ParseOption(config_options, "", "D", - reinterpret_cast(&e1))); + ASSERT_NOK( + opt_info.Parse(config_options, "", "D", reinterpret_cast(&e1))); ASSERT_EQ(e1, TestEnum::kC); } @@ -3360,6 +3353,94 @@ TEST_F(OptionTypeInfoTest, TestBuiltinEnum) { ASSERT_EQ(e1, iter.second); } } + +TEST_F(OptionTypeInfoTest, TestStruct) { + struct Basic { + int i = 42; + std::string s = "Hello"; + }; + + struct Extended { + int j = 11; + Basic b; + }; + + std::unordered_map basic_type_map = { + {"i", {offsetof(struct Basic, i), OptionType::kInt}}, + {"s", {offsetof(struct Basic, s), OptionType::kString}}, + }; + OptionTypeInfo basic_info = OptionTypeInfo::Struct( + "b", &basic_type_map, 0, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, 0); + + std::unordered_map extended_type_map = { + {"j", {offsetof(struct Extended, j), OptionType::kInt}}, + {"b", OptionTypeInfo::Struct( + "b", &basic_type_map, offsetof(struct Extended, b), + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0)}, + {"m", OptionTypeInfo::Struct( + "m", &basic_type_map, offsetof(struct Extended, b), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct Extended, b))}, + }; + OptionTypeInfo extended_info = OptionTypeInfo::Struct( + "e", &extended_type_map, 0, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, 0); + Extended e1, e2; + ConfigOptions config_options; + std::string mismatch; + TestAndCompareOption(config_options, basic_info, "b", "{i=33;s=33}", &e1.b, + &e2.b); + ASSERT_EQ(e1.b.i, 33); + ASSERT_EQ(e1.b.s, "33"); + + TestAndCompareOption(config_options, basic_info, "b.i", "44", &e1.b, &e2.b); + ASSERT_EQ(e1.b.i, 44); + + TestAndCompareOption(config_options, basic_info, "i", "55", &e1.b, &e2.b); + ASSERT_EQ(e1.b.i, 55); + + e1.b.i = 0; + auto e1bc = reinterpret_cast(&e1.b); + auto e2bc = reinterpret_cast(&e2.b); + + ASSERT_FALSE(basic_info.AreEqual(config_options, "b", e1bc, e2bc, &mismatch)); + ASSERT_EQ(mismatch, "b.i"); + mismatch.clear(); + ASSERT_FALSE( + basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); + ASSERT_EQ(mismatch, "b.i"); + mismatch.clear(); + ASSERT_FALSE(basic_info.AreEqual(config_options, "i", e1bc, e2bc, &mismatch)); + ASSERT_EQ(mismatch, "b.i"); + mismatch.clear(); + + e1 = e2; + ASSERT_NOK(basic_info.Parse(config_options, "b", "{i=33;s=33;j=44}", e1bc)); + ASSERT_TRUE( + basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); + ASSERT_NOK(basic_info.Parse(config_options, "b.j", "44", e1bc)); + ASSERT_TRUE( + basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); + ASSERT_NOK(basic_info.Parse(config_options, "j", "44", e1bc)); + ASSERT_TRUE( + basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); + + TestAndCompareOption(config_options, extended_info, "e", + "b={i=55;s=55}; j=22;", &e1, &e2); + ASSERT_EQ(e1.b.i, 55); + ASSERT_EQ(e1.j, 22); + ASSERT_EQ(e1.b.s, "55"); + TestAndCompareOption(config_options, extended_info, "e.b", "{i=66;s=66;}", + &e1, &e2); + ASSERT_EQ(e1.b.i, 66); + ASSERT_EQ(e1.j, 22); + ASSERT_EQ(e1.b.s, "66"); + TestAndCompareOption(config_options, extended_info, "e.b.i", "77", &e1, &e2); + ASSERT_EQ(e1.b.i, 77); + ASSERT_EQ(e1.j, 22); + ASSERT_EQ(e1.b.s, "66"); +} #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/options/options_type.h b/options/options_type.h index 77697c029..e39966e44 100644 --- a/options/options_type.h +++ b/options/options_type.h @@ -36,8 +36,6 @@ enum class OptionType { kComparator, kCompactionFilter, kCompactionFilterFactory, - kCompactionOptionsFIFO, - kCompactionOptionsUniversal, kCompactionStopStyle, kMergeOperator, kMemTableRepFactory, @@ -45,9 +43,9 @@ enum class OptionType { kFlushBlockPolicyFactory, kChecksumType, kEncodingType, - kLRUCacheOptions, kEnv, kEnum, + kStruct, kUnknown, }; @@ -133,7 +131,7 @@ bool SerializeEnum(const std::unordered_map& type_map, // @param name The name of the options being parsed // @param value The string representation of the option // @param addr Pointer to the object -using ParserFunc = std::function; @@ -144,7 +142,7 @@ using ParserFunc = std::function; @@ -164,67 +162,68 @@ using EqualsFunc = std::function static OptionTypeInfo Enum( - int _offset, const std::unordered_map* const map) { + int offset, const std::unordered_map* const map) { return OptionTypeInfo( - _offset, OptionType::kEnum, OptionVerificationType::kNormal, + offset, OptionType::kEnum, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0, // Uses the map argument to convert the input string into // its corresponding enum value. If value is found in the map, @@ -283,7 +282,79 @@ class OptionTypeInfo { }); } // End OptionTypeInfo::Enum - bool IsEnabled(OptionTypeFlags otf) const { return (flags & otf) == otf; } + // Creates an OptionTypeInfo for a Struct type. Structs have a + // map of string-OptionTypeInfo associated with them that describes how + // to process the object for parsing, serializing, and matching. + // Structs also have a struct_name, which is the name of the object + // as registered in the parent map. + // When processing a struct, the option name can be specified as: + // - Meaning to process the entire struct. + // - Meaning to process the single field + // - Process the single fields + // The CompactionOptionsFIFO, CompactionOptionsUniversal, and LRUCacheOptions + // are all examples of Struct options. + // + // To create an OptionTypeInfo that is a Struct, one should: + // - Create a static map of string-OptionTypeInfo corresponding to the + // properties of the object that can be set via the options. + // - Call this method passing the name and map in as parameters. + // Note that it is not necessary to add a new OptionType or make any + // other changes -- the returned object handles parsing, serialization, and + // comparisons. + // + // @param offset The offset in the option object for this enum + // @param map The string to enum mapping for this enum + static OptionTypeInfo Struct( + const std::string& struct_name, + const std::unordered_map* struct_map, + int offset, OptionVerificationType verification, OptionTypeFlags flags, + int mutable_offset) { + return OptionTypeInfo( + offset, OptionType::kStruct, verification, flags, mutable_offset, + // Parses the struct and updates the fields at addr + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, + const std::string& value, char* addr) { + return ParseStruct(opts, struct_name, struct_map, name, value, addr); + }, + // Serializes the struct options into value + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, const char* addr, + std::string* value) { + return SerializeStruct(opts, struct_name, struct_map, name, addr, + value); + }, + // Compares the struct fields of addr1 and addr2 for equality + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, const char* addr1, + const char* addr2, std::string* mismatch) { + return StructsAreEqual(opts, struct_name, struct_map, name, addr1, + addr2, mismatch); + }); + } + static OptionTypeInfo Struct( + const std::string& struct_name, + const std::unordered_map* struct_map, + int offset, OptionVerificationType verification, OptionTypeFlags flags, + int mutable_offset, const ParseFunc& parse_func) { + return OptionTypeInfo( + offset, OptionType::kStruct, verification, flags, mutable_offset, + parse_func, + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, const char* addr, + std::string* value) { + return SerializeStruct(opts, struct_name, struct_map, name, addr, + value); + }, + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, const char* addr1, + const char* addr2, std::string* mismatch) { + return StructsAreEqual(opts, struct_name, struct_map, name, addr1, + addr2, mismatch); + }); + } + + bool IsEnabled(OptionTypeFlags otf) const { return (flags_ & otf) == otf; } bool IsMutable() const { return IsEnabled(OptionTypeFlags::kMutable); } @@ -297,7 +368,7 @@ class OptionTypeInfo { bool IsAlias() const { return IsEnabled(OptionVerificationType::kAlias); } bool IsEnabled(OptionVerificationType ovf) const { - return verification == ovf; + return verification_ == ovf; } // Returns the sanity level for comparing the option. @@ -308,7 +379,7 @@ class OptionTypeInfo { if (IsDeprecated() || IsAlias()) { return ConfigOptions::SanityLevel::kSanityLevelNone; } else { - auto match = (flags & OptionTypeFlags::kCompareExact); + auto match = (flags_ & OptionTypeFlags::kCompareExact); if (match == OptionTypeFlags::kCompareDefault) { return ConfigOptions::SanityLevel::kSanityLevelExactMatch; } else { @@ -331,54 +402,99 @@ class OptionTypeInfo { } bool IsByName() const { - return (verification == OptionVerificationType::kByName || - verification == OptionVerificationType::kByNameAllowNull || - verification == OptionVerificationType::kByNameAllowFromNull); + return (verification_ == OptionVerificationType::kByName || + verification_ == OptionVerificationType::kByNameAllowNull || + verification_ == OptionVerificationType::kByNameAllowFromNull); } + bool IsStruct() const { return (type_ == OptionType::kStruct); } + // Parses the option in "opt_value" according to the rules of this class // and updates the value at "opt_addr". // On success, Status::OK() is returned. On failure: // NotFound means the opt_name is not valid for this option // NotSupported means we do not know how to parse the value for this option // InvalidArgument means the opt_value is not valid for this option. - Status ParseOption(const ConfigOptions& config_options, - const std::string& opt_name, const std::string& opt_value, - char* opt_addr) const; + Status Parse(const ConfigOptions& config_options, const std::string& opt_name, + const std::string& opt_value, char* opt_addr) const; // Serializes the option in "opt_addr" according to the rules of this class // into the value at "opt_value". - Status SerializeOption(const ConfigOptions& config_options, - const std::string& opt_name, const char* opt_addr, - std::string* opt_value) const; + Status Serialize(const ConfigOptions& config_options, + const std::string& opt_name, const char* opt_addr, + std::string* opt_value) const; // Compares the "addr1" and "addr2" values according to the rules of this // class and returns true if they match. On a failed match, mismatch is the // name of the option that failed to match. - bool MatchesOption(const ConfigOptions& config_options, - const std::string& opt_name, const char* addr1, - const char* addr2, std::string* mismatch) const; + bool AreEqual(const ConfigOptions& config_options, + const std::string& opt_name, const char* addr1, + const char* addr2, std::string* mismatch) const; // Used to override the match rules for "ByName" options. - bool MatchesByName(const ConfigOptions& config_options, - const std::string& opt_name, const char* this_offset, - const char* that_offset) const; - bool MatchesByName(const ConfigOptions& config_options, - const std::string& opt_name, const char* this_ptr, - const std::string& that_value) const; + bool AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, const char* this_offset, + const char* that_offset) const; + bool AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, const char* this_ptr, + const std::string& that_value) const; + + // Parses the input value according to the map for the struct at opt_addr + // struct_name is the name of the struct option as registered + // opt_name is the name of the option being evaluated. This may + // be the whole struct or a sub-element of it, based on struct_name and + // opt_name. + static Status ParseStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* map, + const std::string& opt_name, const std::string& value, char* opt_addr); + + // Serializes the input addr according to the map for the struct to value. + // struct_name is the name of the struct option as registered + // opt_name is the name of the option being evaluated. This may + // be the whole struct or a sub-element of it + static Status SerializeStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* map, + const std::string& opt_name, const char* opt_addr, std::string* value); + + // Compares the input offsets according to the map for the struct and returns + // true if they are equivalent, false otherwise. + // struct_name is the name of the struct option as registered + // opt_name is the name of the option being evaluated. This may + // be the whole struct or a sub-element of it + static bool StructsAreEqual( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* map, + const std::string& opt_name, const char* this_offset, + const char* that_offset, std::string* mismatch); + + // Finds the entry for the opt_name in the opt_map, returning + // nullptr if not found. + // If found, elem_name will be the name of option to find. + // This may be opt_name, or a substring of opt_name. + // For "simple" options, opt_name will be equal to elem_name. Given the + // opt_name "opt", elem_name will equal "opt". + // For "embedded" options (like structs), elem_name may be opt_name + // or a field within the opt_name. For example, given the struct "struct", + // and opt_name of "struct.field", elem_name will be "field" + static const OptionTypeInfo* Find( + const std::string& opt_name, + const std::unordered_map& opt_map, + std::string* elem_name); private: // The optional function to convert a string to its representation - ParserFunc parser_func; + ParseFunc parse_func_; // The optional function to convert a value to its string representation - StringFunc string_func; + SerializeFunc serialize_func_; - // The optional function to convert a match to option values - EqualsFunc equals_func; + // The optional function to match two option values + EqualsFunc equals_func_; - OptionType type; - OptionVerificationType verification; - OptionTypeFlags flags; + OptionType type_; + OptionVerificationType verification_; + OptionTypeFlags flags_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index a4c88da53..af2b55676 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -661,9 +661,9 @@ std::string ParseBlockBasedTableOption(const ConfigOptions& config_options, } } const auto& opt_info = iter->second; - Status s = opt_info.ParseOption( - config_options, iter->first, value, - reinterpret_cast(new_options) + opt_info.offset); + Status s = + opt_info.Parse(config_options, iter->first, value, + reinterpret_cast(new_options) + opt_info.offset_); if (s.ok()) { return ""; } else { @@ -757,14 +757,14 @@ Status VerifyBlockBasedTableFactory(const ConfigOptions& config_options, // contain random values since they might not be initialized if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) { const char* base_addr = - reinterpret_cast(&base_opt) + pair.second.offset; + reinterpret_cast(&base_opt) + pair.second.offset_; const char* file_addr = - reinterpret_cast(&file_opt) + pair.second.offset; + reinterpret_cast(&file_opt) + pair.second.offset_; - if (!pair.second.MatchesOption(config_options, pair.first, base_addr, - file_addr, &mismatch) && - !pair.second.MatchesByName(config_options, pair.first, base_addr, - file_addr)) { + if (!pair.second.AreEqual(config_options, pair.first, base_addr, + file_addr, &mismatch) && + !pair.second.AreEqualByName(config_options, pair.first, base_addr, + file_addr)) { return Status::Corruption( "[RocksDBOptionsParser]: " "failed the verification on BlockBasedTableOptions::", diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index 8b498a6da..98133f4bc 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -220,9 +220,9 @@ std::string ParsePlainTableOptions(const ConfigOptions& config_options, } } const auto& opt_info = iter->second; - Status s = opt_info.ParseOption( - config_options, name, value, - reinterpret_cast(new_options) + opt_info.offset); + Status s = + opt_info.Parse(config_options, name, value, + reinterpret_cast(new_options) + opt_info.offset_); if (s.ok()) { return ""; } else { diff --git a/util/string_util.cc b/util/string_util.cc index bb234f239..d098ecb0b 100644 --- a/util/string_util.cc +++ b/util/string_util.cc @@ -263,6 +263,20 @@ std::string trim(const std::string& str) { return std::string(); } +bool EndsWith(const std::string& string, const std::string& pattern) { + size_t plen = pattern.size(); + size_t slen = string.size(); + if (plen <= slen) { + return string.compare(slen - plen, plen, pattern) == 0; + } else { + return false; + } +} + +bool StartsWith(const std::string& string, const std::string& pattern) { + return string.compare(0, pattern.size(), pattern) == 0; +} + #ifndef ROCKSDB_LITE bool ParseBoolean(const std::string& type, const std::string& value) { diff --git a/util/string_util.h b/util/string_util.h index a761be66c..5ff516cac 100644 --- a/util/string_util.h +++ b/util/string_util.h @@ -111,6 +111,12 @@ std::string UnescapeOptionString(const std::string& escaped_string); std::string trim(const std::string& str); +// Returns true if "string" ends with "pattern" +bool EndsWith(const std::string& string, const std::string& pattern); + +// Returns true if "string" starts with "pattern" +bool StartsWith(const std::string& string, const std::string& pattern); + #ifndef ROCKSDB_LITE bool ParseBoolean(const std::string& type, const std::string& value);