From 0bb8ea56bed58603a09a36b50a8566faa54c5c6d Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Sun, 11 Oct 2015 12:17:42 -0700 Subject: [PATCH] [RocksDB Options File] Add TableOptions section and support BlockBasedTable Summary: Introduce TableOptions section and support BlockBasedTable in RocksDB options file. A TableOptions section has the following format: [TableOptions/ ""] which includes information about its TableFactory class and belonging column family. Below is an example TableOptions section of a BlockBasedTableOptions that belongs to the default column family: [TableOptions/BlockBasedTable "default"] format_version=0 whole_key_filtering=true block_size_deviation=10 block_size=4096 block_restart_interval=16 filter_policy=nullptr no_block_cache=false checksum=kCRC32c cache_index_and_filter_blocks=false index_type=kBinarySearch hash_index_allow_collision=true flush_block_policy_factory=FlushBlockBySizePolicyFactory Currently, Cache-type options (i.e., block_cache and block_cache_compressed) are not supported. Test Plan: options_test Reviewers: igor, anthony, IslamAbdelRahman, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D48435 --- examples/rocksdb_option_file_example.ini | 182 ++++++++++++-- include/rocksdb/convenience.h | 3 +- util/options_helper.cc | 292 +++++++++++++++++------ util/options_helper.h | 55 +++++ util/options_parser.cc | 141 ++++++++++- util/options_parser.h | 43 ++-- util/options_test.cc | 17 ++ 7 files changed, 609 insertions(+), 124 deletions(-) diff --git a/examples/rocksdb_option_file_example.ini b/examples/rocksdb_option_file_example.ini index ce74f77fd..8af1b021d 100644 --- a/examples/rocksdb_option_file_example.ini +++ b/examples/rocksdb_option_file_example.ini @@ -29,25 +29,167 @@ # # Below is an example of a RocksDB options file: [Version] - # The Version section stores the version information about rocksdb - # and option file. This is used for handling potential format - # change in the future. - rocksdb_version=4.0.0 # We support "#" style comment. - options_file_version=1.0 + rocksdb_version=4.0.0 + options_file_version=1.1 + [DBOptions] - # Followed by the Version section is the DBOptions section. - # The value of an options can be assigned using a statement. - # Note that for those options that is not set in the options file, - # we will use the default value. - max_open_files=12345 - max_background_flushes=301 + stats_dump_period_sec=600 + max_manifest_file_size=18446744073709551615 + bytes_per_sync=0 + delayed_write_rate=1048576 + WAL_ttl_seconds=0 + WAL_size_limit_MB=0 + max_subcompactions=1 + wal_dir= + wal_bytes_per_sync=0 + db_write_buffer_size=0 + max_total_wal_size=0 + skip_stats_update_on_db_open=false + max_open_files=5000 + max_file_opening_threads=1 + use_fsync=false + max_background_compactions=1 + manifest_preallocation_size=4194304 + max_background_flushes=1 + is_fd_close_on_exec=true + create_if_missing=false + use_adaptive_mutex=false + enable_thread_tracking=false + disableDataSync=false + max_log_file_size=0 + advise_random_on_open=true + create_missing_column_families=false + keep_log_file_num=1000 + table_cache_numshardbits=4 + error_if_exists=false + skip_log_error_on_recovery=false + allow_os_buffer=true + allow_mmap_reads=false + paranoid_checks=true + delete_obsolete_files_period_micros=21600000000 + disable_data_sync=false + log_file_time_to_roll=0 + compaction_readahead_size=0 + db_log_dir= + new_table_reader_for_compaction_inputs=false + allow_mmap_writes=false + [CFOptions "default"] - # ColumnFamilyOptions section must follow the format of - # [CFOptions "cf name"]. If a rocksdb instance - # has multiple column families, then its CFOptions must be - # specified in the same order as column family creation order. -[CFOptions "the second column family"] - # Each column family must have one section in the RocksDB option - # file even all the options of this column family are set to - # default value. -[CFOptions "the third column family"] + compaction_style=kCompactionStyleLevel + compaction_filter=nullptr + num_levels=7 + table_factory=BlockBasedTable + comparator=leveldb.BytewiseComparator + max_sequential_skip_in_iterations=8 + soft_rate_limit=0.000000 + max_bytes_for_level_base=536870912 + memtable_prefix_bloom_probes=6 + memtable_prefix_bloom_bits=0 + memtable_prefix_bloom_huge_page_tlb_size=0 + max_successive_merges=0 + arena_block_size=0 + min_write_buffer_number_to_merge=2 + target_file_size_multiplier=1 + source_compaction_factor=1 + max_bytes_for_level_multiplier=10 + compaction_filter_factory=nullptr + max_write_buffer_number=6 + level0_stop_writes_trigger=24 + compression=kSnappyCompression + level0_file_num_compaction_trigger=2 + purge_redundant_kvs_while_flush=true + max_write_buffer_number_to_maintain=0 + memtable_factory=SkipListFactory + max_grandparent_overlap_factor=10 + expanded_compaction_factor=25 + hard_pending_compaction_bytes_limit=0 + inplace_update_num_locks=10000 + level_compaction_dynamic_level_bytes=false + level0_slowdown_writes_trigger=20 + filter_deletes=false + verify_checksums_in_compaction=true + min_partial_merge_operands=2 + paranoid_file_checks=false + target_file_size_base=67108864 + optimize_filters_for_hits=false + merge_operator=nullptr + compression_per_level=kNoCompression:kNoCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression + compaction_measure_io_stats=false + prefix_extractor=nullptr + bloom_locality=0 + write_buffer_size=134217728 + disable_auto_compactions=false + inplace_update_support=false +[TableOptions/BlockBasedTable "default"] + format_version=0 + whole_key_filtering=true + block_size_deviation=10 + block_size=4096 + block_restart_interval=16 + filter_policy=nullptr + no_block_cache=false + checksum=kCRC32c + cache_index_and_filter_blocks=false + index_type=kBinarySearch + hash_index_allow_collision=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + +[CFOptions "universal"] + compaction_style=kCompactionStyleUniversal + compaction_filter=nullptr + num_levels=7 + table_factory=BlockBasedTable + comparator=leveldb.BytewiseComparator + max_sequential_skip_in_iterations=8 + soft_rate_limit=0.000000 + max_bytes_for_level_base=10485760 + memtable_prefix_bloom_probes=6 + memtable_prefix_bloom_bits=0 + memtable_prefix_bloom_huge_page_tlb_size=0 + max_successive_merges=0 + arena_block_size=0 + min_write_buffer_number_to_merge=2 + target_file_size_multiplier=1 + source_compaction_factor=1 + max_bytes_for_level_multiplier=10 + compaction_filter_factory=nullptr + max_write_buffer_number=6 + level0_stop_writes_trigger=24 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + purge_redundant_kvs_while_flush=true + max_write_buffer_number_to_maintain=0 + memtable_factory=SkipListFactory + max_grandparent_overlap_factor=10 + expanded_compaction_factor=25 + hard_pending_compaction_bytes_limit=0 + inplace_update_num_locks=10000 + level_compaction_dynamic_level_bytes=false + level0_slowdown_writes_trigger=20 + filter_deletes=false + verify_checksums_in_compaction=true + min_partial_merge_operands=2 + paranoid_file_checks=false + target_file_size_base=2097152 + optimize_filters_for_hits=false + merge_operator=nullptr + compression_per_level= + compaction_measure_io_stats=false + prefix_extractor=nullptr + bloom_locality=0 + write_buffer_size=134217728 + disable_auto_compactions=false + inplace_update_support=false +[TableOptions/BlockBasedTable "universal"] + format_version=0 + whole_key_filtering=true + block_size_deviation=10 + block_size=4096 + block_restart_interval=16 + filter_policy=nullptr + no_block_cache=false + checksum=kCRC32c + cache_index_and_filter_blocks=false + index_type=kBinarySearch + hash_index_allow_collision=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index db597279e..f7b845555 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -40,7 +40,8 @@ Status GetDBOptionsFromMap( Status GetBlockBasedTableOptionsFromMap( const BlockBasedTableOptions& table_options, const std::unordered_map& opts_map, - BlockBasedTableOptions* new_table_options); + BlockBasedTableOptions* new_table_options, + bool input_strings_escaped = false); // Take a string representation of option names and values, apply them into the // base_options, and return the new options as a result. The string has the diff --git a/util/options_helper.cc b/util/options_helper.cc index 78ae59992..5809ff3c3 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -174,26 +174,52 @@ bool ParseCompressionType(const std::string& string_value, return true; } -BlockBasedTableOptions::IndexType ParseBlockBasedTableIndexType( - const std::string& type) { - if (type == "kBinarySearch") { - return BlockBasedTableOptions::kBinarySearch; - } else if (type == "kHashSearch") { - return BlockBasedTableOptions::kHashSearch; +bool SerializeBlockBasedTableIndexType( + const BlockBasedTableOptions::IndexType& type, std::string* value) { + switch (type) { + case BlockBasedTableOptions::kBinarySearch: + *value = "kBinarySearch"; + return true; + case BlockBasedTableOptions::kHashSearch: + *value = "kHashSearch"; + return true; + default: + return false; } - throw std::invalid_argument("Unknown index type: " + type); } -ChecksumType ParseBlockBasedTableChecksumType( - const std::string& type) { - if (type == "kNoChecksum") { - return kNoChecksum; - } else if (type == "kCRC32c") { - return kCRC32c; - } else if (type == "kxxHash") { - return kxxHash; +bool ParseBlockBasedTableIndexType(const std::string& type, + BlockBasedTableOptions::IndexType* value) { + if (type == "kBinarySearch") { + *value = BlockBasedTableOptions::kBinarySearch; + } else if (type == "kHashSearch") { + *value = BlockBasedTableOptions::kHashSearch; + } else { + return false; } - throw std::invalid_argument("Unknown checksum type: " + type); + return true; +} + +static std::unordered_map checksum_type_map = { + {"kNoChecksum", kNoChecksum}, {"kCRC32c", kCRC32c}, {"kxxHash", kxxHash}}; + +bool ParseChecksumType(const std::string& type, ChecksumType* value) { + auto iter = checksum_type_map.find(type); + if (iter != checksum_type_map.end()) { + *value = iter->second; + return true; + } + return false; +} + +bool SerializeChecksumType(const ChecksumType& type, std::string* value) { + for (const auto& pair : checksum_type_map) { + if (pair.second == type) { + *value = pair.first; + return true; + } + } + return false; } bool ParseBoolean(const std::string& type, const std::string& value) { @@ -328,6 +354,7 @@ bool ParseSliceTransformHelper( const std::string& kFixedPrefixName, const std::string& kCappedPrefixName, const std::string& value, std::shared_ptr* slice_transform) { + static const std::string kNullptrString = "nullptr"; auto& pe_value = value; if (pe_value.size() > kFixedPrefixName.size() && pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) { @@ -339,7 +366,7 @@ bool ParseSliceTransformHelper( int prefix_length = ParseInt(trim(pe_value.substr(kCappedPrefixName.size()))); slice_transform->reset(NewCappedPrefixTransform(prefix_length)); - } else if (value == "nullptr") { + } else if (value == kNullptrString) { slice_transform->reset(); } else { return false; @@ -414,6 +441,13 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, return ParseSliceTransform( value, reinterpret_cast*>( opt_address)); + case OptionType::kChecksumType: + return ParseChecksumType(value, + reinterpret_cast(opt_address)); + case OptionType::kBlockBasedTableIndexType: + return ParseBlockBasedTableIndexType( + value, + reinterpret_cast(opt_address)); default: return false; } @@ -425,6 +459,7 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, bool SerializeSingleOptionHelper(const char* opt_address, const OptionType opt_type, std::string* value) { + static const std::string kNullptrString = kNullptrString; assert(value); switch (opt_type) { case OptionType::kBoolean: @@ -469,7 +504,7 @@ bool SerializeSingleOptionHelper(const char* opt_address, reinterpret_cast*>( opt_address); *value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name() - : "nullptr"; + : kNullptrString; break; } case OptionType::kTableFactory: { @@ -477,40 +512,61 @@ bool SerializeSingleOptionHelper(const char* opt_address, reinterpret_cast*>( opt_address); *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name() - : "nullptr"; + : kNullptrString; break; } case OptionType::kComparator: { // it's a const pointer of const Comparator* const auto* ptr = reinterpret_cast(opt_address); - *value = *ptr ? (*ptr)->Name() : "nullptr"; + *value = *ptr ? (*ptr)->Name() : kNullptrString; break; } case OptionType::kCompactionFilter: { // it's a const pointer of const CompactionFilter* const auto* ptr = reinterpret_cast(opt_address); - *value = *ptr ? (*ptr)->Name() : "nullptr"; + *value = *ptr ? (*ptr)->Name() : kNullptrString; break; } case OptionType::kCompactionFilterFactory: { const auto* ptr = reinterpret_cast*>( opt_address); - *value = ptr->get() ? ptr->get()->Name() : "nullptr"; + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; break; } case OptionType::kMemTableRepFactory: { const auto* ptr = reinterpret_cast*>( opt_address); - *value = ptr->get() ? ptr->get()->Name() : "nullptr"; + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; break; } case OptionType::kMergeOperator: { const auto* ptr = reinterpret_cast*>(opt_address); - *value = ptr->get() ? ptr->get()->Name() : "nullptr"; + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; + break; + } + case OptionType::kFilterPolicy: { + const auto* ptr = + reinterpret_cast*>(opt_address); + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; + break; + } + case OptionType::kChecksumType: + return SerializeChecksumType( + *reinterpret_cast(opt_address), value); + case OptionType::kBlockBasedTableIndexType: + return SerializeBlockBasedTableIndexType( + *reinterpret_cast( + opt_address), + value); + case OptionType::kFlushBlockPolicyFactory: { + const auto* ptr = + reinterpret_cast*>( + opt_address); + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; break; } default: @@ -881,6 +937,59 @@ Status GetStringFromColumnFamilyOptions(std::string* opt_string, return Status::OK(); } +bool SerializeSingleBlockBasedTableOption( + std::string* opt_string, const BlockBasedTableOptions& bbt_options, + const std::string& name, const std::string& delimiter) { + auto iter = block_based_table_type_info.find(name); + if (iter == block_based_table_type_info.end()) { + return false; + } + auto& opt_info = iter->second; + const char* opt_address = + reinterpret_cast(&bbt_options) + opt_info.offset; + std::string value; + bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); + if (result) { + *opt_string = name + "=" + value + delimiter; + } + return result; +} + +Status GetStringFromBlockBasedTableOptions( + std::string* opt_string, const BlockBasedTableOptions& bbt_options, + const std::string& delimiter) { + assert(opt_string); + opt_string->clear(); + for (auto iter = block_based_table_type_info.begin(); + iter != block_based_table_type_info.end(); ++iter) { + if (iter->second.verification == OptionVerificationType::kDeprecated) { + // If the option is no longer used in rocksdb and marked as deprecated, + // we skip it in the serialization. + continue; + } + std::string single_output; + bool result = SerializeSingleBlockBasedTableOption( + &single_output, bbt_options, iter->first, delimiter); + assert(result); + if (result) { + opt_string->append(single_output); + } + } + return Status::OK(); +} + +Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf, + const std::string& delimiter) { + const auto* bbtf = dynamic_cast(tf); + opts_str->clear(); + if (bbtf != nullptr) { + return GetStringFromBlockBasedTableOptions( + opts_str, bbtf->GetTableOptions(), delimiter); + } + + return Status::OK(); +} + bool ParseDBOption(const std::string& name, const std::string& org_value, DBOptions* new_options, bool input_string_escaped = false) { const std::string& value = @@ -908,67 +1017,73 @@ bool ParseDBOption(const std::string& name, const std::string& org_value, return true; } +std::string ParseBlockBasedTableOption(const std::string& name, + const std::string& org_value, + BlockBasedTableOptions* new_options, + bool input_string_escaped = false) { + const std::string& value = + input_string_escaped ? UnescapeOptionString(org_value) : org_value; + if (!input_string_escaped) { + // if the input string is not escaped, it means this function is + // invoked from SetOptions, which takes the old format. + if (name == "block_cache") { + new_options->block_cache = NewLRUCache(ParseSizeT(value)); + return ""; + } else if (name == "block_cache_compressed") { + new_options->block_cache_compressed = NewLRUCache(ParseSizeT(value)); + return ""; + } else if (name == "filter_policy") { + // Expect the following format + // bloomfilter:int:bool + const std::string kName = "bloomfilter:"; + if (value.compare(0, kName.size(), kName) != 0) { + return "Invalid filter policy name"; + } + size_t pos = value.find(':', kName.size()); + if (pos == std::string::npos) { + return "Invalid filter policy config, missing bits_per_key"; + } + int bits_per_key = + ParseInt(trim(value.substr(kName.size(), pos - kName.size()))); + bool use_block_based_builder = + ParseBoolean("use_block_based_builder", trim(value.substr(pos + 1))); + new_options->filter_policy.reset( + NewBloomFilterPolicy(bits_per_key, use_block_based_builder)); + return ""; + } + } + const auto iter = block_based_table_type_info.find(name); + if (iter == block_based_table_type_info.end()) { + return "Unrecognized option"; + } + const auto& opt_info = iter->second; + if (!ParseOptionHelper(reinterpret_cast(new_options) + opt_info.offset, + opt_info.type, value)) { + return "Invalid value"; + } + return ""; +} + Status GetBlockBasedTableOptionsFromMap( const BlockBasedTableOptions& table_options, const std::unordered_map& opts_map, - BlockBasedTableOptions* new_table_options) { - + BlockBasedTableOptions* new_table_options, bool input_strings_escaped) { assert(new_table_options); *new_table_options = table_options; for (const auto& o : opts_map) { - try { - if (o.first == "cache_index_and_filter_blocks") { - new_table_options->cache_index_and_filter_blocks = - ParseBoolean(o.first, o.second); - } else if (o.first == "index_type") { - new_table_options->index_type = ParseBlockBasedTableIndexType(o.second); - } else if (o.first == "hash_index_allow_collision") { - new_table_options->hash_index_allow_collision = - ParseBoolean(o.first, o.second); - } else if (o.first == "checksum") { - new_table_options->checksum = - ParseBlockBasedTableChecksumType(o.second); - } else if (o.first == "no_block_cache") { - new_table_options->no_block_cache = ParseBoolean(o.first, o.second); - } else if (o.first == "block_cache") { - new_table_options->block_cache = NewLRUCache(ParseSizeT(o.second)); - } else if (o.first == "block_cache_compressed") { - new_table_options->block_cache_compressed = - NewLRUCache(ParseSizeT(o.second)); - } else if (o.first == "block_size") { - new_table_options->block_size = ParseSizeT(o.second); - } else if (o.first == "block_size_deviation") { - new_table_options->block_size_deviation = ParseInt(o.second); - } else if (o.first == "block_restart_interval") { - new_table_options->block_restart_interval = ParseInt(o.second); - } else if (o.first == "filter_policy") { - // Expect the following format - // bloomfilter:int:bool - const std::string kName = "bloomfilter:"; - if (o.second.compare(0, kName.size(), kName) != 0) { - return Status::InvalidArgument("Invalid filter policy name"); - } - size_t pos = o.second.find(':', kName.size()); - if (pos == std::string::npos) { - return Status::InvalidArgument("Invalid filter policy config, " - "missing bits_per_key"); - } - int bits_per_key = ParseInt( - trim(o.second.substr(kName.size(), pos - kName.size()))); - bool use_block_based_builder = - ParseBoolean("use_block_based_builder", - trim(o.second.substr(pos + 1))); - new_table_options->filter_policy.reset( - NewBloomFilterPolicy(bits_per_key, use_block_based_builder)); - } else if (o.first == "whole_key_filtering") { - new_table_options->whole_key_filtering = - ParseBoolean(o.first, o.second); - } else { - return Status::InvalidArgument("Unrecognized option: " + o.first); + auto error_message = ParseBlockBasedTableOption( + o.first, o.second, new_table_options, input_strings_escaped); + if (error_message != "") { + const auto iter = block_based_table_type_info.find(o.first); + if (iter == block_based_table_type_info.end() || + !input_strings_escaped || // !input_strings_escaped indicates + // the old API, where everything is + // parsable. + (iter->second.verification != OptionVerificationType::kByName && + iter->second.verification != OptionVerificationType::kDeprecated)) { + return Status::InvalidArgument("Can't parse BlockBasedTableOptions:", + o.first + " " + error_message); } - } catch (std::exception& e) { - return Status::InvalidArgument("error parsing " + o.first + ":" + - std::string(e.what())); } } return Status::OK(); @@ -1110,5 +1225,26 @@ Status GetOptionsFromString(const Options& base_options, return Status::OK(); } +Status GetTableFactoryFromMap( + const std::string& factory_name, + const std::unordered_map& opt_map, + std::shared_ptr* table_factory) { + Status s; + if (factory_name == BlockBasedTableFactory().Name()) { + BlockBasedTableOptions bbt_opt; + s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map, + &bbt_opt, true); + if (!s.ok()) { + return s; + } + table_factory->reset(new BlockBasedTableFactory(bbt_opt)); + return Status::OK(); + } + // Return OK for not supported table factories as TableFactory + // Deserialization is optional. + table_factory->reset(); + return Status::OK(); +} + #endif // !ROCKSDB_LITE } // namespace rocksdb diff --git a/util/options_helper.h b/util/options_helper.h index d72a375f1..ec567bc7c 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -9,6 +9,7 @@ #include #include "rocksdb/options.h" #include "rocksdb/status.h" +#include "rocksdb/table.h" #include "util/mutable_cf_options.h" #ifndef ROCKSDB_LITE @@ -55,6 +56,14 @@ Status GetMutableOptionsFromStrings( const std::unordered_map& options_map, MutableCFOptions* new_options); +Status GetTableFactoryFromMap( + const std::string& factory_name, + const std::unordered_map& opt_map, + std::shared_ptr* table_factory); + +Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf, + const std::string& delimiter = "; "); + enum class OptionType { kBoolean, kInt, @@ -74,6 +83,10 @@ enum class OptionType { kCompactionFilterFactory, kMergeOperator, kMemTableRepFactory, + kBlockBasedTableIndexType, + kFilterPolicy, + kFlushBlockPolicyFactory, + kChecksumType, kUnknown }; @@ -401,6 +414,48 @@ static std::unordered_map cf_options_type_info = { {offsetof(struct ColumnFamilyOptions, compaction_style), OptionType::kCompactionStyle, OptionVerificationType::kNormal}}}; +static std::unordered_map block_based_table_type_info = { + /* currently not supported + std::shared_ptr block_cache = nullptr; + std::shared_ptr block_cache_compressed = nullptr; + */ + {"flush_block_policy_factory", + {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), + OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName}}, + {"cache_index_and_filter_blocks", + {offsetof(struct BlockBasedTableOptions, cache_index_and_filter_blocks), + OptionType::kBoolean, OptionVerificationType::kNormal}}, + {"index_type", + {offsetof(struct BlockBasedTableOptions, index_type), + OptionType::kBlockBasedTableIndexType, OptionVerificationType::kNormal}}, + {"hash_index_allow_collision", + {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), + OptionType::kBoolean, OptionVerificationType::kNormal}}, + {"checksum", + {offsetof(struct BlockBasedTableOptions, checksum), + OptionType::kChecksumType, OptionVerificationType::kNormal}}, + {"no_block_cache", + {offsetof(struct BlockBasedTableOptions, no_block_cache), + OptionType::kBoolean, OptionVerificationType::kNormal}}, + {"block_size", + {offsetof(struct BlockBasedTableOptions, block_size), OptionType::kSizeT, + OptionVerificationType::kNormal}}, + {"block_size_deviation", + {offsetof(struct BlockBasedTableOptions, block_size_deviation), + OptionType::kInt, OptionVerificationType::kNormal}}, + {"block_restart_interval", + {offsetof(struct BlockBasedTableOptions, block_restart_interval), + OptionType::kInt, OptionVerificationType::kNormal}}, + {"filter_policy", + {offsetof(struct BlockBasedTableOptions, filter_policy), + OptionType::kFilterPolicy, OptionVerificationType::kByName}}, + {"whole_key_filtering", + {offsetof(struct BlockBasedTableOptions, whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal}}, + {"format_version", + {offsetof(struct BlockBasedTableOptions, format_version), + OptionType::kUInt32T, OptionVerificationType::kNormal}}}; } // namespace rocksdb #endif // !ROCKSDB_LITE diff --git a/util/options_parser.cc b/util/options_parser.cc index d79255415..c8439c6fd 100644 --- a/util/options_parser.cc +++ b/util/options_parser.cc @@ -66,6 +66,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, writable->Append(options_file_content + "\n"); for (size_t i = 0; i < cf_opts.size(); ++i) { + // CFOptions section writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i], @@ -75,6 +76,18 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, return s; } writable->Append(options_file_content + "\n"); + // TableOptions section + auto* tf = cf_opts[i].table_factory.get(); + if (tf != nullptr) { + writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] + + tf->Name() + " \"" + EscapeOptionString(cf_names[i]) + + "\"]\n "); + s = GetStringFromTableFactory(&options_file_content, tf, "\n "); + if (!s.ok()) { + return s; + } + writable->Append(options_file_content + "\n"); + } } writable->Flush(); writable->Fsync(); @@ -112,11 +125,11 @@ bool RocksDBOptionsParser::IsSection(const std::string& line) { } Status RocksDBOptionsParser::ParseSection(OptionSection* section, + std::string* title, std::string* argument, const std::string& line, const int line_num) { *section = kOptionSectionUnknown; - std::string sec_string; // A section is of the form [ ""], where // "" is optional. size_t arg_start_pos = line.find("\""); @@ -124,17 +137,30 @@ Status RocksDBOptionsParser::ParseSection(OptionSection* section, // The following if-then check tries to identify whether the input // section has the optional section argument. if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) { - sec_string = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true); + *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true); *argument = UnescapeOptionString( line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1)); } else { - sec_string = TrimAndRemoveComment(line.substr(1, line.size() - 2), true); + *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true); *argument = ""; } for (int i = 0; i < kOptionSectionUnknown; ++i) { - if (opt_section_titles[i] == sec_string) { - *section = static_cast(i); - return CheckSection(*section, *argument, line_num); + if (title->find(opt_section_titles[i]) == 0) { + if (i == kOptionSectionVersion || i == kOptionSectionDBOptions || + i == kOptionSectionCFOptions) { + if (title->size() == opt_section_titles[i].size()) { + // if true, then it indicats equal + *section = static_cast(i); + return CheckSection(*section, *argument, line_num); + } + } else if (i == kOptionSectionTableOptions) { + // This type of sections has a sufffix at the end of the + // section title + if (title->size() > opt_section_titles[i].size()) { + *section = static_cast(i); + return CheckSection(*section, *argument, line_num); + } + } } } return Status::InvalidArgument(std::string("Unknown section ") + line); @@ -215,6 +241,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) { } OptionSection section = kOptionSectionUnknown; + std::string title; std::string argument; std::unordered_map opt_map; std::istringstream iss; @@ -231,12 +258,12 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) { continue; } if (IsSection(line)) { - s = EndSection(section, argument, opt_map); + s = EndSection(section, title, argument, opt_map); opt_map.clear(); if (!s.ok()) { return s; } - s = ParseSection(§ion, &argument, line, line_num); + s = ParseSection(§ion, &title, &argument, line, line_num); if (!s.ok()) { return s; } @@ -251,7 +278,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) { } } - s = EndSection(section, argument, opt_map); + s = EndSection(section, title, argument, opt_map); opt_map.clear(); if (!s.ok()) { return s; @@ -280,13 +307,21 @@ Status RocksDBOptionsParser::CheckSection(const OptionSection section, return InvalidArgument( line_num, "Default column family must be the first CFOptions section " - "in the option config file"); + "in the optio/n config file"); } else if (GetCFOptions(section_arg) != nullptr) { return InvalidArgument( line_num, "Two identical column families found in option config file"); } has_default_cf_options_ |= is_default_cf; + } else if (section == kOptionSectionTableOptions) { + if (GetCFOptions(section_arg) == nullptr) { + return InvalidArgument( + line_num, std::string( + "Does not find a matched column family name in " + "TableOptions section. Column Family Name:") + + section_arg); + } } else if (section == kOptionSectionVersion) { if (has_version_section_) { return InvalidArgument( @@ -350,7 +385,8 @@ Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name, } Status RocksDBOptionsParser::EndSection( - const OptionSection section, const std::string& section_arg, + const OptionSection section, const std::string& section_title, + const std::string& section_arg, const std::unordered_map& opt_map) { Status s; if (section == kOptionSectionDBOptions) { @@ -372,6 +408,23 @@ Status RocksDBOptionsParser::EndSection( } // keep the parsed string. cf_opt_maps_.emplace_back(opt_map); + } else if (section == kOptionSectionTableOptions) { + assert(GetCFOptions(section_arg) != nullptr); + auto* cf_opt = GetCFOptionsImpl(section_arg); + if (cf_opt == nullptr) { + return Status::InvalidArgument( + "The specified column family must be defined before the " + "TableOptions section:", + section_arg); + } + // Ignore error as table factory deserialization is optional + s = GetTableFactoryFromMap( + section_title.substr( + opt_section_titles[kOptionSectionTableOptions].size()), + opt_map, &(cf_opt->table_factory)); + if (!s.ok()) { + return s; + } } else if (section == kOptionSectionVersion) { for (const auto pair : opt_map) { if (pair.first == "rocksdb_version") { @@ -493,6 +546,14 @@ bool AreEqualOptions( reinterpret_cast*>(offset2); return (*vec1 == *vec2); } + case OptionType::kChecksumType: + return (*reinterpret_cast(offset1) == + *reinterpret_cast(offset2)); + case OptionType::kBlockBasedTableIndexType: + return ( + *reinterpret_cast( + offset1) == + *reinterpret_cast(offset2)); default: if (type_info.verification == OptionVerificationType::kByName) { std::string value1; @@ -561,6 +622,11 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( if (!s.ok()) { return s; } + s = VerifyTableFactory(cf_opts[i].table_factory.get(), + parser.cf_opts()->at(i).table_factory.get()); + if (!s.ok()) { + return s; + } } return Status::OK(); @@ -607,6 +673,59 @@ Status RocksDBOptionsParser::VerifyCFOptions( } return Status::OK(); } + +Status RocksDBOptionsParser::VerifyBlockBasedTableFactory( + const BlockBasedTableFactory* base_tf, + const BlockBasedTableFactory* file_tf) { + if ((base_tf != nullptr) != (file_tf != nullptr)) { + return Status::Corruption( + "[RocksDBOptionsParser]: Inconsistent TableFactory class type"); + } + if (base_tf == nullptr) { + return Status::OK(); + } + + const auto& base_opt = base_tf->GetTableOptions(); + const auto& file_opt = file_tf->GetTableOptions(); + + for (auto& pair : block_based_table_type_info) { + if (pair.second.verification == OptionVerificationType::kDeprecated) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + continue; + } + if (!AreEqualOptions(reinterpret_cast(&base_opt), + reinterpret_cast(&file_opt), pair.second, + pair.first, nullptr)) { + return Status::Corruption( + "[RocksDBOptionsParser]: " + "failed the verification on BlockBasedTableOptions::", + pair.first); + } + } + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyTableFactory(const TableFactory* base_tf, + const TableFactory* file_tf) { + if (base_tf && file_tf) { + if (base_tf->Name() != file_tf->Name()) { + return Status::Corruption( + "[RocksDBOptionsParser]: " + "failed the verification on TableFactory->Name()"); + } + auto s = VerifyBlockBasedTableFactory( + dynamic_cast(base_tf), + dynamic_cast(file_tf)); + if (!s.ok()) { + return s; + } + // TODO(yhchiang): add checks for other table factory types + } else { + // TODO(yhchiang): further support sanity check here + } + return Status::OK(); +} } // namespace rocksdb #endif // !ROCKSDB_LITE diff --git a/util/options_parser.h b/util/options_parser.h index f308fcb51..4ceae7254 100644 --- a/util/options_parser.h +++ b/util/options_parser.h @@ -11,23 +11,25 @@ #include "rocksdb/env.h" #include "rocksdb/options.h" +#include "table/block_based_table_factory.h" namespace rocksdb { #ifndef ROCKSDB_LITE #define ROCKSDB_OPTION_FILE_MAJOR 1 -#define ROCKSDB_OPTION_FILE_MINOR 0 +#define ROCKSDB_OPTION_FILE_MINOR 1 enum OptionSection : char { kOptionSectionVersion = 0, kOptionSectionDBOptions, kOptionSectionCFOptions, + kOptionSectionTableOptions, kOptionSectionUnknown }; -static const std::string opt_section_titles[] = {"Version", "DBOptions", - "CFOptions", "Unknown"}; +static const std::string opt_section_titles[] = { + "Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"}; Status PersistRocksDBOptions(const DBOptions& db_opt, const std::vector& cf_names, @@ -55,14 +57,8 @@ class RocksDBOptionsParser { return &cf_opt_maps_; } - const ColumnFamilyOptions* GetCFOptions(const std::string& name) const { - assert(cf_names_.size() == cf_opts_.size()); - for (size_t i = 0; i < cf_names_.size(); ++i) { - if (cf_names_[i] == name) { - return &cf_opts_[i]; - } - } - return nullptr; + const ColumnFamilyOptions* GetCFOptions(const std::string& name) { + return GetCFOptionsImpl(name); } size_t NumColumnFamilies() { return cf_opts_.size(); } @@ -81,12 +77,20 @@ class RocksDBOptionsParser { const std::unordered_map* new_opt_map = nullptr); + static Status VerifyTableFactory(const TableFactory* base_tf, + const TableFactory* file_tf); + + static Status VerifyBlockBasedTableFactory( + const BlockBasedTableFactory* base_tf, + const BlockBasedTableFactory* file_tf); + static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); protected: bool IsSection(const std::string& line); - Status ParseSection(OptionSection* section, std::string* argument, - const std::string& line, const int line_num); + Status ParseSection(OptionSection* section, std::string* title, + std::string* argument, const std::string& line, + const int line_num); Status CheckSection(const OptionSection section, const std::string& section_arg, const int line_num); @@ -95,7 +99,8 @@ class RocksDBOptionsParser { const std::string& line, const int line_num); Status EndSection( - const OptionSection section, const std::string& section_arg, + const OptionSection section, const std::string& title, + const std::string& section_arg, const std::unordered_map& opt_map); Status ValidityCheck(); @@ -106,6 +111,16 @@ class RocksDBOptionsParser { const std::string& ver_string, const int max_count, int* version); + ColumnFamilyOptions* GetCFOptionsImpl(const std::string& name) { + assert(cf_names_.size() == cf_opts_.size()); + for (size_t i = 0; i < cf_names_.size(); ++i) { + if (cf_names_[i] == name) { + return &cf_opts_[i]; + } + } + return nullptr; + } + private: DBOptions db_opt_; std::unordered_map db_opt_map_; diff --git a/util/options_test.cc b/util/options_test.cc index ee1354089..48f56cea3 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -1565,6 +1565,23 @@ TEST_F(OptionsParserTest, DumpAndParse) { } } +TEST_F(OptionsParserTest, DifferentDefault) { + const std::string kOptionsFileName = "test-persisted-options.ini"; + + ColumnFamilyOptions cf_level_opts; + cf_level_opts.OptimizeLevelStyleCompaction(); + + ColumnFamilyOptions cf_univ_opts; + cf_univ_opts.OptimizeUniversalStyleCompaction(); + + ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"}, + {cf_level_opts, cf_univ_opts}, + kOptionsFileName, env_.get())); + + RocksDBOptionsParser parser; + ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); +} + namespace { bool IsEscapedString(const std::string& str) { for (size_t i = 0; i < str.size(); ++i) {