From 7d472accdca996d7d83ae8ce78ad17f799696926 Mon Sep 17 00:00:00 2001 From: mrambacher Date: Mon, 14 Sep 2020 16:59:00 -0700 Subject: [PATCH] Bring the Configurable options together (#5753) Summary: This PR merges the functionality of making the ColumnFamilyOptions, TableFactory, and DBOptions into Configurable into a single PR, resolving any merge conflicts Pull Request resolved: https://github.com/facebook/rocksdb/pull/5753 Reviewed By: ajkr Differential Revision: D23385030 Pulled By: zhichao-cao fbshipit-source-id: 8b977a7731556230b9b8c5a081b98e49ee4f160a --- .circleci/config.yml | 2 +- CMakeLists.txt | 3 + HISTORY.md | 7 +- Makefile | 5 + TARGETS | 9 + cache/cache.cc | 15 +- db/column_family.cc | 12 +- db/db_basic_test.cc | 18 +- db/db_block_cache_test.cc | 22 +- db/db_bloom_filter_test.cc | 17 +- db/db_impl/db_impl_open.cc | 8 +- db/db_iterator_test.cc | 2 +- db/db_options_test.cc | 25 +- db/db_test.cc | 4 +- db/db_test2.cc | 10 +- db/db_test_util.h | 2 - db/flush_job.cc | 2 - db/internal_stats.cc | 18 +- db/listener_test.cc | 2 - db/table_properties_collector_test.cc | 4 +- examples/options_file_example.cc | 5 +- include/rocksdb/configurable.h | 364 ++++++++ include/rocksdb/convenience.h | 6 + include/rocksdb/table.h | 62 +- include/rocksdb/utilities/object_registry.h | 12 +- .../rocksdb/utilities}/options_type.h | 147 ++- options/cf_options.cc | 834 ++++++++++-------- options/cf_options.h | 2 + options/configurable.cc | 610 +++++++++++++ options/configurable_helper.h | 211 +++++ options/configurable_test.cc | 791 +++++++++++++++++ options/configurable_test.h | 127 +++ options/db_options.cc | 551 +++++++----- options/db_options.h | 2 + options/options.cc | 2 +- options/options_helper.cc | 452 +++++----- options/options_helper.h | 72 +- options/options_parser.cc | 203 ++--- options/options_parser.h | 8 +- options/options_settable_test.cc | 3 + options/options_test.cc | 355 ++++---- src.mk | 3 + table/adaptive/adaptive_table_factory.cc | 10 +- table/adaptive/adaptive_table_factory.h | 9 +- .../block_based/block_based_table_factory.cc | 255 +++--- table/block_based/block_based_table_factory.h | 32 +- .../block_based_table_reader_test.cc | 9 +- table/block_based/filter_policy.cc | 2 +- table/block_based/reader_common.h | 1 + table/block_fetcher_test.cc | 9 +- table/cuckoo/cuckoo_table_factory.cc | 36 +- table/cuckoo/cuckoo_table_factory.h | 22 +- table/mock_table.h | 8 +- table/plain/plain_table_factory.cc | 99 +-- table/plain/plain_table_factory.h | 28 +- table/sst_file_dumper.cc | 3 +- table/table_factory.cc | 49 + table/table_test.cc | 4 +- test_util/testutil.cc | 1 + tools/db_bench_tool.cc | 8 +- tools/trace_analyzer_tool.cc | 1 - util/timer_test.cc | 2 +- utilities/memory/memory_test.cc | 10 +- utilities/options/options_util.cc | 10 +- utilities/options/options_util_test.cc | 47 +- utilities/simulator_cache/sim_cache_test.cc | 6 +- 66 files changed, 3958 insertions(+), 1712 deletions(-) create mode 100644 include/rocksdb/configurable.h rename {options => include/rocksdb/utilities}/options_type.h (85%) create mode 100644 options/configurable.cc create mode 100644 options/configurable_helper.h create mode 100644 options/configurable_test.cc create mode 100644 options/configurable_test.h create mode 100644 table/table_factory.cc diff --git a/.circleci/config.yml b/.circleci/config.yml index d656a1961..f486a5a62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: executors: windows-2xlarge: machine: - image: 'windows-server-2019-vs2019:201908-06' + image: 'windows-server-2019-vs2019:stable' resource_class: windows.2xlarge shell: bash.exe diff --git a/CMakeLists.txt b/CMakeLists.txt index bf2050449..91625bd30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -677,6 +677,7 @@ set(SOURCES monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc + options/configurable.cc options/db_options.cc options/options.cc options/options_helper.cc @@ -727,6 +728,7 @@ set(SOURCES table/sst_file_dumper.cc table/sst_file_reader.cc table/sst_file_writer.cc + table/table_factory.cc table/table_properties.cc table/two_level_iterator.cc test_util/sync_point.cc @@ -1125,6 +1127,7 @@ if(WITH_TESTS) monitoring/statistics_test.cc monitoring/stats_dump_scheduler_test.cc monitoring/stats_history_test.cc + options/configurable_test.cc options/options_settable_test.cc options/options_test.cc table/block_based/block_based_filter_block_test.cc diff --git a/HISTORY.md b/HISTORY.md index b70f341e1..808d799c1 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -43,8 +43,7 @@ * `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the LOG file on the line starting with "DB Session ID:". * `DB::OpenForReadOnly()` now returns `Status::NotFound` when the specified DB directory does not exist. Previously the error returned depended on the underlying `Env`. This change is available in all 6.11 releases as well. * A parameter `verify_with_checksum` is added to `BackupEngine::VerifyBackup`, which is false by default. If it is ture, `BackupEngine::VerifyBackup` verifies checksums and file sizes of backup files. Pass `false` for `verify_with_checksum` to maintain the previous behavior and performance of `BackupEngine::VerifyBackup`, by only verifying sizes of backup files. - - +* Methods to configure serialize, and compare -- such as TableFactory -- are exposed directly through the Configurable base class (from which these objects inherity). This change will allow for better and more thorough configuration management and retrieval in the future ### Behavior Changes * Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s). * In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early. @@ -68,12 +67,16 @@ * Added auto resume function to automatically recover the DB from background Retryable IO Error. When retryable IOError happens during flush and WAL write, the error is mapped to Hard Error and DB will be in read mode. When retryable IO Error happens during compaction, the error will be mapped to Soft Error. DB is still in write/read mode. Autoresume function will create a thread for a DB to call DB->ResumeImpl() to try the recover for Retryable IO Error during flush and WAL write. Compaction will be rescheduled by itself if retryable IO Error happens. Auto resume may also cause other Retryable IO Error during the recovery, so the recovery will fail. Retry the auto resume may solve the issue, so we use max_bgerror_resume_count to decide how many resume cycles will be tried in total. If it is <=0, auto resume retryable IO Error is disabled. Default is INT_MAX, which will lead to a infinit auto resume. bgerror_resume_retry_interval decides the time interval between two auto resumes. * Option `max_subcompactions` can be set dynamically using DB::SetDBOptions(). * Added experimental ColumnFamilyOptions::sst_partitioner_factory to define determine the partitioning of sst files. This helps compaction to split the files on interesting boundaries (key prefixes) to make propagation of sst files less write amplifying (covering the whole key space). +* Methods to configure serialize, and compare -- such as TableFactory -- are exposed directly through the Configurable base class (from which these objects inherity). This change will allow for better and more thorough configuration management and retrieval in the future. The options for a Configurable object can be set via the ConfigureFromMap, ConfigureFromString, or ConfigureOption method. The serialized version of the options of an object can be retrieved via the GetOptionString, ToString, or GetOption methods. The list of options supported by an object can be obtained via the GetOptionNames method. The "raw" object (such as the BlockBasedTableOption) for an option may be retrieved via the GetOptions method. Configurable options can be compared via the AreEquivalent method. The settings within a Configurable object may be validated via the ValidateOptions method. The object may be intialized (at which point only mutable options may be updated) via the PrepareOptions method. ### Performance Improvements * Eliminate key copies for internal comparisons while accessing ingested block-based tables. * Reduce key comparisons during random access in all block-based tables. * BackupEngine avoids unnecessary repeated checksum computation for backing up a table file to the `shared_checksum` directory when using `kOptionalChecksumAndDbSessionId`, except on SST files generated before this version of RocksDB, which fall back on using `kChecksumAndFileSize`. +### General Improvements +* The settings of the DBOptions and ColumnFamilyOptions are now managed by Configurable objects (see New Features). The same convenience methods to configure these options still exist but the backend implementation has been unified under a common implementation. + ## 6.11 (6/12/2020) ### Bug Fixes * Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true. diff --git a/Makefile b/Makefile index c08605060..4e1d33c88 100644 --- a/Makefile +++ b/Makefile @@ -555,6 +555,7 @@ PARALLEL_TEST = \ ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) TESTS += folly_synchronization_distributed_mutex_test PARALLEL_TEST += folly_synchronization_distributed_mutex_test + TESTS_PASSING_ASC = folly_synchronization_distributed_mutex_test endif # options_settable_test doesn't pass with UBSAN as we use hack in the test @@ -599,6 +600,7 @@ ifdef ASSERT_STATUS_CHECKED merger_test \ mock_env_test \ object_registry_test \ + configurable_test \ options_settable_test \ options_test \ random_test \ @@ -1724,6 +1726,9 @@ thread_list_test: $(OBJ_DIR)/util/thread_list_test.o $(TEST_LIBRARY) $(LIBRARY) compact_files_test: $(OBJ_DIR)/db/compact_files_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +configurable_test: options/configurable_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + options_test: $(OBJ_DIR)/options/options_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) diff --git a/TARGETS b/TARGETS index caa65f752..3e2f556f7 100644 --- a/TARGETS +++ b/TARGETS @@ -239,6 +239,7 @@ cpp_library( "monitoring/thread_status_util.cc", "monitoring/thread_status_util_debug.cc", "options/cf_options.cc", + "options/configurable.cc", "options/db_options.cc", "options/options.cc", "options/options_helper.cc", @@ -290,6 +291,7 @@ cpp_library( "table/sst_file_dumper.cc", "table/sst_file_reader.cc", "table/sst_file_writer.cc", + "table/table_factory.cc", "table/table_properties.cc", "table/two_level_iterator.cc", "test_util/sync_point.cc", @@ -716,6 +718,13 @@ ROCKS_TESTS = [ [], [], ], + [ + "configurable_test", + "options/configurable_test.cc", + "serial", + [], + [], + ], [ "corruption_test", "db/corruption_test.cc", diff --git a/cache/cache.cc b/cache/cache.cc index f4e49c481..78897c416 100644 --- a/cache/cache.cc +++ b/cache/cache.cc @@ -10,7 +10,7 @@ #include "rocksdb/cache.h" #include "cache/lru_cache.h" -#include "options/options_helper.h" +#include "rocksdb/utilities/options_type.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { @@ -19,22 +19,19 @@ static std::unordered_map lru_cache_options_type_info = { {"capacity", {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, capacity)}}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, {"num_shard_bits", {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, num_shard_bits)}}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, {"strict_capacity_limit", {offsetof(struct LRUCacheOptions, strict_capacity_limit), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, + OptionTypeFlags::kMutable}}, {"high_pri_pool_ratio", {offsetof(struct LRUCacheOptions, high_pri_pool_ratio), OptionType::kDouble, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; + OptionTypeFlags::kMutable}}, +}; #endif // ROCKSDB_LITE Status Cache::CreateFromString(const ConfigOptions& config_options, diff --git a/db/column_family.cc b/db/column_family.cc index d6b9c69b9..a6089f436 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -32,7 +32,7 @@ #include "monitoring/thread_status_util.h" #include "options/options_helper.h" #include "port/port.h" -#include "table/block_based/block_based_table_factory.h" +#include "rocksdb/table.h" #include "table/merging_iterator.h" #include "util/autovector.h" #include "util/cast_util.h" @@ -357,8 +357,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, result.max_compaction_bytes = result.target_file_size_base * 25; } - bool is_block_based_table = - (result.table_factory->Name() == BlockBasedTableFactory::kName); + bool is_block_based_table = (result.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())); const uint64_t kAdjustedTtl = 30 * 24 * 60 * 60; if (result.ttl == kDefaultTtl) { @@ -1316,7 +1316,8 @@ Status ColumnFamilyData::ValidateOptions( } if (cf_options.ttl > 0 && cf_options.ttl != kDefaultTtl) { - if (cf_options.table_factory->Name() != BlockBasedTableFactory::kName) { + if (!cf_options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())) { return Status::NotSupported( "TTL is only supported in Block-Based Table format. "); } @@ -1324,7 +1325,8 @@ Status ColumnFamilyData::ValidateOptions( if (cf_options.periodic_compaction_seconds > 0 && cf_options.periodic_compaction_seconds != kDefaultPeriodicCompSecs) { - if (cf_options.table_factory->Name() != BlockBasedTableFactory::kName) { + if (!cf_options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())) { return Status::NotSupported( "Periodic Compaction is only supported in " "Block-Based Table format. "); diff --git a/db/db_basic_test.cc b/db/db_basic_test.cc index f079be43d..c9483faf9 100644 --- a/db/db_basic_test.cc +++ b/db/db_basic_test.cc @@ -3250,12 +3250,9 @@ TEST_P(DBBasicTestDeadline, PointLookupDeadline) { SetTimeElapseOnlySleepOnReopen(&options); Reopen(options); - if (options.table_factory && - !strcmp(options.table_factory->Name(), - BlockBasedTableFactory::kName.c_str())) { - BlockBasedTableFactory* bbtf = - static_cast(options.table_factory.get()); - block_cache = bbtf->table_options().block_cache.get(); + if (options.table_factory) { + block_cache = options.table_factory->GetOptions( + TableFactory::kBlockCacheOpts()); } Random rnd(301); @@ -3336,12 +3333,9 @@ TEST_P(DBBasicTestDeadline, IteratorDeadline) { SetTimeElapseOnlySleepOnReopen(&options); Reopen(options); - if (options.table_factory && - !strcmp(options.table_factory->Name(), - BlockBasedTableFactory::kName.c_str())) { - BlockBasedTableFactory* bbtf = - static_cast(options.table_factory.get()); - block_cache = bbtf->table_options().block_cache.get(); + if (options.table_factory) { + block_cache = options.table_factory->GetOptions( + TableFactory::kBlockCacheOpts()); } Random rnd(301); diff --git a/db/db_block_cache_test.cc b/db/db_block_cache_test.cc index e58ecde0a..da37babcc 100644 --- a/db/db_block_cache_test.cc +++ b/db/db_block_cache_test.cc @@ -50,7 +50,7 @@ class DBBlockCacheTest : public DBTestBase { options.avoid_flush_during_recovery = false; // options.compression = kNoCompression; options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); return options; } @@ -158,7 +158,7 @@ TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) { std::shared_ptr cache = NewLRUCache(0, 0, false); table_options.block_cache = cache; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); RecordCacheCounters(options); @@ -182,7 +182,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) { std::shared_ptr cache = NewLRUCache(0, 0, false); table_options.block_cache = cache; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); RecordCacheCounters(options); @@ -238,7 +238,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) { std::shared_ptr compressed_cache = NewLRUCache(1 << 25, 0, false); table_options.block_cache = cache; table_options.block_cache_compressed = compressed_cache; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); RecordCacheCounters(options); @@ -299,7 +299,7 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) { BlockBasedTableOptions table_options; table_options.cache_index_and_filter_blocks = true; table_options.filter_policy.reset(NewBloomFilterPolicy(20)); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); CreateAndReopenWithCF({"pikachu"}, options); ASSERT_OK(Put(1, "key", "val")); @@ -355,7 +355,7 @@ TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) { std::shared_ptr cache = NewLRUCache(10, 0, true); table_options.block_cache = cache; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); ASSERT_OK(Put("key1", "val1")); ASSERT_OK(Put("key2", "val2")); @@ -393,7 +393,7 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) { std::shared_ptr cache = NewLRUCache(co); table_options.block_cache = cache; table_options.filter_policy.reset(NewBloomFilterPolicy(20, true)); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); CreateAndReopenWithCF({"pikachu"}, options); ASSERT_OK(Put(1, "longer_key", "val")); @@ -474,7 +474,7 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksCachePriority) { table_options.filter_policy.reset(NewBloomFilterPolicy(20)); table_options.cache_index_and_filter_blocks_with_high_priority = priority == Cache::Priority::HIGH ? true : false; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); DestroyAndReopen(options); MockCache::high_pri_insert_count = 0; @@ -573,7 +573,7 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) { table_options.cache_index_and_filter_blocks = true; table_options.block_cache = cache; table_options.filter_policy.reset(NewBloomFilterPolicy(50)); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); DestroyAndReopen(options); // Create a new table. @@ -662,7 +662,7 @@ TEST_F(DBBlockCacheTest, ParanoidFileChecks) { BlockBasedTableOptions table_options; table_options.cache_index_and_filter_blocks = false; table_options.filter_policy.reset(NewBloomFilterPolicy(20)); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); CreateAndReopenWithCF({"pikachu"}, options); ASSERT_OK(Put(1, "1_key", "val")); @@ -846,7 +846,7 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) { BlockBasedTableOptions table_options; table_options.cache_index_and_filter_blocks = true; table_options.block_cache.reset(new MockCache()); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); DestroyAndReopen(options); RecordCacheCountersForCompressionDict(options); diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index 2231af6da..d46102461 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -83,13 +83,16 @@ TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) { options_override.partition_filters = partition_filters_; options_override.metadata_block_size = 32; Options options = CurrentOptions(options_override); - if (partition_filters_ && - static_cast( - options.table_factory->GetOptions()) - ->index_type != BlockBasedTableOptions::kTwoLevelIndexSearch) { - // In the current implementation partitioned filters depend on partitioned - // indexes - continue; + if (partition_filters_) { + auto* table_options = + options.table_factory->GetOptions(); + if (table_options != nullptr && + table_options->index_type != + BlockBasedTableOptions::kTwoLevelIndexSearch) { + // In the current implementation partitioned filters depend on + // partitioned indexes + continue; + } } options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); CreateAndReopenWithCF({"pikachu"}, options); diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 307ce99e1..72b6a8ef9 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -18,8 +18,8 @@ #include "monitoring/persistent_stats_history.h" #include "monitoring/stats_dump_scheduler.h" #include "options/options_helper.h" +#include "rocksdb/table.h" #include "rocksdb/wal_filter.h" -#include "table/block_based/block_based_table_factory.h" #include "test_util/sync_point.h" #include "util/rate_limiter.h" @@ -185,12 +185,12 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) { } namespace { -Status SanitizeOptionsByTable( +Status ValidateOptionsByTable( const DBOptions& db_opts, const std::vector& column_families) { Status s; for (auto cf : column_families) { - s = cf.options.table_factory->SanitizeOptions(db_opts, cf.options); + s = ValidateOptions(db_opts, cf.options); if (!s.ok()) { return s; } @@ -1451,7 +1451,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, const std::vector& column_families, std::vector* handles, DB** dbptr, const bool seq_per_batch, const bool batch_per_txn) { - Status s = SanitizeOptionsByTable(db_options, column_families); + Status s = ValidateOptionsByTable(db_options, column_families); if (!s.ok()) { return s; } diff --git a/db/db_iterator_test.cc b/db/db_iterator_test.cc index 7b6f26293..8d2b0a7c0 100644 --- a/db/db_iterator_test.cc +++ b/db/db_iterator_test.cc @@ -2146,7 +2146,7 @@ TEST_P(DBIteratorTest, ReadAhead) { BlockBasedTableOptions table_options; table_options.block_size = 1024; table_options.no_block_cache = true; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); std::string value(1024, 'a'); diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 23db6a901..f16703780 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -33,17 +33,13 @@ class DBOptionsTest : public DBTestBase { std::unordered_map GetMutableDBOptionsMap( const DBOptions& options) { std::string options_str; + std::unordered_map mutable_map; ConfigOptions config_options; config_options.delimiter = "; "; - GetStringFromDBOptions(config_options, options, &options_str); - std::unordered_map options_map; - StringToMap(options_str, &options_map); - std::unordered_map mutable_map; - for (const auto& opt : db_options_type_info) { - if (opt.second.IsMutable() && opt.second.ShouldSerialize()) { - mutable_map[opt.first] = options_map[opt.first]; - } - } + + GetStringFromMutableDBOptions(config_options, MutableDBOptions(options), + &options_str); + StringToMap(options_str, &mutable_map); return mutable_map; } @@ -53,15 +49,10 @@ class DBOptionsTest : public DBTestBase { ConfigOptions config_options; config_options.delimiter = "; "; - GetStringFromColumnFamilyOptions(config_options, options, &options_str); - std::unordered_map options_map; - StringToMap(options_str, &options_map); std::unordered_map mutable_map; - for (const auto& opt : cf_options_type_info) { - if (opt.second.IsMutable() && opt.second.ShouldSerialize()) { - mutable_map[opt.first] = options_map[opt.first]; - } - } + GetStringFromMutableCFOptions(config_options, MutableCFOptions(options), + &options_str); + StringToMap(options_str, &mutable_map); return mutable_map; } diff --git a/db/db_test.cc b/db/db_test.cc index 41f1de8e9..c624f1b9e 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -55,9 +55,7 @@ #include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/write_batch_with_index.h" -#include "table/block_based/block_based_table_factory.h" #include "table/mock_table.h" -#include "table/plain/plain_table_factory.h" #include "table/scoped_arena_iterator.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" @@ -3842,7 +3840,7 @@ TEST_F(DBTest, TableOptionsSanitizeTest) { DestroyAndReopen(options); ASSERT_EQ(db_->GetOptions().allow_mmap_reads, false); - options.table_factory.reset(new PlainTableFactory()); + options.table_factory.reset(NewPlainTableFactory()); options.prefix_extractor.reset(NewNoopTransform()); Destroy(options); ASSERT_TRUE(!TryReopen(options).IsNotSupported()); diff --git a/db/db_test2.cc b/db/db_test2.cc index 6e3126745..ed0f469d0 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -2967,11 +2967,11 @@ TEST_F(DBTest2, OptimizeForSmallDB) { options.OptimizeForSmallDb(); // Find the cache object - ASSERT_EQ(std::string(BlockBasedTableFactory::kName), - std::string(options.table_factory->Name())); - BlockBasedTableOptions* table_options = - reinterpret_cast( - options.table_factory->GetOptions()); + ASSERT_TRUE(options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())); + auto table_options = + options.table_factory->GetOptions(); + ASSERT_TRUE(table_options != nullptr); std::shared_ptr cache = table_options->block_cache; diff --git a/db/db_test_util.h b/db/db_test_util.h index d32cc8545..fc5c5929a 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -38,9 +38,7 @@ #include "rocksdb/statistics.h" #include "rocksdb/table.h" #include "rocksdb/utilities/checkpoint.h" -#include "table/block_based/block_based_table_factory.h" #include "table/mock_table.h" -#include "table/plain/plain_table_factory.h" #include "table/scoped_arena_iterator.h" #include "test_util/mock_time_env.h" #include "test_util/sync_point.h" diff --git a/db/flush_job.cc b/db/flush_job.cc index b0e316a1e..088c06723 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -39,8 +39,6 @@ #include "rocksdb/statistics.h" #include "rocksdb/status.h" #include "rocksdb/table.h" -#include "table/block_based/block.h" -#include "table/block_based/block_based_table_factory.h" #include "table/merging_iterator.h" #include "table/table_builder.h" #include "table/two_level_iterator.h" diff --git a/db/internal_stats.cc b/db/internal_stats.cc index ff4c5f46c..57abdcb58 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -19,7 +19,7 @@ #include "db/column_family.h" #include "db/db_impl/db_impl.h" -#include "table/block_based/block_based_table_factory.h" +#include "rocksdb/table.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { @@ -907,19 +907,9 @@ bool InternalStats::HandleBlockCacheStat(Cache** block_cache) { assert(block_cache != nullptr); auto* table_factory = cfd_->ioptions()->table_factory; assert(table_factory != nullptr); - if (BlockBasedTableFactory::kName != table_factory->Name()) { - return false; - } - auto* table_options = - reinterpret_cast(table_factory->GetOptions()); - if (table_options == nullptr) { - return false; - } - *block_cache = table_options->block_cache.get(); - if (table_options->no_block_cache || *block_cache == nullptr) { - return false; - } - return true; + *block_cache = + table_factory->GetOptions(TableFactory::kBlockCacheOpts()); + return *block_cache != nullptr; } bool InternalStats::HandleBlockCacheCapacity(uint64_t* value, DBImpl* /*db*/, diff --git a/db/listener_test.cc b/db/listener_test.cc index 9239068ce..5f0511d78 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -24,8 +24,6 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/table.h" #include "rocksdb/table_properties.h" -#include "table/block_based/block_based_table_factory.h" -#include "table/plain/plain_table_factory.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" #include "test_util/testutil.h" diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index 5c202de81..56d7edefe 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -3,6 +3,8 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). +#include "db/table_properties_collector.h" + #include #include #include @@ -11,11 +13,11 @@ #include "db/db_impl/db_impl.h" #include "db/dbformat.h" -#include "db/table_properties_collector.h" #include "env/composite_env_wrapper.h" #include "file/sequence_file_reader.h" #include "file/writable_file_writer.h" #include "options/cf_options.h" +#include "rocksdb/flush_block_policy.h" #include "rocksdb/table.h" #include "table/block_based/block_based_table_factory.h" #include "table/meta_blocks.h" diff --git a/examples/options_file_example.cc b/examples/options_file_example.cc index 11ec642a4..92c577b15 100644 --- a/examples/options_file_example.cc +++ b/examples/options_file_example.cc @@ -91,8 +91,9 @@ int main() { // Initialize pointer options for each column family for (size_t i = 0; i < loaded_cf_descs.size(); ++i) { - auto* loaded_bbt_opt = reinterpret_cast( - loaded_cf_descs[0].options.table_factory->GetOptions()); + auto* loaded_bbt_opt = + loaded_cf_descs[0] + .options.table_factory->GetOptions(); // Expect the same as BlockBasedTableOptions will be loaded form file. assert(loaded_bbt_opt->block_size == bbt_opts.block_size); // However, block_cache needs to be manually initialized as documented diff --git a/include/rocksdb/configurable.h b/include/rocksdb/configurable.h new file mode 100644 index 000000000..f4bfbf532 --- /dev/null +++ b/include/rocksdb/configurable.h @@ -0,0 +1,364 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +class Logger; +class ObjectRegistry; +class OptionTypeInfo; +struct ColumnFamilyOptions; +struct ConfigOptions; +struct DBOptions; + +// Configurable is a base class used by the rocksdb that describes a +// standard way of configuring objects. A Configurable object can: +// -> Populate itself given: +// - One or more "name/value" pair strings +// - A string repesenting the set of name=value properties +// - A map of name/value properties. +// -> Convert itself into its string representation +// -> Dump itself to a Logger +// -> Compare itself to another Configurable object to see if the two objects +// have equivalent options settings +// +// If a derived class calls RegisterOptions to register (by name) how its +// options objects are to be processed, this functionality can typically be +// handled by this class without additional overrides. Otherwise, the derived +// class will need to implement the methods for handling the corresponding +// functionality. +class Configurable { + protected: + friend class ConfigurableHelper; + struct RegisteredOptions { + // The name of the options being registered + std::string name; + // Pointer to the object being registered + void* opt_ptr; +#ifndef ROCKSDB_LITE + // The map of options being registered + const std::unordered_map* type_map; +#endif + }; + + public: + Configurable() : prepared_(false) {} + virtual ~Configurable() {} + + // Returns the raw pointer of the named options that is used by this + // object, or nullptr if this function is not supported. + // Since the return value is a raw pointer, the object owns the + // pointer and the caller should not delete the pointer. + // + // Note that changing the underlying options while the object + // is currently used by any open DB is undefined behavior. + // Developers should use DB::SetOption() instead to dynamically change + // options while the DB is open. + template + const T* GetOptions() const { + return GetOptions(T::kName()); + } + template + T* GetOptions() { + return GetOptions(T::kName()); + } + template + const T* GetOptions(const std::string& name) const { + return reinterpret_cast(GetOptionsPtr(name)); + } + template + T* GetOptions(const std::string& name) { + return reinterpret_cast(const_cast(GetOptionsPtr(name))); + } + + // Configures the options for this class based on the input parameters. + // On successful completion, the object is updated with the settings from + // the opt_map. + // If this method fails, an attempt is made to revert the object to original + // state. Note that the revert may not be the original state but may be an + // equivalent. For example, if the object contains an option that is a + // shared_ptr, the shared_ptr may not be the original one but a copy (e.g. not + // the Cache object that was passed in, but a Cache object of the same size). + // + // The acceptable values of the name/value pairs are documented with the + // specific class/instance. + // + // @param config_options Controls how the arguments are processed. + // @param opt_map Name/value pairs of the options to update + // @param unused If specified, this value will return the name/value + // pairs from opt_map that were NotFound for this object. + // @return OK If all values in the map were successfully updated + // If invoke_prepare_options is true, OK also implies + // PrepareOptions ran successfully. + // @return NotFound If any of the names in the opt_map were not valid + // for this object. If unused is specified, it will contain the + // collection of NotFound names. + // @return NotSupported If any of the names are valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If any of the values cannot be successfully + // parsed. This can also be returned if PrepareOptions encounters an + // error. + // @see ConfigOptions for a description of the controls. + Status ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opt_map); + Status ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opt_map, + std::unordered_map* unused); + +#ifndef ROCKSDB_LITE + // Updates the named option to the input value, returning OK if successful. + // Note that ConfigureOption does not cause PrepareOptions to be invoked. + // @param config_options Controls how the name/value is processed. + // @param name The name of the option to update + // @param value The value to set for the named option + // @return OK If the named field was successfully updated to value. + // @return NotFound If the name is not valid for this object. + // @return NotSupported If the name is valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If the value cannot be successfully parsed. + Status ConfigureOption(const ConfigOptions& config_options, + const std::string& name, const std::string& value); +#endif // ROCKSDB_LITE + + // Configures the options for this class based on the input parameters. + // On successful completion, the object is updated with the settings from + // the opt_map. If this method fails, an attempt is made to revert the + // object to original state. Note that the revert may not be the original + // state but may be an equivalent. + // @see ConfigureFromMap for more details + // @param config_options Controls how the arguments are processed. + // @param opt_str string containing the values to update. + // @param unused If specified, this value will return the name/value + // pairs from opt_map that were NotFound for this object. + // @return OK If all specified values were successfully updated + // If invoke_prepare_options is true, OK also implies + // PrepareOptions ran successfully. + // @return NotFound If any of the names were not valid for this object. + // If unused is specified, it will contain the collection of NotFound + // names. + // @return NotSupported If any of the names are valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If any of the values cannot be successfully + // parsed. This can also be returned if PrepareOptions encounters an + // error. + Status ConfigureFromString(const ConfigOptions& config_options, + const std::string& opts); + + // Fills in result with the serialized options for this object. + // This is the inverse of ConfigureFromString. + // @param config_options Controls how serialization happens. + // @param result The string representation of this object. + // @return OK If the options for this object wer successfully serialized. + // @return InvalidArgument If one or more of the options could not be + // serialized. + Status GetOptionString(const ConfigOptions& config_options, + std::string* result) const; +#ifndef ROCKSDB_LITE + // Returns the serialized options for this object. + // This method is similar to GetOptionString with no errors. + // @param config_options Controls how serialization happens. + // @param prefix A string to prepend to every option. + // @return The serialized representation of the options for this object + std::string ToString(const ConfigOptions& config_options) const { + return ToString(config_options, ""); + } + std::string ToString(const ConfigOptions& config_options, + const std::string& prefix) const; + + // Returns the list of option names associated with this configurable + // @param config_options Controls how the names are returned + // @param result The set of option names for this object. Note that + // options that are deprecated or aliases are not returned. + // @return OK on success. + Status GetOptionNames(const ConfigOptions& config_options, + std::unordered_set* result) const; + + // Returns the value of the option associated with the input name + // This method is the functional inverse of ConfigureOption + // @param config_options Controls how the value is returned + // @param name The name of the option to return a value for. + // @param value The returned value associated with the named option. + // @return OK If the named field was successfully updated to value. + // @return NotFound If the name is not valid for this object. + // @param InvalidArgument If the name is valid for this object but + // its value cannot be serialized. + virtual Status GetOption(const ConfigOptions& config_options, + const std::string& name, std::string* value) const; +#endif // ROCKSDB_LITE + + // Checks to see if this Configurable is equivalent to other. + // This method assumes that the two objects are of the same class. + // @param config_options Controls how the options are compared. + // @param other The other object to compare to. + // @param mismatch If the objects do not match, this parameter contains + // the name of the option that triggered the match failure. + // @param True if the objects match, false otherwise. + virtual bool AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* name) const; + + // Returns a pretty-printed, human-readable version of the options. + // This method is typically used to dump the options to a log file. + // Classes should override this method + virtual std::string GetPrintableOptions() const { return ""; } + + // Validates that the settings are valid/consistent and performs any object + // initialization required by this object. This method may be called as part + // of Configure (if invoke_prepare_options is set), or may be invoked + // separately. + // + // Once an object has been prepared, non-mutable options can no longer be + // updated. + // + // Classes must override this method to provide any implementation-specific + // initialization, such as opening log files or setting up cache parameters. + // Implementations should be idempotent (e.g. don't re-open the log file or + // reconfigure the cache), as there is the potential this method can be called + // more than once. + // + // By default, this method will also prepare all nested (Inner and + // OptionType::kConfigurable) objects. + // + // @param config_options Controls how the object is prepared. Also contains + // a Logger and Env that can be used to initialize this object. + // @return OK If the object was successfully initialized. + // @return InvalidArgument If this object could not be successfull + // initialized. + virtual Status PrepareOptions(const ConfigOptions& config_options); + + // Checks to see if the settings are valid for this object. + // This method checks to see if the input DBOptions and ColumnFamilyOptions + // are valid for the settings of this object. For example, an Env might not + // support certain mmap modes or a TableFactory might require certain + // settings. + // + // By default, this method will also validate all nested (Inner and + // OptionType::kConfigurable) objects. + // + // @param db_opts The DBOptions to validate + // @param cf_opts The ColumnFamilyOptions to validate + // @return OK if the options are valid + // @return InvalidArgument If the arguments are not valid for the options + // of the current object. + virtual Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const; + + // Returns true if this object has been initialized via PrepareOptions, false + // otherwise. Once an object has been prepared, only mutable options may be + // changed. + virtual bool IsPrepared() const { return prepared_; } + + protected: + // True once the object is prepared. Once the object is prepared, only + // mutable options can be configured. + bool prepared_; + // If this class is a wrapper (has-a), this method should be + // over-written to return the inner configurable (like an EnvWrapper). + // This method should NOT recurse, but should instead return the + // direct Inner object. + virtual Configurable* Inner() const { return nullptr; } + + // Returns the raw pointer for the associated named option. + // The name is typically the name of an option registered via the + // Classes may override this method to provide further specialization (such as + // returning a sub-option) + // + // The default implemntation looks at the registered options. If the + // input name matches that of a registered option, the pointer registered + // with that name is returned. + // e.g,, RegisterOptions("X", &my_ptr, ...); GetOptionsPtr("X") returns + // "my_ptr" + virtual const void* GetOptionsPtr(const std::string& name) const; + + // Method for allowing options to be configured outside of the normal + // registered options framework. Classes may override this method if they + // wish to support non-standard options implementations (such as configuring + // themselves from constant or simple ":"-separated strings. + // + // The default implementation does nothing and returns OK + virtual Status ParseStringOptions(const ConfigOptions& config_options, + const std::string& opts_str); + + // Internal method to configure an object from a map of name-value options. + // This method uses the input config_options to drive the configuration of + // the options in opt_map. Any option name that cannot be found from the + // input set will be returned in "unused". + // + // Classes may override this method to extend the functionality if required. + // @param config_options Controls how the options are configured and errors + // handled. + // @param opts_map The set of options to configure + // @param unused Any options from opt_map that were not configured. + // @returns a Status based on the rules outlined in ConfigureFromMap + virtual Status ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused); + +#ifndef ROCKSDB_LITE + // Method that configures a the specific opt_name from opt_value. + // By default, this method calls opt_info.ParseOption with the + // input parameters. + // Classes may override this method to extend the functionality, or + // change the returned Status. + virtual Status ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& opt_value, void* opt_ptr); + + // Internal method to see if the single option name/info matches for this and + // that Classes may override this value to change its behavior. + // @param config_options Controls how the options are being matched + // @param opt_info The OptionTypeInfo registered for this option name + // that controls what field is matched (offset) and how (type). + // @param name The name associated with this opt_info. + // @param this_ptr The base pointer to compare to. This is the object + // registered for + // for this OptionTypeInfo. + // @param that_ptr The other pointer to compare to. This is the object + // registered for + // for this OptionTypeInfo. + // @param bad_name If the match fails, the name of the option that failed to + // match. + virtual bool OptionsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& name, + const void* const this_ptr, + const void* const that_ptr, + std::string* bad_name) const; +#endif +#ifndef ROCKSDB_LITE + // Internal method to serialize options (ToString) + // Classes may override this value to change its behavior. + virtual std::string SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const; +#endif // ROCKSDB_LITE + + // Given a name (e.g. rocksdb.my.type.opt), returns the short name (opt) + virtual std::string GetOptionName(const std::string& long_name) const; + + private: + // Contains the collection of options (name, opt_ptr, opt_map) associated with + // this object. This collection is typically set in the constructor of the + // Configurable option via + std::vector options_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index 60618374d..f861b2fcf 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -47,9 +47,15 @@ struct ConfigOptions { // When true, any unused options will be ignored and OK will be returned bool ignore_unknown_options = false; + // When true, any unsupported options will be ignored and OK will be returned + bool ignore_unsupported_options = true; + // If the strings are escaped (old-style?) bool input_strings_escaped = true; + // Whether or not to invoke PrepareOptions after configure is called. + bool invoke_prepare_options = true; + // The separator between options when converting to a string std::string delimiter = ";"; diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 95cdb8d21..8e0d144f8 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -22,15 +22,15 @@ #include #include -#include "rocksdb/cache.h" +#include "rocksdb/configurable.h" #include "rocksdb/env.h" -#include "rocksdb/iterator.h" #include "rocksdb/options.h" #include "rocksdb/status.h" namespace ROCKSDB_NAMESPACE { // -- Block-based Table +class Cache; class FilterPolicy; class FlushBlockPolicyFactory; class PersistentCache; @@ -53,6 +53,7 @@ enum ChecksumType : char { // For advanced user only struct BlockBasedTableOptions { + static const char* kName() { return "BlockTableOptions"; }; // @flush_block_policy_factory creates the instances of flush block policy. // which provides a configurable way to determine when to flush a block in // the block based tables. If not set, table builder will use the default @@ -392,6 +393,7 @@ struct PlainTablePropertyNames { const uint32_t kPlainTableVariableLength = 0; struct PlainTableOptions { + static const char* kName() { return "PlainTableOptions"; }; // @user_key_len: plain table has optimization for fix-sized keys, which can // be specified via user_key_len. Alternatively, you can pass // `kPlainTableVariableLength` if your keys have variable @@ -485,6 +487,8 @@ struct CuckooTablePropertyNames { }; struct CuckooTableOptions { + static const char* kName() { return "CuckooTableOptions"; }; + // Determines the utilization of hash tables. Smaller values // result in larger hash tables with fewer collisions. double hash_table_ratio = 0.9; @@ -522,9 +526,19 @@ extern TableFactory* NewCuckooTableFactory( class RandomAccessFileReader; // A base class for table factories. -class TableFactory { +class TableFactory : public Configurable { public: - virtual ~TableFactory() {} + virtual ~TableFactory() override {} + + static const char* kBlockCacheOpts() { return "BlockCache"; }; + static const char* kBlockBasedTableName() { return "BlockBasedTable"; }; + static const char* kPlainTableName() { return "PlainTable"; } + static const char* kCuckooTableName() { return "CuckooTable"; }; + + // Creates and configures a new TableFactory from the input options and id. + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* factory); // The type of the table. // @@ -535,6 +549,13 @@ class TableFactory { // by any clients of this package. virtual const char* Name() const = 0; + // Returns true if the class is an instance of the input name. + // This is typically determined by if the input name matches the + // name of this object. + virtual bool IsInstanceOf(const std::string& name) const { + return name == Name(); + } + // Returns a Table object table that can fetch data from file specified // in parameter file. It's the caller's responsibility to make sure // file is in the correct format. @@ -592,39 +613,6 @@ class TableFactory { const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const = 0; - // Sanitizes the specified DB Options and ColumnFamilyOptions. - // - // If the function cannot find a way to sanitize the input DB Options, - // a non-ok Status will be returned. - virtual Status SanitizeOptions(const DBOptions& db_opts, - const ColumnFamilyOptions& cf_opts) const = 0; - - // Return a string that contains printable format of table configurations. - // RocksDB prints configurations at DB Open(). - virtual std::string GetPrintableTableOptions() const = 0; - - virtual Status GetOptionString(const ConfigOptions& /*config_options*/, - std::string* /*opt_string*/) const { - return Status::NotSupported( - "The table factory doesn't implement GetOptionString()."); - } - - // Returns the raw pointer of the table options that is used by this - // TableFactory, or nullptr if this function is not supported. - // Since the return value is a raw pointer, the TableFactory owns the - // pointer and the caller should not delete the pointer. - // - // In certain case, it is desirable to alter the underlying options when the - // TableFactory is not used by any open DB by casting the returned pointer - // to the right class. For instance, if BlockBasedTableFactory is used, - // then the pointer can be casted to BlockBasedTableOptions. - // - // Note that changing the underlying TableFactory options while the - // TableFactory is currently used by any open DB is undefined behavior. - // Developers should use DB::SetOption() instead to dynamically change - // options while the DB is open. - virtual void* GetOptions() { return nullptr; } - // Return is delete range supported virtual bool IsDeleteRangeSupported() const { return false; } }; diff --git a/include/rocksdb/utilities/object_registry.h b/include/rocksdb/utilities/object_registry.h index 74a49d400..538cb6a8f 100644 --- a/include/rocksdb/utilities/object_registry.h +++ b/include/rocksdb/utilities/object_registry.h @@ -125,7 +125,7 @@ class ObjectRegistry { // Creates a new unique T using the input factory functions. // Returns OK if a new unique T was successfully created - // Returns NotFound if the type/target could not be created + // Returns NotSupported if the type/target could not be created // Returns InvalidArgument if the factory return an unguarded object // (meaning it cannot be managed by a unique ptr) template @@ -134,7 +134,7 @@ class ObjectRegistry { std::string errmsg; T* ptr = NewObject(target, result, &errmsg); if (ptr == nullptr) { - return Status::NotFound(errmsg, target); + return Status::NotSupported(errmsg, target); } else if (*result) { return Status::OK(); } else { @@ -146,7 +146,7 @@ class ObjectRegistry { // Creates a new shared T using the input factory functions. // Returns OK if a new shared T was successfully created - // Returns NotFound if the type/target could not be created + // Returns NotSupported if the type/target could not be created // Returns InvalidArgument if the factory return an unguarded object // (meaning it cannot be managed by a shared ptr) template @@ -156,7 +156,7 @@ class ObjectRegistry { std::unique_ptr guard; T* ptr = NewObject(target, &guard, &errmsg); if (ptr == nullptr) { - return Status::NotFound(errmsg, target); + return Status::NotSupported(errmsg, target); } else if (guard) { result->reset(guard.release()); return Status::OK(); @@ -169,7 +169,7 @@ class ObjectRegistry { // Creates a new static T using the input factory functions. // Returns OK if a new static T was successfully created - // Returns NotFound if the type/target could not be created + // Returns NotSupported if the type/target could not be created // Returns InvalidArgument if the factory return a guarded object // (meaning it is managed by a unique ptr) template @@ -178,7 +178,7 @@ class ObjectRegistry { std::unique_ptr guard; T* ptr = NewObject(target, &guard, &errmsg); if (ptr == nullptr) { - return Status::NotFound(errmsg, target); + return Status::NotSupported(errmsg, target); } else if (guard.get()) { return Status::InvalidArgument(std::string("Cannot make a static ") + T::Type() + " from a guarded one ", diff --git a/options/options_type.h b/include/rocksdb/utilities/options_type.h similarity index 85% rename from options/options_type.h rename to include/rocksdb/utilities/options_type.h index c2df3ea9a..2bd081abf 100644 --- a/options/options_type.h +++ b/include/rocksdb/utilities/options_type.h @@ -16,6 +16,9 @@ namespace ROCKSDB_NAMESPACE { class OptionTypeInfo; +// The underlying "class/type" of the option. +// This enum is used to determine how the option should +// be converted to/from strings and compared. enum class OptionType { kBoolean, kInt, @@ -31,7 +34,6 @@ enum class OptionType { kCompactionPri, kSliceTransform, kCompressionType, - kTableFactory, kComparator, kCompactionFilter, kCompactionFilterFactory, @@ -46,6 +48,7 @@ enum class OptionType { kEnum, kStruct, kVector, + kConfigurable, kUnknown, }; @@ -67,6 +70,22 @@ enum class OptionVerificationType { // independently }; +// A set of modifier flags used to alter how an option is evaluated or +// processed. These flags can be combined together (e.g. kMutable | kShared). +// The kCompare flags can be used to control if/when options are compared. +// If kCompareNever is set, two related options would never be compared (always +// equal) If kCompareExact is set, the options will only be compared if the +// sanity mode +// is exact +// kMutable means the option can be changed after it is prepared +// kShared means the option is contained in a std::shared_ptr +// kUnique means the option is contained in a std::uniqued_ptr +// kRawPointer means the option is a raw pointer value. +// kAllowNull means that an option is allowed to be null for verification +// purposes. +// kDontSerialize means this option should not be serialized and included in +// the string representation. +// kDontPrepare means do not call PrepareOptions for this pointer value. enum class OptionTypeFlags : uint32_t { kNone = 0x00, // No flags kCompareDefault = 0x0, @@ -75,7 +94,12 @@ enum class OptionTypeFlags : uint32_t { kCompareExact = ConfigOptions::kSanityLevelExactMatch, kMutable = 0x0100, // Option is mutable + kRawPointer = 0x0200, // The option is stored as a raw pointer + kShared = 0x0400, // The option is stored as a shared_ptr + kUnique = 0x0800, // The option is stored as a unique_ptr + kAllowNull = 0x1000, // The option can be null kDontSerialize = 0x2000, // Don't serialize the option + kDontPrepare = 0x4000, // Don't prepare or sanitize this option }; inline OptionTypeFlags operator|(const OptionTypeFlags &a, @@ -179,13 +203,9 @@ using EqualsFunc = std::function* const map) { return OptionTypeInfo( offset, OptionType::kEnum, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0, + OptionTypeFlags::kNone, // Uses the map argument to convert the input string into // its corresponding enum value. If value is found in the map, // addr is updated to the corresponding map entry. @@ -324,10 +329,9 @@ class OptionTypeInfo { static OptionTypeInfo Struct( const std::string& struct_name, const std::unordered_map* struct_map, - int offset, OptionVerificationType verification, OptionTypeFlags flags, - int mutable_offset) { + int offset, OptionVerificationType verification, OptionTypeFlags flags) { return OptionTypeInfo( - offset, OptionType::kStruct, verification, flags, mutable_offset, + offset, OptionType::kStruct, verification, flags, // Parses the struct and updates the fields at addr [struct_name, struct_map](const ConfigOptions& opts, const std::string& name, @@ -353,10 +357,9 @@ class OptionTypeInfo { const std::string& struct_name, const std::unordered_map* struct_map, int offset, OptionVerificationType verification, OptionTypeFlags flags, - int mutable_offset, const ParseFunc& parse_func) { + const ParseFunc& parse_func) { return OptionTypeInfo( - offset, OptionType::kStruct, verification, flags, mutable_offset, - parse_func, + offset, OptionType::kStruct, verification, flags, parse_func, [struct_name, struct_map](const ConfigOptions& opts, const std::string& name, const char* addr, std::string* value) { @@ -374,11 +377,11 @@ class OptionTypeInfo { template static OptionTypeInfo Vector(int _offset, OptionVerificationType _verification, - OptionTypeFlags _flags, int _mutable_offset, + OptionTypeFlags _flags, const OptionTypeInfo& elem_info, char separator = ':') { return OptionTypeInfo( - _offset, OptionType::kVector, _verification, _flags, _mutable_offset, + _offset, OptionType::kVector, _verification, _flags, [elem_info, separator](const ConfigOptions& opts, const std::string& name, const std::string& value, char* addr) { @@ -450,6 +453,20 @@ class OptionTypeInfo { } } + // Returns true if the option is allowed to be null. + // Options can be null if the verification type is allow from null + // or if the flags specify allow null. + bool CanBeNull() const { + return (IsEnabled(OptionTypeFlags::kAllowNull) || + IsEnabled(OptionVerificationType::kByNameAllowFromNull)); + } + + bool IsSharedPtr() const { return IsEnabled(OptionTypeFlags::kShared); } + + bool IsUniquePtr() const { return IsEnabled(OptionTypeFlags::kUnique); } + + bool IsRawPtr() const { return IsEnabled(OptionTypeFlags::kRawPointer); } + bool IsByName() const { return (verification_ == OptionVerificationType::kByName || verification_ == OptionVerificationType::kByNameAllowNull || @@ -458,34 +475,82 @@ class OptionTypeInfo { bool IsStruct() const { return (type_ == OptionType::kStruct); } + bool IsConfigurable() const { return (type_ == OptionType::kConfigurable); } + + // Returns the underlying pointer for the type at base_addr + // The value returned is the underlying "raw" pointer, offset from base. + template + const T* AsRawPointer(const void* const base_addr) const { + if (base_addr == nullptr) { + return nullptr; + } + const auto opt_addr = reinterpret_cast(base_addr) + offset_; + if (IsUniquePtr()) { + const std::unique_ptr* ptr = + reinterpret_cast*>(opt_addr); + return ptr->get(); + } else if (IsSharedPtr()) { + const std::shared_ptr* ptr = + reinterpret_cast*>(opt_addr); + return ptr->get(); + } else if (IsRawPtr()) { + const T* const* ptr = reinterpret_cast(opt_addr); + return *ptr; + } else { + return reinterpret_cast(opt_addr); + } + } + + // Returns the underlying pointer for the type at base_addr + // The value returned is the underlying "raw" pointer, offset from base. + template + T* AsRawPointer(void* base_addr) const { + if (base_addr == nullptr) { + return nullptr; + } + auto opt_addr = reinterpret_cast(base_addr) + offset_; + if (IsUniquePtr()) { + std::unique_ptr* ptr = reinterpret_cast*>(opt_addr); + return ptr->get(); + } else if (IsSharedPtr()) { + std::shared_ptr* ptr = reinterpret_cast*>(opt_addr); + return ptr->get(); + } else if (IsRawPtr()) { + T** ptr = reinterpret_cast(opt_addr); + return *ptr; + } else { + return reinterpret_cast(opt_addr); + } + } + // Parses the option in "opt_value" according to the rules of this class - // and updates the value at "opt_addr". + // and updates the value at "opt_ptr". // On success, Status::OK() is returned. On failure: // NotFound means the opt_name is not valid for this option // NotSupported means we do not know how to parse the value for this option // InvalidArgument means the opt_value is not valid for this option. Status Parse(const ConfigOptions& config_options, const std::string& opt_name, - const std::string& opt_value, char* opt_addr) const; + const std::string& opt_value, void* const opt_ptr) const; // Serializes the option in "opt_addr" according to the rules of this class // into the value at "opt_value". Status Serialize(const ConfigOptions& config_options, - const std::string& opt_name, const char* opt_addr, + const std::string& opt_name, const void* const opt_ptr, std::string* opt_value) const; // Compares the "addr1" and "addr2" values according to the rules of this // class and returns true if they match. On a failed match, mismatch is the // name of the option that failed to match. bool AreEqual(const ConfigOptions& config_options, - const std::string& opt_name, const char* addr1, - const char* addr2, std::string* mismatch) const; + const std::string& opt_name, const void* const addr1, + const void* const addr2, std::string* mismatch) const; // Used to override the match rules for "ByName" options. bool AreEqualByName(const ConfigOptions& config_options, - const std::string& opt_name, const char* this_offset, - const char* that_offset) const; + const std::string& opt_name, const void* const this_ptr, + const void* const that_ptr) const; bool AreEqualByName(const ConfigOptions& config_options, - const std::string& opt_name, const char* this_ptr, + const std::string& opt_name, const void* const this_ptr, const std::string& that_value) const; // Parses the input value according to the map for the struct at opt_addr @@ -553,6 +618,8 @@ class OptionTypeInfo { size_t* end, std::string* token); private: + int offset_; + // The optional function to convert a string to its representation ParseFunc parse_func_; diff --git a/options/cf_options.cc b/options/cf_options.cc index bfee67f73..6fadec962 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -10,18 +10,21 @@ #include #include +#include "options/configurable_helper.h" #include "options/db_options.h" #include "options/options_helper.h" +#include "options/options_parser.h" #include "port/port.h" #include "rocksdb/concurrent_task_limiter.h" +#include "rocksdb/configurable.h" #include "rocksdb/convenience.h" #include "rocksdb/env.h" #include "rocksdb/file_system.h" #include "rocksdb/merge_operator.h" #include "rocksdb/options.h" +#include "rocksdb/table.h" #include "rocksdb/utilities/object_registry.h" -#include "table/block_based/block_based_table_factory.h" -#include "table/plain/plain_table_factory.h" +#include "rocksdb/utilities/options_type.h" #include "util/cast_util.h" namespace ROCKSDB_NAMESPACE { @@ -35,16 +38,14 @@ namespace ROCKSDB_NAMESPACE { // http://en.cppreference.com/w/cpp/concept/StandardLayoutType // https://gist.github.com/graphitemaster/494f21190bb2c63c5516 #ifndef ROCKSDB_LITE -ColumnFamilyOptions OptionsHelper::dummy_cf_options; +static ColumnFamilyOptions dummy_cf_options; template int offset_of(T1 ColumnFamilyOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - - size_t(&OptionsHelper::dummy_cf_options)); + return int(size_t(&(dummy_cf_options.*member)) - size_t(&dummy_cf_options)); } template int offset_of(T1 AdvancedColumnFamilyOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - - size_t(&OptionsHelper::dummy_cf_options)); + return int(size_t(&(dummy_cf_options.*member)) - size_t(&dummy_cf_options)); } static Status ParseCompressionOptions(const std::string& value, @@ -139,34 +140,27 @@ static std::unordered_map compression_options_type_info = { {"window_bits", {offsetof(struct CompressionOptions, window_bits), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct CompressionOptions, window_bits)}}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, {"level", {offsetof(struct CompressionOptions, level), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct CompressionOptions, level)}}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, {"strategy", {offsetof(struct CompressionOptions, strategy), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct CompressionOptions, strategy)}}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, {"max_dict_bytes", {offsetof(struct CompressionOptions, max_dict_bytes), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct CompressionOptions, max_dict_bytes)}}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, {"zstd_max_train_bytes", {offsetof(struct CompressionOptions, zstd_max_train_bytes), OptionType::kUInt32T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct CompressionOptions, zstd_max_train_bytes)}}, + OptionTypeFlags::kMutable}}, {"parallel_threads", {offsetof(struct CompressionOptions, parallel_threads), OptionType::kUInt32T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct CompressionOptions, parallel_threads)}}, + OptionTypeFlags::kMutable}}, {"enabled", {offsetof(struct CompressionOptions, enabled), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct CompressionOptions, enabled)}}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, }; static std::unordered_map @@ -174,424 +168,195 @@ static std::unordered_map {"max_table_files_size", {offsetof(struct CompactionOptionsFIFO, max_table_files_size), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, + OptionTypeFlags::kMutable}}, {"ttl", {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"allow_compaction", {offsetof(struct CompactionOptionsFIFO, allow_compaction), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; + OptionTypeFlags::kMutable}}, +}; static std::unordered_map universal_compaction_options_type_info = { {"size_ratio", {offsetof(class CompactionOptionsUniversal, size_ratio), OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, size_ratio)}}, + OptionTypeFlags::kMutable}}, {"min_merge_width", {offsetof(class CompactionOptionsUniversal, min_merge_width), OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, min_merge_width)}}, + OptionTypeFlags::kMutable}}, {"max_merge_width", {offsetof(class CompactionOptionsUniversal, max_merge_width), OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, max_merge_width)}}, + OptionTypeFlags::kMutable}}, {"max_size_amplification_percent", {offsetof(class CompactionOptionsUniversal, max_size_amplification_percent), OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, - max_size_amplification_percent)}}, + OptionTypeFlags::kMutable}}, {"compression_size_percent", {offsetof(class CompactionOptionsUniversal, compression_size_percent), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, - compression_size_percent)}}, + OptionTypeFlags::kMutable}}, {"stop_style", {offsetof(class CompactionOptionsUniversal, stop_style), OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, stop_style)}}, + OptionTypeFlags::kMutable}}, {"allow_trivial_move", {offsetof(class CompactionOptionsUniversal, allow_trivial_move), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}}; + OptionTypeFlags::kMutable}}}; -std::unordered_map - OptionsHelper::cf_options_type_info = { - /* not yet supported - CompressionOptions compression_opts; - TablePropertiesCollectorFactories table_properties_collector_factories; - typedef std::vector> - TablePropertiesCollectorFactories; - UpdateStatus (*inplace_callback)(char* existing_value, - uint34_t* existing_value_size, - Slice delta_value, - std::string* merged_value); - std::vector cf_paths; - */ +static std::unordered_map + cf_mutable_options_type_info = { {"report_bg_io_stats", - {offset_of(&ColumnFamilyOptions::report_bg_io_stats), + {offsetof(struct MutableCFOptions, report_bg_io_stats), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, report_bg_io_stats)}}, - {"compaction_measure_io_stats", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"disable_auto_compactions", - {offset_of(&ColumnFamilyOptions::disable_auto_compactions), + {offsetof(struct MutableCFOptions, disable_auto_compactions), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, disable_auto_compactions)}}, + OptionTypeFlags::kMutable}}, {"filter_deletes", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, - {"inplace_update_support", - {offset_of(&ColumnFamilyOptions::inplace_update_support), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"level_compaction_dynamic_level_bytes", - {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"optimize_filters_for_hits", - {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"paranoid_file_checks", - {offset_of(&ColumnFamilyOptions::paranoid_file_checks), + {offsetof(struct MutableCFOptions, paranoid_file_checks), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, paranoid_file_checks)}}, - {"force_consistency_checks", - {offset_of(&ColumnFamilyOptions::force_consistency_checks), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"purge_redundant_kvs_while_flush", - {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), - OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"verify_checksums_in_compaction", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"soft_pending_compaction_bytes_limit", - {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit), + {offsetof(struct MutableCFOptions, + soft_pending_compaction_bytes_limit), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, - soft_pending_compaction_bytes_limit)}}, + OptionTypeFlags::kMutable}}, {"hard_pending_compaction_bytes_limit", - {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit), + {offsetof(struct MutableCFOptions, + hard_pending_compaction_bytes_limit), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, - hard_pending_compaction_bytes_limit)}}, + OptionTypeFlags::kMutable}}, {"hard_rate_limit", {0, OptionType::kDouble, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"soft_rate_limit", {0, OptionType::kDouble, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"max_compaction_bytes", - {offset_of(&ColumnFamilyOptions::max_compaction_bytes), + {offsetof(struct MutableCFOptions, max_compaction_bytes), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, max_compaction_bytes)}}, + OptionTypeFlags::kMutable}}, {"expanded_compaction_factor", {0, OptionType::kInt, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"level0_file_num_compaction_trigger", - {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger), + {offsetof(struct MutableCFOptions, level0_file_num_compaction_trigger), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, - level0_file_num_compaction_trigger)}}, + OptionTypeFlags::kMutable}}, {"level0_slowdown_writes_trigger", - {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger), + {offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}}, + OptionTypeFlags::kMutable}}, {"level0_stop_writes_trigger", - {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger), + {offsetof(struct MutableCFOptions, level0_stop_writes_trigger), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}}, + OptionTypeFlags::kMutable}}, {"max_grandparent_overlap_factor", {0, OptionType::kInt, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, - {"max_mem_compaction_level", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"max_write_buffer_number", - {offset_of(&ColumnFamilyOptions::max_write_buffer_number), + {offsetof(struct MutableCFOptions, max_write_buffer_number), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, max_write_buffer_number)}}, - {"max_write_buffer_number_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), - OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"max_write_buffer_size_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain), - OptionType::kInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"min_write_buffer_number_to_merge", - {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), - OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"num_levels", - {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"source_compaction_factor", {0, OptionType::kInt, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"target_file_size_multiplier", - {offset_of(&ColumnFamilyOptions::target_file_size_multiplier), + {offsetof(struct MutableCFOptions, target_file_size_multiplier), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, target_file_size_multiplier)}}, + OptionTypeFlags::kMutable}}, {"arena_block_size", - {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, arena_block_size)}}, + {offsetof(struct MutableCFOptions, arena_block_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, {"inplace_update_num_locks", - {offset_of(&ColumnFamilyOptions::inplace_update_num_locks), + {offsetof(struct MutableCFOptions, inplace_update_num_locks), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, inplace_update_num_locks)}}, + OptionTypeFlags::kMutable}}, {"max_successive_merges", - {offset_of(&ColumnFamilyOptions::max_successive_merges), + {offsetof(struct MutableCFOptions, max_successive_merges), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, max_successive_merges)}}, + OptionTypeFlags::kMutable}}, {"memtable_huge_page_size", - {offset_of(&ColumnFamilyOptions::memtable_huge_page_size), + {offsetof(struct MutableCFOptions, memtable_huge_page_size), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, memtable_huge_page_size)}}, + OptionTypeFlags::kMutable}}, {"memtable_prefix_bloom_huge_page_tlb_size", {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"write_buffer_size", - {offset_of(&ColumnFamilyOptions::write_buffer_size), + {offsetof(struct MutableCFOptions, write_buffer_size), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, write_buffer_size)}}, - {"bloom_locality", - {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"memtable_prefix_bloom_bits", {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"memtable_prefix_bloom_size_ratio", - {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio), + {offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio), OptionType::kDouble, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}}, + OptionTypeFlags::kMutable}}, {"memtable_prefix_bloom_probes", {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"memtable_whole_key_filtering", - {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering), + {offsetof(struct MutableCFOptions, memtable_whole_key_filtering), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}}, + OptionTypeFlags::kMutable}}, {"min_partial_merge_operands", {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"max_bytes_for_level_base", - {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base), + {offsetof(struct MutableCFOptions, max_bytes_for_level_base), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, max_bytes_for_level_base)}}, + OptionTypeFlags::kMutable}}, {"snap_refresh_nanos", {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + OptionTypeFlags::kMutable}}, {"max_bytes_for_level_multiplier", - {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier), + {offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier), OptionType::kDouble, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}}, + OptionTypeFlags::kMutable}}, {"max_bytes_for_level_multiplier_additional", OptionTypeInfo::Vector( - offset_of(&ColumnFamilyOptions:: - max_bytes_for_level_multiplier_additional), - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier_additional), - {0, OptionType::kInt, 0})}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + {0, OptionType::kInt})}, {"max_sequential_skip_in_iterations", - {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations), + {offsetof(struct MutableCFOptions, max_sequential_skip_in_iterations), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, - max_sequential_skip_in_iterations)}}, + OptionTypeFlags::kMutable}}, {"target_file_size_base", - {offset_of(&ColumnFamilyOptions::target_file_size_base), + {offsetof(struct MutableCFOptions, target_file_size_base), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, target_file_size_base)}}, - {"rate_limit_delay_max_milliseconds", - {0, OptionType::kUInt, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"compression", - {offset_of(&ColumnFamilyOptions::compression), + {offsetof(struct MutableCFOptions, compression), OptionType::kCompressionType, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, compression)}}, - {"compression_per_level", - OptionTypeInfo::Vector( - offset_of(&ColumnFamilyOptions::compression_per_level), - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0, - {0, OptionType::kCompressionType})}, - {"bottommost_compression", - {offset_of(&ColumnFamilyOptions::bottommost_compression), - OptionType::kCompressionType, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, bottommost_compression)}}, - {"comparator", - {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, - OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose, 0, - // Parses the string and sets the corresponding comparator - [](const ConfigOptions& /*opts*/, const std::string& /*name*/, - const std::string& value, char* addr) { - auto old_comparator = reinterpret_cast(addr); - const Comparator* new_comparator = *old_comparator; - Status status = ObjectRegistry::NewInstance()->NewStaticObject( - value, &new_comparator); - if (status.ok()) { - *old_comparator = new_comparator; - return status; - } - return Status::OK(); - }}}, + OptionTypeFlags::kMutable}}, {"prefix_extractor", - {offset_of(&ColumnFamilyOptions::prefix_extractor), + {offsetof(struct MutableCFOptions, prefix_extractor), OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, prefix_extractor)}}, - {"memtable_insert_with_hint_prefix_extractor", - {offset_of( - &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), - OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, - OptionTypeFlags::kNone, 0}}, - {"memtable_factory", - {offset_of(&ColumnFamilyOptions::memtable_factory), - OptionType::kMemTableRepFactory, OptionVerificationType::kByName, - OptionTypeFlags::kNone, 0}}, - {"memtable", - {offset_of(&ColumnFamilyOptions::memtable_factory), - OptionType::kMemTableRepFactory, OptionVerificationType::kAlias, - OptionTypeFlags::kNone, 0, - // Parses the value string and updates the memtable_factory - [](const ConfigOptions& /*opts*/, const std::string& /*name*/, - const std::string& value, char* addr) { - std::unique_ptr new_mem_factory; - Status s = GetMemTableRepFactoryFromString(value, &new_mem_factory); - if (s.ok()) { - auto memtable_factory = - reinterpret_cast*>(addr); - memtable_factory->reset(new_mem_factory.release()); - } - return s; - }}}, - {"table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kTableFactory, OptionVerificationType::kByName, - OptionTypeFlags::kCompareLoose, 0}}, - {"block_based_table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kTableFactory, OptionVerificationType::kAlias, - OptionTypeFlags::kCompareLoose, 0, - // Parses the input value and creates a BlockBasedTableFactory - [](const ConfigOptions& /*opts*/, const std::string& /*name*/, - const std::string& value, char* addr) { - // Nested options - auto old_table_factory = - reinterpret_cast*>(addr); - BlockBasedTableOptions table_opts, base_opts; - BlockBasedTableFactory* block_based_table_factory = - static_cast_with_check( - old_table_factory->get()); - if (block_based_table_factory != nullptr) { - base_opts = block_based_table_factory->table_options(); - } - Status s = GetBlockBasedTableOptionsFromString(base_opts, value, - &table_opts); - if (s.ok()) { - old_table_factory->reset(NewBlockBasedTableFactory(table_opts)); - } - return s; - }}}, - {"plain_table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kTableFactory, OptionVerificationType::kAlias, - OptionTypeFlags::kCompareLoose, 0, - // Parses the input value and creates a PlainTableFactory - [](const ConfigOptions& /*opts*/, const std::string& /*name*/, - const std::string& value, char* addr) { - // Nested options - auto old_table_factory = - reinterpret_cast*>(addr); - PlainTableOptions table_opts, base_opts; - PlainTableFactory* plain_table_factory = - static_cast_with_check( - old_table_factory->get()); - if (plain_table_factory != nullptr) { - base_opts = plain_table_factory->table_options(); - } - Status s = - GetPlainTableOptionsFromString(base_opts, value, &table_opts); - if (s.ok()) { - old_table_factory->reset(NewPlainTableFactory(table_opts)); - } - return s; - }}}, - {"compaction_filter", - {offset_of(&ColumnFamilyOptions::compaction_filter), - OptionType::kCompactionFilter, OptionVerificationType::kByName, - OptionTypeFlags::kNone, 0}}, - {"compaction_filter_factory", - {offset_of(&ColumnFamilyOptions::compaction_filter_factory), - OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, - OptionTypeFlags::kNone, 0}}, - {"merge_operator", - {offset_of(&ColumnFamilyOptions::merge_operator), - OptionType::kMergeOperator, - OptionVerificationType::kByNameAllowFromNull, - OptionTypeFlags::kCompareLoose, 0, - // Parses the input value as a MergeOperator, updating the value - [](const ConfigOptions& /*opts*/, const std::string& /*name*/, - const std::string& value, char* addr) { - auto mop = reinterpret_cast*>(addr); - ObjectRegistry::NewInstance() - ->NewSharedObject(value, mop) - .PermitUncheckedError(); - return Status::OK(); - }}}, - {"compaction_style", - {offset_of(&ColumnFamilyOptions::compaction_style), - OptionType::kCompactionStyle, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"compaction_pri", - {offset_of(&ColumnFamilyOptions::compaction_pri), - OptionType::kCompactionPri, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"compaction_options_fifo", OptionTypeInfo::Struct( "compaction_options_fifo", &fifo_compaction_options_type_info, - offset_of(&ColumnFamilyOptions::compaction_options_fifo), - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct MutableCFOptions, compaction_options_fifo), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, [](const ConfigOptions& opts, const std::string& name, const std::string& value, char* addr) { // This is to handle backward compatibility, where @@ -614,51 +379,45 @@ std::unordered_map OptionTypeInfo::Struct( "compaction_options_universal", &universal_compaction_options_type_info, - offset_of(&ColumnFamilyOptions::compaction_options_universal), - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, compaction_options_universal))}, + offsetof(struct MutableCFOptions, compaction_options_universal), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable)}, {"ttl", - {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, ttl)}}, + {offsetof(struct MutableCFOptions, ttl), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, {"periodic_compaction_seconds", - {offset_of(&ColumnFamilyOptions::periodic_compaction_seconds), + {offsetof(struct MutableCFOptions, periodic_compaction_seconds), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, periodic_compaction_seconds)}}, - {"sample_for_compression", - {offset_of(&ColumnFamilyOptions::sample_for_compression), - OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, sample_for_compression)}}, + OptionTypeFlags::kMutable}}, {"enable_blob_files", - {offset_of(&ColumnFamilyOptions::enable_blob_files), + {offsetof(struct MutableCFOptions, enable_blob_files), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, enable_blob_files)}}, + OptionTypeFlags::kMutable}}, {"min_blob_size", - {offset_of(&ColumnFamilyOptions::min_blob_size), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, min_blob_size)}}, + {offsetof(struct MutableCFOptions, min_blob_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, {"blob_file_size", - {offset_of(&ColumnFamilyOptions::blob_file_size), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, blob_file_size)}}, + {offsetof(struct MutableCFOptions, blob_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, {"blob_compression_type", - {offset_of(&ColumnFamilyOptions::blob_compression_type), + {offsetof(struct MutableCFOptions, blob_compression_type), OptionType::kCompressionType, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableCFOptions, blob_compression_type)}}, - // The following properties were handled as special cases in ParseOption - // This means that the properties could be read from the options file - // but never written to the file or compared to each other. + OptionTypeFlags::kMutable}}, + {"sample_for_compression", + {offsetof(struct MutableCFOptions, sample_for_compression), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"bottommost_compression", + {offsetof(struct MutableCFOptions, bottommost_compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, {kOptNameCompOpts, OptionTypeInfo::Struct( kOptNameCompOpts, &compression_options_type_info, - offset_of(&ColumnFamilyOptions::compression_opts), + offsetof(struct MutableCFOptions, compression_opts), OptionVerificationType::kNormal, (OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever), - offsetof(struct MutableCFOptions, compression_opts), [](const ConfigOptions& opts, const std::string& name, const std::string& value, char* addr) { // This is to handle backward compatibility, where @@ -677,10 +436,9 @@ std::unordered_map {kOptNameBMCompOpts, OptionTypeInfo::Struct( kOptNameBMCompOpts, &compression_options_type_info, - offset_of(&ColumnFamilyOptions::bottommost_compression_opts), + offsetof(struct MutableCFOptions, bottommost_compression_opts), OptionVerificationType::kNormal, (OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever), - offsetof(struct MutableCFOptions, bottommost_compression_opts), [](const ConfigOptions& opts, const std::string& name, const std::string& value, char* addr) { // This is to handle backward compatibility, where @@ -699,29 +457,337 @@ std::unordered_map // End special case properties }; -Status ParseColumnFamilyOption(const ConfigOptions& config_options, - const std::string& name, - const std::string& org_value, - ColumnFamilyOptions* new_options) { - const std::string& value = config_options.input_strings_escaped - ? UnescapeOptionString(org_value) - : org_value; - try { - std::string elem; - const auto opt_info = - OptionTypeInfo::Find(name, cf_options_type_info, &elem); - if (opt_info != nullptr) { - return opt_info->Parse( - config_options, elem, value, - reinterpret_cast(new_options) + opt_info->offset_); - } else { - return Status::InvalidArgument( - "Unable to parse the specified CF option " + name); - } - } catch (const std::exception&) { - return Status::InvalidArgument("unable to parse the specified option " + - name); +static std::unordered_map + cf_immutable_options_type_info = { + /* not yet supported + CompressionOptions compression_opts; + TablePropertiesCollectorFactories table_properties_collector_factories; + typedef std::vector> + TablePropertiesCollectorFactories; + UpdateStatus (*inplace_callback)(char* existing_value, + uint34_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + std::vector cf_paths; + */ + {"compaction_measure_io_stats", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"inplace_update_support", + {offset_of(&ColumnFamilyOptions::inplace_update_support), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"level_compaction_dynamic_level_bytes", + {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"optimize_filters_for_hits", + {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"force_consistency_checks", + {offset_of(&ColumnFamilyOptions::force_consistency_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"purge_redundant_kvs_while_flush", + {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), + OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"max_mem_compaction_level", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"max_write_buffer_number_to_maintain", + {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_write_buffer_size_to_maintain", + {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain), + OptionType::kInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"min_write_buffer_number_to_merge", + {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"num_levels", + {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bloom_locality", + {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"rate_limit_delay_max_milliseconds", + {0, OptionType::kUInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"compression_per_level", + OptionTypeInfo::Vector( + offset_of(&ColumnFamilyOptions::compression_per_level), + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kCompressionType})}, + {"comparator", + {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, + OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose, + // Parses the string and sets the corresponding comparator + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto old_comparator = reinterpret_cast(addr); + const Comparator* new_comparator = *old_comparator; + Status status = ObjectRegistry::NewInstance()->NewStaticObject( + value, &new_comparator); + if (status.ok()) { + *old_comparator = new_comparator; + return status; + } + return Status::OK(); + }}}, + {"memtable_insert_with_hint_prefix_extractor", + {offset_of( + &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), + OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, + OptionTypeFlags::kNone}}, + {"memtable_factory", + {offset_of(&ColumnFamilyOptions::memtable_factory), + OptionType::kMemTableRepFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone}}, + {"memtable", + {offset_of(&ColumnFamilyOptions::memtable_factory), + OptionType::kMemTableRepFactory, OptionVerificationType::kAlias, + OptionTypeFlags::kNone, + // Parses the value string and updates the memtable_factory + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + std::unique_ptr new_mem_factory; + Status s = GetMemTableRepFactoryFromString(value, &new_mem_factory); + if (s.ok()) { + auto memtable_factory = + reinterpret_cast*>(addr); + memtable_factory->reset(new_mem_factory.release()); + } + return s; + }}}, + {"table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kConfigurable, OptionVerificationType::kByName, + (OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose | + OptionTypeFlags::kDontPrepare), + // Creates a new TableFactory based on value + [](const ConfigOptions& opts, const std::string& /*name*/, + const std::string& value, char* addr) { + auto table_factory = + reinterpret_cast*>(addr); + return TableFactory::CreateFromString(opts, value, table_factory); + }, + // Converts the TableFactory into its string representation + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const char* addr, std::string* value) { + const auto* table_factory = + reinterpret_cast*>(addr); + *value = table_factory->get() ? table_factory->get()->Name() + : kNullptrString; + return Status::OK(); + }, + /* No equals function for table factories */ nullptr}}, + {"block_based_table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kConfigurable, OptionVerificationType::kAlias, + OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, + // Parses the input value and creates a BlockBasedTableFactory + [](const ConfigOptions& opts, const std::string& name, + const std::string& value, char* addr) { + BlockBasedTableOptions* old_opts = nullptr; + auto table_factory = + reinterpret_cast*>(addr); + if (table_factory->get() != nullptr) { + old_opts = + table_factory->get()->GetOptions(); + } + if (name == "block_based_table_factory") { + std::unique_ptr new_factory; + if (old_opts != nullptr) { + new_factory.reset(NewBlockBasedTableFactory(*old_opts)); + } else { + new_factory.reset(NewBlockBasedTableFactory()); + } + Status s = new_factory->ConfigureFromString(opts, value); + if (s.ok()) { + table_factory->reset(new_factory.release()); + } + return s; + } else if (old_opts != nullptr) { + return table_factory->get()->ConfigureOption(opts, name, value); + } else { + return Status::NotFound("Mismatched table option: ", name); + } + }}}, + {"plain_table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kConfigurable, OptionVerificationType::kAlias, + OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, + // Parses the input value and creates a PlainTableFactory + [](const ConfigOptions& opts, const std::string& name, + const std::string& value, char* addr) { + PlainTableOptions* old_opts = nullptr; + auto table_factory = + reinterpret_cast*>(addr); + if (table_factory->get() != nullptr) { + old_opts = table_factory->get()->GetOptions(); + } + if (name == "plain_table_factory") { + std::unique_ptr new_factory; + if (old_opts != nullptr) { + new_factory.reset(NewPlainTableFactory(*old_opts)); + } else { + new_factory.reset(NewPlainTableFactory()); + } + Status s = new_factory->ConfigureFromString(opts, value); + if (s.ok()) { + table_factory->reset(new_factory.release()); + } + return s; + } else if (old_opts != nullptr) { + return table_factory->get()->ConfigureOption(opts, name, value); + } else { + return Status::NotFound("Mismatched table option: ", name); + } + }}}, + {"compaction_filter", + {offset_of(&ColumnFamilyOptions::compaction_filter), + OptionType::kCompactionFilter, OptionVerificationType::kByName, + OptionTypeFlags::kNone}}, + {"compaction_filter_factory", + {offset_of(&ColumnFamilyOptions::compaction_filter_factory), + OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone}}, + {"merge_operator", + {offset_of(&ColumnFamilyOptions::merge_operator), + OptionType::kMergeOperator, + OptionVerificationType::kByNameAllowFromNull, + OptionTypeFlags::kCompareLoose, + // Parses the input value as a MergeOperator, updating the value + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto mop = reinterpret_cast*>(addr); + Status status = + ObjectRegistry::NewInstance()->NewSharedObject( + value, mop); + // Only support static comparator for now. + if (status.ok()) { + return status; + } + return Status::OK(); + }}}, + {"compaction_style", + {offset_of(&ColumnFamilyOptions::compaction_style), + OptionType::kCompactionStyle, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"compaction_pri", + {offset_of(&ColumnFamilyOptions::compaction_pri), + OptionType::kCompactionPri, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions"; + +class ConfigurableMutableCFOptions : public Configurable { + public: + ConfigurableMutableCFOptions(const MutableCFOptions& mcf) { + mutable_ = mcf; + ConfigurableHelper::RegisterOptions(*this, &mutable_, + &cf_mutable_options_type_info); } + + protected: + MutableCFOptions mutable_; +}; + +class ConfigurableCFOptions : public ConfigurableMutableCFOptions { + public: + ConfigurableCFOptions(const ColumnFamilyOptions& opts, + const std::unordered_map* map) + : ConfigurableMutableCFOptions(MutableCFOptions(opts)), + immutable_(opts), + cf_options_(opts), + opt_map_(map) { + ConfigurableHelper::RegisterOptions(*this, OptionsHelper::kCFOptionsName, + &immutable_, + &cf_immutable_options_type_info); + } + + protected: + Status ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) override { + Status s = ConfigurableHelper::ConfigureOptions(config_options, *this, + opts_map, unused); + if (s.ok()) { + cf_options_ = BuildColumnFamilyOptions(immutable_, mutable_); + s = PrepareOptions(config_options); + } + return s; + } + + virtual const void* GetOptionsPtr(const std::string& name) const override { + if (name == OptionsHelper::kCFOptionsName) { + return &cf_options_; + } else { + return ConfigurableMutableCFOptions::GetOptionsPtr(name); + } + } + + bool OptionsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, const void* const this_ptr, + const void* const that_ptr, + std::string* mismatch) const override { + bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr, + that_ptr, mismatch); + if (!equals && opt_info.IsByName()) { + if (opt_map_ == nullptr) { + equals = true; + } else { + const auto& iter = opt_map_->find(opt_name); + if (iter == opt_map_->end()) { + equals = true; + } else { + equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr, + iter->second); + } + } + if (equals) { // False alarm, clear mismatch + *mismatch = ""; + } + } + if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) { + const auto* this_config = opt_info.AsRawPointer(this_ptr); + if (this_config == nullptr) { + const auto& iter = opt_map_->find(opt_name); + // If the name exists in the map and is not empty/null, + // then the this_config should be set. + if (iter != opt_map_->end() && !iter->second.empty() && + iter->second != kNullptrString) { + *mismatch = opt_name; + equals = false; + } + } + } + return equals; + } + + private: + ColumnFamilyOptions immutable_; + ColumnFamilyOptions cf_options_; + const std::unordered_map* opt_map_; +}; + +std::unique_ptr CFOptionsAsConfigurable( + const MutableCFOptions& opts) { + std::unique_ptr ptr(new ConfigurableMutableCFOptions(opts)); + return ptr; +} +std::unique_ptr CFOptionsAsConfigurable( + const ColumnFamilyOptions& opts, + const std::unordered_map* opt_map) { + std::unique_ptr ptr(new ConfigurableCFOptions(opts, opt_map)); + return ptr; } #endif // ROCKSDB_LITE diff --git a/options/cf_options.h b/options/cf_options.h index a2fbf2240..ae5f6d08d 100644 --- a/options/cf_options.h +++ b/options/cf_options.h @@ -20,6 +20,7 @@ namespace ROCKSDB_NAMESPACE { // of DB. Raw pointers defined in this struct do not have ownership to the data // they point to. Options contains std::shared_ptr to these data. struct ImmutableCFOptions { + static const char* kName() { return "ImmutableCFOptions"; } explicit ImmutableCFOptions(const Options& options); ImmutableCFOptions(const ImmutableDBOptions& db_options, @@ -124,6 +125,7 @@ struct ImmutableCFOptions { }; struct MutableCFOptions { + static const char* kName() { return "MutableCFOptions"; } explicit MutableCFOptions(const ColumnFamilyOptions& options) : write_buffer_size(options.write_buffer_size), max_write_buffer_number(options.max_write_buffer_number), diff --git a/options/configurable.cc b/options/configurable.cc new file mode 100644 index 000000000..8c11b0b0e --- /dev/null +++ b/options/configurable.cc @@ -0,0 +1,610 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/configurable.h" + +#include "logging/logging.h" +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "rocksdb/status.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/coding.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +void ConfigurableHelper::RegisterOptions( + Configurable& configurable, const std::string& name, void* opt_ptr, + const std::unordered_map* type_map) { + Configurable::RegisteredOptions opts; + opts.name = name; +#ifndef ROCKSDB_LITE + opts.type_map = type_map; +#else + (void)type_map; +#endif // ROCKSDB_LITE + opts.opt_ptr = opt_ptr; + configurable.options_.emplace_back(opts); +} + +//************************************************************************* +// +// Methods for Initializing and Validating Configurable Objects +// +//************************************************************************* + +Status Configurable::PrepareOptions(const ConfigOptions& opts) { + Status status = Status::OK(); +#ifndef ROCKSDB_LITE + for (auto opt_iter : options_) { + for (auto map_iter : *(opt_iter.type_map)) { + auto& opt_info = map_iter.second; + if (!opt_info.IsDeprecated() && !opt_info.IsAlias() && + opt_info.IsConfigurable()) { + if (!opt_info.IsEnabled(OptionTypeFlags::kDontPrepare)) { + Configurable* config = + opt_info.AsRawPointer(opt_iter.opt_ptr); + if (config != nullptr) { + status = config->PrepareOptions(opts); + if (!status.ok()) { + return status; + } + } + } + } + } + } +#endif // ROCKSDB_LITE + if (status.ok()) { + auto inner = Inner(); + if (inner != nullptr) { + status = inner->PrepareOptions(opts); + } + } + if (status.ok()) { + prepared_ = true; + } + return status; +} + +Status Configurable::ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const { + Status status; +#ifndef ROCKSDB_LITE + for (auto opt_iter : options_) { + for (auto map_iter : *(opt_iter.type_map)) { + auto& opt_info = map_iter.second; + if (!opt_info.IsDeprecated() && !opt_info.IsAlias()) { + if (opt_info.IsConfigurable()) { + const Configurable* config = + opt_info.AsRawPointer(opt_iter.opt_ptr); + if (config != nullptr) { + status = config->ValidateOptions(db_opts, cf_opts); + } else if (!opt_info.CanBeNull()) { + status = + Status::NotFound("Missing configurable object", map_iter.first); + } + if (!status.ok()) { + return status; + } + } + } + } + } +#endif // ROCKSDB_LITE + if (status.ok()) { + const auto inner = Inner(); + if (inner != nullptr) { + status = inner->ValidateOptions(db_opts, cf_opts); + } + } + return status; +} + +/*********************************************************************************/ +/* */ +/* Methods for Retrieving Options from Configurables */ +/* */ +/*********************************************************************************/ + +const void* Configurable::GetOptionsPtr(const std::string& name) const { + for (auto o : options_) { + if (o.name == name) { + return o.opt_ptr; + } + } + auto inner = Inner(); + if (inner != nullptr) { + return inner->GetOptionsPtr(name); + } else { + return nullptr; + } +} + +std::string Configurable::GetOptionName(const std::string& opt_name) const { + return opt_name; +} + +#ifndef ROCKSDB_LITE +const OptionTypeInfo* ConfigurableHelper::FindOption( + const std::vector& options, + const std::string& short_name, std::string* opt_name, void** opt_ptr) { + for (auto iter : options) { + const auto opt_info = + OptionTypeInfo::Find(short_name, *(iter.type_map), opt_name); + if (opt_info != nullptr) { + *opt_ptr = iter.opt_ptr; + return opt_info; + } + } + return nullptr; +} +#endif // ROCKSDB_LITE + +//************************************************************************* +// +// Methods for Configuring Options from Strings/Name-Value Pairs/Maps +// +//************************************************************************* + +Status Configurable::ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opts_map) { + Status s = ConfigureFromMap(config_options, opts_map, nullptr); + return s; +} + +Status Configurable::ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) { + return ConfigureOptions(config_options, opts_map, unused); +} + +Status Configurable::ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) { + std::string curr_opts; +#ifndef ROCKSDB_LITE + if (!config_options.ignore_unknown_options) { + // If we are not ignoring unused, get the defaults in case we need to reset + GetOptionString(config_options, &curr_opts).PermitUncheckedError(); + } +#endif // ROCKSDB_LITE + Status s = ConfigurableHelper::ConfigureOptions(config_options, *this, + opts_map, unused); + if (config_options.invoke_prepare_options && s.ok()) { + s = PrepareOptions(config_options); + } +#ifndef ROCKSDB_LITE + if (!s.ok() && !curr_opts.empty()) { + ConfigOptions reset = config_options; + reset.ignore_unknown_options = true; + reset.invoke_prepare_options = true; + // There are some options to reset from this current error + ConfigureFromString(reset, curr_opts).PermitUncheckedError(); + } +#endif // ROCKSDB_LITE + return s; +} + +Status Configurable::ParseStringOptions(const ConfigOptions& /*config_options*/, + const std::string& /*opts_str*/) { + return Status::OK(); +} + +Status Configurable::ConfigureFromString(const ConfigOptions& config_options, + const std::string& opts_str) { + Status s; + if (!opts_str.empty()) { +#ifndef ROCKSDB_LITE + if (opts_str.find(';') != std::string::npos || + opts_str.find('=') != std::string::npos) { + std::unordered_map opt_map; + s = StringToMap(opts_str, &opt_map); + if (s.ok()) { + s = ConfigureFromMap(config_options, opt_map, nullptr); + } + } else { +#endif // ROCKSDB_LITE + s = ParseStringOptions(config_options, opts_str); + if (s.ok() && config_options.invoke_prepare_options) { + s = PrepareOptions(config_options); + } +#ifndef ROCKSDB_LITE + } +#endif // ROCKSDB_LITE + } else if (config_options.invoke_prepare_options) { + s = PrepareOptions(config_options); + } else { + s = Status::OK(); + } + return s; +} + +#ifndef ROCKSDB_LITE +/** + * Sets the value of the named property to the input value, returning OK on + * succcess. + */ +Status Configurable::ConfigureOption(const ConfigOptions& config_options, + const std::string& name, + const std::string& value) { + const std::string& opt_name = GetOptionName(name); + return ConfigurableHelper::ConfigureSingleOption(config_options, *this, + opt_name, value); +} + +/** + * Looks for the named option amongst the options for this type and sets + * the value for it to be the input value. + * If the name was found, found_option will be set to true and the resulting + * status should be returned. + */ + +Status Configurable::ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& opt_value, void* opt_ptr) { + if (opt_info.IsMutable() || opt_info.IsConfigurable()) { + return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr); + } else if (prepared_) { + return Status::InvalidArgument("Option not changeable: " + opt_name); + } else { + return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr); + } +} + +#endif // ROCKSDB_LITE + +Status ConfigurableHelper::ConfigureOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& opts_map, + std::unordered_map* unused) { + std::unordered_map remaining = opts_map; + Status s = Status::OK(); + if (!opts_map.empty()) { +#ifndef ROCKSDB_LITE + for (const auto& iter : configurable.options_) { + s = ConfigureSomeOptions(config_options, configurable, *(iter.type_map), + &remaining, iter.opt_ptr); + if (remaining.empty()) { // Are there more options left? + break; + } else if (!s.ok()) { + break; + } + } +#else + (void)configurable; + if (!config_options.ignore_unknown_options) { + s = Status::NotSupported("ConfigureFromMap not supported in LITE mode"); + } +#endif // ROCKSDB_LITE + } + if (unused != nullptr && !remaining.empty()) { + unused->insert(remaining.begin(), remaining.end()); + } + if (config_options.ignore_unknown_options) { + s = Status::OK(); + } else if (s.ok() && unused == nullptr && !remaining.empty()) { + s = Status::NotFound("Could not find option: ", remaining.begin()->first); + } + return s; +} + +#ifndef ROCKSDB_LITE +/** + * Updates the object with the named-value property values, returning OK on + * succcess. Any properties that were found are removed from the options list; + * upon return only options that were not found in this opt_map remain. + + * Returns: + * - OK if ignore_unknown_options is set + * - InvalidArgument, if any option was invalid + * - NotSupported, if any option is unsupported and ignore_unsupported_options + is OFF + * - OK, if no option was invalid or not supported (or ignored) + */ +Status ConfigurableHelper::ConfigureSomeOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& type_map, + std::unordered_map* options, void* opt_ptr) { + Status result = Status::OK(); // The last non-OK result (if any) + Status notsup = Status::OK(); // The last NotSupported result (if any) + std::string elem_name; + int found = 1; + std::unordered_set unsupported; + // While there are unused properties and we processed at least one, + // go through the remaining unused properties and attempt to configure them. + while (found > 0 && !options->empty()) { + found = 0; + notsup = Status::OK(); + for (auto it = options->begin(); it != options->end();) { + const std::string& opt_name = configurable.GetOptionName(it->first); + const std::string& opt_value = it->second; + const auto opt_info = + OptionTypeInfo::Find(opt_name, type_map, &elem_name); + if (opt_info == nullptr) { // Did not find the option. Skip it + ++it; + } else { + Status s = ConfigureOption(config_options, configurable, *opt_info, + opt_name, elem_name, opt_value, opt_ptr); + if (s.IsNotFound()) { + ++it; + } else if (s.IsNotSupported()) { + notsup = s; + unsupported.insert(it->first); + ++it; // Skip it for now + } else { + found++; + it = options->erase(it); + if (!s.ok()) { + result = s; + } + } + } + } // End for all remaining options + } // End while found one or options remain + + // Now that we have been through the list, remove any unsupported + for (auto u : unsupported) { + auto it = options->find(u); + if (it != options->end()) { + options->erase(it); + } + } + if (config_options.ignore_unknown_options) { + if (!result.ok()) result.PermitUncheckedError(); + if (!notsup.ok()) notsup.PermitUncheckedError(); + return Status::OK(); + } else if (!result.ok()) { + if (!notsup.ok()) notsup.PermitUncheckedError(); + return result; + } else if (config_options.ignore_unsupported_options) { + if (!notsup.ok()) notsup.PermitUncheckedError(); + return Status::OK(); + } else { + return notsup; + } +} + +Status ConfigurableHelper::ConfigureSingleOption( + const ConfigOptions& config_options, Configurable& configurable, + const std::string& name, const std::string& value) { + std::string opt_name; + void* opt_ptr = nullptr; + const auto opt_info = + FindOption(configurable.options_, name, &opt_name, &opt_ptr); + if (opt_info == nullptr) { + return Status::NotFound("Could not find option: ", name); + } else { + return ConfigureOption(config_options, configurable, *opt_info, name, + opt_name, value, opt_ptr); + } +} + +Status ConfigurableHelper::ConfigureOption( + const ConfigOptions& config_options, Configurable& configurable, + const OptionTypeInfo& opt_info, const std::string& opt_name, + const std::string& name, const std::string& value, void* opt_ptr) { + if (opt_name == name) { + return configurable.ParseOption(config_options, opt_info, opt_name, value, + opt_ptr); + } else if (opt_info.IsStruct() || opt_info.IsConfigurable()) { + return configurable.ParseOption(config_options, opt_info, name, value, + opt_ptr); + } else { + return Status::NotFound("Could not find option: ", name); + } +} +#endif // ROCKSDB_LITE + +//******************************************************************************* +// +// Methods for Converting Options into strings +// +//******************************************************************************* + +Status Configurable::GetOptionString(const ConfigOptions& config_options, + std::string* result) const { + assert(result); + result->clear(); +#ifndef ROCKSDB_LITE + return ConfigurableHelper::SerializeOptions(config_options, *this, "", + result); +#else + (void)config_options; + return Status::NotSupported("GetOptionString not supported in LITE mode"); +#endif // ROCKSDB_LITE +} + +#ifndef ROCKSDB_LITE +std::string Configurable::ToString(const ConfigOptions& config_options, + const std::string& prefix) const { + std::string result = SerializeOptions(config_options, prefix); + if (result.empty() || result.find('=') == std::string::npos) { + return result; + } else { + return "{" + result + "}"; + } +} + +std::string Configurable::SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const { + std::string result; + Status s = ConfigurableHelper::SerializeOptions(config_options, *this, header, + &result); + assert(s.ok()); + return result; +} + +Status Configurable::GetOption(const ConfigOptions& config_options, + const std::string& name, + std::string* value) const { + return ConfigurableHelper::GetOption(config_options, *this, + GetOptionName(name), value); +} + +Status ConfigurableHelper::GetOption(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& short_name, + std::string* value) { + // Look for option directly + assert(value); + value->clear(); + + std::string opt_name; + void* opt_ptr = nullptr; + const auto opt_info = + FindOption(configurable.options_, short_name, &opt_name, &opt_ptr); + if (opt_info != nullptr) { + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + if (short_name == opt_name) { + return opt_info->Serialize(embedded, opt_name, opt_ptr, value); + } else if (opt_info->IsStruct()) { + return opt_info->Serialize(embedded, opt_name, opt_ptr, value); + } else if (opt_info->IsConfigurable()) { + auto const* config = opt_info->AsRawPointer(opt_ptr); + if (config != nullptr) { + return config->GetOption(embedded, opt_name, value); + } + } + } + return Status::NotFound("Cannot find option: ", short_name); +} + +Status ConfigurableHelper::SerializeOptions(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& prefix, + std::string* result) { + assert(result); + for (auto const& opt_iter : configurable.options_) { + for (const auto& map_iter : *(opt_iter.type_map)) { + const auto& opt_name = map_iter.first; + const auto& opt_info = map_iter.second; + if (opt_info.ShouldSerialize()) { + std::string value; + Status s = opt_info.Serialize(config_options, prefix + opt_name, + opt_iter.opt_ptr, &value); + if (!s.ok()) { + return s; + } else if (!value.empty()) { + // = + result->append(prefix + opt_name + "=" + value + + config_options.delimiter); + } + } + } + } + return Status::OK(); +} +#endif // ROCKSDB_LITE + +//******************************************************************************** +// +// Methods for listing the options from Configurables +// +//******************************************************************************** +#ifndef ROCKSDB_LITE +Status Configurable::GetOptionNames( + const ConfigOptions& config_options, + std::unordered_set* result) const { + assert(result); + return ConfigurableHelper::ListOptions(config_options, *this, "", result); +} + +Status ConfigurableHelper::ListOptions( + const ConfigOptions& /*config_options*/, const Configurable& configurable, + const std::string& prefix, std::unordered_set* result) { + Status status; + for (auto const& opt_iter : configurable.options_) { + for (const auto& map_iter : *(opt_iter.type_map)) { + const auto& opt_name = map_iter.first; + const auto& opt_info = map_iter.second; + // If the option is no longer used in rocksdb and marked as deprecated, + // we skip it in the serialization. + if (!opt_info.IsDeprecated() && !opt_info.IsAlias()) { + result->emplace(prefix + opt_name); + } + } + } + return status; +} +#endif // ROCKSDB_LITE + +//******************************************************************************* +// +// Methods for Comparing Configurables +// +//******************************************************************************* + +bool Configurable::AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* name) const { + assert(name); + name->clear(); + if (this == other || config_options.IsCheckDisabled()) { + return true; + } else if (other != nullptr) { +#ifndef ROCKSDB_LITE + return ConfigurableHelper::AreEquivalent(config_options, *this, *other, + name); +#else + return true; +#endif // ROCKSDB_LITE + } else { + return false; + } +} + +#ifndef ROCKSDB_LITE +bool Configurable::OptionsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const void* const this_ptr, + const void* const that_ptr, + std::string* mismatch) const { + if (opt_info.AreEqual(config_options, opt_name, this_ptr, that_ptr, + mismatch)) { + return true; + } else if (opt_info.AreEqualByName(config_options, opt_name, this_ptr, + that_ptr)) { + *mismatch = ""; + return true; + } else { + return false; + } +} + +bool ConfigurableHelper::AreEquivalent(const ConfigOptions& config_options, + const Configurable& this_one, + const Configurable& that_one, + std::string* mismatch) { + assert(mismatch != nullptr); + for (auto const& o : this_one.options_) { + const auto this_offset = this_one.GetOptionsPtr(o.name); + const auto that_offset = that_one.GetOptionsPtr(o.name); + if (this_offset != that_offset) { + if (this_offset == nullptr || that_offset == nullptr) { + return false; + } else { + for (const auto& map_iter : *(o.type_map)) { + if (config_options.IsCheckEnabled(map_iter.second.GetSanityLevel()) && + !this_one.OptionsAreEqual(config_options, map_iter.second, + map_iter.first, this_offset, + that_offset, mismatch)) { + return false; + } + } + } + } + } + return true; +} +#endif // ROCKSDB_LITE +} // namespace ROCKSDB_NAMESPACE diff --git a/options/configurable_helper.h b/options/configurable_helper.h new file mode 100644 index 000000000..6a2454727 --- /dev/null +++ b/options/configurable_helper.h @@ -0,0 +1,211 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/configurable.h" +#include "rocksdb/convenience.h" + +namespace ROCKSDB_NAMESPACE { +// Helper class defining static methods for supporting the Configurable +// class. The purpose of this class is to keep the Configurable class +// as tight as possible and provide methods for doing the actual work +// of configuring the objects. +class ConfigurableHelper { + public: + // Registers the input name with the options and associated map. + // When classes register their options in this manner, most of the + // functionality (excluding unknown options and validate/prepare) is + // implemented by the base class. + // + // This method should be called in the class constructor to register the + // option set for this object. For example, to register the options + // associated with the BlockBasedTableFactory, the constructor calls this + // method passing in: + // - the name of the options ("BlockBasedTableOptions"); + // - the options object (the BlockBasedTableOptions object for this object; + // - the options type map for the BlockBasedTableOptions. + // This registration allows the Configurable class to process the option + // values associated with the BlockBasedTableOptions without further code in + // the derived class. + // + // @param name The name of this set of options (@see GetOptionsPtr) + // @param opt_ptr Pointer to the options to associate with this name + // @param opt_map Options map that controls how this option is configured. + template + static void RegisterOptions( + Configurable& configurable, T* opt_ptr, + const std::unordered_map* opt_map) { + RegisterOptions(configurable, T::kName(), opt_ptr, opt_map); + } + static void RegisterOptions( + Configurable& configurable, const std::string& name, void* opt_ptr, + const std::unordered_map* opt_map); + + // Configures the input Configurable object based on the parameters. + // On successful completion, the Configurable is updated with the settings + // from the opt_map. + // + // The acceptable values of the name/value pairs are documented with the + // specific class/instance. + // + // @param config_options Controls how the arguments are processed. + // @param opt_map Name/value pairs of the options to update + // @param unused If specified, this value will return the name/value + // pairs from opt_map that were NotFound for this object. + // @return OK If all values in the map were successfully updated + // @return NotFound If any of the names in the opt_map were not valid + // for this object. If unused is specified, it will contain the + // collection of NotFound entries + // @return NotSupported If any of the names are valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If any of the values cannot be successfully + // parsed. This can also be returned if PrepareOptions encounters an + // error. + static Status ConfigureOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& options, + std::unordered_map* unused); + +#ifndef ROCKSDB_LITE + // Internal method to configure a set of options for this object. + // Classes may override this value to change its behavior. + // @param config_options Controls how the options are being configured + // @param type_name The name that was registered for this set of options + // @param type_map The map of options for this name + // @param opt_ptr Pointer to the object being configured for this option set. + // @param options The option name/values being updated. On return, any + // option that was found is removed from the list. + // @return OK If all of the options were successfully updated. + // @return InvalidArgument If an option was found but the value could not + // be updated. + // @return NotFound If an option name was not found in type_mape + // @return NotSupported If the option was found but no rule for converting + // the value could be found. + static Status ConfigureSomeOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& type_map, + std::unordered_map* options, void* opt_ptr); + + // Configures a single option in the input Configurable. + // This method will look through the set of option names for this + // Configurable searching for one with the input name. If such an option + // is found, it will be configured via the input value. + // + // @param config_options Controls how the option is being configured + // @param configurable The object to configure + // @param name For options with sub-options (like Structs or + // Configurables), + // this value may be the name of the sub-field of the option being + // updated. For example, if the option is + // "compaction_options_fifo.allow_compaction", then field name would be + // "allow_compaction". For most options, field_name and opt_name will be + // equivalent. + // @param value The new value for this option. + // @param See ConfigureOptions for the possible return values + static Status ConfigureSingleOption(const ConfigOptions& config_options, + Configurable& configurable, + const std::string& name, + const std::string& value); + + // Configures the option referenced by opt_info for this configurable + // This method configures the option based on opt_info for the input + // configurable. + // @param config_options Controls how the option is being configured + // @param configurable The object to configure + // @param opt_name The full option name + // @param name For options with sub-options (like Structs or + // Configurables), + // this value may be the name of the sub-field of the option being + // updated. For example, if the option is + // "compaction_options_fifo.allow_compaction", then field name would be + // "allow_compaction". For most options, field_name and opt_name will be + // equivalent. + // @param value The new value for this option. + // @param See ConfigureOptions for the possible return values + static Status ConfigureOption(const ConfigOptions& config_options, + Configurable& configurable, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& name, + const std::string& value, void* opt_ptr); + + // Returns the value of the option associated with the input name + // This method is the functional inverse of ConfigureOption + // @param config_options Controls how the value is returned + // @param configurable The object from which to get the option. + // @param name The name of the option to return a value for. + // @param value The returned value associated with the named option. + // Note that value will be only the serialized version + // of the option and not "name=value" + // @return OK If the named field was successfully updated to value. + // @return NotFound If the name is not valid for this object. + // @param InvalidArgument If the name is valid for this object but + // its value cannot be serialized. + static Status GetOption(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& name, std::string* value); + + // Serializes the input Configurable into the output result. + // This is the inverse of ConfigureOptions + // @param config_options Controls how serialization happens. + // @param configurable The object to serialize + // @param prefix A prefix to add to the each option as it is serialized. + // @param result The string representation of the configurable. + // @return OK If the options for this object wer successfully serialized. + // @return InvalidArgument If one or more of the options could not be + // serialized. + static Status SerializeOptions(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& prefix, + std::string* result); + + // Internal method to list the option names for this object. + // Classes may override this value to change its behavior. + // @see ListOptions for more details + static Status ListOptions(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& prefix, + std::unordered_set* result); + + // Checks to see if the two configurables are equivalent to one other. + // This method assumes that the two objects are of the same class. + // @param config_options Controls how the options are compared. + // @param this_one The object to compare to. + // @param that_one The other object being compared. + // @param mismatch If the objects do not match, this parameter contains + // the name of the option that triggered the match failure. + // @param True if the objects match, false otherwise. + static bool AreEquivalent(const ConfigOptions& config_options, + const Configurable& this_one, + const Configurable& that_one, + std::string* mismatch); + + private: + // Looks for the option specified by name in the RegisteredOptions. + // This method traverses the types in the input options vector. If an entry + // matching name is found, that entry, opt_name, and pointer are returned. + // @param options The vector of options to search through + // @param name The name of the option to search for in the OptionType map + // @param opt_name If the name was found, this value is set to the option name + // associated with the input name/type. + // @param opt_ptr If the name was found, this value is set to the option + // pointer + // in the RegisteredOptions vector associated with this entry + // @return A pointer to the OptionTypeInfo from the options if found, + // nullptr if the name was not found in the input options + static const OptionTypeInfo* FindOption( + const std::vector& options, + const std::string& name, std::string* opt_name, void** opt_ptr); +#endif // ROCKSDB_LITE +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/options/configurable_test.cc b/options/configurable_test.cc new file mode 100644 index 000000000..b37fa9066 --- /dev/null +++ b/options/configurable_test.cc @@ -0,0 +1,791 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "options/configurable_test.h" + +#include +#include +#include +#include + +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "rocksdb/configurable.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace ROCKSDB_NAMESPACE { +namespace test { +class StringLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char buffer[1000]; + vsnprintf(buffer, sizeof(buffer), format, ap); + string_.append(buffer); + } + const std::string& str() const { return string_; } + void clear() { string_.clear(); } + + private: + std::string string_; +}; + +class SimpleConfigurable : public TestConfigurable { + public: + static SimpleConfigurable* Create( + const std::string& name = "simple", + int mode = TestConfigMode::kDefaultMode, + const std::unordered_map* map = + &simple_option_info) { + return new SimpleConfigurable(name, mode, map); + } + + SimpleConfigurable(const std::string& name, int mode, + const std::unordered_map* + map = &simple_option_info) + : TestConfigurable(name, mode, map) { + if ((mode & TestConfigMode::kUniqueMode) != 0) { + unique_.reset(SimpleConfigurable::Create("Unique" + name_)); + ConfigurableHelper::RegisterOptions(*this, name_ + "Unique", &unique_, + &unique_option_info); + } + if ((mode & TestConfigMode::kSharedMode) != 0) { + shared_.reset(SimpleConfigurable::Create("Shared" + name_)); + ConfigurableHelper::RegisterOptions(*this, name_ + "Shared", &shared_, + &shared_option_info); + } + if ((mode & TestConfigMode::kRawPtrMode) != 0) { + pointer_ = SimpleConfigurable::Create("Pointer" + name_); + ConfigurableHelper::RegisterOptions(*this, name_ + "Pointer", &pointer_, + &pointer_option_info); + } + } + +}; // End class SimpleConfigurable + +static std::unordered_map wrapped_option_info = { +#ifndef ROCKSDB_LITE + {"inner", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + OptionTypeFlags::kShared}}, +#endif // ROCKSDB_LITE +}; +class WrappedConfigurable : public SimpleConfigurable { + public: + WrappedConfigurable(const std::string& name, unsigned char mode, + const std::shared_ptr& t) + : SimpleConfigurable(name, mode, &simple_option_info), inner_(t) { + ConfigurableHelper::RegisterOptions(*this, "WrappedOptions", &inner_, + &wrapped_option_info); + } + + protected: + Configurable* Inner() const override { return inner_.get(); } + + private: + std::shared_ptr inner_; +}; + +using ConfigTestFactoryFunc = std::function; + +class ConfigurableTest : public testing::Test { + public: + ConfigurableTest() { config_options_.invoke_prepare_options = false; } + + ConfigOptions config_options_; +}; + +class ConfigurableParamTest + : public ConfigurableTest, + virtual public ::testing::WithParamInterface< + std::pair> { + public: + ConfigurableParamTest() { + configuration_ = GetParam().first; + factory_ = GetParam().second; + object_.reset(factory_()); + } + void TestConfigureOptions(const ConfigOptions& opts); + ConfigTestFactoryFunc factory_; + std::string configuration_; + std::unique_ptr object_; +}; + +TEST_F(ConfigurableTest, GetOptionsPtrTest) { + std::string opt_str; + std::unique_ptr configurable(SimpleConfigurable::Create()); + ASSERT_NE(configurable->GetOptions("simple"), nullptr); + ASSERT_EQ(configurable->GetOptions("bad-opt"), nullptr); +} + +TEST_F(ConfigurableTest, ConfigureFromMapTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + auto* opts = configurable->GetOptions("simple"); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, {})); + ASSERT_NE(opts, nullptr); +#ifndef ROCKSDB_LITE + std::unordered_map options_map = { + {"int", "1"}, {"bool", "true"}, {"string", "string"}}; + ASSERT_OK(configurable->ConfigureFromMap(config_options_, options_map)); + ASSERT_EQ(opts->i, 1); + ASSERT_EQ(opts->b, true); + ASSERT_EQ(opts->s, "string"); +#endif +} + +TEST_F(ConfigurableTest, ConfigureFromStringTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + auto* opts = configurable->GetOptions("simple"); + ASSERT_OK(configurable->ConfigureFromString(config_options_, "")); + ASSERT_NE(opts, nullptr); +#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE + ASSERT_OK(configurable->ConfigureFromString(config_options_, + "int=1;bool=true;string=s")); + ASSERT_EQ(opts->i, 1); + ASSERT_EQ(opts->b, true); + ASSERT_EQ(opts->s, "s"); +#endif +} + +#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE +TEST_F(ConfigurableTest, ConfigureIgnoreTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + std::unordered_map options_map = {{"unused", "u"}}; + ConfigOptions ignore = config_options_; + ignore.ignore_unknown_options = true; + ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map)); + ASSERT_OK(configurable->ConfigureFromMap(ignore, options_map)); + ASSERT_NOK(configurable->ConfigureFromString(config_options_, "unused=u")); + ASSERT_OK(configurable->ConfigureFromString(ignore, "unused=u")); +} + +TEST_F(ConfigurableTest, ConfigureNestedOptionsTest) { + std::unique_ptr base, copy; + std::string opt_str; + std::string mismatch; + + base.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode)); + copy.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode)); + ASSERT_OK(base->ConfigureFromString(config_options_, + "shared={int=10; string=10};" + "unique={int=20; string=20};" + "pointer={int=30; string=30};")); + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(ConfigurableTest, GetOptionsTest) { + std::unique_ptr simple; + + simple.reset( + SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode)); + int i = 11; + for (auto opt : {"", "shared.", "unique.", "pointer."}) { + std::string value; + std::string expected = ToString(i); + std::string opt_name = opt; + ASSERT_OK( + simple->ConfigureOption(config_options_, opt_name + "int", expected)); + ASSERT_OK(simple->GetOption(config_options_, opt_name + "int", &value)); + ASSERT_EQ(expected, value); + ASSERT_OK(simple->ConfigureOption(config_options_, opt_name + "string", + expected)); + ASSERT_OK(simple->GetOption(config_options_, opt_name + "string", &value)); + ASSERT_EQ(expected, value); + + ASSERT_NOK( + simple->ConfigureOption(config_options_, opt_name + "bad", expected)); + ASSERT_NOK(simple->GetOption(config_options_, "bad option", &value)); + ASSERT_TRUE(value.empty()); + i += 11; + } +} + +TEST_F(ConfigurableTest, ConfigureBadOptionsTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + auto* opts = configurable->GetOptions("simple"); + ASSERT_NE(opts, nullptr); + ASSERT_OK(configurable->ConfigureOption(config_options_, "int", "42")); + ASSERT_EQ(opts->i, 42); + ASSERT_NOK(configurable->ConfigureOption(config_options_, "int", "fred")); + ASSERT_NOK(configurable->ConfigureOption(config_options_, "bool", "fred")); + ASSERT_NOK( + configurable->ConfigureFromString(config_options_, "int=33;unused=u")); + ASSERT_EQ(opts->i, 42); +} + +TEST_F(ConfigurableTest, InvalidOptionTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + std::unordered_map options_map = { + {"bad-option", "bad"}}; + ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map)); + ASSERT_NOK( + configurable->ConfigureFromString(config_options_, "bad-option=bad")); + ASSERT_NOK( + configurable->ConfigureOption(config_options_, "bad-option", "bad")); +} + +static std::unordered_map validated_option_info = { +#ifndef ROCKSDB_LITE + {"validated", + {0, OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; +static std::unordered_map prepared_option_info = { +#ifndef ROCKSDB_LITE + {"prepared", + {0, OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +#endif // ROCKSDB_LITE +}; +static std::unordered_map + dont_prepare_option_info = { +#ifndef ROCKSDB_LITE + {"unique", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + (OptionTypeFlags::kUnique | OptionTypeFlags::kDontPrepare)}}, + +#endif // ROCKSDB_LITE +}; + +class ValidatedConfigurable : public SimpleConfigurable { + public: + ValidatedConfigurable(const std::string& name, unsigned char mode, + bool dont_prepare = false) + : SimpleConfigurable(name, TestConfigMode::kDefaultMode), + validated(false), + prepared(0) { + ConfigurableHelper::RegisterOptions(*this, "Validated", &validated, + &validated_option_info); + ConfigurableHelper::RegisterOptions(*this, "Prepared", &prepared, + &prepared_option_info); + if ((mode & TestConfigMode::kUniqueMode) != 0) { + unique_.reset(new ValidatedConfigurable( + "Unique" + name_, TestConfigMode::kDefaultMode, false)); + if (dont_prepare) { + ConfigurableHelper::RegisterOptions(*this, name_ + "Unique", &unique_, + &dont_prepare_option_info); + } else { + ConfigurableHelper::RegisterOptions(*this, name_ + "Unique", &unique_, + &unique_option_info); + } + } + } + + Status PrepareOptions(const ConfigOptions& config_options) override { + if (++prepared <= 0) { + return Status::InvalidArgument("Cannot prepare option"); + } else { + return SimpleConfigurable::PrepareOptions(config_options); + } + } + + Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const override { + if (!validated) { + return Status::InvalidArgument("Not Validated"); + } else { + return SimpleConfigurable::ValidateOptions(db_opts, cf_opts); + } + } + + private: + bool validated; + int prepared; +}; + +TEST_F(ConfigurableTest, ValidateOptionsTest) { + std::unique_ptr configurable( + new ValidatedConfigurable("validated", TestConfigMode::kDefaultMode)); + ColumnFamilyOptions cf_opts; + DBOptions db_opts; + ASSERT_OK( + configurable->ConfigureOption(config_options_, "validated", "false")); + ASSERT_NOK(configurable->ValidateOptions(db_opts, cf_opts)); + ASSERT_OK( + configurable->ConfigureOption(config_options_, "validated", "true")); + ASSERT_OK(configurable->ValidateOptions(db_opts, cf_opts)); +} + +TEST_F(ConfigurableTest, PrepareOptionsTest) { + std::unique_ptr c( + new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, false)); + auto cp = c->GetOptions("Prepared"); + auto u = c->GetOptions>("SimpleUnique"); + auto up = u->get()->GetOptions("Prepared"); + config_options_.invoke_prepare_options = false; + + ASSERT_NE(cp, nullptr); + ASSERT_NE(up, nullptr); + ASSERT_EQ(*cp, 0); + ASSERT_EQ(*up, 0); + ASSERT_OK(c->ConfigureFromMap(config_options_, {})); + ASSERT_EQ(*cp, 0); + ASSERT_EQ(*up, 0); + config_options_.invoke_prepare_options = true; + ASSERT_OK(c->ConfigureFromMap(config_options_, {})); + ASSERT_EQ(*cp, 1); + ASSERT_EQ(*up, 1); + ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0")); + ASSERT_EQ(*up, 2); + ASSERT_EQ(*cp, 1); + + ASSERT_NOK(c->ConfigureFromString(config_options_, "prepared=-2")); + + c.reset( + new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, true)); + cp = c->GetOptions("Prepared"); + u = c->GetOptions>("SimpleUnique"); + up = u->get()->GetOptions("Prepared"); + + ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0")); + ASSERT_EQ(*cp, 1); + ASSERT_EQ(*up, 0); +} + +TEST_F(ConfigurableTest, DeprecatedOptionsTest) { + static std::unordered_map + deprecated_option_info = { + {"deprecated", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}}; + std::unique_ptr orig; + orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode, + &deprecated_option_info)); + auto* opts = orig->GetOptions("simple"); + ASSERT_NE(opts, nullptr); + opts->d = true; + ASSERT_OK(orig->ConfigureOption(config_options_, "deprecated", "false")); + ASSERT_TRUE(opts->d); + ASSERT_OK(orig->ConfigureFromString(config_options_, "deprecated=false")); + ASSERT_TRUE(opts->d); +} + +TEST_F(ConfigurableTest, AliasOptionsTest) { + static std::unordered_map alias_option_info = { + {"bool", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"alias", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kAlias, OptionTypeFlags::kNone, 0}}}; + std::unique_ptr orig; + orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode, + &alias_option_info)); + auto* opts = orig->GetOptions("simple"); + ASSERT_NE(opts, nullptr); + ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false")); + ASSERT_FALSE(opts->b); + ASSERT_OK(orig->ConfigureOption(config_options_, "alias", "true")); + ASSERT_TRUE(opts->b); + std::string opts_str; + ASSERT_OK(orig->GetOptionString(config_options_, &opts_str)); + ASSERT_EQ(opts_str.find("alias"), std::string::npos); + + ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false")); + ASSERT_FALSE(opts->b); + ASSERT_OK(orig->GetOption(config_options_, "alias", &opts_str)); + ASSERT_EQ(opts_str, "false"); +} + +TEST_F(ConfigurableTest, NestedUniqueConfigTest) { + std::unique_ptr simple; + simple.reset( + SimpleConfigurable::Create("Outer", TestConfigMode::kAllOptMode)); + const auto outer = simple->GetOptions("Outer"); + const auto unique = + simple->GetOptions>("OuterUnique"); + ASSERT_NE(outer, nullptr); + ASSERT_NE(unique, nullptr); + ASSERT_OK( + simple->ConfigureFromString(config_options_, "int=24;string=outer")); + ASSERT_OK(simple->ConfigureFromString(config_options_, + "unique={int=42;string=nested}")); + const auto inner = unique->get()->GetOptions("UniqueOuter"); + ASSERT_NE(inner, nullptr); + ASSERT_EQ(outer->i, 24); + ASSERT_EQ(outer->s, "outer"); + ASSERT_EQ(inner->i, 42); + ASSERT_EQ(inner->s, "nested"); +} + +TEST_F(ConfigurableTest, NestedSharedConfigTest) { + std::unique_ptr simple; + simple.reset(SimpleConfigurable::Create( + "Outer", TestConfigMode::kDefaultMode | TestConfigMode::kSharedMode)); + ASSERT_OK( + simple->ConfigureFromString(config_options_, "int=24;string=outer")); + ASSERT_OK(simple->ConfigureFromString(config_options_, + "shared={int=42;string=nested}")); + const auto outer = simple->GetOptions("Outer"); + const auto shared = + simple->GetOptions>("OuterShared"); + ASSERT_NE(outer, nullptr); + ASSERT_NE(shared, nullptr); + const auto inner = shared->get()->GetOptions("SharedOuter"); + ASSERT_NE(inner, nullptr); + ASSERT_EQ(outer->i, 24); + ASSERT_EQ(outer->s, "outer"); + ASSERT_EQ(inner->i, 42); + ASSERT_EQ(inner->s, "nested"); +} + +TEST_F(ConfigurableTest, NestedRawConfigTest) { + std::unique_ptr simple; + simple.reset(SimpleConfigurable::Create( + "Outer", TestConfigMode::kDefaultMode | TestConfigMode::kRawPtrMode)); + ASSERT_OK( + simple->ConfigureFromString(config_options_, "int=24;string=outer")); + ASSERT_OK(simple->ConfigureFromString(config_options_, + "pointer={int=42;string=nested}")); + const auto outer = simple->GetOptions("Outer"); + const auto pointer = simple->GetOptions("OuterPointer"); + ASSERT_NE(outer, nullptr); + ASSERT_NE(pointer, nullptr); + const auto inner = (*pointer)->GetOptions("PointerOuter"); + ASSERT_NE(inner, nullptr); + ASSERT_EQ(outer->i, 24); + ASSERT_EQ(outer->s, "outer"); + ASSERT_EQ(inner->i, 42); + ASSERT_EQ(inner->s, "nested"); +} + +TEST_F(ConfigurableTest, MatchesTest) { + std::string mismatch; + std::unique_ptr base, copy; + base.reset(SimpleConfigurable::Create( + "simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode)); + copy.reset(SimpleConfigurable::Create( + "simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode)); + ASSERT_OK(base->ConfigureFromString( + config_options_, + "int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}")); + ASSERT_OK(copy->ConfigureFromString( + config_options_, + "int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}")); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->ConfigureOption(config_options_, "shared", "int=44")); + ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_EQ(mismatch, "shared.int"); + std::string c1value, c2value; + ASSERT_OK(base->GetOption(config_options_, mismatch, &c1value)); + ASSERT_OK(copy->GetOption(config_options_, mismatch, &c2value)); + ASSERT_NE(c1value, c2value); +} + +static Configurable* SimpleStructFactory() { + static std::unordered_map struct_option_info = { +#ifndef ROCKSDB_LITE + {"struct", OptionTypeInfo::Struct("struct", &simple_option_info, 0, + OptionVerificationType::kNormal, + OptionTypeFlags::kMutable)}, +#endif // ROCKSDB_LITE + }; + return SimpleConfigurable::Create( + "simple-struct", TestConfigMode::kDefaultMode, &struct_option_info); +} + +TEST_F(ConfigurableTest, ConfigureStructTest) { + std::unique_ptr base(SimpleStructFactory()); + std::unique_ptr copy(SimpleStructFactory()); + std::string opt_str, value; + std::string mismatch; + std::unordered_set names; + + ASSERT_OK( + base->ConfigureFromString(config_options_, "struct={int=10; string=10}")); + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->GetOptionNames(config_options_, &names)); + ASSERT_EQ(names.size(), 1); + ASSERT_EQ(*(names.begin()), "struct"); + ASSERT_OK( + base->ConfigureFromString(config_options_, "struct={int=20; string=20}")); + ASSERT_OK(base->GetOption(config_options_, "struct", &value)); + ASSERT_OK(copy->ConfigureOption(config_options_, "struct", value)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + + ASSERT_NOK(base->ConfigureFromString(config_options_, + "struct={int=10; string=10; bad=11}")); + ASSERT_OK(base->ConfigureOption(config_options_, "struct.int", "42")); + ASSERT_NOK(base->ConfigureOption(config_options_, "struct.bad", "42")); + ASSERT_NOK(base->GetOption(config_options_, "struct.bad", &value)); + ASSERT_OK(base->GetOption(config_options_, "struct.int", &value)); + ASSERT_EQ(value, "42"); +} + +TEST_F(ConfigurableTest, ConfigurableEnumTest) { + std::unique_ptr base, copy; + base.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode)); + copy.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode)); + + std::string opts_str; + std::string mismatch; + + ASSERT_OK(base->ConfigureFromString(config_options_, "enum=B")); + ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->GetOptionString(config_options_, &opts_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opts_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_NOK(base->ConfigureOption(config_options_, "enum", "bad")); + ASSERT_NOK(base->ConfigureOption(config_options_, "unknown", "bad")); +} + +#ifndef ROCKSDB_LITE +static std::unordered_map noserialize_option_info = + { + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kDontSerialize}}, +}; + +TEST_F(ConfigurableTest, TestNoSerialize) { + std::unique_ptr base; + base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, + &noserialize_option_info)); + std::string opts_str, value; + ASSERT_OK(base->ConfigureFromString(config_options_, "int=10")); + ASSERT_OK(base->GetOptionString(config_options_, &opts_str)); + ASSERT_EQ(opts_str, ""); + ASSERT_NOK(base->GetOption(config_options_, "int", &value)); +} + +TEST_F(ConfigurableTest, TestNoCompare) { + std::unordered_map nocomp_option_info = { + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}}, + }; + std::unordered_map normal_option_info = { + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + }; + + std::unique_ptr base, copy; + base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, + &nocomp_option_info)); + copy.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, + &normal_option_info)); + ASSERT_OK(base->ConfigureFromString(config_options_, "int=10")); + ASSERT_OK(copy->ConfigureFromString(config_options_, "int=20")); + std::string bvalue, cvalue, mismatch; + ASSERT_OK(base->GetOption(config_options_, "int", &bvalue)); + ASSERT_OK(copy->GetOption(config_options_, "int", &cvalue)); + ASSERT_EQ(bvalue, "10"); + ASSERT_EQ(cvalue, "20"); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_FALSE(copy->AreEquivalent(config_options_, base.get(), &mismatch)); +} +#endif + +void ConfigurableParamTest::TestConfigureOptions( + const ConfigOptions& config_options) { + std::unique_ptr base, copy; + std::unordered_set names; + std::string opt_str, mismatch; + + base.reset(factory_()); + copy.reset(factory_()); + + ASSERT_OK(base->ConfigureFromString(config_options, configuration_)); + ASSERT_OK(base->GetOptionString(config_options, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options, opt_str)); + ASSERT_OK(copy->GetOptionString(config_options, &opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch)); + + copy.reset(factory_()); + ASSERT_OK(base->GetOptionNames(config_options, &names)); + std::unordered_map unused; + bool found_one = false; + for (auto name : names) { + std::string value; + Status s = base->GetOption(config_options, name, &value); + if (s.ok()) { + s = copy->ConfigureOption(config_options, name, value); + if (s.ok() || s.IsNotSupported()) { + found_one = true; + } else { + unused[name] = value; + } + } else { + ASSERT_TRUE(s.IsNotSupported()); + } + } + ASSERT_TRUE(found_one || names.empty()); + while (found_one && !unused.empty()) { + found_one = false; + for (auto iter = unused.begin(); iter != unused.end();) { + if (copy->ConfigureOption(config_options, iter->first, iter->second) + .ok()) { + found_one = true; + iter = unused.erase(iter); + } else { + ++iter; + } + } + } + ASSERT_EQ(0, unused.size()); + ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch)); +} + +TEST_P(ConfigurableParamTest, GetDefaultOptionsTest) { + TestConfigureOptions(config_options_); +} + +TEST_P(ConfigurableParamTest, ConfigureFromPropsTest) { + std::string opt_str, mismatch; + std::unordered_set names; + std::unique_ptr copy(factory_()); + + ASSERT_OK(object_->ConfigureFromString(config_options_, configuration_)); + config_options_.delimiter = "\n"; + ASSERT_OK(object_->GetOptionString(config_options_, &opt_str)); + std::istringstream iss(opt_str); + std::unordered_map copy_map; + std::string line; + for (int line_num = 0; std::getline(iss, line); line_num++) { + std::string name; + std::string value; + ASSERT_OK( + RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num)); + copy_map[name] = value; + } + ASSERT_OK(copy->ConfigureFromMap(config_options_, copy_map)); + ASSERT_TRUE(object_->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +static Configurable* SimpleFactory() { + return SimpleConfigurable::Create("simple"); +} + +static Configurable* UniqueFactory() { + return SimpleConfigurable::Create( + "simple", TestConfigMode::kSimpleMode | TestConfigMode::kUniqueMode); +} +static Configurable* SharedFactory() { + return SimpleConfigurable::Create( + "simple", TestConfigMode::kSimpleMode | TestConfigMode::kSharedMode); +} + +static Configurable* NestedFactory() { + return SimpleConfigurable::Create( + "simple", TestConfigMode::kSimpleMode | TestConfigMode::kNestedMode); +} + +static Configurable* MutableFactory() { + return SimpleConfigurable::Create("simple", TestConfigMode::kMutableMode | + TestConfigMode::kSimpleMode | + TestConfigMode::kNestedMode); +} + +static Configurable* ThreeWrappedFactory() { + std::shared_ptr child; + child.reset( + SimpleConfigurable::Create("child", TestConfigMode::kDefaultMode)); + std::shared_ptr parent; + parent.reset( + new WrappedConfigurable("parent", TestConfigMode::kDefaultMode, child)); + return new WrappedConfigurable("master", TestConfigMode::kDefaultMode, + parent); +} + +static Configurable* ThreeDeepFactory() { + Configurable* simple = SimpleConfigurable::Create( + "Simple", TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode); + auto* unique = + simple->GetOptions>("SimpleUnique"); + unique->reset(SimpleConfigurable::Create( + "Child", TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode)); + unique = + unique->get()->GetOptions>("ChildUnique"); + unique->reset( + SimpleConfigurable::Create("Child", TestConfigMode::kDefaultMode)); + return simple; +} + +static Configurable* DBOptionsFactory() { + auto config = DBOptionsAsConfigurable(DBOptions()); + return config.release(); +} + +static Configurable* CFOptionsFactory() { + auto config = CFOptionsAsConfigurable(ColumnFamilyOptions()); + return config.release(); +} + +static Configurable* BlockBasedFactory() { return NewBlockBasedTableFactory(); } + +INSTANTIATE_TEST_CASE_P( + ParamTest, ConfigurableParamTest, + testing::Values( + std::pair( + "int=42;bool=true;string=s", SimpleFactory), + std::pair( + "int=42;unique={int=33;string=unique}", MutableFactory), + std::pair( + "struct={int=33;bool=true;string=s;}", SimpleStructFactory), + std::pair( + "int=33;bool=true;string=outer;" + "shared={int=42;string=shared}", + SharedFactory), + std::pair( + "int=33;bool=true;string=outer;" + "unique={int=42;string=unique}", + UniqueFactory), + std::pair( + "int=11;bool=true;string=outer;" + "pointer={int=22;string=pointer};" + "unique={int=33;string=unique};" + "shared={int=44;string=shared}", + NestedFactory), + std::pair( + "int=11;bool=true;string=outer;" + "inner={int=22;string=parent;" + "inner={int=33;string=child}};", + ThreeWrappedFactory), + std::pair( + "int=11;bool=true;string=outer;" + "unique={int=22;string=inner;" + "unique={int=33;string=unique}};", + ThreeDeepFactory), + std::pair("max_background_jobs=100;" + "max_open_files=200;", + DBOptionsFactory), + std::pair( + "table_factory=BlockBasedTable;" + "disable_auto_compactions=true;", + CFOptionsFactory), + std::pair("block_size=1024;" + "no_block_cache=true;", + BlockBasedFactory))); +#endif // ROCKSDB_LITE + +} // namespace test +} // namespace ROCKSDB_NAMESPACE +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/options/configurable_test.h b/options/configurable_test.h new file mode 100644 index 000000000..52c3599f6 --- /dev/null +++ b/options/configurable_test.h @@ -0,0 +1,127 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include + +#include "options/configurable_helper.h" +#include "rocksdb/configurable.h" +#include "rocksdb/utilities/options_type.h" + +namespace ROCKSDB_NAMESPACE { +struct ColumnFamilyOptions; +struct DBOptions; + +namespace test { +enum TestEnum { kTestA, kTestB }; + +static const std::unordered_map test_enum_map = { + {"A", TestEnum::kTestA}, + {"B", TestEnum::kTestB}, +}; + +struct TestOptions { + int i = 0; + bool b = false; + bool d = true; + TestEnum e = TestEnum::kTestA; + std::string s = ""; + std::string u = ""; +}; + +static std::unordered_map simple_option_info = { +#ifndef ROCKSDB_LITE + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"bool", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"string", + {offsetof(struct TestOptions, s), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; + +static std::unordered_map enum_option_info = { +#ifndef ROCKSDB_LITE + {"enum", + OptionTypeInfo::Enum(offsetof(struct TestOptions, e), &test_enum_map)} +#endif +}; + +static std::unordered_map unique_option_info = { +#ifndef ROCKSDB_LITE + {"unique", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + (OptionTypeFlags::kUnique | OptionTypeFlags::kMutable)}}, +#endif // ROCKSDB_LITE +}; + +static std::unordered_map shared_option_info = { +#ifndef ROCKSDB_LITE + {"shared", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + (OptionTypeFlags::kShared)}}, +#endif // ROCKSDB_LITE +}; +static std::unordered_map pointer_option_info = { +#ifndef ROCKSDB_LITE + {"pointer", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + OptionTypeFlags::kRawPointer}}, +#endif // ROCKSDB_LITE +}; + +enum TestConfigMode { + kEmptyMode = 0x0, // Don't register anything + kMutableMode = 0x01, // Configuration is mutable + kSimpleMode = 0x02, // Use the simple options + kEnumMode = 0x04, // Use the enum options + kDefaultMode = kSimpleMode, // Use no inner nested configurations + kSharedMode = 0x10, // Use shared configuration + kUniqueMode = 0x20, // Use unique configuration + kRawPtrMode = 0x40, // Use pointer configuration + kNestedMode = (kSharedMode | kUniqueMode | kRawPtrMode), + kAllOptMode = (kNestedMode | kEnumMode | kSimpleMode), +}; + +template +class TestConfigurable : public Configurable { + protected: + std::string name_; + std::string prefix_; + TestOptions options_; + + public: + std::unique_ptr unique_; + std::shared_ptr shared_; + T* pointer_; + + TestConfigurable(const std::string& name, int mode, + const std::unordered_map* map = + &simple_option_info) + : name_(name), pointer_(nullptr) { + prefix_ = "test." + name + "."; + if ((mode & TestConfigMode::kSimpleMode) != 0) { + ConfigurableHelper::RegisterOptions(*this, name_, &options_, map); + } + if ((mode & TestConfigMode::kEnumMode) != 0) { + ConfigurableHelper::RegisterOptions(*this, name_ + "Enum", &options_, + &enum_option_info); + } + } + + ~TestConfigurable() override { delete pointer_; } +}; + +} // namespace test +} // namespace ROCKSDB_NAMESPACE diff --git a/options/db_options.cc b/options/db_options.cc index 05b76b57a..4f97644c7 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -8,14 +8,18 @@ #include #include "logging/logging.h" +#include "options/configurable_helper.h" #include "options/options_helper.h" +#include "options/options_parser.h" #include "port/port.h" -#include "rocksdb/cache.h" +#include "rocksdb/configurable.h" #include "rocksdb/env.h" #include "rocksdb/file_system.h" #include "rocksdb/rate_limiter.h" #include "rocksdb/sst_file_manager.h" +#include "rocksdb/utilities/options_type.h" #include "rocksdb/wal_filter.h" +#include "util/string_util.h" namespace ROCKSDB_NAMESPACE { #ifndef ROCKSDB_LITE @@ -42,8 +46,87 @@ static std::unordered_map info_log_level_string_map = {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; -std::unordered_map - OptionsHelper::db_options_type_info = { +static std::unordered_map + db_mutable_options_type_info = { + {"allow_os_buffer", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"max_background_jobs", + {offsetof(struct MutableDBOptions, max_background_jobs), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_background_compactions", + {offsetof(struct MutableDBOptions, max_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"base_background_compactions", + {offsetof(struct MutableDBOptions, base_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_subcompactions", + {offsetof(struct MutableDBOptions, max_subcompactions), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"avoid_flush_during_shutdown", + {offsetof(struct MutableDBOptions, avoid_flush_during_shutdown), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"writable_file_max_buffer_size", + {offsetof(struct MutableDBOptions, writable_file_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"delayed_write_rate", + {offsetof(struct MutableDBOptions, delayed_write_rate), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_total_wal_size", + {offsetof(struct MutableDBOptions, max_total_wal_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"delete_obsolete_files_period_micros", + {offsetof(struct MutableDBOptions, + delete_obsolete_files_period_micros), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_dump_period_sec", + {offsetof(struct MutableDBOptions, stats_dump_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_persist_period_sec", + {offsetof(struct MutableDBOptions, stats_persist_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_history_buffer_size", + {offsetof(struct MutableDBOptions, stats_history_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_open_files", + {offsetof(struct MutableDBOptions, max_open_files), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"bytes_per_sync", + {offsetof(struct MutableDBOptions, bytes_per_sync), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"wal_bytes_per_sync", + {offsetof(struct MutableDBOptions, wal_bytes_per_sync), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"strict_bytes_per_sync", + {offsetof(struct MutableDBOptions, strict_bytes_per_sync), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compaction_readahead_size", + {offsetof(struct MutableDBOptions, compaction_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_background_flushes", + {offsetof(struct MutableDBOptions, max_background_flushes), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +}; + +static std::unordered_map + db_immutable_options_type_info = { /* // not yet supported std::shared_ptr row_cache; @@ -55,321 +138,259 @@ std::unordered_map std::vector> listeners; */ {"advise_random_on_open", - {offsetof(struct DBOptions, advise_random_on_open), + {offsetof(struct ImmutableDBOptions, advise_random_on_open), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"allow_mmap_reads", - {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, allow_mmap_reads), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"allow_fallocate", - {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, allow_fallocate), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"allow_mmap_writes", - {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, allow_mmap_writes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"use_direct_reads", - {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, use_direct_reads), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"use_direct_writes", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"use_direct_io_for_flush_and_compaction", - {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction), + {offsetof(struct ImmutableDBOptions, + use_direct_io_for_flush_and_compaction), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"allow_2pc", - {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"allow_os_buffer", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kMutable, 0}}, + {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"create_if_missing", - {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"create_missing_column_families", - {offsetof(struct DBOptions, create_missing_column_families), + {offsetof(struct ImmutableDBOptions, create_if_missing), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, + {"create_missing_column_families", + {offsetof(struct ImmutableDBOptions, create_missing_column_families), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"disableDataSync", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"disable_data_sync", // for compatibility {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"enable_thread_tracking", - {offsetof(struct DBOptions, enable_thread_tracking), + {offsetof(struct ImmutableDBOptions, enable_thread_tracking), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"error_if_exists", - {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, error_if_exists), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"is_fd_close_on_exec", - {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, is_fd_close_on_exec), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"paranoid_checks", - {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, paranoid_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"skip_log_error_on_recovery", - {offsetof(struct DBOptions, skip_log_error_on_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, {"skip_stats_update_on_db_open", - {offsetof(struct DBOptions, skip_stats_update_on_db_open), + {offsetof(struct ImmutableDBOptions, skip_stats_update_on_db_open), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"skip_checking_sst_file_sizes_on_db_open", - {offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open), + {offsetof(struct ImmutableDBOptions, + skip_checking_sst_file_sizes_on_db_open), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"new_table_reader_for_compaction_inputs", - {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), + {offsetof(struct ImmutableDBOptions, + new_table_reader_for_compaction_inputs), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"compaction_readahead_size", - {offsetof(struct DBOptions, compaction_readahead_size), - OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, compaction_readahead_size)}}, + OptionTypeFlags::kNone}}, {"random_access_max_buffer_size", - {offsetof(struct DBOptions, random_access_max_buffer_size), + {offsetof(struct ImmutableDBOptions, random_access_max_buffer_size), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"use_adaptive_mutex", - {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, use_adaptive_mutex), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"use_fsync", - {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"max_background_jobs", - {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, max_background_jobs)}}, - {"max_background_compactions", - {offsetof(struct DBOptions, max_background_compactions), - OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, max_background_compactions)}}, - {"max_subcompactions", - {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, max_subcompactions)}}, - {"base_background_compactions", - {offsetof(struct DBOptions, base_background_compactions), - OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, base_background_compactions)}}, - {"max_background_flushes", - {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, max_background_flushes)}}, + {offsetof(struct ImmutableDBOptions, use_fsync), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"max_file_opening_threads", - {offsetof(struct DBOptions, max_file_opening_threads), + {offsetof(struct ImmutableDBOptions, max_file_opening_threads), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"max_open_files", - {offsetof(struct DBOptions, max_open_files), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, max_open_files)}}, + OptionTypeFlags::kNone}}, {"table_cache_numshardbits", - {offsetof(struct DBOptions, table_cache_numshardbits), + {offsetof(struct ImmutableDBOptions, table_cache_numshardbits), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"db_write_buffer_size", - {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, db_write_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"keep_log_file_num", - {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, keep_log_file_num), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"recycle_log_file_num", - {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, recycle_log_file_num), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"log_file_time_to_roll", - {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, log_file_time_to_roll), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"manifest_preallocation_size", - {offsetof(struct DBOptions, manifest_preallocation_size), + {offsetof(struct ImmutableDBOptions, manifest_preallocation_size), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"max_log_file_size", - {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"db_log_dir", - {offsetof(struct DBOptions, db_log_dir), OptionType::kString, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"wal_dir", - {offsetof(struct DBOptions, wal_dir), OptionType::kString, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"WAL_size_limit_MB", - {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"WAL_ttl_seconds", - {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, - {"bytes_per_sync", - {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, bytes_per_sync)}}, - {"delayed_write_rate", - {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, delayed_write_rate)}}, - {"delete_obsolete_files_period_micros", - {offsetof(struct DBOptions, delete_obsolete_files_period_micros), - OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, - delete_obsolete_files_period_micros)}}, - {"max_manifest_file_size", - {offsetof(struct DBOptions, max_manifest_file_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"max_total_wal_size", - {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, max_total_wal_size)}}, - {"wal_bytes_per_sync", - {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, wal_bytes_per_sync)}}, - {"strict_bytes_per_sync", - {offsetof(struct DBOptions, strict_bytes_per_sync), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, strict_bytes_per_sync)}}, - {"stats_dump_period_sec", - {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt, - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, stats_dump_period_sec)}}, - {"stats_persist_period_sec", - {offsetof(struct DBOptions, stats_persist_period_sec), - OptionType::kUInt, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, stats_persist_period_sec)}}, - {"persist_stats_to_disk", - {offsetof(struct DBOptions, persist_stats_to_disk), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, - offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}}, - {"stats_history_buffer_size", - {offsetof(struct DBOptions, stats_history_buffer_size), + {offsetof(struct ImmutableDBOptions, max_log_file_size), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, stats_history_buffer_size)}}, + OptionTypeFlags::kNone}}, + {"db_log_dir", + {offsetof(struct ImmutableDBOptions, db_log_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"wal_dir", + {offsetof(struct ImmutableDBOptions, wal_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"WAL_size_limit_MB", + {offsetof(struct ImmutableDBOptions, wal_size_limit_mb), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"WAL_ttl_seconds", + {offsetof(struct ImmutableDBOptions, wal_ttl_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_manifest_file_size", + {offsetof(struct ImmutableDBOptions, max_manifest_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"persist_stats_to_disk", + {offsetof(struct ImmutableDBOptions, persist_stats_to_disk), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"fail_if_options_file_error", - {offsetof(struct DBOptions, fail_if_options_file_error), + {offsetof(struct ImmutableDBOptions, fail_if_options_file_error), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"enable_pipelined_write", - {offsetof(struct DBOptions, enable_pipelined_write), + {offsetof(struct ImmutableDBOptions, enable_pipelined_write), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"unordered_write", - {offsetof(struct DBOptions, unordered_write), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, unordered_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"allow_concurrent_memtable_write", - {offsetof(struct DBOptions, allow_concurrent_memtable_write), + {offsetof(struct ImmutableDBOptions, allow_concurrent_memtable_write), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"wal_recovery_mode", OptionTypeInfo::Enum( - offsetof(struct DBOptions, wal_recovery_mode), - &wal_recovery_mode_string_map)}, + OptionTypeFlags::kNone}}, + {"wal_recovery_mode", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, wal_recovery_mode), + &wal_recovery_mode_string_map)}, {"enable_write_thread_adaptive_yield", - {offsetof(struct DBOptions, enable_write_thread_adaptive_yield), + {offsetof(struct ImmutableDBOptions, + enable_write_thread_adaptive_yield), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"write_thread_slow_yield_usec", - {offsetof(struct DBOptions, write_thread_slow_yield_usec), + {offsetof(struct ImmutableDBOptions, write_thread_slow_yield_usec), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"max_write_batch_group_size_bytes", - {offsetof(struct DBOptions, max_write_batch_group_size_bytes), + {offsetof(struct ImmutableDBOptions, max_write_batch_group_size_bytes), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"write_thread_max_yield_usec", - {offsetof(struct DBOptions, write_thread_max_yield_usec), + {offsetof(struct ImmutableDBOptions, write_thread_max_yield_usec), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"access_hint_on_compaction_start", OptionTypeInfo::Enum( - offsetof(struct DBOptions, access_hint_on_compaction_start), + offsetof(struct ImmutableDBOptions, + access_hint_on_compaction_start), &access_hint_string_map)}, - {"info_log_level", OptionTypeInfo::Enum( - offsetof(struct DBOptions, info_log_level), - &info_log_level_string_map)}, + {"info_log_level", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, info_log_level), + &info_log_level_string_map)}, {"dump_malloc_stats", - {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, dump_malloc_stats), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"avoid_flush_during_recovery", - {offsetof(struct DBOptions, avoid_flush_during_recovery), + {offsetof(struct ImmutableDBOptions, avoid_flush_during_recovery), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, - {"avoid_flush_during_shutdown", - {offsetof(struct DBOptions, avoid_flush_during_shutdown), - OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}, - {"writable_file_max_buffer_size", - {offsetof(struct DBOptions, writable_file_max_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, - offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}}, + OptionTypeFlags::kNone}}, {"allow_ingest_behind", - {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, - offsetof(struct ImmutableDBOptions, allow_ingest_behind)}}, + {offsetof(struct ImmutableDBOptions, allow_ingest_behind), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"preserve_deletes", - {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, - offsetof(struct ImmutableDBOptions, preserve_deletes)}}, + {offsetof(struct ImmutableDBOptions, preserve_deletes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"concurrent_prepare", // Deprecated by two_write_queues {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"two_write_queues", - {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, - offsetof(struct ImmutableDBOptions, two_write_queues)}}, + {offsetof(struct ImmutableDBOptions, two_write_queues), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"manual_wal_flush", - {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, - offsetof(struct ImmutableDBOptions, manual_wal_flush)}}, + {offsetof(struct ImmutableDBOptions, manual_wal_flush), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"seq_per_batch", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"atomic_flush", - {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, - offsetof(struct ImmutableDBOptions, atomic_flush)}}, + {offsetof(struct ImmutableDBOptions, atomic_flush), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"avoid_unnecessary_blocking_io", - {offsetof(struct DBOptions, avoid_unnecessary_blocking_io), + {offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, - offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}}, + OptionTypeFlags::kNone}}, {"write_dbid_to_manifest", - {offsetof(struct DBOptions, write_dbid_to_manifest), + {offsetof(struct ImmutableDBOptions, write_dbid_to_manifest), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"log_readahead_size", - {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {offsetof(struct ImmutableDBOptions, log_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"best_efforts_recovery", - {offsetof(struct DBOptions, best_efforts_recovery), + {offsetof(struct ImmutableDBOptions, best_efforts_recovery), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"max_bgerror_resume_count", - {offsetof(struct DBOptions, max_bgerror_resume_count), + {offsetof(struct ImmutableDBOptions, max_bgerror_resume_count), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"bgerror_resume_retry_interval", - {offsetof(struct DBOptions, bgerror_resume_retry_interval), + {offsetof(struct ImmutableDBOptions, bgerror_resume_retry_interval), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, // The following properties were handled as special cases in ParseOption // This means that the properties could be read from the options file // but never written to the file or compared to each other. {"rate_limiter_bytes_per_sec", - {offsetof(struct DBOptions, rate_limiter), OptionType::kUnknown, - OptionVerificationType::kNormal, - (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0, + {offsetof(struct ImmutableDBOptions, rate_limiter), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), // Parse the input value as a RateLimiter [](const ConfigOptions& /*opts*/, const std::string& /*name*/, const std::string& value, char* addr) { @@ -380,9 +401,9 @@ std::unordered_map return Status::OK(); }}}, {"env", - {offsetof(struct DBOptions, env), OptionType::kUnknown, + {offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown, OptionVerificationType::kNormal, - (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), // Parse the input value as an Env [](const ConfigOptions& /*opts*/, const std::string& /*name*/, const std::string& value, char* addr) { @@ -395,6 +416,74 @@ std::unordered_map return s; }}}, }; + +const std::string OptionsHelper::kDBOptionsName = "DBOptions"; + +class MutableDBConfigurable : public Configurable { + public: + MutableDBConfigurable(const MutableDBOptions& mdb) { + mutable_ = mdb; + ConfigurableHelper::RegisterOptions(*this, &mutable_, + &db_mutable_options_type_info); + } + + protected: + MutableDBOptions mutable_; +}; + +class DBOptionsConfigurable : public MutableDBConfigurable { + public: + DBOptionsConfigurable(const DBOptions& opts) + : MutableDBConfigurable(MutableDBOptions(opts)), db_options_(opts) { + // The ImmutableDBOptions currently requires the env to be non-null. Make + // sure it is + if (opts.env != nullptr) { + immutable_ = ImmutableDBOptions(opts); + } else { + DBOptions copy = opts; + copy.env = Env::Default(); + immutable_ = ImmutableDBOptions(copy); + } + ConfigurableHelper::RegisterOptions(*this, &immutable_, + &db_immutable_options_type_info); + } + + protected: + Status ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) override { + Status s = ConfigurableHelper::ConfigureOptions(config_options, *this, + opts_map, unused); + if (s.ok()) { + db_options_ = BuildDBOptions(immutable_, mutable_); + s = PrepareOptions(config_options); + } + return s; + } + + const void* GetOptionsPtr(const std::string& name) const override { + if (name == OptionsHelper::kDBOptionsName) { + return &db_options_; + } else { + return MutableDBConfigurable::GetOptionsPtr(name); + } + } + + private: + ImmutableDBOptions immutable_; + DBOptions db_options_; +}; + +std::unique_ptr DBOptionsAsConfigurable( + const MutableDBOptions& opts) { + std::unique_ptr ptr(new MutableDBConfigurable(opts)); + return ptr; +} +std::unique_ptr DBOptionsAsConfigurable(const DBOptions& opts) { + std::unique_ptr ptr(new DBOptionsConfigurable(opts)); + return ptr; +} #endif // ROCKSDB_LITE ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {} diff --git a/options/db_options.h b/options/db_options.h index 042f06a31..4a3e73677 100644 --- a/options/db_options.h +++ b/options/db_options.h @@ -13,6 +13,7 @@ namespace ROCKSDB_NAMESPACE { struct ImmutableDBOptions { + static const char* kName() { return "ImmutableDBOptions"; } ImmutableDBOptions(); explicit ImmutableDBOptions(const DBOptions& options); @@ -92,6 +93,7 @@ struct ImmutableDBOptions { }; struct MutableDBOptions { + static const char* kName() { return "MutableDBOptions"; } MutableDBOptions(); explicit MutableDBOptions(const MutableDBOptions& options) = default; explicit MutableDBOptions(const DBOptions& options); diff --git a/options/options.cc b/options/options.cc index a1848219a..cf00059b7 100644 --- a/options/options.cc +++ b/options/options.cc @@ -135,7 +135,7 @@ void ColumnFamilyOptions::Dump(Logger* log) const { ROCKS_LOG_HEADER(log, " Options.table_factory: %s", table_factory->Name()); ROCKS_LOG_HEADER(log, " table_factory options: %s", - table_factory->GetPrintableTableOptions().c_str()); + table_factory->GetPrintableOptions().c_str()); ROCKS_LOG_HEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt, write_buffer_size); ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number: %d", diff --git a/options/options_helper.cc b/options/options_helper.cc index dddefeebb..70b980b22 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -10,11 +10,13 @@ #include #include -#include "options/options_type.h" +#include "options/cf_options.h" +#include "options/db_options.h" #include "rocksdb/cache.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/convenience.h" #include "rocksdb/filter_policy.h" +#include "rocksdb/flush_block_policy.h" #include "rocksdb/memtablerep.h" #include "rocksdb/merge_operator.h" #include "rocksdb/options.h" @@ -22,11 +24,23 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/table.h" #include "rocksdb/utilities/object_registry.h" -#include "table/block_based/block_based_table_factory.h" -#include "table/plain/plain_table_factory.h" +#include "rocksdb/utilities/options_type.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { +Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) { + Status s; +#ifndef ROCKSDB_LITE + auto db_cfg = DBOptionsAsConfigurable(db_opts); + auto cf_cfg = CFOptionsAsConfigurable(cf_opts); + s = db_cfg->ValidateOptions(db_opts, cf_opts); + if (s.ok()) s = cf_cfg->ValidateOptions(db_opts, cf_opts); +#else + s = cf_opts.table_factory->ValidateOptions(db_opts, cf_opts); +#endif + return s; +} DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, const MutableDBOptions& mutable_db_options) { @@ -336,8 +350,8 @@ bool ParseSliceTransform( return false; } -bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, - const std::string& value) { +static bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, + const std::string& value) { switch (opt_type) { case OptionType::kBoolean: *reinterpret_cast(opt_address) = ParseBoolean("", value); @@ -472,14 +486,6 @@ bool SerializeSingleOptionHelper(const char* opt_address, : kNullptrString; break; } - case OptionType::kTableFactory: { - const auto* table_factory_ptr = - reinterpret_cast*>( - opt_address); - *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name() - : kNullptrString; - break; - } case OptionType::kComparator: { // it's a const pointer of const Comparator* const auto* ptr = reinterpret_cast(opt_address); @@ -555,36 +561,29 @@ bool SerializeSingleOptionHelper(const char* opt_address, return true; } +template +Status ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opt_map, + const std::string& option_name, Configurable* config, T* new_opts) { + Status s = config->ConfigureFromMap(config_options, opt_map); + if (s.ok()) { + *new_opts = *(config->GetOptions(option_name)); + } + return s; +} + Status GetMutableOptionsFromStrings( const MutableCFOptions& base_options, const std::unordered_map& options_map, - Logger* info_log, MutableCFOptions* new_options) { + Logger* /*info_log*/, MutableCFOptions* new_options) { assert(new_options); *new_options = base_options; ConfigOptions config_options; - for (const auto& o : options_map) { - std::string elem; - const auto opt_info = - OptionTypeInfo::Find(o.first, cf_options_type_info, &elem); - if (opt_info == nullptr) { - return Status::InvalidArgument("Unrecognized option: " + o.first); - } else if (!opt_info->IsMutable()) { - return Status::InvalidArgument("Option not changeable: " + o.first); - } else if (opt_info->IsDeprecated()) { - // log warning when user tries to set a deprecated option but don't fail - // the call for compatibility. - ROCKS_LOG_WARN(info_log, "%s is a deprecated option and cannot be set", - o.first.c_str()); - } else { - Status s = opt_info->Parse( - config_options, elem, o.second, - reinterpret_cast(new_options) + opt_info->mutable_offset_); - if (!s.ok()) { - return s; - } - } - } - return Status::OK(); + const auto config = CFOptionsAsConfigurable(base_options); + return ConfigureFromMap(config_options, options_map, + MutableCFOptions::kName(), + config.get(), new_options); } Status GetMutableDBOptionsFromStrings( @@ -595,29 +594,10 @@ Status GetMutableDBOptionsFromStrings( *new_options = base_options; ConfigOptions config_options; - for (const auto& o : options_map) { - try { - std::string elem; - const auto opt_info = - OptionTypeInfo::Find(o.first, db_options_type_info, &elem); - if (opt_info == nullptr) { - return Status::InvalidArgument("Unrecognized option: " + o.first); - } else if (!opt_info->IsMutable()) { - return Status::InvalidArgument("Option not changeable: " + o.first); - } else { - Status s = opt_info->Parse( - config_options, elem, o.second, - reinterpret_cast(new_options) + opt_info->mutable_offset_); - if (!s.ok()) { - return s; - } - } - } catch (std::exception& e) { - return Status::InvalidArgument("Error parsing " + o.first + ":" + - std::string(e.what())); - } - } - return Status::OK(); + auto config = DBOptionsAsConfigurable(base_options); + return ConfigureFromMap(config_options, options_map, + MutableDBOptions::kName(), + config.get(), new_options); } Status StringToMap(const std::string& opts_str, @@ -660,30 +640,11 @@ Status StringToMap(const std::string& opts_str, return Status::OK(); } -Status GetStringFromStruct( - const ConfigOptions& config_options, const void* const opt_ptr, - const std::unordered_map& type_info, - std::string* opt_string) { - assert(opt_string); - opt_string->clear(); - for (const auto& iter : type_info) { - const auto& opt_info = iter.second; - // If the option is no longer used in rocksdb and marked as deprecated, - // we skip it in the serialization. - if (opt_info.ShouldSerialize()) { - const char* opt_addr = - reinterpret_cast(opt_ptr) + opt_info.offset_; - std::string value; - Status s = - opt_info.Serialize(config_options, iter.first, opt_addr, &value); - if (s.ok()) { - opt_string->append(iter.first + "=" + value + config_options.delimiter); - } else { - return s; - } - } - } - return Status::OK(); +Status GetStringFromMutableDBOptions(const ConfigOptions& config_options, + const MutableDBOptions& mutable_opts, + std::string* opt_string) { + auto config = DBOptionsAsConfigurable(mutable_opts); + return config->GetOptionString(config_options, opt_string); } Status GetStringFromDBOptions(std::string* opt_string, @@ -694,11 +655,22 @@ Status GetStringFromDBOptions(std::string* opt_string, return GetStringFromDBOptions(config_options, db_options, opt_string); } -Status GetStringFromDBOptions(const ConfigOptions& cfg_options, +Status GetStringFromDBOptions(const ConfigOptions& config_options, const DBOptions& db_options, std::string* opt_string) { - return GetStringFromStruct(cfg_options, &db_options, db_options_type_info, - opt_string); + assert(opt_string); + opt_string->clear(); + auto config = DBOptionsAsConfigurable(db_options); + return config->GetOptionString(config_options, opt_string); +} + +Status GetStringFromMutableCFOptions(const ConfigOptions& config_options, + const MutableCFOptions& mutable_opts, + std::string* opt_string) { + assert(opt_string); + opt_string->clear(); + const auto config = CFOptionsAsConfigurable(mutable_opts); + return config->GetOptionString(config_options, opt_string); } Status GetStringFromColumnFamilyOptions(std::string* opt_string, @@ -713,8 +685,8 @@ Status GetStringFromColumnFamilyOptions(std::string* opt_string, Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options, const ColumnFamilyOptions& cf_options, std::string* opt_string) { - return GetStringFromStruct(config_options, &cf_options, cf_options_type_info, - opt_string); + const auto config = CFOptionsAsConfigurable(cf_options); + return config->GetOptionString(config_options, opt_string); } Status GetStringFromCompressionType(std::string* compression_str, @@ -728,24 +700,6 @@ Status GetStringFromCompressionType(std::string* compression_str, } } -static Status ParseDBOption(const ConfigOptions& config_options, - const std::string& name, - const std::string& org_value, - DBOptions* new_options) { - const std::string& value = config_options.input_strings_escaped - ? UnescapeOptionString(org_value) - : org_value; - std::string elem; - const auto opt_info = OptionTypeInfo::Find(name, db_options_type_info, &elem); - if (opt_info == nullptr) { - return Status::InvalidArgument("Unrecognized option DBOptions:", name); - } else { - return opt_info->Parse( - config_options, elem, value, - reinterpret_cast(new_options) + opt_info->offset_); - } -} - Status GetColumnFamilyOptionsFromMap( const ColumnFamilyOptions& base_options, const std::unordered_map& opts_map, @@ -764,24 +718,13 @@ Status GetColumnFamilyOptionsFromMap( const std::unordered_map& opts_map, ColumnFamilyOptions* new_options) { assert(new_options); + *new_options = base_options; - for (const auto& o : opts_map) { - auto s = - ParseColumnFamilyOption(config_options, o.first, o.second, new_options); - if (!s.ok()) { - if (s.IsNotSupported()) { - continue; - } else if (s.IsInvalidArgument() && - config_options.ignore_unknown_options) { - continue; - } else { - // Restore "new_options" to the default "base_options". - *new_options = base_options; - return s; - } - } - } - return Status::OK(); + + const auto config = CFOptionsAsConfigurable(base_options); + return ConfigureFromMap(config_options, opts_map, + OptionsHelper::kCFOptionsName, + config.get(), new_options); } Status GetColumnFamilyOptionsFromString( @@ -825,44 +768,12 @@ Status GetDBOptionsFromMap( const ConfigOptions& config_options, const DBOptions& base_options, const std::unordered_map& opts_map, DBOptions* new_options) { - return GetDBOptionsFromMapInternal(config_options, base_options, opts_map, - new_options, nullptr); -} - -Status GetDBOptionsFromMapInternal( - const ConfigOptions& config_options, const DBOptions& base_options, - const std::unordered_map& opts_map, - DBOptions* new_options, - std::vector* unsupported_options_names) { assert(new_options); *new_options = base_options; - if (unsupported_options_names) { - unsupported_options_names->clear(); - } - for (const auto& o : opts_map) { - auto s = ParseDBOption(config_options, o.first, o.second, new_options); - if (!s.ok()) { - if (s.IsNotSupported()) { - // If the deserialization of the specified option is not supported - // and an output vector of unsupported_options is provided, then - // we log the name of the unsupported option and proceed. - if (unsupported_options_names != nullptr) { - unsupported_options_names->push_back(o.first); - } - // Note that we still return Status::OK in such case to maintain - // the backward compatibility in the old public API defined in - // rocksdb/convenience.h - } else if (s.IsInvalidArgument() && - config_options.ignore_unknown_options) { - continue; - } else { - // Restore "new_options" to the default "base_options". - *new_options = base_options; - return s; - } - } - } - return Status::OK(); + auto config = DBOptionsAsConfigurable(base_options); + return ConfigureFromMap(config_options, opts_map, + OptionsHelper::kDBOptionsName, + config.get(), new_options); } Status GetDBOptionsFromString(const DBOptions& base_options, @@ -903,65 +814,31 @@ Status GetOptionsFromString(const Options& base_options, Status GetOptionsFromString(const ConfigOptions& config_options, const Options& base_options, const std::string& opts_str, Options* new_options) { + ColumnFamilyOptions new_cf_options; + std::unordered_map unused_opts; std::unordered_map opts_map; + + *new_options = base_options; Status s = StringToMap(opts_str, &opts_map); if (!s.ok()) { return s; } - DBOptions new_db_options(base_options); - ColumnFamilyOptions new_cf_options(base_options); - for (const auto& o : opts_map) { - if (ParseDBOption(config_options, o.first, o.second, &new_db_options) - .ok()) { - } else if (ParseColumnFamilyOption(config_options, o.first, o.second, - &new_cf_options) - .ok()) { + auto config = DBOptionsAsConfigurable(base_options); + s = config->ConfigureFromMap(config_options, opts_map, &unused_opts); + + if (s.ok()) { + DBOptions* new_db_options = + config->GetOptions(OptionsHelper::kDBOptionsName); + if (!unused_opts.empty()) { + s = GetColumnFamilyOptionsFromMap(config_options, base_options, + unused_opts, &new_cf_options); + if (s.ok()) { + *new_options = Options(*new_db_options, new_cf_options); + } } else { - return Status::InvalidArgument("Can't parse option " + o.first); + *new_options = Options(*new_db_options, base_options); } } - *new_options = Options(new_db_options, new_cf_options); - return Status::OK(); -} - -Status GetTableFactoryFromMap( - const std::string& factory_name, - const std::unordered_map& opt_map, - std::shared_ptr* table_factory, bool ignore_unknown_options) { - ConfigOptions - config_options; // Use default for escaped(true) and check (exact) - config_options.ignore_unknown_options = ignore_unknown_options; - return GetTableFactoryFromMap(config_options, factory_name, opt_map, - table_factory); -} - -Status GetTableFactoryFromMap( - const ConfigOptions& config_options, const std::string& factory_name, - const std::unordered_map& opt_map, - std::shared_ptr* table_factory) { - Status s; - if (factory_name == BlockBasedTableFactory::kName) { - BlockBasedTableOptions bbt_opt; - s = GetBlockBasedTableOptionsFromMap( - config_options, BlockBasedTableOptions(), opt_map, &bbt_opt); - if (!s.ok()) { - return s; - } - table_factory->reset(new BlockBasedTableFactory(bbt_opt)); - return s; - } else if (factory_name == PlainTableFactory::kName) { - PlainTableOptions pt_opt; - s = GetPlainTableOptionsFromMap(config_options, PlainTableOptions(), - opt_map, &pt_opt); - if (!s.ok()) { - return s; - } - table_factory->reset(new PlainTableFactory(pt_opt)); - return s; - } - // Return OK for not supported table factories as TableFactory - // Deserialization is optional. - table_factory->reset(); return s; } @@ -1043,18 +920,41 @@ Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter, Status OptionTypeInfo::Parse(const ConfigOptions& config_options, const std::string& opt_name, - const std::string& opt_value, - char* opt_addr) const { + const std::string& value, void* opt_ptr) const { if (IsDeprecated()) { return Status::OK(); } try { + char* opt_addr = reinterpret_cast(opt_ptr) + offset_; + const std::string& opt_value = config_options.input_strings_escaped + ? UnescapeOptionString(value) + : value; + if (opt_addr == nullptr) { return Status::NotFound("Could not find option", opt_name); } else if (parse_func_ != nullptr) { - return parse_func_(config_options, opt_name, opt_value, opt_addr); + ConfigOptions copy = config_options; + copy.invoke_prepare_options = false; + return parse_func_(copy, opt_name, opt_value, opt_addr); } else if (ParseOptionHelper(opt_addr, type_, opt_value)) { return Status::OK(); + } else if (IsConfigurable()) { + // The option is . + Configurable* config = AsRawPointer(opt_ptr); + if (opt_value.empty()) { + return Status::OK(); + } else if (config == nullptr) { + return Status::NotFound("Could not find configurable: ", opt_name); + } else { + ConfigOptions copy = config_options; + copy.ignore_unknown_options = false; + copy.invoke_prepare_options = false; + if (opt_value.find("=") != std::string::npos) { + return config->ConfigureFromString(copy, opt_value); + } else { + return config->ConfigureOption(copy, opt_name, opt_value); + } + } } else if (IsByName()) { return Status::NotSupported("Deserializing the option " + opt_name + " is not supported"); @@ -1083,9 +983,8 @@ Status OptionTypeInfo::ParseStruct( } const auto iter = struct_map->find(map_iter.first); if (iter != struct_map->end()) { - status = - iter->second.Parse(config_options, map_iter.first, map_iter.second, - opt_addr + iter->second.offset_); + status = iter->second.Parse(config_options, map_iter.first, + map_iter.second, opt_addr); } else { status = Status::InvalidArgument("Unrecognized option", struct_name + "." + map_iter.first); @@ -1097,8 +996,7 @@ Status OptionTypeInfo::ParseStruct( const auto opt_info = Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); if (opt_info != nullptr) { - status = opt_info->Parse(config_options, elem_name, opt_value, - opt_addr + opt_info->offset_); + status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr); } else { status = Status::InvalidArgument("Unrecognized option", opt_name); } @@ -1107,8 +1005,7 @@ Status OptionTypeInfo::ParseStruct( std::string elem_name; const auto opt_info = Find(opt_name, *struct_map, &elem_name); if (opt_info != nullptr) { - status = opt_info->Parse(config_options, elem_name, opt_value, - opt_addr + opt_info->offset_); + status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr); } else { status = Status::InvalidArgument("Unrecognized option", struct_name + "." + opt_name); @@ -1119,18 +1016,30 @@ Status OptionTypeInfo::ParseStruct( Status OptionTypeInfo::Serialize(const ConfigOptions& config_options, const std::string& opt_name, - const char* opt_addr, + const void* const opt_ptr, std::string* opt_value) const { // If the option is no longer used in rocksdb and marked as deprecated, // we skip it in the serialization. - if (opt_addr != nullptr && ShouldSerialize()) { - if (serialize_func_ != nullptr) { - return serialize_func_(config_options, opt_name, opt_addr, opt_value); - } else if (!SerializeSingleOptionHelper(opt_addr, type_, opt_value)) { - return Status::InvalidArgument("Cannot serialize option", opt_name); + const char* opt_addr = reinterpret_cast(opt_ptr) + offset_; + if (opt_addr == nullptr || IsDeprecated()) { + return Status::OK(); + } else if (IsEnabled(OptionTypeFlags::kDontSerialize)) { + return Status::NotSupported("Cannot serialize option: ", opt_name); + } else if (serialize_func_ != nullptr) { + return serialize_func_(config_options, opt_name, opt_addr, opt_value); + } else if (SerializeSingleOptionHelper(opt_addr, type_, opt_value)) { + return Status::OK(); + } else if (IsConfigurable()) { + const Configurable* config = AsRawPointer(opt_ptr); + if (config != nullptr) { + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + *opt_value = config->ToString(embedded); } + return Status::OK(); + } else { + return Status::InvalidArgument("Cannot serialize option: ", opt_name); } - return Status::OK(); } Status OptionTypeInfo::SerializeStruct( @@ -1151,8 +1060,7 @@ Status OptionTypeInfo::SerializeStruct( std::string single; const auto& opt_info = iter.second; if (opt_info.ShouldSerialize()) { - status = opt_info.Serialize(embedded, iter.first, - opt_addr + opt_info.offset_, &single); + status = opt_info.Serialize(embedded, iter.first, opt_addr, &single); if (!status.ok()) { return status; } else { @@ -1167,8 +1075,7 @@ Status OptionTypeInfo::SerializeStruct( const auto opt_info = Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); if (opt_info != nullptr) { - status = opt_info->Serialize(config_options, elem_name, - opt_addr + opt_info->offset_, value); + status = opt_info->Serialize(config_options, elem_name, opt_addr, value); } else { status = Status::InvalidArgument("Unrecognized option", opt_name); } @@ -1180,7 +1087,7 @@ Status OptionTypeInfo::SerializeStruct( status = Status::InvalidArgument("Unrecognized option", opt_name); } else if (opt_info->ShouldSerialize()) { status = opt_info->Serialize(config_options, opt_name + "." + elem_name, - opt_addr + opt_info->offset_, value); + opt_addr, value); } } return status; @@ -1251,11 +1158,15 @@ static bool AreOptionsEqual(OptionType type, const char* this_offset, bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options, const std::string& opt_name, - const char* this_addr, const char* that_addr, + const void* const this_ptr, + const void* const that_ptr, std::string* mismatch) const { - if (!config_options.IsCheckEnabled(GetSanityLevel())) { + auto level = GetSanityLevel(); + if (!config_options.IsCheckEnabled(level)) { return true; // If the sanity level is not being checked, skip it } + const auto this_addr = reinterpret_cast(this_ptr) + offset_; + const auto that_addr = reinterpret_cast(that_ptr) + offset_; if (this_addr == nullptr || that_addr == nullptr) { if (this_addr == that_addr) { return true; @@ -1267,6 +1178,27 @@ bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options, } } else if (AreOptionsEqual(type_, this_addr, that_addr)) { return true; + } else if (IsConfigurable()) { + const auto* this_config = AsRawPointer(this_ptr); + const auto* that_config = AsRawPointer(that_ptr); + if (this_config == that_config) { + return true; + } else if (this_config != nullptr && that_config != nullptr) { + std::string bad_name; + bool matches; + if (level < config_options.sanity_level) { + ConfigOptions copy = config_options; + copy.sanity_level = level; + matches = this_config->AreEquivalent(copy, that_config, &bad_name); + } else { + matches = + this_config->AreEquivalent(config_options, that_config, &bad_name); + } + if (!matches) { + *mismatch = opt_name + "." + bad_name; + } + return matches; + } } if (mismatch->empty()) { *mismatch = opt_name; @@ -1287,9 +1219,8 @@ bool OptionTypeInfo::StructsAreEqual( for (const auto& iter : *struct_map) { const auto& opt_info = iter.second; - matches = opt_info.AreEqual(config_options, iter.first, - this_addr + opt_info.offset_, - that_addr + opt_info.offset_, &result); + matches = opt_info.AreEqual(config_options, iter.first, this_addr, + that_addr, &result); if (!matches) { *mismatch = struct_name + "." + result; return false; @@ -1304,9 +1235,8 @@ bool OptionTypeInfo::StructsAreEqual( if (opt_info == nullptr) { *mismatch = opt_name; matches = false; - } else if (!opt_info->AreEqual(config_options, elem_name, - this_addr + opt_info->offset_, - that_addr + opt_info->offset_, &result)) { + } else if (!opt_info->AreEqual(config_options, elem_name, this_addr, + that_addr, &result)) { matches = false; *mismatch = struct_name + "." + result; } @@ -1318,9 +1248,8 @@ bool OptionTypeInfo::StructsAreEqual( if (opt_info == nullptr) { *mismatch = struct_name + "." + opt_name; matches = false; - } else if (!opt_info->AreEqual(config_options, elem_name, - this_addr + opt_info->offset_, - that_addr + opt_info->offset_, &result)) { + } else if (!opt_info->AreEqual(config_options, elem_name, this_addr, + that_addr, &result)) { matches = false; *mismatch = struct_name + "." + result; } @@ -1328,14 +1257,34 @@ bool OptionTypeInfo::StructsAreEqual( return matches; } +bool MatchesOptionsTypeFromMap( + const ConfigOptions& config_options, + const std::unordered_map& type_map, + const void* const this_ptr, const void* const that_ptr, + std::string* mismatch) { + for (auto& pair : type_map) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) { + if (!pair.second.AreEqual(config_options, pair.first, this_ptr, that_ptr, + mismatch) && + !pair.second.AreEqualByName(config_options, pair.first, this_ptr, + that_ptr)) { + return false; + } + } + } + return true; +} + bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, const std::string& opt_name, - const char* this_addr, - const char* that_addr) const { + const void* const this_ptr, + const void* const that_ptr) const { if (IsByName()) { std::string that_value; - if (Serialize(config_options, opt_name, that_addr, &that_value).ok()) { - return AreEqualByName(config_options, opt_name, this_addr, that_value); + if (Serialize(config_options, opt_name, that_ptr, &that_value).ok()) { + return AreEqualByName(config_options, opt_name, this_ptr, that_value); } } return false; @@ -1343,12 +1292,12 @@ bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, const std::string& opt_name, - const char* opt_addr, + const void* const opt_ptr, const std::string& that_value) const { std::string this_value; if (!IsByName()) { return false; - } else if (!Serialize(config_options, opt_name, opt_addr, &this_value).ok()) { + } else if (!Serialize(config_options, opt_name, opt_ptr, &this_value).ok()) { return false; } else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) { if (that_value == kNullptrString) { @@ -1376,7 +1325,8 @@ const OptionTypeInfo* OptionTypeInfo::Find( auto siter = opt_map.find(opt_name.substr(0, idx)); // Look for the short name if (siter != opt_map.end()) { // We found the short name - if (siter->second.IsStruct()) { // If the object is a struct + if (siter->second.IsStruct() || // If the object is a struct + siter->second.IsConfigurable()) { // or a Configurable *elem_name = opt_name.substr(idx + 1); // Return the rest return &(siter->second); // Return the contents of the iterator } diff --git a/options/options_helper.h b/options/options_helper.h index 7fbc6095e..4323d5f8e 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -10,19 +10,25 @@ #include #include -#include "options/cf_options.h" -#include "options/db_options.h" -#include "options/options_type.h" #include "rocksdb/options.h" #include "rocksdb/status.h" #include "rocksdb/table.h" -#include "rocksdb/universal_compaction.h" namespace ROCKSDB_NAMESPACE { +struct ColumnFamilyOptions; struct ConfigOptions; +struct DBOptions; +struct ImmutableDBOptions; +struct MutableDBOptions; +struct MutableCFOptions; +struct Options; std::vector GetSupportedCompressions(); +// Checks that the combination of DBOptions and ColumnFamilyOptions are valid +Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts); + DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, const MutableDBOptions& mutable_db_options); @@ -31,15 +37,22 @@ ColumnFamilyOptions BuildColumnFamilyOptions( const MutableCFOptions& mutable_cf_options); #ifndef ROCKSDB_LITE -Status GetStringFromStruct( - const ConfigOptions& config_options, const void* const opt_ptr, - const std::unordered_map& type_info, - std::string* opt_string); +std::unique_ptr DBOptionsAsConfigurable( + const MutableDBOptions& opts); +std::unique_ptr DBOptionsAsConfigurable(const DBOptions& opts); +std::unique_ptr CFOptionsAsConfigurable( + const MutableCFOptions& opts); +std::unique_ptr CFOptionsAsConfigurable( + const ColumnFamilyOptions& opts, + const std::unordered_map* opt_map = nullptr); -Status ParseColumnFamilyOption(const ConfigOptions& config_options, - const std::string& name, - const std::string& org_value, - ColumnFamilyOptions* new_options); +Status GetStringFromMutableCFOptions(const ConfigOptions& config_options, + const MutableCFOptions& mutable_opts, + std::string* opt_string); + +Status GetStringFromMutableDBOptions(const ConfigOptions& config_options, + const MutableDBOptions& mutable_opts, + std::string* opt_string); Status GetMutableOptionsFromStrings( const MutableCFOptions& base_options, @@ -51,31 +64,6 @@ Status GetMutableDBOptionsFromStrings( const std::unordered_map& options_map, MutableDBOptions* new_options); -Status GetTableFactoryFromMap( - const std::string& factory_name, - const std::unordered_map& opt_map, - std::shared_ptr* table_factory, - bool ignore_unknown_options = false); - -Status GetTableFactoryFromMap( - const ConfigOptions& config_options, const std::string& factory_name, - const std::unordered_map& opt_map, - std::shared_ptr* table_factory); - -// A helper function that converts "opt_address" to a std::string -// based on the specified OptionType. -bool SerializeSingleOptionHelper(const char* opt_address, - const OptionType opt_type, std::string* value); - -// In addition to its public version defined in rocksdb/convenience.h, -// this further takes an optional output vector "unsupported_options_names", -// which stores the name of all the unsupported options specified in "opts_map". -Status GetDBOptionsFromMapInternal( - const ConfigOptions& config_options, const DBOptions& base_options, - const std::unordered_map& opts_map, - DBOptions* new_options, - std::vector* unsupported_options_names = nullptr); - bool ParseSliceTransform( const std::string& value, std::shared_ptr* slice_transform); @@ -83,12 +71,11 @@ bool ParseSliceTransform( extern Status StringToMap( const std::string& opts_str, std::unordered_map* opts_map); - -extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, - const std::string& value); #endif // !ROCKSDB_LITE struct OptionsHelper { + static const std::string kCFOptionsName /*= "ColumnFamilyOptions"*/; + static const std::string kDBOptionsName /*= "DBOptions" */; static std::map compaction_style_to_string; static std::map compaction_pri_to_string; static std::map @@ -97,16 +84,13 @@ struct OptionsHelper { static std::unordered_map compression_type_string_map; #ifndef ROCKSDB_LITE - static std::unordered_map cf_options_type_info; static std::unordered_map compaction_stop_style_string_map; - static std::unordered_map db_options_type_info; static std::unordered_map encoding_type_string_map; static std::unordered_map compaction_style_string_map; static std::unordered_map compaction_pri_string_map; - static ColumnFamilyOptions dummy_cf_options; #endif // !ROCKSDB_LITE }; @@ -118,10 +102,8 @@ static auto& compaction_stop_style_to_string = OptionsHelper::compaction_stop_style_to_string; static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map; #ifndef ROCKSDB_LITE -static auto& cf_options_type_info = OptionsHelper::cf_options_type_info; static auto& compaction_stop_style_string_map = OptionsHelper::compaction_stop_style_string_map; -static auto& db_options_type_info = OptionsHelper::db_options_type_info; static auto& compression_type_string_map = OptionsHelper::compression_type_string_map; static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map; diff --git a/options/options_parser.cc b/options/options_parser.cc index 36db390ed..4ce578b41 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -15,11 +15,13 @@ #include "file/read_write_util.h" #include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "options/db_options.h" #include "options/options_helper.h" #include "port/port.h" #include "rocksdb/convenience.h" #include "rocksdb/db.h" -#include "table/block_based/block_based_table_factory.h" +#include "rocksdb/utilities/options_type.h" #include "test_util/sync_point.h" #include "util/cast_util.h" #include "util/string_util.h" @@ -41,6 +43,8 @@ Status PersistRocksDBOptions(const DBOptions& db_opt, ConfigOptions config_options; // Use default for escaped(true) and check (exact) config_options.delimiter = "\n "; + // Do not invoke PrepareOptions when we are doing validation. + config_options.invoke_prepare_options = false; // If a readahead size was set in the input options, use it if (db_opt.log_readahead_size > 0) { config_options.file_readahead_size = db_opt.log_readahead_size; @@ -450,13 +454,18 @@ Status RocksDBOptionsParser::EndSection( section_arg); } // Ignore error as table factory deserialization is optional - s = GetTableFactoryFromMap( + s = TableFactory::CreateFromString( config_options, section_title.substr( opt_section_titles[kOptionSectionTableOptions].size()), - opt_map, &(cf_opt->table_factory)); - if (!s.ok()) { - return s; + &(cf_opt->table_factory)); + if (s.ok()) { + return cf_opt->table_factory->ConfigureFromMap(config_options, opt_map); + } else { + // Return OK for not supported table factories as TableFactory + // Deserialization is optional. + cf_opt->table_factory.reset(); + return Status::OK(); } } else if (section == kOptionSectionVersion) { for (const auto& pair : opt_map) { @@ -531,11 +540,14 @@ std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line, } Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( - const ConfigOptions& config_options, const DBOptions& db_opt, + const ConfigOptions& config_options_in, const DBOptions& db_opt, const std::vector& cf_names, const std::vector& cf_opts, const std::string& file_name, FileSystem* fs) { RocksDBOptionsParser parser; + ConfigOptions config_options = config_options_in; + config_options.invoke_prepare_options = + false; // No need to do a prepare for verify Status s = parser.Parse(config_options, file_name, fs); if (!s.ok()) { return s; @@ -606,54 +618,35 @@ Status RocksDBOptionsParser::VerifyDBOptions( const ConfigOptions& config_options, const DBOptions& base_opt, const DBOptions& file_opt, const std::unordered_map* /*opt_map*/) { - for (const auto& pair : db_options_type_info) { - const auto& opt_info = pair.second; - if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) { - const char* base_addr = - reinterpret_cast(&base_opt) + opt_info.offset_; - const char* file_addr = - reinterpret_cast(&file_opt) + opt_info.offset_; - std::string mismatch; - if (!opt_info.AreEqual(config_options, pair.first, base_addr, file_addr, - &mismatch) && - !opt_info.AreEqualByName(config_options, pair.first, base_addr, - file_addr)) { - const size_t kBufferSize = 2048; - char buffer[kBufferSize]; - std::string base_value; - std::string file_value; - int offset = - snprintf(buffer, sizeof(buffer), - "[RocksDBOptionsParser]: " - "failed the verification on ColumnFamilyOptions::%s", - pair.first.c_str()); - Status s = opt_info.Serialize(config_options, pair.first, base_addr, - &base_value); - if (s.ok()) { - s = opt_info.Serialize(config_options, pair.first, file_addr, - &file_value); - } - snprintf(buffer, sizeof(buffer), - "[RocksDBOptionsParser]: " - "failed the verification on DBOptions::%s --- " - "The specified one is %s while the persisted one is %s.\n", - pair.first.c_str(), base_value.c_str(), file_value.c_str()); - assert(offset >= 0); - assert(static_cast(offset) < sizeof(buffer)); - if (s.ok()) { - snprintf( - buffer + offset, sizeof(buffer) - static_cast(offset), - "--- The specified one is %s while the persisted one is %s.\n", - base_value.c_str(), file_value.c_str()); - } else { - snprintf(buffer + offset, - sizeof(buffer) - static_cast(offset), - "--- Unable to re-serialize an option: %s.\n", - s.ToString().c_str()); - } - return Status::InvalidArgument(Slice(buffer, strlen(buffer))); - } + auto base_config = DBOptionsAsConfigurable(base_opt); + auto file_config = DBOptionsAsConfigurable(file_opt); + std::string mismatch; + if (!base_config->AreEquivalent(config_options, file_config.get(), + &mismatch)) { + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + std::string base_value; + std::string file_value; + int offset = snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on DBOptions::%s -- ", + mismatch.c_str()); + Status s = base_config->GetOption(config_options, mismatch, &base_value); + if (s.ok()) { + s = file_config->GetOption(config_options, mismatch, &file_value); } + assert(offset >= 0); + assert(static_cast(offset) < sizeof(buffer)); + if (s.ok()) { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "-- The specified one is %s while the persisted one is %s.\n", + base_value.c_str(), file_value.c_str()); + } else { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "-- Unable to re-serialize an option: %s.\n", + s.ToString().c_str()); + } + return Status::InvalidArgument(Slice(buffer, strlen(buffer))); } return Status::OK(); } @@ -662,86 +655,56 @@ Status RocksDBOptionsParser::VerifyCFOptions( const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& file_opt, const std::unordered_map* opt_map) { - for (const auto& pair : cf_options_type_info) { - const auto& opt_info = pair.second; - - if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) { - std::string mismatch; - const char* base_addr = - reinterpret_cast(&base_opt) + opt_info.offset_; - const char* file_addr = - reinterpret_cast(&file_opt) + opt_info.offset_; - bool matches = opt_info.AreEqual(config_options, pair.first, base_addr, - file_addr, &mismatch); - if (!matches && opt_info.IsByName()) { - if (opt_map == nullptr) { - matches = true; - } else { - auto iter = opt_map->find(pair.first); - if (iter == opt_map->end()) { - matches = true; - } else { - matches = opt_info.AreEqualByName(config_options, pair.first, - base_addr, iter->second); - } - } - } - if (!matches) { - // The options do not match - const size_t kBufferSize = 2048; - char buffer[kBufferSize]; - std::string base_value; - std::string file_value; - Status s = opt_info.Serialize(config_options, pair.first, base_addr, - &base_value); - if (s.ok()) { - s = opt_info.Serialize(config_options, pair.first, file_addr, - &file_value); - } - int offset = - snprintf(buffer, sizeof(buffer), - "[RocksDBOptionsParser]: " - "failed the verification on ColumnFamilyOptions::%s", - pair.first.c_str()); - assert(offset >= 0); - assert(static_cast(offset) < sizeof(buffer)); - if (s.ok()) { - snprintf( - buffer + offset, sizeof(buffer) - static_cast(offset), - "--- The specified one is %s while the persisted one is %s.\n", - base_value.c_str(), file_value.c_str()); - } else { - snprintf(buffer + offset, - sizeof(buffer) - static_cast(offset), - "--- Unable to re-serialize an option: %s.\n", - s.ToString().c_str()); - } - return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); - } // if (! matches) - } // CheckSanityLevel - } // For each option + auto base_config = CFOptionsAsConfigurable(base_opt, opt_map); + auto file_config = CFOptionsAsConfigurable(file_opt, opt_map); + std::string mismatch; + if (!base_config->AreEquivalent(config_options, file_config.get(), + &mismatch)) { + std::string base_value; + std::string file_value; + // The options do not match + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + Status s = base_config->GetOption(config_options, mismatch, &base_value); + if (s.ok()) { + s = file_config->GetOption(config_options, mismatch, &file_value); + } + int offset = snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on ColumnFamilyOptions::%s", + mismatch.c_str()); + assert(offset >= 0); + assert(static_cast(offset) < sizeof(buffer)); + if (s.ok()) { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "--- The specified one is %s while the persisted one is %s.\n", + base_value.c_str(), file_value.c_str()); + } else { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "--- Unable to re-serialize an option: %s.\n", + s.ToString().c_str()); + } + return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); + } // For each option return Status::OK(); } Status RocksDBOptionsParser::VerifyTableFactory( const ConfigOptions& config_options, const TableFactory* base_tf, const TableFactory* file_tf) { + std::string mismatch; if (base_tf && file_tf) { if (config_options.sanity_level > ConfigOptions::kSanityLevelNone && std::string(base_tf->Name()) != std::string(file_tf->Name())) { return Status::Corruption( "[RocksDBOptionsParser]: " "failed the verification on TableFactory->Name()"); + } else if (!base_tf->AreEquivalent(config_options, file_tf, &mismatch)) { + return Status::Corruption(std::string("[RocksDBOptionsParser]:" + "failed the verification on ") + + base_tf->Name() + "::", + mismatch); } - if (base_tf->Name() == BlockBasedTableFactory::kName) { - return VerifyBlockBasedTableFactory( - config_options, - static_cast_with_check(base_tf), - static_cast_with_check(file_tf)); - } - // TODO(yhchiang): add checks for other table factory types } else { // TODO(yhchiang): further support sanity check here } diff --git a/options/options_parser.h b/options/options_parser.h index ec5d4308a..20e3d772d 100644 --- a/options/options_parser.h +++ b/options/options_parser.h @@ -98,6 +98,9 @@ class RocksDBOptionsParser { static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); + static Status ParseStatement(std::string* name, std::string* value, + const std::string& line, const int line_num); + protected: bool IsSection(const std::string& line); Status ParseSection(OptionSection* section, std::string* title, @@ -107,9 +110,6 @@ class RocksDBOptionsParser { Status CheckSection(const OptionSection section, const std::string& section_arg, const int line_num); - Status ParseStatement(std::string* name, std::string* value, - const std::string& line, const int line_num); - Status EndSection( const ConfigOptions& config_options, const OptionSection section, const std::string& title, const std::string& section_arg, @@ -117,7 +117,7 @@ class RocksDBOptionsParser { Status ValidityCheck(); - Status InvalidArgument(const int line_num, const std::string& message); + static Status InvalidArgument(const int line_num, const std::string& message); Status ParseVersionNumber(const std::string& ver_name, const std::string& ver_string, const int max_count, diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index 70e0bdb51..bf0958610 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -9,6 +9,8 @@ #include +#include "options/cf_options.h" +#include "options/db_options.h" #include "options/options_helper.h" #include "rocksdb/convenience.h" #include "test_util/testharness.h" @@ -424,6 +426,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { options->compaction_options_universal = CompactionOptionsUniversal(); options->hard_rate_limit = 0; options->soft_rate_limit = 0; + options->num_levels = 42; // Initialize options for MutableCF options->purge_redundant_kvs_while_flush = false; options->max_mem_compaction_level = 0; options->compaction_filter = nullptr; diff --git a/options/options_test.cc b/options/options_test.cc index 8a981c182..2ad551eed 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -22,6 +22,7 @@ #include "rocksdb/memtablerep.h" #include "rocksdb/utilities/leveldb_options.h" #include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" #include "table/block_based/filter_policy_internal.h" #include "test_util/testharness.h" #include "test_util/testutil.h" @@ -817,7 +818,7 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { ASSERT_OK(GetBlockBasedTableOptionsFromString( config_options, table_opt, "cache_index_and_filter_blocks=1;index_type=kHashSearch;" - "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" + "checksum=kxxHash;hash_index_allow_collision=1;" "block_cache=1M;block_cache_compressed=1k;block_size=1024;" "block_size_deviation=8;block_restart_interval=4;" "format_version=5;whole_key_filtering=1;" @@ -827,7 +828,6 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch); ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash); ASSERT_TRUE(new_opt.hash_index_allow_collision); - ASSERT_TRUE(new_opt.no_block_cache); ASSERT_TRUE(new_opt.block_cache != nullptr); ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); @@ -1127,13 +1127,13 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) { ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); ASSERT_EQ(new_options.write_buffer_size, 10U); ASSERT_EQ(new_options.max_write_buffer_number, 16); - BlockBasedTableOptions new_block_based_table_options = - dynamic_cast(new_options.table_factory.get()) - ->table_options(); - ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20); - ASSERT_EQ(new_block_based_table_options.block_size, 4U); + const auto new_bbto = + new_options.table_factory->GetOptions(); + ASSERT_NE(new_bbto, nullptr); + ASSERT_EQ(new_bbto->block_cache->GetCapacity(), 1U << 20); + ASSERT_EQ(new_bbto->block_size, 4U); // don't overwrite block based table options - ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks); + ASSERT_TRUE(new_bbto->cache_index_and_filter_blocks); ASSERT_EQ(new_options.create_if_missing, true); ASSERT_EQ(new_options.max_open_files, 1); @@ -1142,7 +1142,42 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) { ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv)); ASSERT_EQ(newEnv, new_options.env); - // Test the old interfaxe + config_options.ignore_unknown_options = false; + // Test a bad value for a DBOption returns a failure + base_options.dump_malloc_stats = false; + base_options.write_buffer_size = 1024; + Options bad_options = new_options; + ASSERT_NOK(GetOptionsFromString(config_options, base_options, + "create_if_missing=XX;dump_malloc_stats=true", + &bad_options)); + ASSERT_EQ(bad_options.dump_malloc_stats, false); + + bad_options = new_options; + ASSERT_NOK(GetOptionsFromString(config_options, base_options, + "write_buffer_size=XX;dump_malloc_stats=true", + &bad_options)); + ASSERT_EQ(bad_options.dump_malloc_stats, false); + + // Test a bad value for a TableFactory Option returns a failure + bad_options = new_options; + ASSERT_NOK(GetOptionsFromString(config_options, base_options, + "write_buffer_size=16;dump_malloc_stats=true" + "block_based_table_factory={block_size=XX;};", + &bad_options)); + ASSERT_EQ(bad_options.dump_malloc_stats, false); + ASSERT_EQ(bad_options.write_buffer_size, 1024); + + config_options.ignore_unknown_options = true; + ASSERT_OK(GetOptionsFromString(config_options, base_options, + "create_if_missing=XX;dump_malloc_stats=true;" + "write_buffer_size=XX;" + "block_based_table_factory={block_size=XX;};", + &bad_options)); + ASSERT_EQ(bad_options.create_if_missing, base_options.create_if_missing); + ASSERT_EQ(bad_options.dump_malloc_stats, true); + ASSERT_EQ(bad_options.write_buffer_size, base_options.write_buffer_size); + + // Test the old interface ASSERT_OK(GetOptionsFromString( base_options, "write_buffer_size=22;max_write_buffer_number=33;max_open_files=44;", @@ -1227,6 +1262,76 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { } } +TEST_F(OptionsTest, CheckBlockBasedTableOptions) { + ColumnFamilyOptions cf_opts; + DBOptions db_opts; + ConfigOptions config_opts; + + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_opts, cf_opts, "prefix_extractor=capped:8", &cf_opts)); + ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable", + &cf_opts.table_factory)); + ASSERT_NE(cf_opts.table_factory.get(), nullptr); + ASSERT_TRUE(cf_opts.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())); + auto bbto = cf_opts.table_factory->GetOptions(); + ASSERT_OK(cf_opts.table_factory->ConfigureFromString( + config_opts, + "block_cache={capacity=1M;num_shard_bits=4;};" + "block_size_deviation=101;" + "block_restart_interval=0;" + "index_block_restart_interval=5;" + "partition_filters=true;" + "index_type=kHashSearch;" + "no_block_cache=1;")); + ASSERT_NE(bbto, nullptr); + ASSERT_EQ(bbto->block_cache.get(), nullptr); + ASSERT_EQ(bbto->block_size_deviation, 0); + ASSERT_EQ(bbto->block_restart_interval, 1); + ASSERT_EQ(bbto->index_block_restart_interval, 1); + ASSERT_FALSE(bbto->partition_filters); + ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable", + &cf_opts.table_factory)); + bbto = cf_opts.table_factory->GetOptions(); + + ASSERT_OK(cf_opts.table_factory->ConfigureFromString(config_opts, + "no_block_cache=0;")); + ASSERT_NE(bbto->block_cache.get(), nullptr); + ASSERT_OK(cf_opts.table_factory->ValidateOptions(db_opts, cf_opts)); +} + +TEST_F(OptionsTest, MutableTableOptions) { + ConfigOptions config_options; + std::shared_ptr bbtf; + bbtf.reset(NewBlockBasedTableFactory()); + auto bbto = bbtf->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_FALSE(bbtf->IsPrepared()); + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_align", "true")); + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024")); + ASSERT_EQ(bbto->block_align, true); + ASSERT_EQ(bbto->block_size, 1024); + ASSERT_OK(bbtf->PrepareOptions(config_options)); + ASSERT_TRUE(bbtf->IsPrepared()); + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024")); + ASSERT_EQ(bbto->block_align, true); + ASSERT_NOK(bbtf->ConfigureOption(config_options, "block_align", "false")); + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "2048")); + ASSERT_EQ(bbto->block_align, true); + ASSERT_EQ(bbto->block_size, 2048); + + ColumnFamilyOptions cf_opts; + cf_opts.table_factory = bbtf; + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, cf_opts, "block_based_table_factory.block_align=false", + &cf_opts)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, cf_opts, "block_based_table_factory.block_size=8192", + &cf_opts)); + ASSERT_EQ(bbto->block_align, true); + ASSERT_EQ(bbto->block_size, 8192); +} + #endif // !ROCKSDB_LITE Status StringToMap( @@ -1434,19 +1539,15 @@ TEST_F(OptionsTest, ConvertOptionsTest) { ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); - std::shared_ptr tb_guard = converted_opt.table_factory; - BlockBasedTableFactory* table_factory = - dynamic_cast(converted_opt.table_factory.get()); + std::shared_ptr table_factory = converted_opt.table_factory; + const auto table_opt = table_factory->GetOptions(); + ASSERT_NE(table_opt, nullptr); - ASSERT_TRUE(table_factory != nullptr); - - const BlockBasedTableOptions table_opt = table_factory->table_options(); - - ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20); - ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size); - ASSERT_EQ(table_opt.block_restart_interval, + ASSERT_EQ(table_opt->block_cache->GetCapacity(), 8UL << 20); + ASSERT_EQ(table_opt->block_size, leveldb_opt.block_size); + ASSERT_EQ(table_opt->block_restart_interval, leveldb_opt.block_restart_interval); - ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy); + ASSERT_EQ(table_opt->filter_policy.get(), leveldb_opt.filter_policy); } #ifndef ROCKSDB_LITE @@ -2184,13 +2285,15 @@ TEST_F(OptionsOldApiTest, GetOptionsFromStringTest) { ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); ASSERT_EQ(new_options.write_buffer_size, 10U); ASSERT_EQ(new_options.max_write_buffer_number, 16); - BlockBasedTableOptions new_block_based_table_options = - dynamic_cast(new_options.table_factory.get()) - ->table_options(); - ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20); - ASSERT_EQ(new_block_based_table_options.block_size, 4U); + + auto new_block_based_table_options = + new_options.table_factory->GetOptions(); + ASSERT_NE(new_block_based_table_options, nullptr); + ASSERT_EQ(new_block_based_table_options->block_cache->GetCapacity(), + 1U << 20); + ASSERT_EQ(new_block_based_table_options->block_size, 4U); // don't overwrite block based table options - ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks); + ASSERT_TRUE(new_block_based_table_options->cache_index_and_filter_blocks); ASSERT_EQ(new_options.create_if_missing, true); ASSERT_EQ(new_options.max_open_files, 1); @@ -2763,12 +2866,12 @@ TEST_F(OptionsParserTest, DumpAndParse) { // Make sure block-based table factory options was deserialized correctly std::shared_ptr ttf = (*parser.cf_opts())[4].table_factory; - ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name())); - const BlockBasedTableOptions& parsed_bbto = - static_cast(ttf.get())->table_options(); - ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size); + ASSERT_EQ(TableFactory::kBlockBasedTableName(), std::string(ttf->Name())); + const auto parsed_bbto = ttf->GetOptions(); + ASSERT_NE(parsed_bbto, nullptr); + ASSERT_EQ(special_bbto.block_size, parsed_bbto->block_size); ASSERT_EQ(special_bbto.cache_index_and_filter_blocks, - parsed_bbto.cache_index_and_filter_blocks); + parsed_bbto->cache_index_and_filter_blocks); ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName, @@ -3181,11 +3284,9 @@ static void TestAndCompareOption(const ConfigOptions& config_options, const std::string& opt_name, void* base_ptr, void* comp_ptr) { std::string result, mismatch; - char* base_addr = reinterpret_cast(base_ptr) + opt_info.offset_; - char* comp_addr = reinterpret_cast(comp_ptr) + opt_info.offset_; - ASSERT_OK(opt_info.Serialize(config_options, opt_name, base_addr, &result)); - ASSERT_OK(opt_info.Parse(config_options, opt_name, result, comp_addr)); - ASSERT_TRUE(opt_info.AreEqual(config_options, opt_name, base_addr, comp_addr, + ASSERT_OK(opt_info.Serialize(config_options, opt_name, base_ptr, &result)); + ASSERT_OK(opt_info.Parse(config_options, opt_name, result, comp_ptr)); + ASSERT_TRUE(opt_info.AreEqual(config_options, opt_name, base_ptr, comp_ptr, &mismatch)); } @@ -3194,8 +3295,7 @@ static void TestAndCompareOption(const ConfigOptions& config_options, const std::string& opt_name, const std::string& opt_value, void* base_ptr, void* comp_ptr) { - char* base_addr = reinterpret_cast(base_ptr) + opt_info.offset_; - ASSERT_OK(opt_info.Parse(config_options, opt_name, opt_value, base_addr)); + ASSERT_OK(opt_info.Parse(config_options, opt_name, opt_value, base_ptr)); TestAndCompareOption(config_options, opt_info, opt_name, base_ptr, comp_ptr); } @@ -3204,13 +3304,10 @@ void TestOptInfo(const ConfigOptions& config_options, OptionType opt_type, T* base, T* comp) { std::string result; OptionTypeInfo opt_info(0, opt_type); - char* base_addr = reinterpret_cast(base); - char* comp_addr = reinterpret_cast(comp); - ASSERT_FALSE( - opt_info.AreEqual(config_options, "base", base_addr, comp_addr, &result)); + ASSERT_FALSE(opt_info.AreEqual(config_options, "base", base, comp, &result)); ASSERT_EQ(result, "base"); ASSERT_NE(*base, *comp); - TestAndCompareOption(config_options, opt_info, "base", base_addr, comp_addr); + TestAndCompareOption(config_options, opt_info, "base", base, comp); ASSERT_EQ(*base, *comp); } @@ -3273,53 +3370,46 @@ TEST_F(OptionTypeInfoTest, TestInvalidArgs) { double d; ASSERT_NOK(OptionTypeInfo(0, OptionType::kBoolean) - .Parse(config_options, "b", "x", reinterpret_cast(&b))); - ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt) - .Parse(config_options, "b", "x", reinterpret_cast(&i))); + .Parse(config_options, "b", "x", &b)); ASSERT_NOK( - OptionTypeInfo(0, OptionType::kInt32T) - .Parse(config_options, "b", "x", reinterpret_cast(&i32))); + OptionTypeInfo(0, OptionType::kInt).Parse(config_options, "b", "x", &i)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt32T) + .Parse(config_options, "b", "x", &i32)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt64T) + .Parse(config_options, "b", "x", &i64)); ASSERT_NOK( - OptionTypeInfo(0, OptionType::kInt64T) - .Parse(config_options, "b", "x", reinterpret_cast(&i64))); - ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt) - .Parse(config_options, "b", "x", reinterpret_cast(&u))); - ASSERT_NOK( - OptionTypeInfo(0, OptionType::kUInt32T) - .Parse(config_options, "b", "x", reinterpret_cast(&u32))); - ASSERT_NOK( - OptionTypeInfo(0, OptionType::kUInt64T) - .Parse(config_options, "b", "x", reinterpret_cast(&u64))); - ASSERT_NOK( - OptionTypeInfo(0, OptionType::kSizeT) - .Parse(config_options, "b", "x", reinterpret_cast(&sz))); + OptionTypeInfo(0, OptionType::kUInt).Parse(config_options, "b", "x", &u)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt32T) + .Parse(config_options, "b", "x", &u32)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt64T) + .Parse(config_options, "b", "x", &u64)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kSizeT) + .Parse(config_options, "b", "x", &sz)); ASSERT_NOK(OptionTypeInfo(0, OptionType::kDouble) - .Parse(config_options, "b", "x", reinterpret_cast(&d))); + .Parse(config_options, "b", "x", &d)); // Don't know how to convert Unknowns to anything else ASSERT_NOK(OptionTypeInfo(0, OptionType::kUnknown) - .Parse(config_options, "b", "x", reinterpret_cast(&d))); + .Parse(config_options, "b", "x", &d)); // Verify that if the parse function throws an exception, it is also trapped OptionTypeInfo func_info(0, OptionType::kUnknown, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0, + OptionTypeFlags::kNone, [](const ConfigOptions&, const std::string&, const std::string& value, char* addr) { auto ptr = reinterpret_cast(addr); *ptr = ParseInt(value); return Status::OK(); }); - ASSERT_OK( - func_info.Parse(config_options, "b", "1", reinterpret_cast(&i))); - ASSERT_NOK( - func_info.Parse(config_options, "b", "x", reinterpret_cast(&i))); + ASSERT_OK(func_info.Parse(config_options, "b", "1", &i)); + ASSERT_NOK(func_info.Parse(config_options, "b", "x", &i)); } TEST_F(OptionTypeInfoTest, TestParseFunc) { OptionTypeInfo opt_info( 0, OptionType::kUnknown, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0, + OptionTypeFlags::kNone, [](const ConfigOptions& /*opts*/, const std::string& name, const std::string& value, char* addr) { auto ptr = reinterpret_cast(addr); @@ -3332,17 +3422,15 @@ TEST_F(OptionTypeInfoTest, TestParseFunc) { }); ConfigOptions config_options; std::string base; - ASSERT_OK(opt_info.Parse(config_options, "World", "Hello", - reinterpret_cast(&base))); + ASSERT_OK(opt_info.Parse(config_options, "World", "Hello", &base)); ASSERT_EQ(base, "Hello World"); - ASSERT_NOK(opt_info.Parse(config_options, "Oops", "Hello", - reinterpret_cast(&base))); + ASSERT_NOK(opt_info.Parse(config_options, "Oops", "Hello", &base)); } TEST_F(OptionTypeInfoTest, TestSerializeFunc) { OptionTypeInfo opt_info( 0, OptionType::kString, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0, nullptr, + OptionTypeFlags::kNone, nullptr, [](const ConfigOptions& /*opts*/, const std::string& name, const char* /*addr*/, std::string* value) { if (name == "Oops") { @@ -3356,17 +3444,15 @@ TEST_F(OptionTypeInfoTest, TestSerializeFunc) { ConfigOptions config_options; std::string base; std::string value; - ASSERT_OK(opt_info.Serialize(config_options, "Hello", - reinterpret_cast(&base), &value)); + ASSERT_OK(opt_info.Serialize(config_options, "Hello", &base, &value)); ASSERT_EQ(value, "Hello"); - ASSERT_NOK(opt_info.Serialize(config_options, "Oops", - reinterpret_cast(&base), &value)); + ASSERT_NOK(opt_info.Serialize(config_options, "Oops", &base, &value)); } TEST_F(OptionTypeInfoTest, TestEqualsFunc) { OptionTypeInfo opt_info( 0, OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0, nullptr, nullptr, + OptionTypeFlags::kNone, nullptr, nullptr, [](const ConfigOptions& /*opts*/, const std::string& name, const char* addr1, const char* addr2, std::string* mismatch) { auto i1 = *(reinterpret_cast(addr1)); @@ -3387,69 +3473,56 @@ TEST_F(OptionTypeInfoTest, TestEqualsFunc) { int int1 = 100; int int2 = 200; std::string mismatch; - ASSERT_TRUE(opt_info.AreEqual( - config_options, "LT", reinterpret_cast(&int1), - reinterpret_cast(&int2), &mismatch)); + ASSERT_TRUE(opt_info.AreEqual(config_options, "LT", &int1, &int2, &mismatch)); ASSERT_EQ(mismatch, ""); - ASSERT_FALSE(opt_info.AreEqual(config_options, "GT", - reinterpret_cast(&int1), - reinterpret_cast(&int2), &mismatch)); + ASSERT_FALSE( + opt_info.AreEqual(config_options, "GT", &int1, &int2, &mismatch)); ASSERT_EQ(mismatch, "GT"); - ASSERT_FALSE(opt_info.AreEqual(config_options, "NO", - reinterpret_cast(&int1), - reinterpret_cast(&int2), &mismatch)); + ASSERT_FALSE( + opt_info.AreEqual(config_options, "NO", &int1, &int2, &mismatch)); ASSERT_EQ(mismatch, "NO???"); } TEST_F(OptionTypeInfoTest, TestOptionFlags) { OptionTypeInfo opt_none(0, OptionType::kString, OptionVerificationType::kNormal, - OptionTypeFlags::kDontSerialize, 0); + OptionTypeFlags::kDontSerialize); OptionTypeInfo opt_never(0, OptionType::kString, OptionVerificationType::kNormal, - OptionTypeFlags::kCompareNever, 0); + OptionTypeFlags::kCompareNever); OptionTypeInfo opt_alias(0, OptionType::kString, OptionVerificationType::kAlias, - OptionTypeFlags::kNone, 0); + OptionTypeFlags::kNone); OptionTypeInfo opt_deprecated(0, OptionType::kString, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0); + OptionTypeFlags::kNone); ConfigOptions config_options; + std::string opts_str; std::string base = "base"; std::string comp = "comp"; - // If marked string none, the serialization returns okay but does nothing - ASSERT_OK(opt_none.Serialize(config_options, "None", - reinterpret_cast(&base), &base)); + // If marked string none, the serialization returns not supported + ASSERT_NOK(opt_none.Serialize(config_options, "None", &base, &opts_str)); // If marked never compare, they match even when they do not - ASSERT_TRUE(opt_never.AreEqual(config_options, "Never", - reinterpret_cast(&base), - reinterpret_cast(&comp), &base)); - ASSERT_FALSE(opt_none.AreEqual(config_options, "Never", - reinterpret_cast(&base), - reinterpret_cast(&comp), &base)); + ASSERT_TRUE(opt_never.AreEqual(config_options, "Never", &base, &comp, &base)); + ASSERT_FALSE(opt_none.AreEqual(config_options, "Never", &base, &comp, &base)); // An alias can change the value via parse, but does nothing on serialize on // match std::string result; ASSERT_OK(opt_alias.Parse(config_options, "Alias", "Alias", reinterpret_cast(&base))); - ASSERT_OK(opt_alias.Serialize(config_options, "Alias", - reinterpret_cast(&base), &result)); - ASSERT_TRUE(opt_alias.AreEqual(config_options, "Alias", - reinterpret_cast(&base), - reinterpret_cast(&comp), &result)); + ASSERT_OK(opt_alias.Serialize(config_options, "Alias", &base, &result)); + ASSERT_TRUE( + opt_alias.AreEqual(config_options, "Alias", &base, &comp, &result)); ASSERT_EQ(base, "Alias"); ASSERT_NE(base, comp); // Deprecated options do nothing on any of the commands - ASSERT_OK(opt_deprecated.Parse(config_options, "Alias", "Deprecated", - reinterpret_cast(&base))); - ASSERT_OK(opt_deprecated.Serialize(config_options, "Alias", - reinterpret_cast(&base), &result)); - ASSERT_TRUE(opt_deprecated.AreEqual(config_options, "Alias", - reinterpret_cast(&base), - reinterpret_cast(&comp), &result)); + ASSERT_OK(opt_deprecated.Parse(config_options, "Alias", "Deprecated", &base)); + ASSERT_OK(opt_deprecated.Serialize(config_options, "Alias", &base, &result)); + ASSERT_TRUE( + opt_deprecated.AreEqual(config_options, "Alias", &base, &comp, &result)); ASSERT_EQ(base, "Alias"); ASSERT_NE(base, comp); } @@ -3468,25 +3541,18 @@ TEST_F(OptionTypeInfoTest, TestCustomEnum) { e2 = TestEnum::kA; - ASSERT_OK( - opt_info.Parse(config_options, "", "B", reinterpret_cast(&e1))); - ASSERT_OK(opt_info.Serialize(config_options, "", reinterpret_cast(&e1), - &result)); + ASSERT_OK(opt_info.Parse(config_options, "", "B", &e1)); + ASSERT_OK(opt_info.Serialize(config_options, "", &e1, &result)); ASSERT_EQ(e1, TestEnum::kB); ASSERT_EQ(result, "B"); - ASSERT_FALSE(opt_info.AreEqual(config_options, "Enum", - reinterpret_cast(&e1), - reinterpret_cast(&e2), &mismatch)); + ASSERT_FALSE(opt_info.AreEqual(config_options, "Enum", &e1, &e2, &mismatch)); ASSERT_EQ(mismatch, "Enum"); - TestAndCompareOption(config_options, opt_info, "", "C", - reinterpret_cast(&e1), - reinterpret_cast(&e2)); + TestAndCompareOption(config_options, opt_info, "", "C", &e1, &e2); ASSERT_EQ(e2, TestEnum::kC); - ASSERT_NOK( - opt_info.Parse(config_options, "", "D", reinterpret_cast(&e1))); + ASSERT_NOK(opt_info.Parse(config_options, "", "D", &e1)); ASSERT_EQ(e1, TestEnum::kC); } @@ -3553,21 +3619,20 @@ TEST_F(OptionTypeInfoTest, TestStruct) { }; OptionTypeInfo basic_info = OptionTypeInfo::Struct( "b", &basic_type_map, 0, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, 0); + OptionTypeFlags::kMutable); std::unordered_map extended_type_map = { {"j", {offsetof(struct Extended, j), OptionType::kInt}}, {"b", OptionTypeInfo::Struct( "b", &basic_type_map, offsetof(struct Extended, b), - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0)}, + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, {"m", OptionTypeInfo::Struct( "m", &basic_type_map, offsetof(struct Extended, b), - OptionVerificationType::kNormal, OptionTypeFlags::kMutable, - offsetof(struct Extended, b))}, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable)}, }; OptionTypeInfo extended_info = OptionTypeInfo::Struct( "e", &extended_type_map, 0, OptionVerificationType::kNormal, - OptionTypeFlags::kMutable, 0); + OptionTypeFlags::kMutable); Extended e1, e2; ConfigOptions config_options; std::string mismatch; @@ -3583,30 +3648,24 @@ TEST_F(OptionTypeInfoTest, TestStruct) { ASSERT_EQ(e1.b.i, 55); e1.b.i = 0; - auto e1bc = reinterpret_cast(&e1.b); - auto e2bc = reinterpret_cast(&e2.b); - ASSERT_FALSE(basic_info.AreEqual(config_options, "b", e1bc, e2bc, &mismatch)); + ASSERT_FALSE( + basic_info.AreEqual(config_options, "b", &e1.b, &e2.b, &mismatch)); ASSERT_EQ(mismatch, "b.i"); mismatch.clear(); ASSERT_FALSE( - basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); + basic_info.AreEqual(config_options, "b.i", &e1.b, &e2.b, &mismatch)); ASSERT_EQ(mismatch, "b.i"); mismatch.clear(); - ASSERT_FALSE(basic_info.AreEqual(config_options, "i", e1bc, e2bc, &mismatch)); + ASSERT_FALSE( + basic_info.AreEqual(config_options, "i", &e1.b, &e2.b, &mismatch)); ASSERT_EQ(mismatch, "b.i"); mismatch.clear(); e1 = e2; - ASSERT_NOK(basic_info.Parse(config_options, "b", "{i=33;s=33;j=44}", e1bc)); - ASSERT_TRUE( - basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); - ASSERT_NOK(basic_info.Parse(config_options, "b.j", "44", e1bc)); - ASSERT_TRUE( - basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); - ASSERT_NOK(basic_info.Parse(config_options, "j", "44", e1bc)); - ASSERT_TRUE( - basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); + ASSERT_NOK(basic_info.Parse(config_options, "b", "{i=33;s=33;j=44}", &e1.b)); + ASSERT_NOK(basic_info.Parse(config_options, "b.j", "44", &e1.b)); + ASSERT_NOK(basic_info.Parse(config_options, "j", "44", &e1.b)); TestAndCompareOption(config_options, extended_info, "e", "b={i=55;s=55}; j=22;", &e1, &e2); @@ -3626,7 +3685,7 @@ TEST_F(OptionTypeInfoTest, TestStruct) { TEST_F(OptionTypeInfoTest, TestVectorType) { OptionTypeInfo vec_info = OptionTypeInfo::Vector( - 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0, + 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, {0, OptionType::kString}); std::vector vec1, vec2; std::string mismatch; @@ -3639,9 +3698,7 @@ TEST_F(OptionTypeInfoTest, TestVectorType) { ASSERT_EQ(vec1[2], "c"); ASSERT_EQ(vec1[3], "d"); vec1[3] = "e"; - ASSERT_FALSE(vec_info.AreEqual(config_options, "v", - reinterpret_cast(&vec1), - reinterpret_cast(&vec2), &mismatch)); + ASSERT_FALSE(vec_info.AreEqual(config_options, "v", &vec1, &vec2, &mismatch)); ASSERT_EQ(mismatch, "v"); // Test vectors with inner brackets @@ -3654,7 +3711,7 @@ TEST_F(OptionTypeInfoTest, TestVectorType) { ASSERT_EQ(vec1[3], "d"); OptionTypeInfo bar_info = OptionTypeInfo::Vector( - 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0, + 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, {0, OptionType::kString}, '|'); TestAndCompareOption(config_options, vec_info, "v", "x|y|z", &vec1, &vec2); // Test vectors with inner vector diff --git a/src.mk b/src.mk index 1ec998dd6..8da6f5efc 100644 --- a/src.mk +++ b/src.mk @@ -123,6 +123,7 @@ LIB_SOURCES = \ monitoring/thread_status_util.cc \ monitoring/thread_status_util_debug.cc \ options/cf_options.cc \ + options/configurable.cc \ options/db_options.cc \ options/options.cc \ options/options_helper.cc \ @@ -174,6 +175,7 @@ LIB_SOURCES = \ table/sst_file_dumper.cc \ table/sst_file_reader.cc \ table/sst_file_writer.cc \ + table/table_factory.cc \ table/table_properties.cc \ table/two_level_iterator.cc \ test_util/sync_point.cc \ @@ -439,6 +441,7 @@ TEST_MAIN_SOURCES = \ monitoring/statistics_test.cc \ monitoring/stats_dump_scheduler_test.cc \ monitoring/stats_history_test.cc \ + options/configurable_test.cc \ options/options_settable_test.cc \ options/options_test.cc \ table/block_based/block_based_filter_block_test.cc \ diff --git a/table/adaptive/adaptive_table_factory.cc b/table/adaptive/adaptive_table_factory.cc index 98381ee81..480c4c9a6 100644 --- a/table/adaptive/adaptive_table_factory.cc +++ b/table/adaptive/adaptive_table_factory.cc @@ -77,7 +77,7 @@ TableBuilder* AdaptiveTableFactory::NewTableBuilder( column_family_id, file); } -std::string AdaptiveTableFactory::GetPrintableTableOptions() const { +std::string AdaptiveTableFactory::GetPrintableOptions() const { std::string ret; ret.reserve(20000); const int kBufferSize = 200; @@ -87,13 +87,13 @@ std::string AdaptiveTableFactory::GetPrintableTableOptions() const { snprintf(buffer, kBufferSize, " write factory (%s) options:\n%s\n", (table_factory_to_write_->Name() ? table_factory_to_write_->Name() : ""), - table_factory_to_write_->GetPrintableTableOptions().c_str()); + table_factory_to_write_->GetPrintableOptions().c_str()); ret.append(buffer); } if (plain_table_factory_) { snprintf(buffer, kBufferSize, " %s options:\n%s\n", plain_table_factory_->Name() ? plain_table_factory_->Name() : "", - plain_table_factory_->GetPrintableTableOptions().c_str()); + plain_table_factory_->GetPrintableOptions().c_str()); ret.append(buffer); } if (block_based_table_factory_) { @@ -101,13 +101,13 @@ std::string AdaptiveTableFactory::GetPrintableTableOptions() const { buffer, kBufferSize, " %s options:\n%s\n", (block_based_table_factory_->Name() ? block_based_table_factory_->Name() : ""), - block_based_table_factory_->GetPrintableTableOptions().c_str()); + block_based_table_factory_->GetPrintableOptions().c_str()); ret.append(buffer); } if (cuckoo_table_factory_) { snprintf(buffer, kBufferSize, " %s options:\n%s\n", cuckoo_table_factory_->Name() ? cuckoo_table_factory_->Name() : "", - cuckoo_table_factory_->GetPrintableTableOptions().c_str()); + cuckoo_table_factory_->GetPrintableOptions().c_str()); ret.append(buffer); } return ret; diff --git a/table/adaptive/adaptive_table_factory.h b/table/adaptive/adaptive_table_factory.h index 74d10dba0..cbc81868c 100644 --- a/table/adaptive/adaptive_table_factory.h +++ b/table/adaptive/adaptive_table_factory.h @@ -44,14 +44,7 @@ class AdaptiveTableFactory : public TableFactory { const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const override; - // Sanitizes the specified DB Options. - Status SanitizeOptions( - const DBOptions& /*db_opts*/, - const ColumnFamilyOptions& /*cf_opts*/) const override { - return Status::OK(); - } - - std::string GetPrintableTableOptions() const override; + std::string GetPrintableOptions() const override; private: std::shared_ptr table_factory_to_write_; diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 267e6163b..6dcca8ea5 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -15,12 +15,12 @@ #include #include -#include "options/options_helper.h" -#include "options/options_parser.h" +#include "options/configurable_helper.h" #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/convenience.h" #include "rocksdb/flush_block_policy.h" +#include "rocksdb/utilities/options_type.h" #include "table/block_based/block_based_table_builder.h" #include "table/block_based/block_based_table_reader.h" #include "table/format.h" @@ -187,9 +187,11 @@ static std::unordered_map block_based_table_type_info = { +#ifndef ROCKSDB_LITE /* currently not supported std::shared_ptr block_cache = nullptr; std::shared_ptr block_cache_compressed = nullptr; @@ -197,29 +199,29 @@ static std::unordered_map {"flush_block_policy_factory", {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName, - OptionTypeFlags::kCompareNever, 0}}, + OptionTypeFlags::kCompareNever}}, {"cache_index_and_filter_blocks", {offsetof(struct BlockBasedTableOptions, cache_index_and_filter_blocks), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"cache_index_and_filter_blocks_with_high_priority", {offsetof(struct BlockBasedTableOptions, cache_index_and_filter_blocks_with_high_priority), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"pin_l0_filter_and_index_blocks_in_cache", {offsetof(struct BlockBasedTableOptions, pin_l0_filter_and_index_blocks_in_cache), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"index_type", OptionTypeInfo::Enum( offsetof(struct BlockBasedTableOptions, index_type), &block_base_table_index_type_string_map)}, {"hash_index_allow_collision", {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"data_block_index_type", OptionTypeInfo::Enum( offsetof(struct BlockBasedTableOptions, data_block_index_type), @@ -232,50 +234,50 @@ static std::unordered_map {offsetof(struct BlockBasedTableOptions, data_block_hash_table_util_ratio), OptionType::kDouble, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"checksum", {offsetof(struct BlockBasedTableOptions, checksum), OptionType::kChecksumType, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"no_block_cache", {offsetof(struct BlockBasedTableOptions, no_block_cache), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"block_size", {offsetof(struct BlockBasedTableOptions, block_size), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"block_size_deviation", {offsetof(struct BlockBasedTableOptions, block_size_deviation), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"block_restart_interval", {offsetof(struct BlockBasedTableOptions, block_restart_interval), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kMutable}}, {"index_block_restart_interval", {offsetof(struct BlockBasedTableOptions, index_block_restart_interval), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"index_per_partition", {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"metadata_block_size", {offsetof(struct BlockBasedTableOptions, metadata_block_size), OptionType::kUInt64T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"partition_filters", {offsetof(struct BlockBasedTableOptions, partition_filters), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"optimize_filters_for_memory", {offsetof(struct BlockBasedTableOptions, optimize_filters_for_memory), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"filter_policy", {offsetof(struct BlockBasedTableOptions, filter_policy), OptionType::kUnknown, OptionVerificationType::kByNameAllowFromNull, - OptionTypeFlags::kNone, 0, + OptionTypeFlags::kNone, // Parses the Filter policy [](const ConfigOptions& opts, const std::string&, const std::string& value, char* addr) { @@ -317,39 +319,39 @@ static std::unordered_map {"whole_key_filtering", {offsetof(struct BlockBasedTableOptions, whole_key_filtering), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"skip_table_builder_flush", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"format_version", {offsetof(struct BlockBasedTableOptions, format_version), OptionType::kUInt32T, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"verify_compression", {offsetof(struct BlockBasedTableOptions, verify_compression), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"read_amp_bytes_per_bit", {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"enable_index_compression", {offsetof(struct BlockBasedTableOptions, enable_index_compression), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"block_align", {offsetof(struct BlockBasedTableOptions, block_align), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"pin_top_level_index_and_filter", {offsetof(struct BlockBasedTableOptions, pin_top_level_index_and_filter), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"block_cache", {offsetof(struct BlockBasedTableOptions, block_cache), OptionType::kUnknown, OptionVerificationType::kNormal, - (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), 0, + (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), // Parses the input vsalue as a Cache [](const ConfigOptions& opts, const std::string&, const std::string& value, char* addr) { @@ -359,21 +361,27 @@ static std::unordered_map {"block_cache_compressed", {offsetof(struct BlockBasedTableOptions, block_cache_compressed), OptionType::kUnknown, OptionVerificationType::kNormal, - (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), 0, + (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), // Parses the input vsalue as a Cache [](const ConfigOptions& opts, const std::string&, const std::string& value, char* addr) { auto* cache = reinterpret_cast*>(addr); return Cache::CreateFromString(opts, value, cache); }}}, -}; #endif // ROCKSDB_LITE +}; // TODO(myabandeh): We should return an error instead of silently changing the // options BlockBasedTableFactory::BlockBasedTableFactory( const BlockBasedTableOptions& _table_options) : table_options_(_table_options) { + InitializeOptions(); + ConfigurableHelper::RegisterOptions(*this, &table_options_, + &block_based_table_type_info); +} + +void BlockBasedTableFactory::InitializeOptions() { if (table_options_.flush_block_policy_factory == nullptr) { table_options_.flush_block_policy_factory.reset( new FlushBlockBySizePolicyFactory()); @@ -411,6 +419,11 @@ BlockBasedTableFactory::BlockBasedTableFactory( } } +Status BlockBasedTableFactory::PrepareOptions(const ConfigOptions& opts) { + InitializeOptions(); + return TableFactory::PrepareOptions(opts); +} + Status BlockBasedTableFactory::NewTableReader( const ReadOptions& ro, const TableReaderOptions& table_reader_options, std::unique_ptr&& file, uint64_t file_size, @@ -449,7 +462,7 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder( return table_builder; } -Status BlockBasedTableFactory::SanitizeOptions( +Status BlockBasedTableFactory::ValidateOptions( const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const { if (table_options_.index_type == BlockBasedTableOptions::kHashSearch && cf_opts.prefix_extractor == nullptr) { @@ -501,10 +514,10 @@ Status BlockBasedTableFactory::SanitizeOptions( "max_successive_merges larger than 0 is currently inconsistent with " "unordered_write"); } - return Status::OK(); + return TableFactory::ValidateOptions(db_opts, cf_opts); } -std::string BlockBasedTableFactory::GetPrintableTableOptions() const { +std::string BlockBasedTableFactory::GetPrintableOptions() const { std::string ret; ret.reserve(20000); const int kBufferSize = 200; @@ -630,53 +643,74 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const { return ret; } -#ifndef ROCKSDB_LITE -Status BlockBasedTableFactory::GetOptionString( - const ConfigOptions& config_options, std::string* opt_string) const { - assert(opt_string); - opt_string->clear(); - return GetStringFromStruct(config_options, &table_options_, - block_based_table_type_info, opt_string); -} -#else -Status BlockBasedTableFactory::GetOptionString( - const ConfigOptions& /*opts*/, std::string* /*opt_string*/) const { - return Status::OK(); -} -#endif // !ROCKSDB_LITE - -const BlockBasedTableOptions& BlockBasedTableFactory::table_options() const { - return table_options_; -} - -#ifndef ROCKSDB_LITE -namespace { -std::string ParseBlockBasedTableOption(const ConfigOptions& config_options, - const std::string& name, - const std::string& org_value, - BlockBasedTableOptions* new_options) { - const std::string& value = config_options.input_strings_escaped - ? UnescapeOptionString(org_value) - : org_value; - const auto iter = block_based_table_type_info.find(name); - if (iter == block_based_table_type_info.end()) { - if (config_options.ignore_unknown_options) { - return ""; +const void* BlockBasedTableFactory::GetOptionsPtr( + const std::string& name) const { + if (name == kBlockCacheOpts()) { + if (table_options_.no_block_cache) { + return nullptr; } else { - return "Unrecognized option"; + return table_options_.block_cache.get(); + } + } else { + return TableFactory::GetOptionsPtr(name); + } +} + +#ifndef ROCKSDB_LITE +// Take a default BlockBasedTableOptions "table_options" in addition to a +// map "opts_map" of option name to option value to construct the new +// BlockBasedTableOptions "new_table_options". +// +// Below are the instructions of how to config some non-primitive-typed +// options in BlockBasedTableOptions: +// +// * filter_policy: +// We currently only support the following FilterPolicy in the convenience +// functions: +// - BloomFilter: use "bloomfilter:[bits_per_key]:[use_block_based_builder]" +// to specify BloomFilter. The above string is equivalent to calling +// NewBloomFilterPolicy(bits_per_key, use_block_based_builder). +// [Example]: +// - Pass {"filter_policy", "bloomfilter:4:true"} in +// GetBlockBasedTableOptionsFromMap to use a BloomFilter with 4-bits +// per key and use_block_based_builder enabled. +// +// * block_cache / block_cache_compressed: +// We currently only support LRU cache in the GetOptions API. The LRU +// cache can be set by directly specifying its size. +// [Example]: +// - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is +// equivalent to setting block_cache using NewLRUCache(1024 * 1024). +// +// @param table_options the default options of the output "new_table_options". +// @param opts_map an option name to value map for specifying how +// "new_table_options" should be set. +// @param new_table_options the resulting options based on "table_options" +// with the change specified in "opts_map". +// @param input_strings_escaped when set to true, each escaped characters +// prefixed by '\' in the values of the opts_map will be further converted +// back to the raw string before assigning to the associated options. +// @param ignore_unknown_options when set to true, unknown options are ignored +// instead of resulting in an unknown-option error. +// @return Status::OK() on success. Otherwise, a non-ok status indicating +// error will be returned, and "new_table_options" will be set to +// "table_options". +Status BlockBasedTableFactory::ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& opt_value, + void* opt_ptr) { + Status status = TableFactory::ParseOption(config_options, opt_info, opt_name, + opt_value, opt_ptr); + if (config_options.input_strings_escaped && !status.ok()) { // Got an error + // !input_strings_escaped indicates the old API, where everything is + // parsable. + if (opt_info.IsByName()) { + status = Status::OK(); } } - const auto& opt_info = iter->second; - Status s = - opt_info.Parse(config_options, iter->first, value, - reinterpret_cast(new_options) + opt_info.offset_); - if (s.ok()) { - return ""; - } else { - return s.ToString(); - } + return status; } -} // namespace Status GetBlockBasedTableOptionsFromString( const BlockBasedTableOptions& table_options, const std::string& opts_str, @@ -684,6 +718,7 @@ Status GetBlockBasedTableOptionsFromString( ConfigOptions config_options; config_options.input_strings_escaped = false; config_options.ignore_unknown_options = false; + config_options.invoke_prepare_options = false; return GetBlockBasedTableOptionsFromString(config_options, table_options, opts_str, new_table_options); } @@ -708,6 +743,7 @@ Status GetBlockBasedTableOptionsFromMap( ConfigOptions config_options; config_options.input_strings_escaped = input_strings_escaped; config_options.ignore_unknown_options = ignore_unknown_options; + config_options.invoke_prepare_options = false; return GetBlockBasedTableOptionsFromMap(config_options, table_options, opts_map, new_table_options); @@ -719,66 +755,14 @@ Status GetBlockBasedTableOptionsFromMap( const std::unordered_map& opts_map, BlockBasedTableOptions* new_table_options) { assert(new_table_options); - *new_table_options = table_options; - for (const auto& o : opts_map) { - auto error_message = ParseBlockBasedTableOption( - config_options, o.first, o.second, new_table_options); - if (error_message != "") { - const auto iter = block_based_table_type_info.find(o.first); - if (iter == block_based_table_type_info.end() || - !config_options - .input_strings_escaped || // !input_strings_escaped indicates - // the old API, where everything is - // parsable. - (!iter->second.IsByName() && !iter->second.IsDeprecated())) { - // Restore "new_options" to the default "base_options". - *new_table_options = table_options; - return Status::InvalidArgument("Can't parse BlockBasedTableOptions:", - o.first + " " + error_message); - } - } + BlockBasedTableFactory bbtf(table_options); + Status s = bbtf.ConfigureFromMap(config_options, opts_map); + if (s.ok()) { + *new_table_options = *(bbtf.GetOptions()); + } else { + *new_table_options = table_options; } - return Status::OK(); -} - -Status VerifyBlockBasedTableFactory(const ConfigOptions& config_options, - const BlockBasedTableFactory* base_tf, - const BlockBasedTableFactory* file_tf) { - if ((base_tf != nullptr) != (file_tf != nullptr) && - config_options.sanity_level > ConfigOptions::kSanityLevelNone) { - return Status::Corruption( - "[RocksDBOptionsParser]: Inconsistent TableFactory class type"); - } - if (base_tf == nullptr) { - return Status::OK(); - } - assert(file_tf != nullptr); - - const auto& base_opt = base_tf->table_options(); - const auto& file_opt = file_tf->table_options(); - - std::string mismatch; - for (auto& pair : block_based_table_type_info) { - // We skip checking deprecated variables as they might - // contain random values since they might not be initialized - if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) { - const char* base_addr = - reinterpret_cast(&base_opt) + pair.second.offset_; - const char* file_addr = - reinterpret_cast(&file_opt) + pair.second.offset_; - - if (!pair.second.AreEqual(config_options, pair.first, base_addr, - file_addr, &mismatch) && - !pair.second.AreEqualByName(config_options, pair.first, base_addr, - file_addr)) { - return Status::Corruption( - "[RocksDBOptionsParser]: " - "failed the verification on BlockBasedTableOptions::", - pair.first); - } - } - } - return Status::OK(); + return s; } #endif // !ROCKSDB_LITE @@ -787,7 +771,6 @@ TableFactory* NewBlockBasedTableFactory( return new BlockBasedTableFactory(_table_options); } -const std::string BlockBasedTableFactory::kName = "BlockBasedTable"; const std::string BlockBasedTablePropertyNames::kIndexType = "rocksdb.block.based.table.index.type"; const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering = diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h index 4aff7c862..a7120f854 100644 --- a/table/block_based/block_based_table_factory.h +++ b/table/block_based/block_based_table_factory.h @@ -46,7 +46,7 @@ class BlockBasedTableFactory : public TableFactory { ~BlockBasedTableFactory() {} - const char* Name() const override { return kName.c_str(); } + const char* Name() const override { return kBlockBasedTableName(); } using TableFactory::NewTableReader; Status NewTableReader( @@ -59,24 +59,26 @@ class BlockBasedTableFactory : public TableFactory { const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const override; - // Sanitizes the specified DB Options. - Status SanitizeOptions(const DBOptions& db_opts, + // Valdates the specified DB Options. + Status ValidateOptions(const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const override; + Status PrepareOptions(const ConfigOptions& opts) override; - std::string GetPrintableTableOptions() const override; - - Status GetOptionString(const ConfigOptions& config_options, - std::string* opt_string) const override; - - const BlockBasedTableOptions& table_options() const; - - void* GetOptions() override { return &table_options_; } + std::string GetPrintableOptions() const override; bool IsDeleteRangeSupported() const override { return true; } TailPrefetchStats* tail_prefetch_stats() { return &tail_prefetch_stats_; } - static const std::string kName; + protected: + const void* GetOptionsPtr(const std::string& name) const override; +#ifndef ROCKSDB_LITE + Status ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, const std::string& opt_value, + void* opt_ptr) override; +#endif + void InitializeOptions(); private: BlockBasedTableOptions table_options_; @@ -87,10 +89,4 @@ extern const std::string kHashIndexPrefixesBlock; extern const std::string kHashIndexPrefixesMetadataBlock; extern const std::string kPropTrue; extern const std::string kPropFalse; - -#ifndef ROCKSDB_LITE -extern Status VerifyBlockBasedTableFactory( - const ConfigOptions& config_options, const BlockBasedTableFactory* base_tf, - const BlockBasedTableFactory* file_tf); -#endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_reader_test.cc b/table/block_based/block_based_table_reader_test.cc index e46e3de11..f5ddc1e60 100644 --- a/table/block_based/block_based_table_reader_test.cc +++ b/table/block_based/block_based_table_reader_test.cc @@ -93,9 +93,12 @@ class BlockBasedTableReaderTest std::unique_ptr table_reader; ReadOptions ro; - ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), - table_factory_->table_options(), comparator, - std::move(file), file_size, &table_reader)); + const auto* table_options = + table_factory_->GetOptions(); + ASSERT_NE(table_options, nullptr); + ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), *table_options, + comparator, std::move(file), file_size, + &table_reader)); table->reset(reinterpret_cast(table_reader.release())); } diff --git a/table/block_based/filter_policy.cc b/table/block_based/filter_policy.cc index 9520e2d8a..31eb6b90d 100644 --- a/table/block_based/filter_policy.cc +++ b/table/block_based/filter_policy.cc @@ -925,7 +925,7 @@ Status FilterPolicy::CreateFromString( NewBloomFilterPolicy(bits_per_key, use_block_based_builder)); } } else { - return Status::InvalidArgument("Invalid filter policy name ", value); + return Status::NotFound("Invalid filter policy name ", value); #else } else { return Status::NotSupported("Cannot load filter policy in LITE mode ", diff --git a/table/block_based/reader_common.h b/table/block_based/reader_common.h index 6bd4b532f..5bb199f28 100644 --- a/table/block_based/reader_common.h +++ b/table/block_based/reader_common.h @@ -8,6 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once +#include "rocksdb/cache.h" #include "rocksdb/table.h" namespace ROCKSDB_NAMESPACE { diff --git a/table/block_fetcher_test.cc b/table/block_fetcher_test.cc index e3c17b292..f8cbad6d0 100644 --- a/table/block_fetcher_test.cc +++ b/table/block_fetcher_test.cc @@ -276,9 +276,12 @@ class BlockFetcherTest : public testing::Test { std::unique_ptr table_reader; ReadOptions ro; - ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), - table_factory_.table_options(), comparator, - std::move(file), file_size, &table_reader)); + const auto* table_options = + table_factory_.GetOptions(); + ASSERT_NE(table_options, nullptr); + ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), *table_options, + comparator, std::move(file), file_size, + &table_reader)); table->reset(reinterpret_cast(table_reader.release())); } diff --git a/table/cuckoo/cuckoo_table_factory.cc b/table/cuckoo/cuckoo_table_factory.cc index 5ab8b2f42..c6d3c377c 100644 --- a/table/cuckoo/cuckoo_table_factory.cc +++ b/table/cuckoo/cuckoo_table_factory.cc @@ -7,6 +7,8 @@ #include "table/cuckoo/cuckoo_table_factory.h" #include "db/dbformat.h" +#include "options/configurable_helper.h" +#include "rocksdb/utilities/options_type.h" #include "table/cuckoo/cuckoo_table_builder.h" #include "table/cuckoo/cuckoo_table_reader.h" @@ -44,7 +46,7 @@ TableBuilder* CuckooTableFactory::NewTableBuilder( table_builder_options.db_id, table_builder_options.db_session_id); } -std::string CuckooTableFactory::GetPrintableTableOptions() const { +std::string CuckooTableFactory::GetPrintableOptions() const { std::string ret; ret.reserve(2000); const int kBufferSize = 200; @@ -65,6 +67,38 @@ std::string CuckooTableFactory::GetPrintableTableOptions() const { return ret; } +static std::unordered_map cuckoo_table_type_info = + { +#ifndef ROCKSDB_LITE + {"hash_table_ratio", + {offsetof(struct CuckooTableOptions, hash_table_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_search_depth", + {offsetof(struct CuckooTableOptions, max_search_depth), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"cuckoo_block_size", + {offsetof(struct CuckooTableOptions, cuckoo_block_size), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"identity_as_first_hash", + {offsetof(struct CuckooTableOptions, identity_as_first_hash), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_module_hash", + {offsetof(struct CuckooTableOptions, use_module_hash), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; + +CuckooTableFactory::CuckooTableFactory(const CuckooTableOptions& table_options) + : table_options_(table_options) { + ConfigurableHelper::RegisterOptions(*this, &table_options_, + &cuckoo_table_type_info); +} + TableFactory* NewCuckooTableFactory(const CuckooTableOptions& table_options) { return new CuckooTableFactory(table_options); } diff --git a/table/cuckoo/cuckoo_table_factory.h b/table/cuckoo/cuckoo_table_factory.h index d868a2db4..30d4155e1 100644 --- a/table/cuckoo/cuckoo_table_factory.h +++ b/table/cuckoo/cuckoo_table_factory.h @@ -52,11 +52,11 @@ static inline uint64_t CuckooHash( // - Does not support prefix bloom filters. class CuckooTableFactory : public TableFactory { public: - explicit CuckooTableFactory(const CuckooTableOptions& table_options) - : table_options_(table_options) {} + explicit CuckooTableFactory( + const CuckooTableOptions& table_option = CuckooTableOptions()); ~CuckooTableFactory() {} - const char* Name() const override { return "CuckooTable"; } + const char* Name() const override { return kCuckooTableName(); } using TableFactory::NewTableReader; Status NewTableReader( @@ -69,21 +69,7 @@ class CuckooTableFactory : public TableFactory { const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const override; - // Sanitizes the specified DB Options. - Status SanitizeOptions( - const DBOptions& /*db_opts*/, - const ColumnFamilyOptions& /*cf_opts*/) const override { - return Status::OK(); - } - - std::string GetPrintableTableOptions() const override; - - void* GetOptions() override { return &table_options_; } - - Status GetOptionString(const ConfigOptions& /*config_options*/, - std::string* /*opt_string*/) const override { - return Status::OK(); - } + std::string GetPrintableOptions() const override; private: CuckooTableOptions table_options_; diff --git a/table/mock_table.h b/table/mock_table.h index 3484dbd2d..6f8232240 100644 --- a/table/mock_table.h +++ b/table/mock_table.h @@ -62,13 +62,7 @@ class MockTableFactory : public TableFactory { Status CreateMockTable(Env* env, const std::string& fname, stl_wrappers::KVMap file_contents); - virtual Status SanitizeOptions( - const DBOptions& /*db_opts*/, - const ColumnFamilyOptions& /*cf_opts*/) const override { - return Status::OK(); - } - - virtual std::string GetPrintableTableOptions() const override { + virtual std::string GetPrintableOptions() const override { return std::string(); } diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index e0f24bf53..f465de66f 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -11,9 +11,10 @@ #include #include "db/dbformat.h" -#include "options/options_helper.h" +#include "options/configurable_helper.h" #include "port/port.h" #include "rocksdb/convenience.h" +#include "rocksdb/utilities/options_type.h" #include "table/plain/plain_table_builder.h" #include "table/plain/plain_table_reader.h" #include "util/string_util.h" @@ -22,31 +23,38 @@ namespace ROCKSDB_NAMESPACE { static std::unordered_map plain_table_type_info = { {"user_key_len", {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"bloom_bits_per_key", {offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"hash_table_ratio", {offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"index_sparseness", {offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"huge_page_tlb_size", {offsetof(struct PlainTableOptions, huge_page_tlb_size), OptionType::kSizeT, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone}}, {"encoding_type", {offsetof(struct PlainTableOptions, encoding_type), - OptionType::kEncodingType, OptionVerificationType::kByName, - OptionTypeFlags::kNone, 0}}, + OptionType::kEncodingType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"full_scan_mode", {offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean, - OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"store_index_in_file", {offsetof(struct PlainTableOptions, store_index_in_file), OptionType::kBoolean, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}}; + OptionTypeFlags::kNone}}, +}; + +PlainTableFactory::PlainTableFactory(const PlainTableOptions& options) + : table_options_(options) { + ConfigurableHelper::RegisterOptions(*this, &table_options_, + &plain_table_type_info); +} Status PlainTableFactory::NewTableReader( const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options, @@ -80,7 +88,7 @@ TableBuilder* PlainTableFactory::NewTableBuilder( table_builder_options.db_session_id); } -std::string PlainTableFactory::GetPrintableTableOptions() const { +std::string PlainTableFactory::GetPrintableOptions() const { std::string ret; ret.reserve(20000); const int kBufferSize = 200; @@ -113,16 +121,13 @@ std::string PlainTableFactory::GetPrintableTableOptions() const { return ret; } -const PlainTableOptions& PlainTableFactory::table_options() const { - return table_options_; -} - Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options, const std::string& opts_str, PlainTableOptions* new_table_options) { ConfigOptions config_options; config_options.input_strings_escaped = false; config_options.ignore_unknown_options = false; + config_options.invoke_prepare_options = false; return GetPlainTableOptionsFromString(config_options, table_options, opts_str, new_table_options); } @@ -154,7 +159,7 @@ Status GetMemTableRepFactoryFromString( MemTableRepFactory* mem_factory = nullptr; - if (opts_list[0] == "skip_list") { + if (opts_list[0] == "skip_list" || opts_list[0] == "SkipListFactory") { // Expecting format // skip_list: if (2 == len) { @@ -163,7 +168,8 @@ Status GetMemTableRepFactoryFromString( } else if (1 == len) { mem_factory = new SkipListFactory(); } - } else if (opts_list[0] == "prefix_hash") { + } else if (opts_list[0] == "prefix_hash" || + opts_list[0] == "HashSkipListRepFactory") { // Expecting format // prfix_hash: if (2 == len) { @@ -172,7 +178,8 @@ Status GetMemTableRepFactoryFromString( } else if (1 == len) { mem_factory = NewHashSkipListRepFactory(); } - } else if (opts_list[0] == "hash_linkedlist") { + } else if (opts_list[0] == "hash_linkedlist" || + opts_list[0] == "HashLinkListRepFactory") { // Expecting format // hash_linkedlist: if (2 == len) { @@ -181,7 +188,7 @@ Status GetMemTableRepFactoryFromString( } else if (1 == len) { mem_factory = NewHashLinkListRepFactory(); } - } else if (opts_list[0] == "vector") { + } else if (opts_list[0] == "vector" || opts_list[0] == "VectorRepFactory") { // Expecting format // vector: if (2 == len) { @@ -205,32 +212,6 @@ Status GetMemTableRepFactoryFromString( return Status::OK(); } -std::string ParsePlainTableOptions(const ConfigOptions& config_options, - const std::string& name, - const std::string& org_value, - PlainTableOptions* new_options) { - const std::string& value = config_options.input_strings_escaped - ? UnescapeOptionString(org_value) - : org_value; - const auto iter = plain_table_type_info.find(name); - if (iter == plain_table_type_info.end()) { - if (config_options.ignore_unknown_options) { - return ""; - } else { - return "Unrecognized option"; - } - } - const auto& opt_info = iter->second; - Status s = - opt_info.Parse(config_options, name, value, - reinterpret_cast(new_options) + opt_info.offset_); - if (s.ok()) { - return ""; - } else { - return s.ToString(); - } -} - Status GetPlainTableOptionsFromMap( const PlainTableOptions& table_options, const std::unordered_map& opts_map, @@ -248,33 +229,21 @@ Status GetPlainTableOptionsFromMap( const std::unordered_map& opts_map, PlainTableOptions* new_table_options) { assert(new_table_options); - *new_table_options = table_options; - for (const auto& o : opts_map) { - auto error_message = ParsePlainTableOptions(config_options, o.first, - o.second, new_table_options); - if (error_message != "") { - const auto iter = plain_table_type_info.find(o.first); - if (iter == plain_table_type_info.end() || - !config_options - .input_strings_escaped || // !input_strings_escaped indicates - // the old API, where everything is - // parsable. - (!iter->second.IsByName() && !iter->second.IsDeprecated())) { - // Restore "new_options" to the default "base_options". - *new_table_options = table_options; - return Status::InvalidArgument("Can't parse PlainTableOptions:", - o.first + " " + error_message); - } - } + PlainTableFactory ptf(table_options); + Status s = ptf.ConfigureFromMap(config_options, opts_map); + if (s.ok()) { + *new_table_options = *(ptf.GetOptions()); + } else { + // Restore "new_options" to the default "base_options". + *new_table_options = table_options; } - return Status::OK(); + return s; } extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) { return new PlainTableFactory(options); } -const std::string PlainTableFactory::kName = "PlainTable"; const std::string PlainTablePropertyNames::kEncodingType = "rocksdb.plain.table.encoding.type"; diff --git a/table/plain/plain_table_factory.h b/table/plain/plain_table_factory.h index 82523a23f..61a1ed935 100644 --- a/table/plain/plain_table_factory.h +++ b/table/plain/plain_table_factory.h @@ -10,8 +10,6 @@ #include #include -#include "options/options_helper.h" -#include "rocksdb/options.h" #include "rocksdb/table.h" namespace ROCKSDB_NAMESPACE { @@ -156,10 +154,9 @@ class PlainTableFactory : public TableFactory { // page TLB and the page size if allocating from there. See comments of // Arena::AllocateAligned() for details. explicit PlainTableFactory( - const PlainTableOptions& _table_options = PlainTableOptions()) - : table_options_(_table_options) {} + const PlainTableOptions& _table_options = PlainTableOptions()); - const char* Name() const override { return kName.c_str(); } + const char* Name() const override { return kPlainTableName(); } using TableFactory::NewTableReader; Status NewTableReader(const ReadOptions& ro, const TableReaderOptions& table_reader_options, @@ -171,28 +168,9 @@ class PlainTableFactory : public TableFactory { const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const override; - std::string GetPrintableTableOptions() const override; - - const PlainTableOptions& table_options() const; - + std::string GetPrintableOptions() const override; static const char kValueTypeSeqId0 = char(~0); - // Sanitizes the specified DB Options. - Status SanitizeOptions( - const DBOptions& /*db_opts*/, - const ColumnFamilyOptions& /*cf_opts*/) const override { - return Status::OK(); - } - - void* GetOptions() override { return &table_options_; } - - Status GetOptionString(const ConfigOptions& /*config_options*/, - std::string* /*opt_string*/) const override { - return Status::OK(); - } - - static const std::string kName; - private: PlainTableOptions table_options_; }; diff --git a/table/sst_file_dumper.cc b/table/sst_file_dumper.cc index 686c390d3..af96154c4 100644 --- a/table/sst_file_dumper.cc +++ b/table/sst_file_dumper.cc @@ -156,7 +156,8 @@ Status SstFileDumper::NewTableReader( // We need to turn off pre-fetching of index and filter nodes for // BlockBasedTable - if (BlockBasedTableFactory::kName == options_.table_factory->Name()) { + if (options_.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())) { return options_.table_factory->NewTableReader(t_opt, std::move(file_), file_size, &table_reader_, /*enable_prefetch=*/false); diff --git a/table/table_factory.cc b/table/table_factory.cc new file mode 100644 index 000000000..ebf5cfaf7 --- /dev/null +++ b/table/table_factory.cc @@ -0,0 +1,49 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/convenience.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/cuckoo/cuckoo_table_factory.h" +#include "table/plain/plain_table_factory.h" + +namespace ROCKSDB_NAMESPACE { + +Status TableFactory::CreateFromString(const ConfigOptions& config_options_in, + const std::string& id, + std::shared_ptr* factory) { + Status status; + std::string name = id; + + std::string existing_opts; + + ConfigOptions config_options = config_options_in; + if (factory->get() != nullptr && name == factory->get()->Name()) { + config_options.delimiter = ";"; + + status = factory->get()->GetOptionString(config_options, &existing_opts); + if (!status.ok()) { + return status; + } + } + if (name == TableFactory::kBlockBasedTableName()) { + factory->reset(new BlockBasedTableFactory()); +#ifndef ROCKSDB_LITE + } else if (name == TableFactory::kPlainTableName()) { + factory->reset(new PlainTableFactory()); + } else if (name == TableFactory::kCuckooTableName()) { + factory->reset(new CuckooTableFactory()); +#endif // ROCKSDB_LITE + } else { + return Status::NotSupported("Could not load table factory: ", name); + } + if (!existing_opts.empty()) { + config_options.invoke_prepare_options = false; + status = factory->get()->ConfigureFromString(config_options, existing_opts); + } + return status; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/table/table_test.cc b/table/table_test.cc index 7e9e7098a..80dba1271 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -3008,7 +3008,7 @@ void ValidateBlockSizeDeviation(int value, int expected) { BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options); const BlockBasedTableOptions* normalized_table_options = - (const BlockBasedTableOptions*)factory->GetOptions(); + factory->GetOptions(); ASSERT_EQ(normalized_table_options->block_size_deviation, expected); delete factory; @@ -3020,7 +3020,7 @@ void ValidateBlockRestartInterval(int value, int expected) { BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options); const BlockBasedTableOptions* normalized_table_options = - (const BlockBasedTableOptions*)factory->GetOptions(); + factory->GetOptions(); ASSERT_EQ(normalized_table_options->block_restart_interval, expected); delete factory; diff --git a/test_util/testutil.cc b/test_util/testutil.cc index d643b1d8b..d05bb766e 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -23,6 +23,7 @@ #include "file/sequence_file_reader.h" #include "file/writable_file_writer.h" #include "port/port.h" +#include "rocksdb/convenience.h" #include "test_util/sync_point.h" #include "util/random.h" diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 63b0f6074..73fa35ac6 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -4074,11 +4074,9 @@ class Benchmark { options.compression_opts.parallel_threads = FLAGS_compression_parallel_threads; // If this is a block based table, set some related options - if (options.table_factory->Name() == BlockBasedTableFactory::kName && - options.table_factory->GetOptions() != nullptr) { - BlockBasedTableOptions* table_options = - reinterpret_cast( - options.table_factory->GetOptions()); + auto table_options = + options.table_factory->GetOptions(); + if (table_options != nullptr) { if (FLAGS_cache_size) { table_options->block_cache = cache_; } diff --git a/tools/trace_analyzer_tool.cc b/tools/trace_analyzer_tool.cc index e75845c29..7d240f6b9 100644 --- a/tools/trace_analyzer_tool.cc +++ b/tools/trace_analyzer_tool.cc @@ -40,7 +40,6 @@ #include "rocksdb/utilities/ldb_cmd.h" #include "rocksdb/write_batch.h" #include "table/meta_blocks.h" -#include "table/plain/plain_table_factory.h" #include "table/table_reader.h" #include "tools/trace_analyzer_tool.h" #include "trace_replay/trace_replay.h" diff --git a/util/timer_test.cc b/util/timer_test.cc index 32c0132bf..761143ad3 100644 --- a/util/timer_test.cc +++ b/util/timer_test.cc @@ -188,8 +188,8 @@ TEST_F(TimerTest, AddAfterStartTest) { } TEST_F(TimerTest, CancelRunningTask) { + static constexpr char kTestFuncName[] = "test_func"; const int kRepeatUs = 1 * kUsPerSec; - constexpr char kTestFuncName[] = "test_func"; Timer timer(mock_env_.get()); ASSERT_TRUE(timer.Start()); int* value = new int; diff --git a/utilities/memory/memory_test.cc b/utilities/memory/memory_test.cc index 914900362..d035afc03 100644 --- a/utilities/memory/memory_test.cc +++ b/utilities/memory/memory_test.cc @@ -38,12 +38,10 @@ class MemoryTest : public testing::Test { void GetCachePointersFromTableFactory( const TableFactory* factory, std::unordered_set* cache_set) { - const BlockBasedTableFactory* bbtf = - dynamic_cast(factory); - if (bbtf != nullptr) { - const auto bbt_opts = bbtf->table_options(); - cache_set->insert(bbt_opts.block_cache.get()); - cache_set->insert(bbt_opts.block_cache_compressed.get()); + const auto bbto = factory->GetOptions(); + if (bbto != nullptr) { + cache_set->insert(bbto->block_cache.get()); + cache_set->insert(bbto->block_cache_compressed.get()); } } diff --git a/utilities/options/options_util.cc b/utilities/options/options_util.cc index 63feacdfd..2eb398ed5 100644 --- a/utilities/options/options_util.cc +++ b/utilities/options/options_util.cc @@ -47,11 +47,11 @@ Status LoadOptionsFromFile(const ConfigOptions& config_options, cf_descs->push_back({cf_names[i], cf_opts[i]}); if (cache != nullptr) { TableFactory* tf = cf_opts[i].table_factory.get(); - if (tf != nullptr && tf->GetOptions() != nullptr && - tf->Name() == BlockBasedTableFactory::kName) { - auto* loaded_bbt_opt = - reinterpret_cast(tf->GetOptions()); - loaded_bbt_opt->block_cache = *cache; + if (tf != nullptr) { + auto* opts = tf->GetOptions(); + if (opts != nullptr) { + opts->block_cache = *cache; + } } } } diff --git a/utilities/options/options_util_test.cc b/utilities/options/options_util_test.cc index e9789e389..9bd736c1f 100644 --- a/utilities/options/options_util_test.cc +++ b/utilities/options/options_util_test.cc @@ -43,10 +43,6 @@ class OptionsUtilTest : public testing::Test { Random rnd_; }; -bool IsBlockBasedTableFactory(TableFactory* tf) { - return tf->Name() == BlockBasedTableFactory::kName; -} - TEST_F(OptionsUtilTest, SaveAndLoad) { const size_t kCFCount = 5; @@ -80,11 +76,9 @@ TEST_F(OptionsUtilTest, SaveAndLoad) { ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name); ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( exact, cf_opts[i], loaded_cf_descs[i].options)); - if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { - ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory( - exact, cf_opts[i].table_factory.get(), - loaded_cf_descs[i].options.table_factory.get())); - } + ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory( + exact, cf_opts[i].table_factory.get(), + loaded_cf_descs[i].options.table_factory.get())); test::RandomInitCFOptions(&cf_opts[i], db_opt, &rnd_); ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( exact, cf_opts[i], loaded_cf_descs[i].options)); @@ -137,13 +131,12 @@ TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) { ASSERT_OK(LoadOptionsFromFile(config_options, kFileName, &loaded_db_opt, &loaded_cf_descs, &cache)); for (size_t i = 0; i < loaded_cf_descs.size(); i++) { - if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { - auto* loaded_bbt_opt = reinterpret_cast( - loaded_cf_descs[i].options.table_factory->GetOptions()); - // Expect the same cache will be loaded - if (loaded_bbt_opt != nullptr) { - ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); - } + auto* loaded_bbt_opt = + loaded_cf_descs[i] + .options.table_factory->GetOptions(); + // Expect the same cache will be loaded + if (loaded_bbt_opt != nullptr) { + ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); } } @@ -151,13 +144,12 @@ TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) { ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, &loaded_cf_descs, false, &cache)); for (size_t i = 0; i < loaded_cf_descs.size(); i++) { - if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { - auto* loaded_bbt_opt = reinterpret_cast( - loaded_cf_descs[i].options.table_factory->GetOptions()); - // Expect the same cache will be loaded - if (loaded_bbt_opt != nullptr) { - ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); - } + auto* loaded_bbt_opt = + loaded_cf_descs[i] + .options.table_factory->GetOptions(); + // Expect the same cache will be loaded + if (loaded_bbt_opt != nullptr) { + ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); } } } @@ -187,18 +179,13 @@ class DummyTableFactory : public TableFactory { return nullptr; } - Status SanitizeOptions( + Status ValidateOptions( const DBOptions& /*db_opts*/, const ColumnFamilyOptions& /*cf_opts*/) const override { return Status::NotSupported(); } - std::string GetPrintableTableOptions() const override { return ""; } - - Status GetOptionString(const ConfigOptions& /*opts*/, - std::string* /*opt_string*/) const override { - return Status::OK(); - } + std::string GetPrintableOptions() const override { return ""; } }; class DummyMergeOperator : public MergeOperator { diff --git a/utilities/simulator_cache/sim_cache_test.cc b/utilities/simulator_cache/sim_cache_test.cc index 7b181913a..2c21c66b6 100644 --- a/utilities/simulator_cache/sim_cache_test.cc +++ b/utilities/simulator_cache/sim_cache_test.cc @@ -35,7 +35,7 @@ class SimCacheTest : public DBTestBase { options.create_if_missing = true; // options.compression = kNoCompression; options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); return options; } @@ -84,7 +84,7 @@ TEST_F(SimCacheTest, SimCache) { co.metadata_charge_policy = kDontChargeCacheMetadata; std::shared_ptr simCache = NewSimCache(NewLRUCache(co), 20000, 0); table_options.block_cache = simCache; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); RecordCacheCounters(options); @@ -151,7 +151,7 @@ TEST_F(SimCacheTest, SimCacheLogging) { co.metadata_charge_policy = kDontChargeCacheMetadata; std::shared_ptr sim_cache = NewSimCache(NewLRUCache(co), 20000, 0); table_options.block_cache = sim_cache; - options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); Reopen(options); int num_block_entries = 20;