From e29d3b67c2fba95341861f3307fe37b44924ae89 Mon Sep 17 00:00:00 2001 From: Yi Wu Date: Fri, 14 Oct 2016 12:25:39 -0700 Subject: [PATCH] Make max_background_compactions and base_background_compactions dynamic changeable Summary: Add DB::SetDBOptions to dynamic change max_background_compactions and base_background_compactions. I'll add more dynamic changeable options soon. Test Plan: unit test. Reviewers: yhchiang, IslamAbdelRahman, sdong Reviewed By: sdong Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D64749 --- HISTORY.md | 3 + db/column_family_test.cc | 48 ++++++------ db/db_impl.cc | 93 ++++++++++++++++++++---- db/db_impl.h | 10 +++ db/db_impl_debug.cc | 5 ++ db/db_options_test.cc | 52 ++++++++++++- db/db_test.cc | 6 ++ include/rocksdb/db.h | 3 + include/rocksdb/utilities/stackable_db.h | 6 ++ util/db_options.cc | 24 +++--- util/db_options.h | 6 +- util/options_helper.cc | 34 ++++++++- util/options_helper.h | 11 ++- 13 files changed, 246 insertions(+), 55 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index b3d1dbb5c..3857eac81 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,9 @@ * DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference. * Added Statistics::getAndResetTickerCount(). +### New Features +* Add DB::SetDBOptions() to dynamic change base_background_compactions and max_background_compactions. + ## 4.12.0 (9/12/2016) ### Public API Change * CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads. diff --git a/db/column_family_test.cc b/db/column_family_test.cc index d0088f386..85f285021 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -2458,7 +2458,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) { ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate()); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(400); cfd->RecalculateWriteStallConditions(mutable_cf_options); @@ -2466,7 +2466,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) { ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.2, dbfull()->TEST_write_controler().delayed_write_rate()); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(500); cfd->RecalculateWriteStallConditions(mutable_cf_options); @@ -2522,7 +2522,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) { cfd->RecalculateWriteStallConditions(mutable_cf_options); ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(3001); cfd->RecalculateWriteStallConditions(mutable_cf_options); @@ -2547,7 +2547,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) { ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.2, dbfull()->TEST_write_controler().delayed_write_rate()); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(101); cfd->RecalculateWriteStallConditions(mutable_cf_options); @@ -2641,31 +2641,31 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) { vstorage->TEST_set_estimated_compaction_needed_bytes(40); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(50); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(300); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(45); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(7); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(9); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(6); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); // Speed up threshold = min(4 * 2, 4 + (12 - 4)/4) = 6 mutable_cf_options.level0_file_num_compaction_trigger = 4; @@ -2674,15 +2674,15 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) { vstorage->set_l0_delay_trigger_count(5); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(7); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(3); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); } TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) { @@ -2794,41 +2794,41 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) { vstorage->TEST_set_estimated_compaction_needed_bytes(40); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(60); cfd1->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->TEST_set_estimated_compaction_needed_bytes(30); cfd1->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->TEST_set_estimated_compaction_needed_bytes(70); cfd1->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(20); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->TEST_set_estimated_compaction_needed_bytes(3); cfd1->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(9); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->set_l0_delay_trigger_count(2); cfd1->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(6, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(0); cfd->RecalculateWriteStallConditions(mutable_cf_options); - ASSERT_EQ(2, dbfull()->BGCompactionsAllowed()); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); } #ifndef ROCKSDB_LITE diff --git a/db/db_impl.cc b/db/db_impl.cc index 0e6d08521..228621a24 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -307,8 +307,9 @@ void DumpSupportInfo(Logger* logger) { DBImpl::DBImpl(const DBOptions& options, const std::string& dbname) : env_(options.env), dbname_(dbname), - immutable_db_options_(SanitizeOptions(dbname, options)), - mutable_db_options_(options), + initial_db_options_(SanitizeOptions(dbname, options)), + immutable_db_options_(initial_db_options_), + mutable_db_options_(initial_db_options_), stats_(immutable_db_options_.statistics.get()), db_lock_(nullptr), mutex_(stats_, env_, DB_MUTEX_WAIT_MICROS, @@ -2434,13 +2435,7 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family, InstallSuperVersionAndScheduleWork(cfd, nullptr, new_options); delete old_sv; - // Persist RocksDB options under the single write thread - WriteThread::Writer w; - write_thread_.EnterUnbatched(&w, &mutex_); - - persist_options_status = WriteOptionsFile(); - - write_thread_.ExitUnbatched(&w); + persist_options_status = PersistOptions(); } } @@ -2452,12 +2447,12 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family, } if (s.ok()) { Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log, - "[%s] SetOptions succeeded", cfd->GetName().c_str()); + "[%s] SetOptions() succeeded", cfd->GetName().c_str()); new_options.Dump(immutable_db_options_.info_log.get()); if (!persist_options_status.ok()) { if (immutable_db_options_.fail_if_options_file_error) { s = Status::IOError( - "SetOptions succeeded, but unable to persist options", + "SetOptions() succeeded, but unable to persist options", persist_options_status.ToString()); } Warn(immutable_db_options_.info_log, @@ -2466,13 +2461,82 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family, } } else { Log(InfoLogLevel::WARN_LEVEL, immutable_db_options_.info_log, - "[%s] SetOptions failed", cfd->GetName().c_str()); + "[%s] SetOptions() failed", cfd->GetName().c_str()); } LogFlush(immutable_db_options_.info_log); return s; #endif // ROCKSDB_LITE } +Status DBImpl::SetDBOptions( + const std::unordered_map& options_map) { +#ifdef ROCKSDB_LITE + return Status::NotSupported("Not supported in ROCKSDB LITE"); +#else + if (options_map.empty()) { + Log(InfoLogLevel::WARN_LEVEL, immutable_db_options_.info_log, + "SetDBOptions(), empty input."); + return Status::InvalidArgument("empty input"); + } + + MutableDBOptions new_options; + Status s; + Status persist_options_status; + { + InstrumentedMutexLock l(&mutex_); + s = GetMutableDBOptionsFromStrings(mutable_db_options_, options_map, + &new_options); + if (s.ok()) { + if (new_options.max_background_compactions > + mutable_db_options_.max_background_compactions) { + env_->IncBackgroundThreadsIfNeeded( + new_options.max_background_compactions, Env::Priority::LOW); + MaybeScheduleFlushOrCompaction(); + } + + mutable_db_options_ = new_options; + + persist_options_status = PersistOptions(); + } + } + Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log, + "SetDBOptions(), inputs:"); + for (const auto& o : options_map) { + Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log, "%s: %s\n", + o.first.c_str(), o.second.c_str()); + } + if (s.ok()) { + Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log, + "SetDBOptions() succeeded"); + new_options.Dump(immutable_db_options_.info_log.get()); + if (!persist_options_status.ok()) { + if (immutable_db_options_.fail_if_options_file_error) { + s = Status::IOError( + "SetDBOptions() succeeded, but unable to persist options", + persist_options_status.ToString()); + } + Warn(immutable_db_options_.info_log, + "Unable to persist options in SetDBOptions() -- %s", + persist_options_status.ToString().c_str()); + } + } else { + Log(InfoLogLevel::WARN_LEVEL, immutable_db_options_.info_log, + "SetDBOptions failed"); + } + LogFlush(immutable_db_options_.info_log); + return s; +#endif // ROCKSDB_LITE +} + +Status DBImpl::PersistOptions() { + mutex_.AssertHeld(); + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + Status s = WriteOptionsFile(); + write_thread_.ExitUnbatched(&w); + return s; +} + // return the same level if it cannot be moved int DBImpl::FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, int level) { @@ -2957,10 +3021,11 @@ void DBImpl::SchedulePurge() { } int DBImpl::BGCompactionsAllowed() const { + mutex_.AssertHeld(); if (write_controller_.NeedSpeedupCompaction()) { - return immutable_db_options_.max_background_compactions; + return mutable_db_options_.max_background_compactions; } else { - return immutable_db_options_.base_background_compactions; + return mutable_db_options_.base_background_compactions; } } diff --git a/db/db_impl.h b/db/db_impl.h index e018fdb5a..b1a4479ce 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -156,6 +156,9 @@ class DBImpl : public DB { ColumnFamilyHandle* column_family, const std::unordered_map& options_map) override; + virtual Status SetDBOptions( + const std::unordered_map& options_map) override; + using DB::NumberLevels; virtual int NumberLevels(ColumnFamilyHandle* column_family) override; using DB::MaxMemCompactionLevel; @@ -360,6 +363,8 @@ class DBImpl : public DB { uint64_t TEST_FindMinLogContainingOutstandingPrep(); uint64_t TEST_FindMinPrepLogReferencedByMemTable(); + int TEST_BGCompactionsAllowed() const; + #endif // NDEBUG // Return maximum background compaction allowed to be scheduled based on @@ -508,6 +513,7 @@ class DBImpl : public DB { Env* const env_; const std::string dbname_; unique_ptr versions_; + const DBOptions initial_db_options_; const ImmutableDBOptions immutable_db_options_; MutableDBOptions mutable_db_options_; Statistics* stats_; @@ -716,6 +722,10 @@ class DBImpl : public DB { const Snapshot* GetSnapshotImpl(bool is_write_conflict_boundary); + // Persist RocksDB options under the single write thread + // REQUIRES: mutex locked + Status PersistOptions(); + // table_cache_ provides its own synchronization std::shared_ptr table_cache_; diff --git a/db/db_impl_debug.cc b/db/db_impl_debug.cc index 37a58a307..dbf2391bd 100644 --- a/db/db_impl_debug.cc +++ b/db/db_impl_debug.cc @@ -178,5 +178,10 @@ Status DBImpl::TEST_GetLatestMutableCFOptions( return Status::OK(); } +int DBImpl::TEST_BGCompactionsAllowed() const { + InstrumentedMutexLock l(&mutex_); + return BGCompactionsAllowed(); +} + } // namespace rocksdb #endif // NDEBUG diff --git a/db/db_options_test.cc b/db/db_options_test.cc index cd6c7f71f..a348bffc7 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -11,6 +11,7 @@ #include #include "db/column_family.h" +#include "db/db_impl.h" #include "db/db_test_util.h" #include "port/stack_trace.h" #include "rocksdb/convenience.h" @@ -26,6 +27,22 @@ class DBOptionsTest : public DBTestBase { DBOptionsTest() : DBTestBase("/db_options_test") {} #ifndef ROCKSDB_LITE + std::unordered_map GetMutableDBOptionsMap( + const DBOptions& options) { + std::string options_str; + GetStringFromDBOptions(&options_str, options); + std::unordered_map options_map; + StringToMap(options_str, &options_map); + std::unordered_map mutable_map; + for (const auto opt : db_options_type_info) { + if (opt.second.is_mutable && + opt.second.verification != OptionVerificationType::kDeprecated) { + mutable_map[opt.first] = options_map[opt.first]; + } + } + return mutable_map; + } + std::unordered_map GetMutableCFOptionsMap( const ColumnFamilyOptions& options) { std::string options_str; @@ -52,13 +69,32 @@ class DBOptionsTest : public DBTestBase { delete options.compaction_filter; return opt_map; } + + std::unordered_map GetRandomizedMutableDBOptionsMap( + Random* rnd) { + DBOptions db_options; + test::RandomInitDBOptions(&db_options, rnd); + auto sanitized_options = SanitizeOptions(dbname_, db_options); + return GetMutableDBOptionsMap(sanitized_options); + } #endif // ROCKSDB_LITE }; // RocksDB lite don't support dynamic options. #ifndef ROCKSDB_LITE -TEST_F(DBOptionsTest, GetLatestOptions) { +TEST_F(DBOptionsTest, GetLatestDBOptions) { + // GetOptions should be able to get latest option changed by SetOptions. + Options options; + options.create_if_missing = true; + Random rnd(228); + Reopen(options); + auto new_options = GetRandomizedMutableDBOptionsMap(&rnd); + ASSERT_OK(dbfull()->SetDBOptions(new_options)); + ASSERT_EQ(new_options, GetMutableDBOptionsMap(dbfull()->GetDBOptions())); +} + +TEST_F(DBOptionsTest, GetLatestCFOptions) { // GetOptions should be able to get latest option changed by SetOptions. Options options; options.create_if_missing = true; @@ -206,6 +242,20 @@ TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) { ASSERT_EQ("0,1", FilesPerLevel()); } +TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) { + Options options; + options.create_if_missing = true; + options.base_background_compactions = 1; // default value + options.max_background_compactions = 1; // default value + Reopen(options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + ASSERT_OK(dbfull()->SetDBOptions({{"base_background_compactions", "2"}, + {"max_background_compactions", "3"}})); + ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed()); + auto stop_token = dbfull()->TEST_write_controler().GetStopToken(); + ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed()); +} + #endif // ROCKSDB_LITE } // namespace rocksdb diff --git a/db/db_test.cc b/db/db_test.cc index d019c41ea..04ec66bb8 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2762,6 +2762,12 @@ class ModelDB : public DB { return Status::NotSupported("Not supported operation."); } + virtual Status SetDBOptions( + const std::unordered_map& new_options) + override { + return Status::NotSupported("Not supported operation."); + } + using DB::CompactFiles; virtual Status CompactFiles(const CompactionOptions& compact_options, ColumnFamilyHandle* column_family, diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 7b6ce0aad..39111f5d2 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -629,6 +629,9 @@ class DB { return SetOptions(DefaultColumnFamily(), new_options); } + virtual Status SetDBOptions( + const std::unordered_map& new_options) = 0; + // CompactFiles() inputs a list of files specified by file numbers and // compacts them to the specified level. Note that the behavior is different // from CompactRange() in that CompactFiles() performs the compaction job diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index c91ac8ff6..ed1903e2b 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -289,6 +289,12 @@ class StackableDB : public DB { return db_->SetOptions(column_family_handle, new_options); } + virtual Status SetDBOptions( + const std::unordered_map& new_options) + override { + return db_->SetDBOptions(new_options); + } + using DB::GetPropertiesOfAllTables; virtual Status GetPropertiesOfAllTables( ColumnFamilyHandle* column_family, diff --git a/util/db_options.cc b/util/db_options.cc index 19a263ea2..c844b1f7c 100644 --- a/util/db_options.cc +++ b/util/db_options.cc @@ -42,8 +42,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) wal_dir(options.wal_dir), delete_obsolete_files_period_micros( options.delete_obsolete_files_period_micros), - base_background_compactions(options.base_background_compactions), - max_background_compactions(options.max_background_compactions), max_subcompactions(options.max_subcompactions), max_background_flushes(options.max_background_flushes), max_log_file_size(options.max_log_file_size), @@ -144,10 +142,6 @@ void ImmutableDBOptions::Dump(Logger* log) const { table_cache_numshardbits); Header(log, " Options.delete_obsolete_files_period_micros: %" PRIu64, delete_obsolete_files_period_micros); - Header(log, " Options.base_background_compactions: %d", - base_background_compactions); - Header(log, " Options.max_background_compactions: %d", - max_background_compactions); Header(log, " Options.max_subcompactions: %" PRIu32, max_subcompactions); Header(log, " Options.max_background_flushes: %d", @@ -202,8 +196,8 @@ void ImmutableDBOptions::Dump(Logger* log) const { wal_recovery_mode); Header(log, " Options.enable_thread_tracking: %d", enable_thread_tracking); - Log(log, " Options.delayed_write_rate : %" PRIu64, - delayed_write_rate); + Header(log, " Options.delayed_write_rate : %" PRIu64, + delayed_write_rate); Header(log, " Options.allow_concurrent_memtable_write: %d", allow_concurrent_memtable_write); Header(log, " Options.enable_write_thread_adaptive_yield: %d", @@ -226,8 +220,18 @@ void ImmutableDBOptions::Dump(Logger* log) const { avoid_flush_during_recovery); } -MutableDBOptions::MutableDBOptions(const DBOptions& options) {} +MutableDBOptions::MutableDBOptions() + : base_background_compactions(1), max_background_compactions(1) {} -void MutableDBOptions::Dump(Logger* log) const {} +MutableDBOptions::MutableDBOptions(const DBOptions& options) + : base_background_compactions(options.base_background_compactions), + max_background_compactions(options.max_background_compactions) {} + +void MutableDBOptions::Dump(Logger* log) const { + Header(log, " Options.base_background_compactions: %d", + base_background_compactions); + Header(log, " Options.max_background_compactions: %d", + max_background_compactions); +} } // namespace rocksdb diff --git a/util/db_options.h b/util/db_options.h index dbe17b485..e8fc90fd0 100644 --- a/util/db_options.h +++ b/util/db_options.h @@ -37,8 +37,6 @@ struct ImmutableDBOptions { std::string db_log_dir; std::string wal_dir; uint64_t delete_obsolete_files_period_micros; - int base_background_compactions; - int max_background_compactions; uint32_t max_subcompactions; int max_background_flushes; size_t max_log_file_size; @@ -87,10 +85,14 @@ struct ImmutableDBOptions { }; struct MutableDBOptions { + MutableDBOptions(); explicit MutableDBOptions(const MutableDBOptions& options) = default; explicit MutableDBOptions(const DBOptions& options); void Dump(Logger* log) const; + + int base_background_compactions; + int max_background_compactions; }; } // namespace rocksdb diff --git a/util/options_helper.cc b/util/options_helper.cc index 4240e319b..e9eb20f3b 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -53,9 +53,9 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.delete_obsolete_files_period_micros = immutable_db_options.delete_obsolete_files_period_micros; options.base_background_compactions = - immutable_db_options.base_background_compactions; + mutable_db_options.base_background_compactions; options.max_background_compactions = - immutable_db_options.max_background_compactions; + mutable_db_options.max_background_compactions; options.max_subcompactions = immutable_db_options.max_subcompactions; options.max_background_flushes = immutable_db_options.max_background_flushes; options.max_log_file_size = immutable_db_options.max_log_file_size; @@ -754,6 +754,36 @@ Status GetMutableOptionsFromStrings( return Status::OK(); } +Status GetMutableDBOptionsFromStrings( + const MutableDBOptions& base_options, + const std::unordered_map& options_map, + MutableDBOptions* new_options) { + assert(new_options); + *new_options = base_options; + for (const auto& o : options_map) { + try { + auto iter = db_options_type_info.find(o.first); + if (iter == db_options_type_info.end()) { + return Status::InvalidArgument("Unrecognized option: " + o.first); + } + const auto& opt_info = iter->second; + if (!opt_info.is_mutable) { + return Status::InvalidArgument("Option not changeable: " + o.first); + } + bool is_ok = ParseOptionHelper( + reinterpret_cast(new_options) + opt_info.mutable_offset, + opt_info.type, o.second); + if (!is_ok) { + return Status::InvalidArgument("Error parsing " + o.first); + } + } catch (std::exception& e) { + return Status::InvalidArgument("Error parsing " + o.first + ":" + + std::string(e.what())); + } + } + return Status::OK(); +} + Status StringToMap(const std::string& opts_str, std::unordered_map* opts_map) { assert(opts_map); diff --git a/util/options_helper.h b/util/options_helper.h index 3f856fee9..f87e28997 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -69,6 +69,11 @@ Status GetMutableOptionsFromStrings( const std::unordered_map& options_map, MutableCFOptions* new_options); +Status GetMutableDBOptionsFromStrings( + const MutableDBOptions& base_options, + const std::unordered_map& options_map, + MutableDBOptions* new_options); + Status GetTableFactoryFromMap( const std::string& factory_name, const std::unordered_map& opt_map, @@ -234,10 +239,12 @@ static std::unordered_map db_options_type_info = { OptionVerificationType::kNormal, false, 0}}, {"max_background_compactions", {offsetof(struct DBOptions, max_background_compactions), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, max_background_compactions)}}, {"base_background_compactions", {offsetof(struct DBOptions, base_background_compactions), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, base_background_compactions)}}, {"max_background_flushes", {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, OptionVerificationType::kNormal, false, 0}},