From 9053fe2a5c9b1c4b8ffdfbc59f3855360d97b0ec Mon Sep 17 00:00:00 2001 From: Anton Safonov Date: Mon, 5 Dec 2016 14:09:35 -0800 Subject: [PATCH] Made delete_obsolete_files_period_micros option dynamic Summary: Made delete_obsolete_files_period_micros option dynamic. It can be updating using DB::SetDBOptions(). Closes https://github.com/facebook/rocksdb/pull/1595 Differential Revision: D4246569 Pulled By: tonek fbshipit-source-id: d23f560 --- HISTORY.md | 1 + db/db_impl.cc | 16 +++++++--------- db/db_impl.h | 5 +++-- db/db_options_test.cc | 41 +++++++++++++++++++++++++++++++++++++++++ util/db_options.cc | 14 ++++++++------ util/db_options.h | 2 +- util/options_helper.cc | 2 +- util/options_helper.h | 3 ++- 8 files changed, 64 insertions(+), 20 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 2d091fadf..35b7a7453 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,7 @@ * Options.level0_stop_writes_trigger default value changes from 24 to 32. * New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys. * Removed flashcache support. +* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions(). ## 5.0.0 (11/17/2016) ### Public API Change diff --git a/db/db_impl.cc b/db/db_impl.cc index 4e3d7dcb9..331d115ee 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -332,9 +332,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname) num_running_flushes_(0), bg_purge_scheduled_(0), disable_delete_obsolete_files_(0), - delete_obsolete_files_next_run_( - env_->NowMicros() + - immutable_db_options_.delete_obsolete_files_period_micros), + delete_obsolete_files_last_run_(env_->NowMicros()), last_stats_dump_time_microsec_(0), next_job_id_(1), has_unpersisted_data_(false), @@ -743,7 +741,7 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) { // Otherwise, gets obsolete files from VersionSet. // no_full_scan = true -- never do the full scan using GetChildren() // force = false -- don't force the full scan, except every -// immutable_db_options_.delete_obsolete_files_period_micros +// mutable_db_options_.delete_obsolete_files_period_micros // force = true -- force the full scan void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force, bool no_full_scan) { @@ -760,15 +758,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force, if (no_full_scan) { doing_the_full_scan = false; } else if (force || - immutable_db_options_.delete_obsolete_files_period_micros == 0) { + mutable_db_options_.delete_obsolete_files_period_micros == 0) { doing_the_full_scan = true; } else { const uint64_t now_micros = env_->NowMicros(); - if (delete_obsolete_files_next_run_ < now_micros) { + if ((delete_obsolete_files_last_run_ + + mutable_db_options_.delete_obsolete_files_period_micros) < + now_micros) { doing_the_full_scan = true; - delete_obsolete_files_next_run_ = - now_micros + - immutable_db_options_.delete_obsolete_files_period_micros; + delete_obsolete_files_last_run_ = now_micros; } } diff --git a/db/db_impl.h b/db/db_impl.h index cfedd8103..3e18e3420 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -972,8 +972,9 @@ class DBImpl : public DB { // without any synchronization int disable_delete_obsolete_files_; - // next time when we should run DeleteObsoleteFiles with full scan - uint64_t delete_obsolete_files_next_run_; + // last time when DeleteObsoleteFiles with full scan was executed. Originaly + // initialized with startup time. + uint64_t delete_obsolete_files_last_run_; // last time stats were dumped to LOG std::atomic last_stats_dump_time_microsec_; diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 92325d5fe..294792159 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -316,6 +316,47 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) { } } +static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) { + dbfull->TEST_LockMutex(); + JobContext job_context(0); + dbfull->FindObsoleteFiles(&job_context, false); + ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty()); + job_context.Clean(); + dbfull->TEST_UnlockMutex(); +} + +TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) { + SpecialEnv env(Env::Default()); + env.time_elapse_only_sleep_ = true; + Options options; + options.env = &env; + options.create_if_missing = true; + ASSERT_OK(TryReopen(options)); + + // Verify that candidate files set is empty when no full scan requested. + assert_candidate_files_empty(dbfull(), true); + + ASSERT_OK( + dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}})); + + // After delete_obsolete_files_period_micros updated to 0, the next call + // to FindObsoleteFiles should make a full scan + assert_candidate_files_empty(dbfull(), false); + + ASSERT_OK( + dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}})); + + assert_candidate_files_empty(dbfull(), true); + + env.addon_time_.store(20); + assert_candidate_files_empty(dbfull(), true); + + env.addon_time_.store(21); + assert_candidate_files_empty(dbfull(), false); + + Close(); +} + #endif // ROCKSDB_LITE } // namespace rocksdb diff --git a/util/db_options.cc b/util/db_options.cc index 033a7bc52..123c87af7 100644 --- a/util/db_options.cc +++ b/util/db_options.cc @@ -39,8 +39,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) db_paths(options.db_paths), db_log_dir(options.db_log_dir), wal_dir(options.wal_dir), - delete_obsolete_files_period_micros( - options.delete_obsolete_files_period_micros), max_subcompactions(options.max_subcompactions), max_background_flushes(options.max_background_flushes), max_log_file_size(options.max_log_file_size), @@ -139,8 +137,6 @@ void ImmutableDBOptions::Dump(Logger* log) const { wal_dir.c_str()); Header(log, " Options.table_cache_numshardbits: %d", table_cache_numshardbits); - Header(log, " Options.delete_obsolete_files_period_micros: %" PRIu64, - delete_obsolete_files_period_micros); Header(log, " Options.max_subcompactions: %" PRIu32, max_subcompactions); Header(log, " Options.max_background_flushes: %d", @@ -222,14 +218,17 @@ MutableDBOptions::MutableDBOptions() max_background_compactions(1), avoid_flush_during_shutdown(false), delayed_write_rate(2 * 1024U * 1024U), - max_total_wal_size(0) {} + max_total_wal_size(0), + delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000) {} MutableDBOptions::MutableDBOptions(const DBOptions& options) : base_background_compactions(options.base_background_compactions), max_background_compactions(options.max_background_compactions), avoid_flush_during_shutdown(options.avoid_flush_during_shutdown), delayed_write_rate(options.delayed_write_rate), - max_total_wal_size(options.max_total_wal_size) {} + max_total_wal_size(options.max_total_wal_size), + delete_obsolete_files_period_micros( + options.delete_obsolete_files_period_micros) {} void MutableDBOptions::Dump(Logger* log) const { Header(log, " Options.base_background_compactions: %d", @@ -242,6 +241,9 @@ void MutableDBOptions::Dump(Logger* log) const { delayed_write_rate); Header(log, " Options.max_total_wal_size: %" PRIu64, max_total_wal_size); + Header(log, + " Options.delete_obsolete_files_period_micros: %" PRIu64, + delete_obsolete_files_period_micros); } } // namespace rocksdb diff --git a/util/db_options.h b/util/db_options.h index 76ebdee99..728e0dd2d 100644 --- a/util/db_options.h +++ b/util/db_options.h @@ -35,7 +35,6 @@ struct ImmutableDBOptions { std::vector db_paths; std::string db_log_dir; std::string wal_dir; - uint64_t delete_obsolete_files_period_micros; uint32_t max_subcompactions; int max_background_flushes; size_t max_log_file_size; @@ -95,6 +94,7 @@ struct MutableDBOptions { bool avoid_flush_during_shutdown; uint64_t delayed_write_rate; uint64_t max_total_wal_size; + uint64_t delete_obsolete_files_period_micros; }; } // namespace rocksdb diff --git a/util/options_helper.cc b/util/options_helper.cc index 0a051a594..b68e777b5 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -53,7 +53,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.db_log_dir = immutable_db_options.db_log_dir; options.wal_dir = immutable_db_options.wal_dir; options.delete_obsolete_files_period_micros = - immutable_db_options.delete_obsolete_files_period_micros; + mutable_db_options.delete_obsolete_files_period_micros; options.base_background_compactions = mutable_db_options.base_background_compactions; options.max_background_compactions = diff --git a/util/options_helper.h b/util/options_helper.h index afc9d6632..164b11f97 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -302,7 +302,8 @@ static std::unordered_map db_options_type_info = { offsetof(struct MutableDBOptions, delayed_write_rate)}}, {"delete_obsolete_files_period_micros", {offsetof(struct DBOptions, delete_obsolete_files_period_micros), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, delete_obsolete_files_period_micros)}}, {"max_manifest_file_size", {offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},