Made delete_obsolete_files_period_micros option dynamic

Summary:
Made delete_obsolete_files_period_micros option dynamic. It can be updating using DB::SetDBOptions().
Closes https://github.com/facebook/rocksdb/pull/1595

Differential Revision: D4246569

Pulled By: tonek

fbshipit-source-id: d23f560
This commit is contained in:
Anton Safonov 2016-12-05 14:09:35 -08:00 committed by Facebook Github Bot
parent edde954e7b
commit 9053fe2a5c
8 changed files with 64 additions and 20 deletions

View File

@ -4,6 +4,7 @@
* Options.level0_stop_writes_trigger default value changes from 24 to 32.
* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
* Removed flashcache support.
* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
## 5.0.0 (11/17/2016)
### Public API Change

View File

@ -332,9 +332,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
num_running_flushes_(0),
bg_purge_scheduled_(0),
disable_delete_obsolete_files_(0),
delete_obsolete_files_next_run_(
env_->NowMicros() +
immutable_db_options_.delete_obsolete_files_period_micros),
delete_obsolete_files_last_run_(env_->NowMicros()),
last_stats_dump_time_microsec_(0),
next_job_id_(1),
has_unpersisted_data_(false),
@ -743,7 +741,7 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
// Otherwise, gets obsolete files from VersionSet.
// no_full_scan = true -- never do the full scan using GetChildren()
// force = false -- don't force the full scan, except every
// immutable_db_options_.delete_obsolete_files_period_micros
// mutable_db_options_.delete_obsolete_files_period_micros
// force = true -- force the full scan
void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
bool no_full_scan) {
@ -760,15 +758,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
if (no_full_scan) {
doing_the_full_scan = false;
} else if (force ||
immutable_db_options_.delete_obsolete_files_period_micros == 0) {
mutable_db_options_.delete_obsolete_files_period_micros == 0) {
doing_the_full_scan = true;
} else {
const uint64_t now_micros = env_->NowMicros();
if (delete_obsolete_files_next_run_ < now_micros) {
if ((delete_obsolete_files_last_run_ +
mutable_db_options_.delete_obsolete_files_period_micros) <
now_micros) {
doing_the_full_scan = true;
delete_obsolete_files_next_run_ =
now_micros +
immutable_db_options_.delete_obsolete_files_period_micros;
delete_obsolete_files_last_run_ = now_micros;
}
}

View File

@ -972,8 +972,9 @@ class DBImpl : public DB {
// without any synchronization
int disable_delete_obsolete_files_;
// next time when we should run DeleteObsoleteFiles with full scan
uint64_t delete_obsolete_files_next_run_;
// last time when DeleteObsoleteFiles with full scan was executed. Originaly
// initialized with startup time.
uint64_t delete_obsolete_files_last_run_;
// last time stats were dumped to LOG
std::atomic<uint64_t> last_stats_dump_time_microsec_;

View File

@ -316,6 +316,47 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
}
}
static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
dbfull->TEST_LockMutex();
JobContext job_context(0);
dbfull->FindObsoleteFiles(&job_context, false);
ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
job_context.Clean();
dbfull->TEST_UnlockMutex();
}
TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
SpecialEnv env(Env::Default());
env.time_elapse_only_sleep_ = true;
Options options;
options.env = &env;
options.create_if_missing = true;
ASSERT_OK(TryReopen(options));
// Verify that candidate files set is empty when no full scan requested.
assert_candidate_files_empty(dbfull(), true);
ASSERT_OK(
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}}));
// After delete_obsolete_files_period_micros updated to 0, the next call
// to FindObsoleteFiles should make a full scan
assert_candidate_files_empty(dbfull(), false);
ASSERT_OK(
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}}));
assert_candidate_files_empty(dbfull(), true);
env.addon_time_.store(20);
assert_candidate_files_empty(dbfull(), true);
env.addon_time_.store(21);
assert_candidate_files_empty(dbfull(), false);
Close();
}
#endif // ROCKSDB_LITE
} // namespace rocksdb

View File

@ -39,8 +39,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
db_paths(options.db_paths),
db_log_dir(options.db_log_dir),
wal_dir(options.wal_dir),
delete_obsolete_files_period_micros(
options.delete_obsolete_files_period_micros),
max_subcompactions(options.max_subcompactions),
max_background_flushes(options.max_background_flushes),
max_log_file_size(options.max_log_file_size),
@ -139,8 +137,6 @@ void ImmutableDBOptions::Dump(Logger* log) const {
wal_dir.c_str());
Header(log, " Options.table_cache_numshardbits: %d",
table_cache_numshardbits);
Header(log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
delete_obsolete_files_period_micros);
Header(log, " Options.max_subcompactions: %" PRIu32,
max_subcompactions);
Header(log, " Options.max_background_flushes: %d",
@ -222,14 +218,17 @@ MutableDBOptions::MutableDBOptions()
max_background_compactions(1),
avoid_flush_during_shutdown(false),
delayed_write_rate(2 * 1024U * 1024U),
max_total_wal_size(0) {}
max_total_wal_size(0),
delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000) {}
MutableDBOptions::MutableDBOptions(const DBOptions& options)
: base_background_compactions(options.base_background_compactions),
max_background_compactions(options.max_background_compactions),
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
delayed_write_rate(options.delayed_write_rate),
max_total_wal_size(options.max_total_wal_size) {}
max_total_wal_size(options.max_total_wal_size),
delete_obsolete_files_period_micros(
options.delete_obsolete_files_period_micros) {}
void MutableDBOptions::Dump(Logger* log) const {
Header(log, " Options.base_background_compactions: %d",
@ -242,6 +241,9 @@ void MutableDBOptions::Dump(Logger* log) const {
delayed_write_rate);
Header(log, " Options.max_total_wal_size: %" PRIu64,
max_total_wal_size);
Header(log,
" Options.delete_obsolete_files_period_micros: %" PRIu64,
delete_obsolete_files_period_micros);
}
} // namespace rocksdb

View File

@ -35,7 +35,6 @@ struct ImmutableDBOptions {
std::vector<DbPath> db_paths;
std::string db_log_dir;
std::string wal_dir;
uint64_t delete_obsolete_files_period_micros;
uint32_t max_subcompactions;
int max_background_flushes;
size_t max_log_file_size;
@ -95,6 +94,7 @@ struct MutableDBOptions {
bool avoid_flush_during_shutdown;
uint64_t delayed_write_rate;
uint64_t max_total_wal_size;
uint64_t delete_obsolete_files_period_micros;
};
} // namespace rocksdb

View File

@ -53,7 +53,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
options.db_log_dir = immutable_db_options.db_log_dir;
options.wal_dir = immutable_db_options.wal_dir;
options.delete_obsolete_files_period_micros =
immutable_db_options.delete_obsolete_files_period_micros;
mutable_db_options.delete_obsolete_files_period_micros;
options.base_background_compactions =
mutable_db_options.base_background_compactions;
options.max_background_compactions =

View File

@ -302,7 +302,8 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
offsetof(struct MutableDBOptions, delayed_write_rate)}},
{"delete_obsolete_files_period_micros",
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
OptionType::kUInt64T, OptionVerificationType::kNormal, true,
offsetof(struct MutableDBOptions, delete_obsolete_files_period_micros)}},
{"max_manifest_file_size",
{offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
OptionVerificationType::kNormal, false, 0}},