Made delete_obsolete_files_period_micros option dynamic
Summary: Made delete_obsolete_files_period_micros option dynamic. It can be updating using DB::SetDBOptions(). Closes https://github.com/facebook/rocksdb/pull/1595 Differential Revision: D4246569 Pulled By: tonek fbshipit-source-id: d23f560
This commit is contained in:
parent
edde954e7b
commit
9053fe2a5c
|
@ -4,6 +4,7 @@
|
||||||
* Options.level0_stop_writes_trigger default value changes from 24 to 32.
|
* Options.level0_stop_writes_trigger default value changes from 24 to 32.
|
||||||
* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
|
* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
|
||||||
* Removed flashcache support.
|
* Removed flashcache support.
|
||||||
|
* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
|
||||||
|
|
||||||
## 5.0.0 (11/17/2016)
|
## 5.0.0 (11/17/2016)
|
||||||
### Public API Change
|
### Public API Change
|
||||||
|
|
|
@ -332,9 +332,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
|
||||||
num_running_flushes_(0),
|
num_running_flushes_(0),
|
||||||
bg_purge_scheduled_(0),
|
bg_purge_scheduled_(0),
|
||||||
disable_delete_obsolete_files_(0),
|
disable_delete_obsolete_files_(0),
|
||||||
delete_obsolete_files_next_run_(
|
delete_obsolete_files_last_run_(env_->NowMicros()),
|
||||||
env_->NowMicros() +
|
|
||||||
immutable_db_options_.delete_obsolete_files_period_micros),
|
|
||||||
last_stats_dump_time_microsec_(0),
|
last_stats_dump_time_microsec_(0),
|
||||||
next_job_id_(1),
|
next_job_id_(1),
|
||||||
has_unpersisted_data_(false),
|
has_unpersisted_data_(false),
|
||||||
|
@ -743,7 +741,7 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
|
||||||
// Otherwise, gets obsolete files from VersionSet.
|
// Otherwise, gets obsolete files from VersionSet.
|
||||||
// no_full_scan = true -- never do the full scan using GetChildren()
|
// no_full_scan = true -- never do the full scan using GetChildren()
|
||||||
// force = false -- don't force the full scan, except every
|
// force = false -- don't force the full scan, except every
|
||||||
// immutable_db_options_.delete_obsolete_files_period_micros
|
// mutable_db_options_.delete_obsolete_files_period_micros
|
||||||
// force = true -- force the full scan
|
// force = true -- force the full scan
|
||||||
void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
||||||
bool no_full_scan) {
|
bool no_full_scan) {
|
||||||
|
@ -760,15 +758,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
||||||
if (no_full_scan) {
|
if (no_full_scan) {
|
||||||
doing_the_full_scan = false;
|
doing_the_full_scan = false;
|
||||||
} else if (force ||
|
} else if (force ||
|
||||||
immutable_db_options_.delete_obsolete_files_period_micros == 0) {
|
mutable_db_options_.delete_obsolete_files_period_micros == 0) {
|
||||||
doing_the_full_scan = true;
|
doing_the_full_scan = true;
|
||||||
} else {
|
} else {
|
||||||
const uint64_t now_micros = env_->NowMicros();
|
const uint64_t now_micros = env_->NowMicros();
|
||||||
if (delete_obsolete_files_next_run_ < now_micros) {
|
if ((delete_obsolete_files_last_run_ +
|
||||||
|
mutable_db_options_.delete_obsolete_files_period_micros) <
|
||||||
|
now_micros) {
|
||||||
doing_the_full_scan = true;
|
doing_the_full_scan = true;
|
||||||
delete_obsolete_files_next_run_ =
|
delete_obsolete_files_last_run_ = now_micros;
|
||||||
now_micros +
|
|
||||||
immutable_db_options_.delete_obsolete_files_period_micros;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -972,8 +972,9 @@ class DBImpl : public DB {
|
||||||
// without any synchronization
|
// without any synchronization
|
||||||
int disable_delete_obsolete_files_;
|
int disable_delete_obsolete_files_;
|
||||||
|
|
||||||
// next time when we should run DeleteObsoleteFiles with full scan
|
// last time when DeleteObsoleteFiles with full scan was executed. Originaly
|
||||||
uint64_t delete_obsolete_files_next_run_;
|
// initialized with startup time.
|
||||||
|
uint64_t delete_obsolete_files_last_run_;
|
||||||
|
|
||||||
// last time stats were dumped to LOG
|
// last time stats were dumped to LOG
|
||||||
std::atomic<uint64_t> last_stats_dump_time_microsec_;
|
std::atomic<uint64_t> last_stats_dump_time_microsec_;
|
||||||
|
|
|
@ -316,6 +316,47 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
|
||||||
|
dbfull->TEST_LockMutex();
|
||||||
|
JobContext job_context(0);
|
||||||
|
dbfull->FindObsoleteFiles(&job_context, false);
|
||||||
|
ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
|
||||||
|
job_context.Clean();
|
||||||
|
dbfull->TEST_UnlockMutex();
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
|
||||||
|
SpecialEnv env(Env::Default());
|
||||||
|
env.time_elapse_only_sleep_ = true;
|
||||||
|
Options options;
|
||||||
|
options.env = &env;
|
||||||
|
options.create_if_missing = true;
|
||||||
|
ASSERT_OK(TryReopen(options));
|
||||||
|
|
||||||
|
// Verify that candidate files set is empty when no full scan requested.
|
||||||
|
assert_candidate_files_empty(dbfull(), true);
|
||||||
|
|
||||||
|
ASSERT_OK(
|
||||||
|
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}}));
|
||||||
|
|
||||||
|
// After delete_obsolete_files_period_micros updated to 0, the next call
|
||||||
|
// to FindObsoleteFiles should make a full scan
|
||||||
|
assert_candidate_files_empty(dbfull(), false);
|
||||||
|
|
||||||
|
ASSERT_OK(
|
||||||
|
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}}));
|
||||||
|
|
||||||
|
assert_candidate_files_empty(dbfull(), true);
|
||||||
|
|
||||||
|
env.addon_time_.store(20);
|
||||||
|
assert_candidate_files_empty(dbfull(), true);
|
||||||
|
|
||||||
|
env.addon_time_.store(21);
|
||||||
|
assert_candidate_files_empty(dbfull(), false);
|
||||||
|
|
||||||
|
Close();
|
||||||
|
}
|
||||||
|
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
} // namespace rocksdb
|
} // namespace rocksdb
|
||||||
|
|
|
@ -39,8 +39,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
|
||||||
db_paths(options.db_paths),
|
db_paths(options.db_paths),
|
||||||
db_log_dir(options.db_log_dir),
|
db_log_dir(options.db_log_dir),
|
||||||
wal_dir(options.wal_dir),
|
wal_dir(options.wal_dir),
|
||||||
delete_obsolete_files_period_micros(
|
|
||||||
options.delete_obsolete_files_period_micros),
|
|
||||||
max_subcompactions(options.max_subcompactions),
|
max_subcompactions(options.max_subcompactions),
|
||||||
max_background_flushes(options.max_background_flushes),
|
max_background_flushes(options.max_background_flushes),
|
||||||
max_log_file_size(options.max_log_file_size),
|
max_log_file_size(options.max_log_file_size),
|
||||||
|
@ -139,8 +137,6 @@ void ImmutableDBOptions::Dump(Logger* log) const {
|
||||||
wal_dir.c_str());
|
wal_dir.c_str());
|
||||||
Header(log, " Options.table_cache_numshardbits: %d",
|
Header(log, " Options.table_cache_numshardbits: %d",
|
||||||
table_cache_numshardbits);
|
table_cache_numshardbits);
|
||||||
Header(log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
|
|
||||||
delete_obsolete_files_period_micros);
|
|
||||||
Header(log, " Options.max_subcompactions: %" PRIu32,
|
Header(log, " Options.max_subcompactions: %" PRIu32,
|
||||||
max_subcompactions);
|
max_subcompactions);
|
||||||
Header(log, " Options.max_background_flushes: %d",
|
Header(log, " Options.max_background_flushes: %d",
|
||||||
|
@ -222,14 +218,17 @@ MutableDBOptions::MutableDBOptions()
|
||||||
max_background_compactions(1),
|
max_background_compactions(1),
|
||||||
avoid_flush_during_shutdown(false),
|
avoid_flush_during_shutdown(false),
|
||||||
delayed_write_rate(2 * 1024U * 1024U),
|
delayed_write_rate(2 * 1024U * 1024U),
|
||||||
max_total_wal_size(0) {}
|
max_total_wal_size(0),
|
||||||
|
delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000) {}
|
||||||
|
|
||||||
MutableDBOptions::MutableDBOptions(const DBOptions& options)
|
MutableDBOptions::MutableDBOptions(const DBOptions& options)
|
||||||
: base_background_compactions(options.base_background_compactions),
|
: base_background_compactions(options.base_background_compactions),
|
||||||
max_background_compactions(options.max_background_compactions),
|
max_background_compactions(options.max_background_compactions),
|
||||||
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
|
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
|
||||||
delayed_write_rate(options.delayed_write_rate),
|
delayed_write_rate(options.delayed_write_rate),
|
||||||
max_total_wal_size(options.max_total_wal_size) {}
|
max_total_wal_size(options.max_total_wal_size),
|
||||||
|
delete_obsolete_files_period_micros(
|
||||||
|
options.delete_obsolete_files_period_micros) {}
|
||||||
|
|
||||||
void MutableDBOptions::Dump(Logger* log) const {
|
void MutableDBOptions::Dump(Logger* log) const {
|
||||||
Header(log, " Options.base_background_compactions: %d",
|
Header(log, " Options.base_background_compactions: %d",
|
||||||
|
@ -242,6 +241,9 @@ void MutableDBOptions::Dump(Logger* log) const {
|
||||||
delayed_write_rate);
|
delayed_write_rate);
|
||||||
Header(log, " Options.max_total_wal_size: %" PRIu64,
|
Header(log, " Options.max_total_wal_size: %" PRIu64,
|
||||||
max_total_wal_size);
|
max_total_wal_size);
|
||||||
|
Header(log,
|
||||||
|
" Options.delete_obsolete_files_period_micros: %" PRIu64,
|
||||||
|
delete_obsolete_files_period_micros);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace rocksdb
|
} // namespace rocksdb
|
||||||
|
|
|
@ -35,7 +35,6 @@ struct ImmutableDBOptions {
|
||||||
std::vector<DbPath> db_paths;
|
std::vector<DbPath> db_paths;
|
||||||
std::string db_log_dir;
|
std::string db_log_dir;
|
||||||
std::string wal_dir;
|
std::string wal_dir;
|
||||||
uint64_t delete_obsolete_files_period_micros;
|
|
||||||
uint32_t max_subcompactions;
|
uint32_t max_subcompactions;
|
||||||
int max_background_flushes;
|
int max_background_flushes;
|
||||||
size_t max_log_file_size;
|
size_t max_log_file_size;
|
||||||
|
@ -95,6 +94,7 @@ struct MutableDBOptions {
|
||||||
bool avoid_flush_during_shutdown;
|
bool avoid_flush_during_shutdown;
|
||||||
uint64_t delayed_write_rate;
|
uint64_t delayed_write_rate;
|
||||||
uint64_t max_total_wal_size;
|
uint64_t max_total_wal_size;
|
||||||
|
uint64_t delete_obsolete_files_period_micros;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace rocksdb
|
} // namespace rocksdb
|
||||||
|
|
|
@ -53,7 +53,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
|
||||||
options.db_log_dir = immutable_db_options.db_log_dir;
|
options.db_log_dir = immutable_db_options.db_log_dir;
|
||||||
options.wal_dir = immutable_db_options.wal_dir;
|
options.wal_dir = immutable_db_options.wal_dir;
|
||||||
options.delete_obsolete_files_period_micros =
|
options.delete_obsolete_files_period_micros =
|
||||||
immutable_db_options.delete_obsolete_files_period_micros;
|
mutable_db_options.delete_obsolete_files_period_micros;
|
||||||
options.base_background_compactions =
|
options.base_background_compactions =
|
||||||
mutable_db_options.base_background_compactions;
|
mutable_db_options.base_background_compactions;
|
||||||
options.max_background_compactions =
|
options.max_background_compactions =
|
||||||
|
|
|
@ -302,7 +302,8 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
|
||||||
offsetof(struct MutableDBOptions, delayed_write_rate)}},
|
offsetof(struct MutableDBOptions, delayed_write_rate)}},
|
||||||
{"delete_obsolete_files_period_micros",
|
{"delete_obsolete_files_period_micros",
|
||||||
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
|
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
|
||||||
OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
|
OptionType::kUInt64T, OptionVerificationType::kNormal, true,
|
||||||
|
offsetof(struct MutableDBOptions, delete_obsolete_files_period_micros)}},
|
||||||
{"max_manifest_file_size",
|
{"max_manifest_file_size",
|
||||||
{offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
|
{offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
|
||||||
OptionVerificationType::kNormal, false, 0}},
|
OptionVerificationType::kNormal, false, 0}},
|
||||||
|
|
Loading…
Reference in New Issue
Block a user