Add avoid_flush_during_shutdown DB option
Summary: Add avoid_flush_during_shutdown DB option. Closes https://github.com/facebook/rocksdb/pull/1451 Differential Revision: D4108643 Pulled By: yiwu-arbug fbshipit-source-id: abdaf4d
This commit is contained in:
parent
2b16d664cb
commit
437942e481
@ -3,6 +3,9 @@
|
||||
### Public API Change
|
||||
* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
|
||||
|
||||
### New Features
|
||||
* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable.
|
||||
|
||||
## 4.13.0 (10/18/2016)
|
||||
### Public API Change
|
||||
* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.
|
||||
|
@ -375,7 +375,8 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
|
||||
InstrumentedMutexLock l(&mutex_);
|
||||
|
||||
if (!shutting_down_.load(std::memory_order_acquire) &&
|
||||
has_unpersisted_data_) {
|
||||
has_unpersisted_data_ &&
|
||||
!mutable_db_options_.avoid_flush_during_shutdown) {
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) {
|
||||
cfd->Ref();
|
||||
|
@ -256,6 +256,28 @@ TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
|
||||
ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed());
|
||||
}
|
||||
|
||||
TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.disable_auto_compactions = true;
|
||||
WriteOptions write_without_wal;
|
||||
write_without_wal.disableWAL = true;
|
||||
|
||||
ASSERT_FALSE(options.avoid_flush_during_shutdown);
|
||||
DestroyAndReopen(options);
|
||||
ASSERT_OK(Put("foo", "v1", write_without_wal));
|
||||
Reopen(options);
|
||||
ASSERT_EQ("v1", Get("foo"));
|
||||
ASSERT_EQ("1", FilesPerLevel());
|
||||
|
||||
DestroyAndReopen(options);
|
||||
ASSERT_OK(Put("foo", "v2", write_without_wal));
|
||||
ASSERT_OK(dbfull()->SetDBOptions({{"avoid_flush_during_shutdown", "true"}}));
|
||||
Reopen(options);
|
||||
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
||||
ASSERT_EQ("", FilesPerLevel());
|
||||
}
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
|
@ -1354,6 +1354,15 @@ struct DBOptions {
|
||||
//
|
||||
// DEFAULT: false
|
||||
bool avoid_flush_during_recovery;
|
||||
|
||||
// By default RocksDB will flush all memtables on DB close if there are
|
||||
// unpersisted data (i.e. with WAL disabled) The flush can be skip to speedup
|
||||
// DB close. Unpersisted data WILL BE LOST.
|
||||
//
|
||||
// DEFAULT: false
|
||||
//
|
||||
// Dynamically changeable through SetDBOptions() API.
|
||||
bool avoid_flush_during_shutdown;
|
||||
};
|
||||
|
||||
// Options to control the behavior of a database (passed to DB::Open)
|
||||
|
@ -224,17 +224,22 @@ void ImmutableDBOptions::Dump(Logger* log) const {
|
||||
}
|
||||
|
||||
MutableDBOptions::MutableDBOptions()
|
||||
: base_background_compactions(1), max_background_compactions(1) {}
|
||||
: base_background_compactions(1),
|
||||
max_background_compactions(1),
|
||||
avoid_flush_during_shutdown(false) {}
|
||||
|
||||
MutableDBOptions::MutableDBOptions(const DBOptions& options)
|
||||
: base_background_compactions(options.base_background_compactions),
|
||||
max_background_compactions(options.max_background_compactions) {}
|
||||
max_background_compactions(options.max_background_compactions),
|
||||
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown) {}
|
||||
|
||||
void MutableDBOptions::Dump(Logger* log) const {
|
||||
Header(log, " Options.base_background_compactions: %d",
|
||||
base_background_compactions);
|
||||
Header(log, " Options.max_background_compactions: %d",
|
||||
max_background_compactions);
|
||||
Header(log, " Options.avoid_flush_during_shutdown: %d",
|
||||
avoid_flush_during_shutdown);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
@ -94,6 +94,7 @@ struct MutableDBOptions {
|
||||
|
||||
int base_background_compactions;
|
||||
int max_background_compactions;
|
||||
bool avoid_flush_during_shutdown;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
|
@ -228,7 +228,8 @@ DBOptions::DBOptions()
|
||||
#endif // ROCKSDB_LITE
|
||||
fail_if_options_file_error(false),
|
||||
dump_malloc_stats(false),
|
||||
avoid_flush_during_recovery(false) {
|
||||
avoid_flush_during_recovery(false),
|
||||
avoid_flush_during_shutdown(false) {
|
||||
}
|
||||
|
||||
DBOptions::DBOptions(const Options& options)
|
||||
@ -301,7 +302,8 @@ DBOptions::DBOptions(const Options& options)
|
||||
#endif // ROCKSDB_LITE
|
||||
fail_if_options_file_error(options.fail_if_options_file_error),
|
||||
dump_malloc_stats(options.dump_malloc_stats),
|
||||
avoid_flush_during_recovery(options.avoid_flush_during_recovery) {
|
||||
avoid_flush_during_recovery(options.avoid_flush_during_recovery),
|
||||
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown) {
|
||||
}
|
||||
|
||||
static const char* const access_hints[] = {
|
||||
|
@ -118,6 +118,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
|
||||
options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
|
||||
options.avoid_flush_during_recovery =
|
||||
immutable_db_options.avoid_flush_during_recovery;
|
||||
options.avoid_flush_during_shutdown =
|
||||
mutable_db_options.avoid_flush_during_shutdown;
|
||||
|
||||
return options;
|
||||
}
|
||||
|
@ -343,7 +343,11 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
|
||||
OptionVerificationType::kNormal, false, 0}},
|
||||
{"avoid_flush_during_recovery",
|
||||
{offsetof(struct DBOptions, avoid_flush_during_recovery),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}};
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
|
||||
{"avoid_flush_during_shutdown",
|
||||
{offsetof(struct DBOptions, avoid_flush_during_shutdown),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal, true,
|
||||
offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}};
|
||||
|
||||
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
|
||||
/* not yet supported
|
||||
|
@ -288,7 +288,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
|
||||
"info_log_level=DEBUG_LEVEL;"
|
||||
"dump_malloc_stats=false;"
|
||||
"allow_2pc=false;"
|
||||
"avoid_flush_during_recovery=false;",
|
||||
"avoid_flush_during_recovery=false;"
|
||||
"avoid_flush_during_shutdown=false;",
|
||||
new_options));
|
||||
|
||||
ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
|
||||
|
@ -253,6 +253,8 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
|
||||
db_opt->use_adaptive_mutex = rnd->Uniform(2);
|
||||
db_opt->use_fsync = rnd->Uniform(2);
|
||||
db_opt->recycle_log_file_num = rnd->Uniform(2);
|
||||
db_opt->avoid_flush_during_recovery = rnd->Uniform(2);
|
||||
db_opt->avoid_flush_during_shutdown = rnd->Uniform(2);
|
||||
|
||||
// int options
|
||||
db_opt->max_background_compactions = rnd->Uniform(100);
|
||||
|
Loading…
Reference in New Issue
Block a user