andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Handle WAL deletion when using avoid_flush_during_recovery

Browse Source 
Summary:
Previously the WAL files that were avoided during recovery would never
be considered for deletion. That was because alive_log_files_ was only
populated when log files are created. This diff further populates
alive_log_files_ with existing log files that aren't flushed during recovery,
such that FindObsoleteFiles() can find them later.

Depends on D64053.

Test Plan: new unit test, verifies it fails before this change and passes after

Reviewers: sdong, IslamAbdelRahman, yiwu

Reviewed By: yiwu

Subscribers: leveldb, dhruba, andrewkr

Differential Revision: https://reviews.facebook.net/D64059
This commit is contained in:
Andrew Kryczka 2016-10-14 12:59:51 -07:00
parent e29d3b67c2
commit f47054015d
2 changed files with 38 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
db/db_impl.cc
View File

@ -859,7 +859,9 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
} }
} }
if (!alive_log_files_.empty()) { // logs_ is empty when called during recovery, in which case there can't yet
// be any tracked obsolete logs
if (!alive_log_files_.empty() && !logs_.empty()) {
uint64_t min_log_number = job_context->log_number; uint64_t min_log_number = job_context->log_number;
size_t num_alive_log_files = alive_log_files_.size(); size_t num_alive_log_files = alive_log_files_.size();
// find newly obsoleted log files // find newly obsoleted log files
@ -990,8 +992,7 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
for (auto file_num : state.log_delete_files) { for (auto file_num : state.log_delete_files) {
if (file_num > 0) { if (file_num > 0) {
candidate_files.emplace_back(LogFileName(kDumbDbName, file_num).substr(1), candidate_files.emplace_back(LogFileName(kDumbDbName, file_num), 0);
0);
} }
} }
for (const auto& filename : state.manifest_delete_files) { for (const auto& filename : state.manifest_delete_files) {
@ -1746,6 +1747,14 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
} }
} }
if (!flushed) {
// Mark these as alive so they'll be considered for deletion later by
// FindObsoleteFiles()
for (auto log_number : log_numbers) {
alive_log_files_.push_back(LogFileNumberSize(log_number));
}
}
event_logger_.Log() << "job" << job_id << "event" event_logger_.Log() << "job" << job_id << "event"
<< "recovery_finished"; << "recovery_finished";

26
db/db_wal_test.cc
View File

@ -1006,6 +1006,32 @@ TEST_F(DBWALTest, AvoidFlushDuringRecovery) {
ASSERT_EQ(2, TotalTableFiles()); ASSERT_EQ(2, TotalTableFiles());
} }
TEST_F(DBWALTest, WalCleanupAfterAvoidFlushDuringRecovery) {
// Verifies WAL files that were present during recovery, but not flushed due
// to avoid_flush_during_recovery, will be considered for deletion at a later
// stage. We check at least one such file is deleted during Flush().
Options options = CurrentOptions();
options.disable_auto_compactions = true;
options.avoid_flush_during_recovery = true;
Reopen(options);
ASSERT_OK(Put("foo", "v1"));
Reopen(options);
for (int i = 0; i < 2; ++i) {
if (i > 0) {
// Flush() triggers deletion of obsolete tracked files
Flush();
}
VectorLogPtr log_files;
ASSERT_OK(dbfull()->GetSortedWalFiles(log_files));
if (i == 0) {
ASSERT_GT(log_files.size(), 0);
} else {
ASSERT_EQ(0, log_files.size());
}
}
}
TEST_F(DBWALTest, RecoverWithoutFlush) { TEST_F(DBWALTest, RecoverWithoutFlush) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.avoid_flush_during_recovery = true; options.avoid_flush_during_recovery = true;