From b703a56e5cd722aaf169baa3e28127426776b6a9 Mon Sep 17 00:00:00 2001 From: anand76 Date: Fri, 7 Jun 2019 15:31:40 -0700 Subject: [PATCH] Potential fix for stress test failure due to "SST file ahead of WAL" error (#5412) Summary: I'm not able to prove it, but the stress test failure may be caused by the following sequence of events - 1. Crash db_stress while writing the log file. This should result in a corrupted WAL. 2. Run db_stress with recycle_log_file_num=1. Crash during recovery immediately after writing manifest and updating the current file. The old log from the previous run is left behind, but the memtable would have been flushed during recovery and the CF log number will point to the newer log 3. Run db_stress with recycle_log_file_num=0. During recovery, the old log file will be processed and the corruption will be detected. Since the CF has moved ahead, we get the "SST file is ahead of WAL" error Test - 1. stress_crash 2. make check Pull Request resolved: https://github.com/facebook/rocksdb/pull/5412 Differential Revision: D15699120 Pulled By: anand1976 fbshipit-source-id: 9092ce81e7c4a0b4b4e66560c23ea4812a4d9cbe --- db/db_impl/db_impl_compaction_flush.cc | 7 +++++++ db/db_impl/db_impl_open.cc | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index bd1a8e74f..8cb37484c 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -107,6 +107,13 @@ Status DBImpl::SyncClosedLogs(JobContext* job_context) { if (!s.ok()) { break; } + + if (immutable_db_options_.recycle_log_file_num > 0) { + s = log->Close(); + if (!s.ok()) { + break; + } + } } if (s.ok()) { s = directories_.GetWalDir()->Fsync(); diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 69c9c4117..baa4fe707 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -555,12 +555,13 @@ Status DBImpl::RecoverLogFiles(const std::vector& log_numbers, bool stop_replay_for_corruption = false; bool flushed = false; uint64_t corrupted_log_number = kMaxSequenceNumber; + uint64_t min_log_number = MinLogNumberToKeep(); for (auto log_number : log_numbers) { - if (log_number < versions_->min_log_number_to_keep_2pc()) { + if (log_number < min_log_number) { ROCKS_LOG_INFO(immutable_db_options_.info_log, "Skipping log #%" PRIu64 " since it is older than min log to keep #%" PRIu64, - log_number, versions_->min_log_number_to_keep_2pc()); + log_number, min_log_number); continue; } // The previous incarnation may not have written any MANIFEST