Avoid deadlock between mutex_ and log_write_mutex_ (#5437)

Summary:
To avoid deadlock mutex_ should never be acquired before log_write_mutex_. The patch documents that and also fixes one case in ::FlushWAL that acquires mutex_ through ::WriteStatusCheck when it already holds lock on log_write_mutex_.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5437

Differential Revision: D15749722

Pulled By: maysamyabandeh

fbshipit-source-id: f57b69c44b4b80cc6d7ddf3d3fdf4a9eb5a5a45a
(cherry picked from commit c8c1a549f0)
This commit is contained in:
Maysam Yabandeh 2019-06-10 17:02:23 -07:00 committed by Yi Zhang
parent 2c3eaeb4db
commit f7e0545692
2 changed files with 9 additions and 4 deletions

View File

@ -1026,10 +1026,13 @@ int DBImpl::FindMinimumEmptyLevelFitting(
Status DBImpl::FlushWAL(bool sync) { Status DBImpl::FlushWAL(bool sync) {
if (manual_wal_flush_) { if (manual_wal_flush_) {
Status s;
{
// We need to lock log_write_mutex_ since logs_ might change concurrently // We need to lock log_write_mutex_ since logs_ might change concurrently
InstrumentedMutexLock wl(&log_write_mutex_); InstrumentedMutexLock wl(&log_write_mutex_);
log::Writer* cur_log_writer = logs_.back().writer; log::Writer* cur_log_writer = logs_.back().writer;
auto s = cur_log_writer->WriteBuffer(); s = cur_log_writer->WriteBuffer();
}
if (!s.ok()) { if (!s.ok()) {
ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL flush error %s", ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL flush error %s",
s.ToString().c_str()); s.ToString().c_str());

View File

@ -1261,6 +1261,8 @@ class DBImpl : public DB {
// logfile_number_. With two_write_queues it also protects alive_log_files_, // logfile_number_. With two_write_queues it also protects alive_log_files_,
// and log_empty_. Refer to the definition of each variable below for more // and log_empty_. Refer to the definition of each variable below for more
// details. // details.
// Note: to avoid dealock, if needed to acquire both log_write_mutex_ and
// mutex_, the order should be first mutex_ and then log_write_mutex_.
InstrumentedMutex log_write_mutex_; InstrumentedMutex log_write_mutex_;
std::atomic<bool> shutting_down_; std::atomic<bool> shutting_down_;