From 3ac13c99d1094c10357ff2a13ad1d09b218f41f7 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 8 Oct 2015 13:06:16 -0400 Subject: [PATCH] log_reader: pass log_number and optional info_log to ctor We will need the log number to validate the recycle-style CRCs. The log is helpful for debugging, but optional, as not all callers have it. Signed-off-by: Sage Weil --- db/db_impl.cc | 4 ++-- db/log_reader.cc | 12 ++++++++---- db/log_reader.h | 11 +++++++++-- db/log_test.cc | 13 +++++++------ db/repair.cc | 4 ++-- db/transaction_log_impl.cc | 6 ++++-- db/version_set.cc | 15 ++++++++------- db/wal_manager.cc | 4 ++-- tools/ldb_cmd.cc | 16 +++++++++++++++- 9 files changed, 57 insertions(+), 28 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 402df5316..fc39e15c6 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1119,8 +1119,8 @@ Status DBImpl::RecoverLogFiles(const std::vector& log_numbers, // paranoid_checks==false so that corruptions cause entire commits // to be skipped instead of propagating bad information (like overly // large sequence numbers). - log::Reader reader(std::move(file_reader), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(db_options_.info_log, std::move(file_reader), &reporter, + true /*checksum*/, 0 /*initial_offset*/, log_number); Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, "Recovering log #%" PRIu64 " mode %d skip-recovery %d", log_number, db_options_.wal_recovery_mode, !continue_replay_log); diff --git a/db/log_reader.cc b/db/log_reader.cc index 296f1d50c..44396eb7a 100644 --- a/db/log_reader.cc +++ b/db/log_reader.cc @@ -21,9 +21,12 @@ namespace log { Reader::Reporter::~Reporter() { } -Reader::Reader(unique_ptr&& _file, Reporter* reporter, - bool checksum, uint64_t initial_offset) - : file_(std::move(_file)), +Reader::Reader(std::shared_ptr info_log, + unique_ptr&& _file, + Reporter* reporter, bool checksum, uint64_t initial_offset, + uint64_t log_num) + : info_log_(info_log), + file_(std::move(_file)), reporter_(reporter), checksum_(checksum), backing_store_(new char[kBlockSize]), @@ -33,7 +36,8 @@ Reader::Reader(unique_ptr&& _file, Reporter* reporter, eof_offset_(0), last_record_offset_(0), end_of_buffer_offset_(0), - initial_offset_(initial_offset) {} + initial_offset_(initial_offset), + log_number_(log_num) {} Reader::~Reader() { delete[] backing_store_; diff --git a/db/log_reader.h b/db/log_reader.h index 390696b85..1b0a592c9 100644 --- a/db/log_reader.h +++ b/db/log_reader.h @@ -18,6 +18,7 @@ namespace rocksdb { class SequentialFileReader; +class Logger; using std::unique_ptr; namespace log { @@ -51,8 +52,10 @@ class Reader { // // The Reader will start reading at the first record located at physical // position >= initial_offset within the file. - Reader(unique_ptr&& file, Reporter* reporter, - bool checksum, uint64_t initial_offset); + Reader(std::shared_ptr info_log, + unique_ptr&& file, + Reporter* reporter, bool checksum, uint64_t initial_offset, + uint64_t log_num); ~Reader(); @@ -84,6 +87,7 @@ class Reader { SequentialFileReader* file() { return file_.get(); } private: + std::shared_ptr info_log_; const unique_ptr file_; Reporter* const reporter_; bool const checksum_; @@ -104,6 +108,9 @@ class Reader { // Offset at which to start looking for the first record to return uint64_t const initial_offset_; + // which log number this is + uint64_t const log_number_; + // Extend record types with the following special values enum { kEof = kMaxRecordType + 1, diff --git a/db/log_test.cc b/db/log_test.cc index 55ecd5f3f..55f8a9682 100644 --- a/db/log_test.cc +++ b/db/log_test.cc @@ -163,8 +163,8 @@ class LogTest : public ::testing::TestWithParam { source_holder_( test::GetSequentialFileReader(new StringSource(reader_contents_))), writer_(std::move(dest_holder_), 123, GetParam()), - reader_(std::move(source_holder_), &report_, true /*checksum*/, - 0 /*initial_offset*/) {} + reader_(NULL, std::move(source_holder_), &report_, + true /*checksum*/, 0 /*initial_offset*/, 123) {} void Write(const std::string& msg) { writer_.AddRecord(Slice(msg)); @@ -258,8 +258,8 @@ class LogTest : public ::testing::TestWithParam { unique_ptr file_reader( test::GetSequentialFileReader(new StringSource(reader_contents_))); unique_ptr offset_reader( - new Reader(std::move(file_reader), &report_, true /*checksum*/, - WrittenBytes() + offset_past_end)); + new Reader(NULL, std::move(file_reader), &report_, + true /*checksum*/, WrittenBytes() + offset_past_end, 123)); Slice record; std::string scratch; ASSERT_TRUE(!offset_reader->ReadRecord(&record, &scratch)); @@ -270,8 +270,9 @@ class LogTest : public ::testing::TestWithParam { WriteInitialOffsetLog(); unique_ptr file_reader( test::GetSequentialFileReader(new StringSource(reader_contents_))); - unique_ptr offset_reader(new Reader( - std::move(file_reader), &report_, true /*checksum*/, initial_offset)); + unique_ptr offset_reader( + new Reader(NULL, std::move(file_reader), &report_, + true /*checksum*/, initial_offset, 123)); Slice record; std::string scratch; ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch)); diff --git a/db/repair.cc b/db/repair.cc index d5375f5dd..c27db6e49 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -249,8 +249,8 @@ class Repairer { // corruptions cause entire commits to be skipped instead of // propagating bad information (like overly large sequence // numbers). - log::Reader reader(std::move(lfile_reader), &reporter, - true /*enable checksum*/, 0 /*initial_offset*/); + log::Reader reader(options_.info_log, std::move(lfile_reader), &reporter, + true /*enable checksum*/, 0 /*initial_offset*/, log); // Read all the records and add to a memtable std::string scratch; diff --git a/db/transaction_log_impl.cc b/db/transaction_log_impl.cc index 23bd6672b..28c4490f5 100644 --- a/db/transaction_log_impl.cc +++ b/db/transaction_log_impl.cc @@ -262,8 +262,10 @@ Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) { return s; } assert(file); - currentLogReader_.reset(new log::Reader(std::move(file), &reporter_, - read_options_.verify_checksums_, 0)); + currentLogReader_.reset(new log::Reader(options_->info_log, + std::move(file), &reporter_, + read_options_.verify_checksums_, 0, + logFile->LogNumber())); return Status::OK(); } } // namespace rocksdb diff --git a/db/version_set.cc b/db/version_set.cc index 146e93ad0..478adbcb4 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -2376,8 +2376,8 @@ Status VersionSet::Recover( { VersionSet::LogReporter reporter; reporter.status = &s; - log::Reader reader(std::move(manifest_file_reader), &reporter, - true /*checksum*/, 0 /*initial_offset*/); + log::Reader reader(NULL, std::move(manifest_file_reader), &reporter, + true /*checksum*/, 0 /*initial_offset*/, 0); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { @@ -2629,8 +2629,8 @@ Status VersionSet::ListColumnFamilies(std::vector* column_families, column_family_names.insert({0, kDefaultColumnFamilyName}); VersionSet::LogReporter reporter; reporter.status = &s; - log::Reader reader(std::move(file_reader), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(NULL, std::move(file_reader), &reporter, true /*checksum*/, + 0 /*initial_offset*/, 0); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { @@ -2787,8 +2787,8 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname, { VersionSet::LogReporter reporter; reporter.status = &s; - log::Reader reader(std::move(file_reader), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(NULL, std::move(file_reader), &reporter, + true /*checksum*/, 0 /*initial_offset*/, 0); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { @@ -3040,7 +3040,8 @@ bool VersionSet::ManifestContains(uint64_t manifest_file_num, } file_reader.reset(new SequentialFileReader(std::move(file))); } - log::Reader reader(std::move(file_reader), nullptr, true /*checksum*/, 0); + log::Reader reader(NULL, std::move(file_reader), nullptr, + true /*checksum*/, 0, 0); Slice r; std::string scratch; bool result = false; diff --git a/db/wal_manager.cc b/db/wal_manager.cc index 37861ab45..58b0b1643 100644 --- a/db/wal_manager.cc +++ b/db/wal_manager.cc @@ -448,8 +448,8 @@ Status WalManager::ReadFirstLine(const std::string& fname, reporter.fname = fname.c_str(); reporter.status = &status; reporter.ignore_error = !db_options_.paranoid_checks; - log::Reader reader(std::move(file_reader), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(db_options_.info_log, std::move(file_reader), &reporter, + true /*checksum*/, 0 /*initial_offset*/, *sequence); std::string scratch; Slice record; diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 112014351..60687f98d 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -1438,7 +1438,21 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values, } } else { StdErrReporter reporter; - log::Reader reader(move(wal_file_reader), &reporter, true, 0); + uint64_t log_number; + FileType type; + + // we need the log number, but ParseFilename expects dbname/NNN.log. + string sanitized = wal_file; + size_t lastslash = sanitized.rfind('/'); + if (lastslash != std::string::npos) + sanitized = sanitized.substr(lastslash + 1); + if (!ParseFileName(sanitized, &log_number, &type)) { + // bogus input, carry on as best we can + log_number = 0; + } + DBOptions db_options; + log::Reader reader(db_options.info_log, move(wal_file_reader), &reporter, + true, 0, log_number); string scratch; WriteBatch batch; Slice record;