Some fixes and enhancements to ldb repair (#8544)

Summary:
* Basic handling of SST file with just range tombstones rather than
failing assertion about smallest_seqno <= largest_seqno
* Adds --verbose option so that there exists a way to see the INFO
output from Repairer.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8544

Test Plan: unit test added, manual testing for --verbose

Reviewed By: ajkr

Differential Revision: D29954805

Pulled By: pdillinger

fbshipit-source-id: 696af25805fc36cc178b04ba6045922a22625fd9
This commit is contained in:
Peter Dillinger 2021-07-28 16:43:16 -07:00 committed by Facebook GitHub Bot
parent a4b8ac9a73
commit 0804b44fb6
4 changed files with 70 additions and 2 deletions

View File

@ -300,6 +300,8 @@ class Repairer {
}
for (size_t path_id = 0; path_id < to_search_paths.size(); path_id++) {
ROCKS_LOG_INFO(db_options_.info_log, "Searching path %s\n",
to_search_paths[path_id].c_str());
status = env_->GetChildren(to_search_paths[path_id], &filenames);
if (!status.ok()) {
return status;
@ -587,6 +589,30 @@ class Repairer {
t->meta.fd.GetNumber(), counter,
status.ToString().c_str());
}
if (status.ok()) {
// XXX/FIXME: This is just basic, naive handling of range tombstones,
// like call to UpdateBoundariesForRange in builder.cc where we assume
// an SST file is a full sorted run. This probably needs the extra logic
// from compaction_job.cc around call to UpdateBoundariesForRange (to
// handle range tombstones extendingg beyond range of other entries).
ReadOptions ropts;
std::unique_ptr<FragmentedRangeTombstoneIterator> r_iter;
status = table_cache_->GetRangeTombstoneIterator(
ropts, cfd->internal_comparator(), t->meta, &r_iter);
if (r_iter) {
r_iter->SeekToFirst();
while (r_iter->Valid()) {
auto tombstone = r_iter->Tombstone();
auto kv = tombstone.Serialize();
t->meta.UpdateBoundariesForRange(
kv.first, tombstone.SerializeEndKey(), tombstone.seq_,
cfd->internal_comparator());
r_iter->Next();
}
}
}
return status;
}

View File

@ -3,6 +3,7 @@
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "rocksdb/options.h"
#ifndef ROCKSDB_LITE
#include <algorithm>
@ -66,6 +67,35 @@ TEST_F(RepairTest, LostManifest) {
ASSERT_EQ(Get("key2"), "val2");
}
TEST_F(RepairTest, LostManifestMoreDbFeatures) {
// Add a couple SST files, delete the manifest, and verify RepairDB() saves
// the day.
ASSERT_OK(Put("key", "val"));
ASSERT_OK(Put("key2", "val2"));
ASSERT_OK(Put("key3", "val3"));
ASSERT_OK(Put("key4", "val4"));
ASSERT_OK(Flush());
// Test an SST file containing only a range tombstone
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key2",
"key3z"));
ASSERT_OK(Flush());
// Need to get path before Close() deletes db_, but delete it after Close() to
// ensure Close() didn't change the manifest.
std::string manifest_path =
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
Close();
ASSERT_OK(env_->FileExists(manifest_path));
ASSERT_OK(env_->DeleteFile(manifest_path));
ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
Reopen(CurrentOptions());
ASSERT_EQ(Get("key"), "val");
ASSERT_EQ(Get("key2"), "NOT_FOUND");
ASSERT_EQ(Get("key3"), "NOT_FOUND");
ASSERT_EQ(Get("key4"), "val4");
}
TEST_F(RepairTest, CorruptManifest) {
// Manifest is in an invalid format. Expect a full recovery.
ASSERT_OK(Put("key", "val"));

View File

@ -3079,20 +3079,26 @@ void CheckPointCommand::DoCommand() {
// ----------------------------------------------------------------------------
const std::string RepairCommand::ARG_VERBOSE = "verbose";
RepairCommand::RepairCommand(const std::vector<std::string>& /*params*/,
const std::map<std::string, std::string>& options,
const std::vector<std::string>& flags)
: LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
: LDBCommand(options, flags, false, BuildCmdLineOptions({ARG_VERBOSE})) {
verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
}
void RepairCommand::Help(std::string& ret) {
ret.append(" ");
ret.append(RepairCommand::Name());
ret.append(" [--" + ARG_VERBOSE + "]");
ret.append("\n");
}
void RepairCommand::OverrideBaseOptions() {
LDBCommand::OverrideBaseOptions();
options_.info_log.reset(new StderrLogger(InfoLogLevel::WARN_LEVEL));
auto level = verbose_ ? InfoLogLevel::INFO_LEVEL : InfoLogLevel::WARN_LEVEL;
options_.info_log.reset(new StderrLogger(level));
}
void RepairCommand::DoCommand() {

View File

@ -549,6 +549,12 @@ class RepairCommand : public LDBCommand {
virtual void OverrideBaseOptions() override;
static void Help(std::string& ret);
protected:
bool verbose_;
private:
static const std::string ARG_VERBOSE;
};
class BackupableCommand : public LDBCommand {