andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Likely fix flaky TableFileCorruptedBeforeBackup (#8151)

Browse Source 
Summary:
Before corrupting a file in the DB and expecting corruption to
be detected, open DB read-only to ensure file is not made obsolete by
compaction. Also, to avoid obsolete files not yet deleted, only select
live files to corrupt.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8151

Test Plan: watch CI

Reviewed By: akankshamahajan15

Differential Revision: D27568849

Pulled By: pdillinger

fbshipit-source-id: 39a69a2eafde0482b20a197949d24abe21952f27
This commit is contained in:
Peter Dillinger 2021-04-05 11:39:39 -07:00 committed by Facebook GitHub Bot
parent bd7ddf58cb
commit 96205baa63
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
utilities/backupable/backupable_db_test.cc
View File

@ -658,14 +658,18 @@ class BackupableDBTest : public testing::Test {
return db;
}
void CloseAndReopenDB() {
void CloseAndReopenDB(bool read_only = false) {
// Close DB
db_.reset();
// Open DB
test_db_env_->SetLimitWrittenFiles(1000000);
DB* db;
ASSERT_OK(DB::Open(options_, dbname_, &db));
if (read_only) {
ASSERT_OK(DB::OpenForReadOnly(options_, dbname_, &db));
} else {
ASSERT_OK(DB::Open(options_, dbname_, &db));
}
db_.reset(db);
}
@ -827,12 +831,19 @@ class BackupableDBTest : public testing::Test {
Status GetDataFilesInDB(const FileType& file_type,
std::vector<FileAttributes>* files) {
std::vector<std::string> live;
uint64_t ignore_manifest_size;
Status s = db_->GetLiveFiles(live, &ignore_manifest_size, /*flush*/ false);
if (!s.ok()) {
return s;
}
std::vector<FileAttributes> children;
Status s = test_db_env_->GetChildrenFileAttributes(dbname_, &children);
s = test_db_env_->GetChildrenFileAttributes(dbname_, &children);
for (const auto& child : children) {
FileType type;
uint64_t number = 0;
if (ParseFileName(child.name, &number, &type) && type == file_type) {
if (ParseFileName(child.name, &number, &type) && type == file_type &&
std::find(live.begin(), live.end(), "/" + child.name) != live.end()) {
files->push_back(child);
}
}
@ -1469,7 +1480,7 @@ TEST_F(BackupableDBTest, TableFileCorruptedBeforeBackup) {
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
kNoShare);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random table file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kTableFile));
// file_checksum_gen_factory is null, and thus table checksum is not
@ -1485,7 +1496,7 @@ TEST_F(BackupableDBTest, TableFileCorruptedBeforeBackup) {
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
kNoShare);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random table file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kTableFile));
// table file checksum is enabled so we should be able to detect any
@ -1503,7 +1514,7 @@ TEST_F(BackupableDBTest, BlobFileCorruptedBeforeBackup) {
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
kNoShare);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random blob file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kBlobFile));
// file_checksum_gen_factory is null, and thus blob checksum is not
@ -1519,7 +1530,7 @@ TEST_F(BackupableDBTest, BlobFileCorruptedBeforeBackup) {
OpenDBAndBackupEngine(true /* destroy_old_data */, false /* dummy */,
kNoShare);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random blob file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kBlobFile));
@ -1537,7 +1548,7 @@ TEST_P(BackupableDBTestWithParam, TableFileCorruptedBeforeBackup) {
OpenDBAndBackupEngine(true /* destroy_old_data */);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random table file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kTableFile));
// cannot detect corruption since DB manifest has no table checksums
@ -1551,7 +1562,7 @@ TEST_P(BackupableDBTestWithParam, TableFileCorruptedBeforeBackup) {
options_.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
OpenDBAndBackupEngine(true /* destroy_old_data */);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random table file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kTableFile));
// corruption is detected
@ -1567,7 +1578,7 @@ TEST_P(BackupableDBTestWithParam, BlobFileCorruptedBeforeBackup) {
options_.enable_blob_files = true;
OpenDBAndBackupEngine(true /* destroy_old_data */);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random blob file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kBlobFile));
// cannot detect corruption since DB manifest has no blob file checksums
@ -1581,7 +1592,7 @@ TEST_P(BackupableDBTestWithParam, BlobFileCorruptedBeforeBackup) {
options_.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
OpenDBAndBackupEngine(true /* destroy_old_data */);
FillDB(db_.get(), 0, keys_iteration);
CloseAndReopenDB();
CloseAndReopenDB(/*read_only*/ true);
// corrupt a random blob file in the DB directory
ASSERT_OK(CorruptRandomDataFileInDB(kBlobFile));
// corruption is detected
@ -2050,7 +2061,7 @@ TEST_F(BackupableDBTest, TableFileCorruptionBeforeIncremental) {
ASSERT_OK(dbi->Put(WriteOptions(), "y", Random(42).RandomString(500)));
ASSERT_OK(dbi->Flush(FlushOptions()));
ASSERT_OK(dbi->TEST_WaitForFlushMemTable());
CloseDBAndBackupEngine();
CloseAndReopenDB(/*read_only*/ true);
std::vector<FileAttributes> table_files;
ASSERT_OK(GetDataFilesInDB(kTableFile, &table_files));
@ -2058,6 +2069,8 @@ TEST_F(BackupableDBTest, TableFileCorruptionBeforeIncremental) {
std::string tf0 = dbname_ + "/" + table_files[0].name;
std::string tf1 = dbname_ + "/" + table_files[1].name;
CloseDBAndBackupEngine();
if (corrupt_before_first_backup) {
// This corrupts a data block, which does not cause DB open
// failure, only failure on accessing the block.