andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make CompactionInputErrorParanoid less flakey

Browse Source 
Summary:
I'm getting lots of e-mails with CompactionInputErrorParanoid failing. Most recent example early morning today was: http://ci-builds.fb.com/job/rocksdb_valgrind/562/consoleFull

I'm putting a stop to these e-mails. I investigated why the test is flakey and it turns out it's because of non-determinsim of compaction scheduling. If there is a compaction after the last flush, CorruptFile will corrupt the compacted file instead of file at level 0 (as it assumes). That makes `Check(9, 9)` fail big time.

I also saw some errors with table file getting outputed to >= 1 levels instead of 0. Also fixed that.

Test Plan: Ran corruption_test 100 times without a failure. Previously it usually failed at 10th occurrence.

Reviewers: dhruba, haobo, ljin

Reviewed By: ljin

CC: leveldb

Differential Revision: https://reviews.facebook.net/D18285
This commit is contained in:
Igor Canadi 2014-04-24 11:13:28 -07:00
parent fc3127e8de
commit 478990c81b
1 changed files with 43 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
db/corruption_test.cc
View File

@ -40,7 +40,7 @@ class CorruptionTest {
CorruptionTest() { CorruptionTest() {
tiny_cache_ = NewLRUCache(100); tiny_cache_ = NewLRUCache(100);
options_.env = &env_; options_.env = &env_;
dbname_ = test::TmpDir() + "/db_test"; dbname_ = test::TmpDir() + "/corruption_test";
DestroyDB(dbname_, options_); DestroyDB(dbname_, options_);
db_ = nullptr; db_ = nullptr;
@ -127,24 +127,7 @@ class CorruptionTest {
ASSERT_GE(max_expected, correct); ASSERT_GE(max_expected, correct);
} }
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { void CorruptFile(const std::string fname, int offset, int bytes_to_corrupt) {
// Pick file to corrupt
std::vector<std::string> filenames;
ASSERT_OK(env_.GetChildren(dbname_, &filenames));
uint64_t number;
FileType type;
std::string fname;
int picked_number = -1;
for (unsigned int i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) &&
type == filetype &&
int(number) > picked_number) { // Pick latest file
fname = dbname_ + "/" + filenames[i];
picked_number = number;
}
}
ASSERT_TRUE(!fname.empty()) << filetype;
struct stat sbuf; struct stat sbuf;
if (stat(fname.c_str(), &sbuf) != 0) { if (stat(fname.c_str(), &sbuf) != 0) {
const char* msg = strerror(errno); const char* msg = strerror(errno);
@ -177,6 +160,42 @@ class CorruptionTest {
ASSERT_TRUE(s.ok()) << s.ToString(); ASSERT_TRUE(s.ok()) << s.ToString();
} }
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
// Pick file to corrupt
std::vector<std::string> filenames;
ASSERT_OK(env_.GetChildren(dbname_, &filenames));
uint64_t number;
FileType type;
std::string fname;
int picked_number = -1;
for (unsigned int i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) &&
type == filetype &&
static_cast<int>(number) > picked_number) { // Pick latest file
fname = dbname_ + "/" + filenames[i];
picked_number = number;
}
}
ASSERT_TRUE(!fname.empty()) << filetype;
CorruptFile(fname, offset, bytes_to_corrupt);
}
// corrupts exactly one file at level `level`. if no file found at level,
// asserts
void CorruptTableFileAtLevel(int level, int offset, int bytes_to_corrupt) {
std::vector<LiveFileMetaData> metadata;
db_->GetLiveFilesMetaData(&metadata);
for (const auto& m : metadata) {
if (m.level == level) {
CorruptFile(dbname_ + "/" + m.name, offset, bytes_to_corrupt);
return;
}
}
ASSERT_TRUE(false) << "no file found at level";
}
int Property(const std::string& name) { int Property(const std::string& name) {
std::string property; std::string property;
int result; int result;
@ -331,19 +350,22 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
Reopen(&options); Reopen(&options);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
// Fill levels >= 1 so memtable compaction outputs to level 1 // Fill levels >= 1 so memtable flush outputs to level 0
for (int level = 1; level < dbi->NumberLevels(); level++) { for (int level = 1; level < dbi->NumberLevels(); level++) {
dbi->Put(WriteOptions(), "", "begin"); dbi->Put(WriteOptions(), "", "begin");
dbi->Put(WriteOptions(), "~", "end"); dbi->Put(WriteOptions(), "~", "end");
dbi->TEST_FlushMemTable(); dbi->TEST_FlushMemTable();
} }
options.max_mem_compaction_level = 0;
Reopen(&options);
Build(10); Build(10);
dbi->TEST_FlushMemTable(); dbi->TEST_FlushMemTable();
dbi->TEST_WaitForCompact(); dbi->TEST_WaitForCompact();
ASSERT_EQ(1, Property("rocksdb.num-files-at-level0")); ASSERT_EQ(1, Property("rocksdb.num-files-at-level0"));
Corrupt(kTableFile, 100, 1); CorruptTableFileAtLevel(0, 100, 1);
Check(9, 9); Check(9, 9);
// Write must eventually fail because of corrupted table // Write must eventually fail because of corrupted table