andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make db_wal_test slightly faster

Browse Source 
Summary:
Avoid to run db_wal_test in all the DB test options, and some small changes.
Closes https://github.com/facebook/rocksdb/pull/1921

Differential Revision: D4622054

Pulled By: siying

fbshipit-source-id: 890fd64
This commit is contained in:
Siying Dong 2017-02-28 17:32:58 -08:00 committed by Facebook Github Bot
parent ba4c77bd6b
commit d5b607a43f
3 changed files with 61 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
db/db_test_util.cc
View File

@ -191,6 +191,36 @@ bool DBTestBase::ChangeCompactOptions() {
}
}
// Switch between different WAL settings
bool DBTestBase::ChangeWalOptions() {
if (option_config_ == kDefault) {
option_config_ = kDBLogDir;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
options.create_if_missing = true;
TryReopen(options);
return true;
} else if (option_config_ == kDBLogDir) {
option_config_ = kWalDirAndMmapReads;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
options.create_if_missing = true;
TryReopen(options);
return true;
} else if (option_config_ == kWalDirAndMmapReads) {
option_config_ = kRecycleLogFiles;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
TryReopen(options);
return true;
} else {
return false;
}
}
// Switch between different filter policy
// Jump from kDefault to kFilter to kFullFilter
bool DBTestBase::ChangeFilterOptions() {

5
db/db_test_util.h
View File

@ -671,9 +671,12 @@ class DBTestBase : public testing::Test {
// test. Return false if there are no more configurations to test.
bool ChangeOptions(int skip_mask = kNoSkip);
// Switch between different compaction styles (we have only 2 now).
// Switch between different compaction styles.
bool ChangeCompactOptions();
// Switch between different WAL-realted options.
bool ChangeWalOptions();
// Switch between different filter policy
// Jump from kDefault to kFilter to kFullFilter
bool ChangeFilterOptions();

53
db/db_wal_test.cc
View File

@ -51,7 +51,7 @@ TEST_F(DBWALTest, WAL) {
// again both values should be present.
ASSERT_EQ("v3", Get(1, "foo"));
ASSERT_EQ("v3", Get(1, "bar"));
} while (ChangeCompactOptions());
} while (ChangeWalOptions());
}
TEST_F(DBWALTest, RollLog) {
@ -68,7 +68,7 @@ TEST_F(DBWALTest, RollLog) {
for (int i = 0; i < 10; i++) {
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
}
} while (ChangeOptions());
} while (ChangeWalOptions());
}
TEST_F(DBWALTest, SyncWALNotBlockWrite) {
@ -151,7 +151,7 @@ TEST_F(DBWALTest, Recover) {
ASSERT_EQ("v4", Get(1, "foo"));
ASSERT_EQ("v2", Get(1, "bar"));
ASSERT_EQ("v5", Get(1, "baz"));
} while (ChangeOptions());
} while (ChangeWalOptions());
}
TEST_F(DBWALTest, RecoverWithTableHandle) {
@ -188,23 +188,22 @@ TEST_F(DBWALTest, RecoverWithTableHandle) {
}
}
}
} while (ChangeOptions());
} while (ChangeWalOptions());
}
TEST_F(DBWALTest, IgnoreRecoveredLog) {
std::string backup_logs = dbname_ + "/backup_logs";
// delete old files in backup_logs directory
env_->CreateDirIfMissing(backup_logs);
std::vector<std::string> old_files;
env_->GetChildren(backup_logs, &old_files);
for (auto& file : old_files) {
if (file != "." && file != "..") {
env_->DeleteFile(backup_logs + "/" + file);
}
}
do {
// delete old files in backup_logs directory
env_->CreateDirIfMissing(backup_logs);
std::vector<std::string> old_files;
env_->GetChildren(backup_logs, &old_files);
for (auto& file : old_files) {
if (file != "." && file != "..") {
env_->DeleteFile(backup_logs + "/" + file);
}
}
Options options = CurrentOptions();
options.create_if_missing = true;
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
@ -277,7 +276,8 @@ TEST_F(DBWALTest, IgnoreRecoveredLog) {
}
Status s = TryReopen(options);
ASSERT_TRUE(!s.ok());
} while (ChangeOptions(kSkipHashCuckoo));
Destroy(options);
} while (ChangeWalOptions());
}
TEST_F(DBWALTest, RecoveryWithEmptyLog) {
@ -290,7 +290,7 @@ TEST_F(DBWALTest, RecoveryWithEmptyLog) {
ASSERT_OK(Put(1, "foo", "v3"));
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
ASSERT_EQ("v3", Get(1, "foo"));
} while (ChangeOptions());
} while (ChangeWalOptions());
}
#if !(defined NDEBUG) || !defined(OS_WIN)
@ -429,7 +429,7 @@ TEST_F(DBWALTest, GetSortedWalFiles) {
ASSERT_OK(Put(1, "foo", "v1"));
ASSERT_OK(dbfull()->GetSortedWalFiles(log_files));
ASSERT_EQ(1, log_files.size());
} while (ChangeOptions());
} while (ChangeWalOptions());
}
TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
@ -454,7 +454,7 @@ TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
}
// Check at least the first WAL was cleaned up during the recovery.
ASSERT_LT(earliest_log_nums[0], earliest_log_nums[1]);
} while (ChangeOptions());
} while (ChangeWalOptions());
}
TEST_F(DBWALTest, RecoverWithLargeLog) {
@ -481,7 +481,7 @@ TEST_F(DBWALTest, RecoverWithLargeLog) {
ASSERT_EQ(std::string(10, '3'), Get(1, "small3"));
ASSERT_EQ(std::string(10, '4'), Get(1, "small4"));
ASSERT_GT(NumTableFilesAtLevel(0, 1), 1);
} while (ChangeCompactOptions());
} while (ChangeWalOptions());
}
// In https://reviews.facebook.net/D20661 we change
@ -679,9 +679,9 @@ class RecoveryTestHelper {
// Starting number for the WAL file name like 00010.log
static const int kWALFileOffset = 10;
// Keys to be written per WAL file
static const int kKeysPerWALFile = 1024;
static const int kKeysPerWALFile = 133;
// Size of the value
static const int kValueSize = 10;
static const int kValueSize = 96;
// Create WAL files with values filled in
static void FillData(DBWALTest* test, const Options& options,
@ -690,7 +690,7 @@ class RecoveryTestHelper {
*count = 0;
shared_ptr<Cache> table_cache = NewLRUCache(50000, 16);
shared_ptr<Cache> table_cache = NewLRUCache(50, 0);
EnvOptions env_options;
WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size);
@ -717,12 +717,13 @@ class RecoveryTestHelper {
new log::Writer(std::move(file_writer), current_log_number,
db_options.recycle_log_file_num > 0));
WriteBatch batch;
for (int i = 0; i < kKeysPerWALFile; i++) {
std::string key = "key" + ToString((*count)++);
std::string value = test->DummyString(kValueSize);
assert(current_log_writer.get() != nullptr);
uint64_t seq = versions->LastSequence() + 1;
WriteBatch batch;
batch.Clear();
batch.Put(key, value);
WriteBatchInternal::SetSequence(&batch, seq);
current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch));
@ -773,7 +774,7 @@ class RecoveryTestHelper {
if (trunc) {
ASSERT_EQ(0, truncate(fname.c_str(), static_cast<int64_t>(size * off)));
} else {
InduceCorruption(fname, static_cast<size_t>(size * off),
InduceCorruption(fname, static_cast<size_t>(size * off + 8),
static_cast<size_t>(size * len));
}
}
@ -792,7 +793,7 @@ class RecoveryTestHelper {
ASSERT_EQ(offset, lseek(fd, static_cast<long>(offset), SEEK_SET));
void* buf = alloca(len);
memset(buf, 'a', len);
memset(buf, 'b', len);
ASSERT_EQ(len, write(fd, buf, static_cast<unsigned int>(len)));
close(fd);
@ -808,7 +809,7 @@ TEST_F(DBWALTest, kTolerateCorruptedTailRecords) {
const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
for (auto trunc : {true, false}) { /* Corruption style */
for (int i = 0; i < 4; i++) { /* Corruption offset position */
for (int i = 0; i < 3; i++) { /* Corruption offset position */
for (int j = jstart; j < jend; j++) { /* WAL file */
// Fill data for testing
Options options = CurrentOptions();