andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make db_wal_test slightly faster

Browse Source 
Summary:
Avoid to run db_wal_test in all the DB test options, and some small changes.
Closes https://github.com/facebook/rocksdb/pull/1921

Differential Revision: D4622054

Pulled By: siying

fbshipit-source-id: 890fd64
This commit is contained in:
Siying Dong 2017-02-28 17:32:58 -08:00 committed by Facebook Github Bot
parent ba4c77bd6b
commit d5b607a43f
3 changed files with 61 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
db/db_test_util.cc
View File

@ -191,6 +191,36 @@ bool DBTestBase::ChangeCompactOptions() {
} }
} }
// Switch between different WAL settings
bool DBTestBase::ChangeWalOptions() {
if (option_config_ == kDefault) {
option_config_ = kDBLogDir;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
options.create_if_missing = true;
TryReopen(options);
return true;
} else if (option_config_ == kDBLogDir) {
option_config_ = kWalDirAndMmapReads;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
options.create_if_missing = true;
TryReopen(options);
return true;
} else if (option_config_ == kWalDirAndMmapReads) {
option_config_ = kRecycleLogFiles;
Destroy(last_options_);
auto options = CurrentOptions();
Destroy(options);
TryReopen(options);
return true;
} else {
return false;
}
}
// Switch between different filter policy // Switch between different filter policy
// Jump from kDefault to kFilter to kFullFilter // Jump from kDefault to kFilter to kFullFilter
bool DBTestBase::ChangeFilterOptions() { bool DBTestBase::ChangeFilterOptions() {

5
db/db_test_util.h
View File

@ -671,9 +671,12 @@ class DBTestBase : public testing::Test {
// test. Return false if there are no more configurations to test. // test. Return false if there are no more configurations to test.
bool ChangeOptions(int skip_mask = kNoSkip); bool ChangeOptions(int skip_mask = kNoSkip);
// Switch between different compaction styles (we have only 2 now). // Switch between different compaction styles.
bool ChangeCompactOptions(); bool ChangeCompactOptions();
// Switch between different WAL-realted options.
bool ChangeWalOptions();
// Switch between different filter policy // Switch between different filter policy
// Jump from kDefault to kFilter to kFullFilter // Jump from kDefault to kFilter to kFullFilter
bool ChangeFilterOptions(); bool ChangeFilterOptions();

37
db/db_wal_test.cc
View File

@ -51,7 +51,7 @@ TEST_F(DBWALTest, WAL) {
// again both values should be present. // again both values should be present.
ASSERT_EQ("v3", Get(1, "foo")); ASSERT_EQ("v3", Get(1, "foo"));
ASSERT_EQ("v3", Get(1, "bar")); ASSERT_EQ("v3", Get(1, "bar"));
} while (ChangeCompactOptions()); } while (ChangeWalOptions());
} }
TEST_F(DBWALTest, RollLog) { TEST_F(DBWALTest, RollLog) {
@ -68,7 +68,7 @@ TEST_F(DBWALTest, RollLog) {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
} }
} while (ChangeOptions()); } while (ChangeWalOptions());
} }
TEST_F(DBWALTest, SyncWALNotBlockWrite) { TEST_F(DBWALTest, SyncWALNotBlockWrite) {
@ -151,7 +151,7 @@ TEST_F(DBWALTest, Recover) {
ASSERT_EQ("v4", Get(1, "foo")); ASSERT_EQ("v4", Get(1, "foo"));
ASSERT_EQ("v2", Get(1, "bar")); ASSERT_EQ("v2", Get(1, "bar"));
ASSERT_EQ("v5", Get(1, "baz")); ASSERT_EQ("v5", Get(1, "baz"));
} while (ChangeOptions()); } while (ChangeWalOptions());
} }
TEST_F(DBWALTest, RecoverWithTableHandle) { TEST_F(DBWALTest, RecoverWithTableHandle) {
@ -188,12 +188,13 @@ TEST_F(DBWALTest, RecoverWithTableHandle) {
} }
} }
} }
} while (ChangeOptions()); } while (ChangeWalOptions());
} }
TEST_F(DBWALTest, IgnoreRecoveredLog) { TEST_F(DBWALTest, IgnoreRecoveredLog) {
std::string backup_logs = dbname_ + "/backup_logs"; std::string backup_logs = dbname_ + "/backup_logs";
do {
// delete old files in backup_logs directory // delete old files in backup_logs directory
env_->CreateDirIfMissing(backup_logs); env_->CreateDirIfMissing(backup_logs);
std::vector<std::string> old_files; std::vector<std::string> old_files;
@ -203,8 +204,6 @@ TEST_F(DBWALTest, IgnoreRecoveredLog) {
env_->DeleteFile(backup_logs + "/" + file); env_->DeleteFile(backup_logs + "/" + file);
} }
} }
do {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.merge_operator = MergeOperators::CreateUInt64AddOperator(); options.merge_operator = MergeOperators::CreateUInt64AddOperator();
@ -277,7 +276,8 @@ TEST_F(DBWALTest, IgnoreRecoveredLog) {
} }
Status s = TryReopen(options); Status s = TryReopen(options);
ASSERT_TRUE(!s.ok()); ASSERT_TRUE(!s.ok());
} while (ChangeOptions(kSkipHashCuckoo)); Destroy(options);
} while (ChangeWalOptions());
} }
TEST_F(DBWALTest, RecoveryWithEmptyLog) { TEST_F(DBWALTest, RecoveryWithEmptyLog) {
@ -290,7 +290,7 @@ TEST_F(DBWALTest, RecoveryWithEmptyLog) {
ASSERT_OK(Put(1, "foo", "v3")); ASSERT_OK(Put(1, "foo", "v3"));
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
ASSERT_EQ("v3", Get(1, "foo")); ASSERT_EQ("v3", Get(1, "foo"));
} while (ChangeOptions()); } while (ChangeWalOptions());
} }
#if !(defined NDEBUG) || !defined(OS_WIN) #if !(defined NDEBUG) || !defined(OS_WIN)
@ -429,7 +429,7 @@ TEST_F(DBWALTest, GetSortedWalFiles) {
ASSERT_OK(Put(1, "foo", "v1")); ASSERT_OK(Put(1, "foo", "v1"));
ASSERT_OK(dbfull()->GetSortedWalFiles(log_files)); ASSERT_OK(dbfull()->GetSortedWalFiles(log_files));
ASSERT_EQ(1, log_files.size()); ASSERT_EQ(1, log_files.size());
} while (ChangeOptions()); } while (ChangeWalOptions());
} }
TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) { TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
@ -454,7 +454,7 @@ TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
} }
// Check at least the first WAL was cleaned up during the recovery. // Check at least the first WAL was cleaned up during the recovery.
ASSERT_LT(earliest_log_nums[0], earliest_log_nums[1]); ASSERT_LT(earliest_log_nums[0], earliest_log_nums[1]);
} while (ChangeOptions()); } while (ChangeWalOptions());
} }
TEST_F(DBWALTest, RecoverWithLargeLog) { TEST_F(DBWALTest, RecoverWithLargeLog) {
@ -481,7 +481,7 @@ TEST_F(DBWALTest, RecoverWithLargeLog) {
ASSERT_EQ(std::string(10, '3'), Get(1, "small3")); ASSERT_EQ(std::string(10, '3'), Get(1, "small3"));
ASSERT_EQ(std::string(10, '4'), Get(1, "small4")); ASSERT_EQ(std::string(10, '4'), Get(1, "small4"));
ASSERT_GT(NumTableFilesAtLevel(0, 1), 1); ASSERT_GT(NumTableFilesAtLevel(0, 1), 1);
} while (ChangeCompactOptions()); } while (ChangeWalOptions());
} }
// In https://reviews.facebook.net/D20661 we change // In https://reviews.facebook.net/D20661 we change
@ -679,9 +679,9 @@ class RecoveryTestHelper {
// Starting number for the WAL file name like 00010.log // Starting number for the WAL file name like 00010.log
static const int kWALFileOffset = 10; static const int kWALFileOffset = 10;
// Keys to be written per WAL file // Keys to be written per WAL file
static const int kKeysPerWALFile = 1024; static const int kKeysPerWALFile = 133;
// Size of the value // Size of the value
static const int kValueSize = 10; static const int kValueSize = 96;
// Create WAL files with values filled in // Create WAL files with values filled in
static void FillData(DBWALTest* test, const Options& options, static void FillData(DBWALTest* test, const Options& options,
@ -690,7 +690,7 @@ class RecoveryTestHelper {
*count = 0; *count = 0;
shared_ptr<Cache> table_cache = NewLRUCache(50000, 16); shared_ptr<Cache> table_cache = NewLRUCache(50, 0);
EnvOptions env_options; EnvOptions env_options;
WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size); WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size);
@ -717,12 +717,13 @@ class RecoveryTestHelper {
new log::Writer(std::move(file_writer), current_log_number, new log::Writer(std::move(file_writer), current_log_number,
db_options.recycle_log_file_num > 0)); db_options.recycle_log_file_num > 0));
WriteBatch batch;
for (int i = 0; i < kKeysPerWALFile; i++) { for (int i = 0; i < kKeysPerWALFile; i++) {
std::string key = "key" + ToString((*count)++); std::string key = "key" + ToString((*count)++);
std::string value = test->DummyString(kValueSize); std::string value = test->DummyString(kValueSize);
assert(current_log_writer.get() != nullptr); assert(current_log_writer.get() != nullptr);
uint64_t seq = versions->LastSequence() + 1; uint64_t seq = versions->LastSequence() + 1;
WriteBatch batch; batch.Clear();
batch.Put(key, value); batch.Put(key, value);
WriteBatchInternal::SetSequence(&batch, seq); WriteBatchInternal::SetSequence(&batch, seq);
current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch)); current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch));
@ -773,7 +774,7 @@ class RecoveryTestHelper {
if (trunc) { if (trunc) {
ASSERT_EQ(0, truncate(fname.c_str(), static_cast<int64_t>(size * off))); ASSERT_EQ(0, truncate(fname.c_str(), static_cast<int64_t>(size * off)));
} else { } else {
InduceCorruption(fname, static_cast<size_t>(size * off), InduceCorruption(fname, static_cast<size_t>(size * off + 8),
static_cast<size_t>(size * len)); static_cast<size_t>(size * len));
} }
} }
@ -792,7 +793,7 @@ class RecoveryTestHelper {
ASSERT_EQ(offset, lseek(fd, static_cast<long>(offset), SEEK_SET)); ASSERT_EQ(offset, lseek(fd, static_cast<long>(offset), SEEK_SET));
void* buf = alloca(len); void* buf = alloca(len);
memset(buf, 'a', len); memset(buf, 'b', len);
ASSERT_EQ(len, write(fd, buf, static_cast<unsigned int>(len))); ASSERT_EQ(len, write(fd, buf, static_cast<unsigned int>(len)));
close(fd); close(fd);
@ -808,7 +809,7 @@ TEST_F(DBWALTest, kTolerateCorruptedTailRecords) {
const int jend = jstart + RecoveryTestHelper::kWALFilesCount; const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
for (auto trunc : {true, false}) { /* Corruption style */ for (auto trunc : {true, false}) { /* Corruption style */
for (int i = 0; i < 4; i++) { /* Corruption offset position */ for (int i = 0; i < 3; i++) { /* Corruption offset position */
for (int j = jstart; j < jend; j++) { /* WAL file */ for (int j = jstart; j < jend; j++) { /* WAL file */
// Fill data for testing // Fill data for testing
Options options = CurrentOptions(); Options options = CurrentOptions();