andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update Tests To Enable Subcompactions

Browse Source 
Summary:
Updated DBTest DBCompactionTest and CompactionJobStatsTest
to run compaction-related tests once with subcompactions enabled and
once disabled using the TEST_P test type in the Google Test suite.

Test Plan: ./db_test  ./db_compaction-test  ./compaction_job_stats_test

Reviewers: sdong, igor, anthony, yhchiang

Reviewed By: yhchiang

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D43443
This commit is contained in:
Ari Ekmekji 2015-08-04 22:19:07 -07:00
parent c45a57b41e
commit 5dc3e6881a
5 changed files with 113 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
db/compaction_job_stats_test.cc
View File

@ -92,7 +92,7 @@ class CompactionJobStatsTest : public testing::Test,
Env* env_; Env* env_;
DB* db_; DB* db_;
std::vector<ColumnFamilyHandle*> handles_; std::vector<ColumnFamilyHandle*> handles_;
bool subcompactions_enabled_; uint32_t num_subcompactions_;
Options last_options_; Options last_options_;
@ -103,8 +103,8 @@ class CompactionJobStatsTest : public testing::Test,
alternative_wal_dir_ = dbname_ + "/wal"; alternative_wal_dir_ = dbname_ + "/wal";
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
subcompactions_enabled_ = GetParam(); num_subcompactions_ = GetParam();
options.num_subcompactions = subcompactions_enabled_ ? 2 : 1; options.num_subcompactions = num_subcompactions_;
auto delete_options = options; auto delete_options = options;
delete_options.wal_dir = alternative_wal_dir_; delete_options.wal_dir = alternative_wal_dir_;
EXPECT_OK(DestroyDB(dbname_, delete_options)); EXPECT_OK(DestroyDB(dbname_, delete_options));
@ -127,6 +127,7 @@ class CompactionJobStatsTest : public testing::Test,
EXPECT_OK(DestroyDB(dbname_, options)); EXPECT_OK(DestroyDB(dbname_, options));
} }
// Required if inheriting from testing::WithParamInterface<>
static void SetUpTestCase() {} static void SetUpTestCase() {}
static void TearDownTestCase() {} static void TearDownTestCase() {}
@ -641,7 +642,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
options.level0_file_num_compaction_trigger = kTestScale + 1; options.level0_file_num_compaction_trigger = kTestScale + 1;
options.num_levels = 3; options.num_levels = 3;
options.compression = kNoCompression; options.compression = kNoCompression;
options.num_subcompactions = subcompactions_enabled_ ? 2 : 1; options.num_subcompactions = num_subcompactions_;
for (int test = 0; test < 2; ++test) { for (int test = 0; test < 2; ++test) {
DestroyAndReopen(options); DestroyAndReopen(options);
@ -807,7 +808,7 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
options.num_levels = 3; options.num_levels = 3;
options.compression = kNoCompression; options.compression = kNoCompression;
options.max_bytes_for_level_multiplier = 2; options.max_bytes_for_level_multiplier = 2;
options.num_subcompactions = subcompactions_enabled_ ? 2 : 1; options.num_subcompactions = num_subcompactions_;
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -899,7 +900,7 @@ TEST_P(CompactionJobStatsTest, UniversalCompactionTest) {
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 1; options.compaction_options_universal.size_ratio = 1;
options.compaction_options_universal.max_size_amplification_percent = 1000; options.compaction_options_universal.max_size_amplification_percent = 1000;
options.num_subcompactions = subcompactions_enabled_ ? 2 : 1; options.num_subcompactions = num_subcompactions_;
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -949,7 +950,7 @@ TEST_P(CompactionJobStatsTest, UniversalCompactionTest) {
} }
INSTANTIATE_TEST_CASE_P(CompactionJobStatsTest, CompactionJobStatsTest, INSTANTIATE_TEST_CASE_P(CompactionJobStatsTest, CompactionJobStatsTest,
::testing::Bool()); ::testing::Values(1, 4));
} // namespace rocksdb } // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {

86
db/db_compaction_test.cc
View File

@ -18,6 +18,20 @@ class DBCompactionTest : public DBTestBase {
DBCompactionTest() : DBTestBase("/db_compaction_test") {} DBCompactionTest() : DBTestBase("/db_compaction_test") {}
}; };
class DBCompactionTestWithParam : public DBTestBase,
public testing::WithParamInterface<uint32_t> {
public:
DBCompactionTestWithParam() : DBTestBase("/db_compaction_test") {
num_subcompactions_ = GetParam();
}
// Required if inheriting from testing::WithParamInterface<>
static void SetUpTestCase() {}
static void TearDownTestCase() {}
uint32_t num_subcompactions_;
};
namespace { namespace {
class OnFileDeletionListener : public EventListener { class OnFileDeletionListener : public EventListener {
public: public:
@ -195,10 +209,13 @@ const SstFileMetaData* PickFileRandomly(
} }
} // anonymous namespace } // anonymous namespace
TEST_F(DBCompactionTest, CompactionDeletionTrigger) { // All the TEST_P tests run once with sub_compactions disabled (i.e.
// options.num_subcompactions = 1) and once with it enabled
TEST_P(DBCompactionTestWithParam, CompactionDeletionTrigger) {
for (int tid = 0; tid < 3; ++tid) { for (int tid = 0; tid < 3; ++tid) {
uint64_t db_size[2]; uint64_t db_size[2];
Options options = CurrentOptions(DeletionTriggerOptions()); Options options = CurrentOptions(DeletionTriggerOptions());
options.num_subcompactions = num_subcompactions_;
if (tid == 1) { if (tid == 1) {
// the following only disable stats update in DB::Open() // the following only disable stats update in DB::Open()
@ -279,10 +296,12 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
ASSERT_GT(env_->random_file_open_counter_.load(), 5); ASSERT_GT(env_->random_file_open_counter_.load(), 5);
} }
TEST_F(DBCompactionTest, CompactionDeletionTriggerReopen) {
TEST_P(DBCompactionTestWithParam, CompactionDeletionTriggerReopen) {
for (int tid = 0; tid < 2; ++tid) { for (int tid = 0; tid < 2; ++tid) {
uint64_t db_size[3]; uint64_t db_size[3];
Options options = CurrentOptions(DeletionTriggerOptions()); Options options = CurrentOptions(DeletionTriggerOptions());
options.num_subcompactions = num_subcompactions_;
if (tid == 1) { if (tid == 1) {
// second pass with universal compaction // second pass with universal compaction
@ -392,11 +411,13 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
} }
} }
TEST_F(DBCompactionTest, CompactionTrigger) {
TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
Options options; Options options;
options.write_buffer_size = 100 << 10; // 100KB options.write_buffer_size = 100 << 10; // 100KB
options.num_levels = 3; options.num_levels = 3;
options.level0_file_num_compaction_trigger = 3; options.level0_file_num_compaction_trigger = 3;
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -426,9 +447,10 @@ TEST_F(DBCompactionTest, CompactionTrigger) {
ASSERT_EQ(NumTableFilesAtLevel(1, 1), 1); ASSERT_EQ(NumTableFilesAtLevel(1, 1), 1);
} }
TEST_F(DBCompactionTest, CompactionsGenerateMultipleFiles) { TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
Options options; Options options;
options.write_buffer_size = 100000000; // Large write buffer options.write_buffer_size = 100000000; // Large write buffer
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -506,7 +528,7 @@ TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
} while (ChangeOptions()); } while (ChangeOptions());
} }
TEST_F(DBCompactionTest, TrivialMoveOneFile) { TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
int32_t trivial_move = 0; int32_t trivial_move = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"DBImpl::BackgroundCompaction:TrivialMove", "DBImpl::BackgroundCompaction:TrivialMove",
@ -515,6 +537,7 @@ TEST_F(DBCompactionTest, TrivialMoveOneFile) {
Options options; Options options;
options.write_buffer_size = 100000000; options.write_buffer_size = 100000000;
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -559,7 +582,7 @@ TEST_F(DBCompactionTest, TrivialMoveOneFile) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }
TEST_F(DBCompactionTest, TrivialMoveNonOverlappingFiles) { TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
int32_t trivial_move = 0; int32_t trivial_move = 0;
int32_t non_trivial_move = 0; int32_t non_trivial_move = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
@ -573,6 +596,7 @@ TEST_F(DBCompactionTest, TrivialMoveNonOverlappingFiles) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.write_buffer_size = 10 * 1024 * 1024; options.write_buffer_size = 10 * 1024 * 1024;
options.num_subcompactions = num_subcompactions_;
DestroyAndReopen(options); DestroyAndReopen(options);
// non overlapping ranges // non overlapping ranges
@ -654,7 +678,7 @@ TEST_F(DBCompactionTest, TrivialMoveNonOverlappingFiles) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }
TEST_F(DBCompactionTest, TrivialMoveTargetLevel) { TEST_P(DBCompactionTestWithParam, TrivialMoveTargetLevel) {
int32_t trivial_move = 0; int32_t trivial_move = 0;
int32_t non_trivial_move = 0; int32_t non_trivial_move = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
@ -669,6 +693,7 @@ TEST_F(DBCompactionTest, TrivialMoveTargetLevel) {
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.write_buffer_size = 10 * 1024 * 1024; options.write_buffer_size = 10 * 1024 * 1024;
options.num_levels = 7; options.num_levels = 7;
options.num_subcompactions = num_subcompactions_;
DestroyAndReopen(options); DestroyAndReopen(options);
int32_t value_size = 10 * 1024; // 10 KB int32_t value_size = 10 * 1024; // 10 KB
@ -711,7 +736,7 @@ TEST_F(DBCompactionTest, TrivialMoveTargetLevel) {
} }
} }
TEST_F(DBCompactionTest, TrivialMoveToLastLevelWithFiles) { TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
int32_t trivial_move = 0; int32_t trivial_move = 0;
int32_t non_trivial_move = 0; int32_t non_trivial_move = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
@ -724,6 +749,7 @@ TEST_F(DBCompactionTest, TrivialMoveToLastLevelWithFiles) {
Options options; Options options;
options.write_buffer_size = 100000000; options.write_buffer_size = 100000000;
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -769,7 +795,7 @@ TEST_F(DBCompactionTest, TrivialMoveToLastLevelWithFiles) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }
TEST_F(DBCompactionTest, LevelCompactionThirdPath) { TEST_P(DBCompactionTestWithParam, LevelCompactionThirdPath) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.db_paths.emplace_back(dbname_, 500 * 1024); options.db_paths.emplace_back(dbname_, 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
@ -779,6 +805,7 @@ TEST_F(DBCompactionTest, LevelCompactionThirdPath) {
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = 4; options.num_levels = 4;
options.max_bytes_for_level_base = 400 * 1024; options.max_bytes_for_level_base = 400 * 1024;
options.num_subcompactions = num_subcompactions_;
// options = CurrentOptions(options); // options = CurrentOptions(options);
std::vector<std::string> filenames; std::vector<std::string> filenames;
@ -882,7 +909,7 @@ TEST_F(DBCompactionTest, LevelCompactionThirdPath) {
Destroy(options); Destroy(options);
} }
TEST_F(DBCompactionTest, LevelCompactionPathUse) { TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.db_paths.emplace_back(dbname_, 500 * 1024); options.db_paths.emplace_back(dbname_, 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
@ -892,6 +919,7 @@ TEST_F(DBCompactionTest, LevelCompactionPathUse) {
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = 4; options.num_levels = 4;
options.max_bytes_for_level_base = 400 * 1024; options.max_bytes_for_level_base = 400 * 1024;
options.num_subcompactions = num_subcompactions_;
// options = CurrentOptions(options); // options = CurrentOptions(options);
std::vector<std::string> filenames; std::vector<std::string> filenames;
@ -996,7 +1024,7 @@ TEST_F(DBCompactionTest, LevelCompactionPathUse) {
Destroy(options); Destroy(options);
} }
TEST_F(DBCompactionTest, ConvertCompactionStyle) { TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
Random rnd(301); Random rnd(301);
int max_key_level_insert = 200; int max_key_level_insert = 200;
int max_key_universal_insert = 600; int max_key_universal_insert = 600;
@ -1010,6 +1038,7 @@ TEST_F(DBCompactionTest, ConvertCompactionStyle) {
options.max_bytes_for_level_multiplier = 1; options.max_bytes_for_level_multiplier = 1;
options.target_file_size_base = 200 << 10; // 200KB options.target_file_size_base = 200 << 10; // 200KB
options.target_file_size_multiplier = 1; options.target_file_size_multiplier = 1;
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -1145,8 +1174,9 @@ TEST_F(DBCompactionTest, L0_CompactionBug_Issue44_b) {
} while (ChangeCompactOptions()); } while (ChangeCompactOptions());
} }
TEST_F(DBCompactionTest, ManualCompaction) { TEST_P(DBCompactionTestWithParam, ManualCompaction) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.num_subcompactions = num_subcompactions_;
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// iter - 0 with 7 levels // iter - 0 with 7 levels
@ -1193,11 +1223,12 @@ TEST_F(DBCompactionTest, ManualCompaction) {
} }
TEST_F(DBCompactionTest, ManualLevelCompactionOutputPathId) { TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760); options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760);
options.db_paths.emplace_back(dbname_ + "_3", 100 * 10485760); options.db_paths.emplace_back(dbname_ + "_3", 100 * 10485760);
options.db_paths.emplace_back(dbname_ + "_4", 120 * 10485760); options.db_paths.emplace_back(dbname_ + "_4", 120 * 10485760);
options.num_subcompactions = num_subcompactions_;
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// iter - 0 with 7 levels // iter - 0 with 7 levels
@ -1252,8 +1283,12 @@ TEST_F(DBCompactionTest, ManualLevelCompactionOutputPathId) {
CompactRangeOptions compact_options; CompactRangeOptions compact_options;
compact_options.target_path_id = 1; compact_options.target_path_id = 1;
db_->CompactRange(compact_options, handles_[1], nullptr, nullptr); db_->CompactRange(compact_options, handles_[1], nullptr, nullptr);
ASSERT_EQ("0,1", FilesPerLevel(1));
ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); int num_files = options.num_subcompactions > 1 ? 2 : 1;
std::string files_string = options.num_subcompactions > 1 ? "0,2" : "0,1";
ASSERT_EQ(files_string, FilesPerLevel(1));
ASSERT_EQ(num_files, GetSstFileCount(options.db_paths[1].path));
ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path)); ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path));
ASSERT_EQ(0, GetSstFileCount(dbname_)); ASSERT_EQ(0, GetSstFileCount(dbname_));
@ -1286,7 +1321,7 @@ TEST_F(DBCompactionTest, FilesDeletedAfterCompaction) {
} }
// Check level comapction with compact files // Check level comapction with compact files
TEST_F(DBCompactionTest, CompactFilesOnLevelCompaction) { TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
const int kTestKeySize = 16; const int kTestKeySize = 16;
const int kTestValueSize = 984; const int kTestValueSize = 984;
const int kEntrySize = kTestKeySize + kTestValueSize; const int kEntrySize = kTestKeySize + kTestValueSize;
@ -1300,6 +1335,7 @@ TEST_F(DBCompactionTest, CompactFilesOnLevelCompaction) {
options.level0_stop_writes_trigger = 2; options.level0_stop_writes_trigger = 2;
options.max_bytes_for_level_multiplier = 2; options.max_bytes_for_level_multiplier = 2;
options.compression = kNoCompression; options.compression = kNoCompression;
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -1341,7 +1377,7 @@ TEST_F(DBCompactionTest, CompactFilesOnLevelCompaction) {
} }
} }
TEST_F(DBCompactionTest, PartialCompactionFailure) { TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
Options options; Options options;
const int kKeySize = 16; const int kKeySize = 16;
const int kKvSize = 1000; const int kKvSize = 1000;
@ -1359,6 +1395,7 @@ TEST_F(DBCompactionTest, PartialCompactionFailure) {
options.target_file_size_base; options.target_file_size_base;
options.max_bytes_for_level_multiplier = 2; options.max_bytes_for_level_multiplier = 2;
options.compression = kNoCompression; options.compression = kNoCompression;
options.num_subcompactions = num_subcompactions_;
env_->SetBackgroundThreads(1, Env::HIGH); env_->SetBackgroundThreads(1, Env::HIGH);
env_->SetBackgroundThreads(1, Env::LOW); env_->SetBackgroundThreads(1, Env::LOW);
@ -1424,7 +1461,7 @@ TEST_F(DBCompactionTest, PartialCompactionFailure) {
} }
} }
TEST_F(DBCompactionTest, DeleteMovedFileAfterCompaction) { TEST_P(DBCompactionTestWithParam, DeleteMovedFileAfterCompaction) {
// iter 1 -- delete_obsolete_files_period_micros == 0 // iter 1 -- delete_obsolete_files_period_micros == 0
for (int iter = 0; iter < 2; ++iter) { for (int iter = 0; iter < 2; ++iter) {
// This test triggers move compaction and verifies that the file is not // This test triggers move compaction and verifies that the file is not
@ -1439,6 +1476,7 @@ TEST_F(DBCompactionTest, DeleteMovedFileAfterCompaction) {
2; // trigger compaction when we have 2 files 2; // trigger compaction when we have 2 files
OnFileDeletionListener* listener = new OnFileDeletionListener(); OnFileDeletionListener* listener = new OnFileDeletionListener();
options.listeners.emplace_back(listener); options.listeners.emplace_back(listener);
options.num_subcompactions = num_subcompactions_;
DestroyAndReopen(options); DestroyAndReopen(options);
Random rnd(301); Random rnd(301);
@ -1502,7 +1540,7 @@ TEST_F(DBCompactionTest, DeleteMovedFileAfterCompaction) {
} }
} }
TEST_F(DBCompactionTest, CompressLevelCompaction) { TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
if (!Zlib_Supported()) { if (!Zlib_Supported()) {
return; return;
} }
@ -1512,6 +1550,7 @@ TEST_F(DBCompactionTest, CompressLevelCompaction) {
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = 4; options.num_levels = 4;
options.max_bytes_for_level_base = 400 * 1024; options.max_bytes_for_level_base = 400 * 1024;
options.num_subcompactions = num_subcompactions_;
// First two levels have no compression, so that a trivial move between // First two levels have no compression, so that a trivial move between
// them will be allowed. Level 2 has Zlib compression so that a trivial // them will be allowed. Level 2 has Zlib compression so that a trivial
// move to level 3 will not be allowed // move to level 3 will not be allowed
@ -1615,7 +1654,7 @@ TEST_F(DBCompactionTest, CompressLevelCompaction) {
#if !(defined NDEBUG) || !defined(OS_WIN) #if !(defined NDEBUG) || !defined(OS_WIN)
// This tests for a bug that could cause two level0 compactions running // This tests for a bug that could cause two level0 compactions running
// concurrently // concurrently
TEST_F(DBCompactionTest, SuggestCompactRangeNoTwoLevel0Compactions) { TEST_P(DBCompactionTestWithParam, SuggestCompactRangeNoTwoLevel0Compactions) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.compaction_style = kCompactionStyleLevel; options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; options.write_buffer_size = 110 << 10;
@ -1626,6 +1665,7 @@ TEST_F(DBCompactionTest, SuggestCompactRangeNoTwoLevel0Compactions) {
options.target_file_size_base = 98 << 10; options.target_file_size_base = 98 << 10;
options.max_write_buffer_number = 2; options.max_write_buffer_number = 2;
options.max_background_compactions = 2; options.max_background_compactions = 2;
options.num_subcompactions = num_subcompactions_;
DestroyAndReopen(options); DestroyAndReopen(options);
@ -1670,7 +1710,7 @@ TEST_F(DBCompactionTest, SuggestCompactRangeNoTwoLevel0Compactions) {
#endif // !(defined NDEBUG) || !defined(OS_WIN) #endif // !(defined NDEBUG) || !defined(OS_WIN)
TEST_F(DBCompactionTest, ForceBottommostLevelCompaction) { TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
int32_t trivial_move = 0; int32_t trivial_move = 0;
int32_t non_trivial_move = 0; int32_t non_trivial_move = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
@ -1683,6 +1723,7 @@ TEST_F(DBCompactionTest, ForceBottommostLevelCompaction) {
Options options; Options options;
options.write_buffer_size = 100000000; options.write_buffer_size = 100000000;
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -1752,6 +1793,9 @@ TEST_F(DBCompactionTest, ForceBottommostLevelCompaction) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }
INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
::testing::Values(1, 4));
} // namespace rocksdb } // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {

40
db/db_test.cc
View File

@ -117,6 +117,20 @@ class DBTest : public DBTestBase {
DBTest() : DBTestBase("/db_test") {} DBTest() : DBTestBase("/db_test") {}
}; };
class DBTestWithParam : public DBTest,
public testing::WithParamInterface<uint32_t> {
public:
DBTestWithParam() {
num_subcompactions_ = GetParam();
}
// Required if inheriting from testing::WithParamInterface<>
static void SetUpTestCase() {}
static void TearDownTestCase() {}
uint32_t num_subcompactions_;
};
TEST_F(DBTest, Empty) { TEST_F(DBTest, Empty) {
do { do {
Options options; Options options;
@ -5613,7 +5627,7 @@ TEST_F(DBTest, ChecksumTest) {
ASSERT_EQ("h", Get("g")); ASSERT_EQ("h", Get("g"));
} }
TEST_F(DBTest, FIFOCompactionTest) { TEST_P(DBTestWithParam, FIFOCompactionTest) {
for (int iter = 0; iter < 2; ++iter) { for (int iter = 0; iter < 2; ++iter) {
// first iteration -- auto compaction // first iteration -- auto compaction
// second iteration -- manual compaction // second iteration -- manual compaction
@ -5623,6 +5637,7 @@ TEST_F(DBTest, FIFOCompactionTest) {
options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB
options.compression = kNoCompression; options.compression = kNoCompression;
options.create_if_missing = true; options.create_if_missing = true;
options.num_subcompactions = num_subcompactions_;
if (iter == 1) { if (iter == 1) {
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
} }
@ -6238,7 +6253,7 @@ TEST_F(DBTest, ThreadStatusFlush) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }
TEST_F(DBTest, ThreadStatusSingleCompaction) { TEST_P(DBTestWithParam, ThreadStatusSingleCompaction) {
const int kTestKeySize = 16; const int kTestKeySize = 16;
const int kTestValueSize = 984; const int kTestValueSize = 984;
const int kEntrySize = kTestKeySize + kTestValueSize; const int kEntrySize = kTestKeySize + kTestValueSize;
@ -6256,6 +6271,7 @@ TEST_F(DBTest, ThreadStatusSingleCompaction) {
options.enable_thread_tracking = true; options.enable_thread_tracking = true;
const int kNumL0Files = 4; const int kNumL0Files = 4;
options.level0_file_num_compaction_trigger = kNumL0Files; options.level0_file_num_compaction_trigger = kNumL0Files;
options.num_subcompactions = num_subcompactions_;
rocksdb::SyncPoint::GetInstance()->LoadDependency({ rocksdb::SyncPoint::GetInstance()->LoadDependency({
{"DBTest::ThreadStatusSingleCompaction:0", "DBImpl::BGWorkCompaction"}, {"DBTest::ThreadStatusSingleCompaction:0", "DBImpl::BGWorkCompaction"},
@ -6301,9 +6317,10 @@ TEST_F(DBTest, ThreadStatusSingleCompaction) {
} }
} }
TEST_F(DBTest, PreShutdownManualCompaction) { TEST_P(DBTestWithParam, PreShutdownManualCompaction) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.max_background_flushes = 0; options.max_background_flushes = 0;
options.num_subcompactions = num_subcompactions_;
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// iter - 0 with 7 levels // iter - 0 with 7 levels
@ -6361,7 +6378,7 @@ TEST_F(DBTest, PreShutdownFlush) {
ASSERT_TRUE(s.IsShutdownInProgress()); ASSERT_TRUE(s.IsShutdownInProgress());
} }
TEST_F(DBTest, PreShutdownMultipleCompaction) { TEST_P(DBTestWithParam, PreShutdownMultipleCompaction) {
const int kTestKeySize = 16; const int kTestKeySize = 16;
const int kTestValueSize = 984; const int kTestValueSize = 984;
const int kEntrySize = kTestKeySize + kTestValueSize; const int kEntrySize = kTestKeySize + kTestValueSize;
@ -6389,6 +6406,7 @@ TEST_F(DBTest, PreShutdownMultipleCompaction) {
options.max_background_compactions = kLowPriCount; options.max_background_compactions = kLowPriCount;
options.level0_stop_writes_trigger = 1 << 10; options.level0_stop_writes_trigger = 1 << 10;
options.level0_slowdown_writes_trigger = 1 << 10; options.level0_slowdown_writes_trigger = 1 << 10;
options.num_subcompactions = num_subcompactions_;
TryReopen(options); TryReopen(options);
Random rnd(301); Random rnd(301);
@ -6449,7 +6467,7 @@ TEST_F(DBTest, PreShutdownMultipleCompaction) {
ASSERT_EQ(operation_count[ThreadStatus::OP_COMPACTION], 0); ASSERT_EQ(operation_count[ThreadStatus::OP_COMPACTION], 0);
} }
TEST_F(DBTest, PreShutdownCompactionMiddle) { TEST_P(DBTestWithParam, PreShutdownCompactionMiddle) {
const int kTestKeySize = 16; const int kTestKeySize = 16;
const int kTestValueSize = 984; const int kTestValueSize = 984;
const int kEntrySize = kTestKeySize + kTestValueSize; const int kEntrySize = kTestKeySize + kTestValueSize;
@ -6477,6 +6495,7 @@ TEST_F(DBTest, PreShutdownCompactionMiddle) {
options.max_background_compactions = kLowPriCount; options.max_background_compactions = kLowPriCount;
options.level0_stop_writes_trigger = 1 << 10; options.level0_stop_writes_trigger = 1 << 10;
options.level0_slowdown_writes_trigger = 1 << 10; options.level0_slowdown_writes_trigger = 1 << 10;
options.num_subcompactions = num_subcompactions_;
TryReopen(options); TryReopen(options);
Random rnd(301); Random rnd(301);
@ -6776,7 +6795,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
ASSERT_GT(num_zlib.load(), 0); ASSERT_GT(num_zlib.load(), 0);
} }
TEST_F(DBTest, DynamicCompactionOptions) { TEST_P(DBTestWithParam, DynamicCompactionOptions) {
// minimum write buffer size is enforced at 64KB // minimum write buffer size is enforced at 64KB
const uint64_t k32KB = 1 << 15; const uint64_t k32KB = 1 << 15;
const uint64_t k64KB = 1 << 16; const uint64_t k64KB = 1 << 16;
@ -6801,6 +6820,7 @@ TEST_F(DBTest, DynamicCompactionOptions) {
options.target_file_size_multiplier = 1; options.target_file_size_multiplier = 1;
options.max_bytes_for_level_base = k128KB; options.max_bytes_for_level_base = k128KB;
options.max_bytes_for_level_multiplier = 4; options.max_bytes_for_level_multiplier = 4;
options.num_subcompactions = num_subcompactions_;
// Block flush thread and disable compaction thread // Block flush thread and disable compaction thread
env_->SetBackgroundThreads(1, Env::LOW); env_->SetBackgroundThreads(1, Env::LOW);
@ -7524,7 +7544,7 @@ TEST_F(DBTest, MergeTestTime) {
ASSERT_GT(TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME), 3200000); ASSERT_GT(TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME), 3200000);
} }
TEST_F(DBTest, MergeCompactionTimeTest) { TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
SetPerfLevel(kEnableTime); SetPerfLevel(kEnableTime);
Options options; Options options;
options = CurrentOptions(options); options = CurrentOptions(options);
@ -7532,6 +7552,7 @@ TEST_F(DBTest, MergeCompactionTimeTest) {
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options.merge_operator.reset(new DelayedMergeOperator(this)); options.merge_operator.reset(new DelayedMergeOperator(this));
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.num_subcompactions = num_subcompactions_;
DestroyAndReopen(options); DestroyAndReopen(options);
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
@ -7544,13 +7565,14 @@ TEST_F(DBTest, MergeCompactionTimeTest) {
ASSERT_NE(TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME), 0); ASSERT_NE(TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME), 0);
} }
TEST_F(DBTest, FilterCompactionTimeTest) { TEST_P(DBTestWithParam, FilterCompactionTimeTest) {
Options options; Options options;
options.compaction_filter_factory = options.compaction_filter_factory =
std::make_shared<DelayFilterFactory>(this); std::make_shared<DelayFilterFactory>(this);
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.create_if_missing = true; options.create_if_missing = true;
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options.num_subcompactions = num_subcompactions_;
options = CurrentOptions(options); options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
@ -8396,6 +8418,8 @@ TEST_F(DBTest, DeleteSchedulerMultipleDBPaths) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }
INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam,
::testing::Values(1, 4));
} // namespace rocksdb } // namespace rocksdb
#endif #endif

2
include/rocksdb/options.h
View File

@ -900,7 +900,7 @@ struct DBOptions {
// into multiple, smaller ones that are run simultaneously. This is still // into multiple, smaller ones that are run simultaneously. This is still
// under development and is only available for level-based compaction. // under development and is only available for level-based compaction.
// Default: 1 // Default: 1
int num_subcompactions; uint32_t num_subcompactions;
// Maximum number of concurrent background memtable flush jobs, submitted to // Maximum number of concurrent background memtable flush jobs, submitted to
// the HIGH priority thread pool. // the HIGH priority thread pool.

7
util/db_test_util.cc
View File

@ -148,6 +148,13 @@ bool DBTestBase::ChangeCompactOptions() {
options.create_if_missing = true; options.create_if_missing = true;
TryReopen(options); TryReopen(options);
return true; return true;
} else if (option_config_ == kUniversalCompactionMultiLevel) {
option_config_ = kLevelSubcompactions;
Destroy(last_options_);
auto options = CurrentOptions();
options.num_subcompactions = 4;
TryReopen(options);
return true;
} else { } else {
return false; return false;
} }