andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix caching of compaction picker's next index

Browse Source 
Summary:
The previous implementation of caching `file_size` index made no sense. It only remembered the original span of locked files starting from beginning of `file_size`. We should remember the index after all compactions that have been considered but rejected. This will reduce the work we do while holding the db mutex.
Closes https://github.com/facebook/rocksdb/pull/2624

Differential Revision: D5468152

Pulled By: ajkr

fbshipit-source-id: ab92a4bffe76f9f174d861bb5812b974d1013400
This commit is contained in:
Andrew Kryczka 2017-07-21 20:56:32 -07:00 committed by Facebook Github Bot
parent 72502cf227
commit a34b2e388e
2 changed files with 48 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
db/compaction_picker.cc
View File

@ -1323,12 +1323,10 @@ bool LevelCompactionBuilder::PickFileToCompact() {
const std::vector<FileMetaData*>& level_files =
vstorage_->LevelFiles(start_level_);
// record the first file that is not yet compacted
int nextIndex = -1;
for (unsigned int i = vstorage_->NextCompactionIndex(start_level_);
i < file_size.size(); i++) {
int index = file_size[i];
unsigned int cmp_idx;
for (cmp_idx = vstorage_->NextCompactionIndex(start_level_);
cmp_idx < file_size.size(); cmp_idx++) {
int index = file_size[cmp_idx];
auto* f = level_files[index];
// do not pick a file to compact if it is being compacted
@ -1337,11 +1335,6 @@ bool LevelCompactionBuilder::PickFileToCompact() {
continue;
}
// remember the startIndex for the next call to PickCompaction
if (nextIndex == -1) {
nextIndex = i;
}
start_level_inputs_.files.push_back(f);
start_level_inputs_.level = start_level_;
if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
@ -1377,7 +1370,7 @@ bool LevelCompactionBuilder::PickFileToCompact() {
}
// store where to start the iteration in the next call to PickCompaction
vstorage_->SetNextCompactionIndex(start_level_, nextIndex);
vstorage_->SetNextCompactionIndex(start_level_, cmp_idx);
return start_level_inputs_.size() > 0;
}

43
db/compaction_picker_test.cc
View File

@ -1390,6 +1390,49 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
ASSERT_FALSE(compaction->IsTrivialMove());
}
TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.max_compaction_bytes = 100000000000u;
Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
"149" /* largest */, 1000000000U /* file_size */);
file_map_[1U].first->being_compacted = true;
Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
"199" /* largest */, 900000000U /* file_size */);
Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
"249" /* largest */, 800000000U /* file_size */);
Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
"299" /* largest */, 700000000U /* file_size */);
Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
"199" /* largest */, 1U /* file_size */);
file_map_[5U].first->being_compacted = true;
UpdateVersionStorageInfo();
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_levels());
ASSERT_EQ(1U, compaction->num_input_files(0));
ASSERT_EQ(0U, compaction->num_input_files(1));
ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
compaction.reset(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_levels());
ASSERT_EQ(1U, compaction->num_input_files(0));
ASSERT_EQ(0U, compaction->num_input_files(1));
ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
compaction.reset(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() == nullptr);
ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
}
} // namespace rocksdb
int main(int argc, char** argv) {