andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve fallocate size in compaction output

Browse Source 
Summary:
Now in leveled compaction, we allocate solely based on output target file size. If the total input size is smaller than the number, we should use the total input size instead. Also, cap the allocate size to 1GB.
Closes https://github.com/facebook/rocksdb/pull/3385

Differential Revision: D6762363

Pulled By: siying

fbshipit-source-id: e30906f6e9bff3ec847d2166e44cb49c92f98a13
This commit is contained in:
Siying Dong 2018-01-22 16:36:00 -08:00 committed by Facebook Github Bot
parent c615689bb5
commit 7291a3f813
2 changed files with 24 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
db/compaction.cc
View File

@ -405,20 +405,23 @@ void Compaction::Summary(char* output, int len) {
uint64_t Compaction::OutputFilePreallocationSize() const { uint64_t Compaction::OutputFilePreallocationSize() const {
uint64_t preallocation_size = 0; uint64_t preallocation_size = 0;
if (max_output_file_size_ != port::kMaxUint64 && for (const auto& level_files : inputs_) {
(cfd_->ioptions()->compaction_style == kCompactionStyleLevel || for (const auto& file : level_files.files) {
output_level() > 0)) { preallocation_size += file->fd.GetFileSize();
preallocation_size = max_output_file_size_;
} else {
for (const auto& level_files : inputs_) {
for (const auto& file : level_files.files) {
preallocation_size += file->fd.GetFileSize();
}
} }
} }
if (max_output_file_size_ != port::kMaxUint64 &&
(immutable_cf_options_.compaction_style == kCompactionStyleLevel ||
output_level() > 0)) {
preallocation_size = std::min(max_output_file_size_, preallocation_size);
}
// Over-estimate slightly so we don't end up just barely crossing // Over-estimate slightly so we don't end up just barely crossing
// the threshold // the threshold
return preallocation_size + (preallocation_size / 10); // No point to prellocate more than 1GB.
return std::min(uint64_t{1073741824},
preallocation_size + (preallocation_size / 10));
} }
std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const { std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {

13
db/compaction_picker_test.cc
View File

@ -175,6 +175,8 @@ TEST_F(CompactionPickerTest, Level1Trigger) {
} }
TEST_F(CompactionPickerTest, Level1Trigger2) { TEST_F(CompactionPickerTest, Level1Trigger2) {
mutable_cf_options_.target_file_size_base = 10000000000;
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
NewVersionStorage(6, kCompactionStyleLevel); NewVersionStorage(6, kCompactionStyleLevel);
Add(1, 66U, "150", "200", 1000000001U); Add(1, 66U, "150", "200", 1000000001U);
Add(1, 88U, "201", "300", 1000000000U); Add(1, 88U, "201", "300", 1000000000U);
@ -191,13 +193,14 @@ TEST_F(CompactionPickerTest, Level1Trigger2) {
ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize());
} }
TEST_F(CompactionPickerTest, LevelMaxScore) { TEST_F(CompactionPickerTest, LevelMaxScore) {
NewVersionStorage(6, kCompactionStyleLevel); NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.target_file_size_base = 10000000;
mutable_cf_options_.target_file_size_multiplier = 10;
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
Add(0, 1U, "150", "200", 1000000U); Add(0, 1U, "150", "200", 1000000U);
// Level 1 score 1.2 // Level 1 score 1.2
Add(1, 66U, "150", "200", 6000000U); Add(1, 66U, "150", "200", 6000000U);
@ -218,6 +221,9 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
ASSERT_TRUE(compaction.get() != nullptr); ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_files(0)); ASSERT_EQ(1U, compaction->num_input_files(0));
ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
ASSERT_EQ(mutable_cf_options_.target_file_size_base +
mutable_cf_options_.target_file_size_base / 10,
compaction->OutputFilePreallocationSize());
} }
TEST_F(CompactionPickerTest, NeedsCompactionLevel) { TEST_F(CompactionPickerTest, NeedsCompactionLevel) {
@ -521,9 +527,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
NewVersionStorage(6, kCompactionStyleLevel); NewVersionStorage(6, kCompactionStyleLevel);
ioptions_.compaction_pri = kMinOverlappingRatio; ioptions_.compaction_pri = kMinOverlappingRatio;
mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.target_file_size_base = 100000000000;
mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.target_file_size_multiplier = 10;
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
Add(2, 6U, "150", "179", 50000000U); Add(2, 6U, "150", "179", 50000000U);
Add(2, 7U, "180", "220", 50000000U); Add(2, 7U, "180", "220", 50000000U);
@ -543,6 +550,8 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
ASSERT_EQ(1U, compaction->num_input_files(0)); ASSERT_EQ(1U, compaction->num_input_files(0));
// Pick file 8 because it overlaps with 0 files on level 3. // Pick file 8 because it overlaps with 0 files on level 3.
ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
// Compaction input size * 1.1
ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize());
} }
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {