From 7291a3f813e563efbd6870465b1063a115480373 Mon Sep 17 00:00:00 2001 From: Siying Dong Date: Mon, 22 Jan 2018 16:36:00 -0800 Subject: [PATCH] Improve fallocate size in compaction output Summary: Now in leveled compaction, we allocate solely based on output target file size. If the total input size is smaller than the number, we should use the total input size instead. Also, cap the allocate size to 1GB. Closes https://github.com/facebook/rocksdb/pull/3385 Differential Revision: D6762363 Pulled By: siying fbshipit-source-id: e30906f6e9bff3ec847d2166e44cb49c92f98a13 --- db/compaction.cc | 23 +++++++++++++---------- db/compaction_picker_test.cc | 13 +++++++++++-- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/db/compaction.cc b/db/compaction.cc index 6126b4642..c2785adee 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -405,20 +405,23 @@ void Compaction::Summary(char* output, int len) { uint64_t Compaction::OutputFilePreallocationSize() const { uint64_t preallocation_size = 0; - if (max_output_file_size_ != port::kMaxUint64 && - (cfd_->ioptions()->compaction_style == kCompactionStyleLevel || - output_level() > 0)) { - preallocation_size = max_output_file_size_; - } else { - for (const auto& level_files : inputs_) { - for (const auto& file : level_files.files) { - preallocation_size += file->fd.GetFileSize(); - } + for (const auto& level_files : inputs_) { + for (const auto& file : level_files.files) { + preallocation_size += file->fd.GetFileSize(); } } + + if (max_output_file_size_ != port::kMaxUint64 && + (immutable_cf_options_.compaction_style == kCompactionStyleLevel || + output_level() > 0)) { + preallocation_size = std::min(max_output_file_size_, preallocation_size); + } + // Over-estimate slightly so we don't end up just barely crossing // the threshold - return preallocation_size + (preallocation_size / 10); + // No point to prellocate more than 1GB. + return std::min(uint64_t{1073741824}, + preallocation_size + (preallocation_size / 10)); } std::unique_ptr Compaction::CreateCompactionFilter() const { diff --git a/db/compaction_picker_test.cc b/db/compaction_picker_test.cc index 069308bcc..4752d3428 100644 --- a/db/compaction_picker_test.cc +++ b/db/compaction_picker_test.cc @@ -175,6 +175,8 @@ TEST_F(CompactionPickerTest, Level1Trigger) { } TEST_F(CompactionPickerTest, Level1Trigger2) { + mutable_cf_options_.target_file_size_base = 10000000000; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); NewVersionStorage(6, kCompactionStyleLevel); Add(1, 66U, "150", "200", 1000000001U); Add(1, 88U, "201", "300", 1000000000U); @@ -191,13 +193,14 @@ TEST_F(CompactionPickerTest, Level1Trigger2) { ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize()); } TEST_F(CompactionPickerTest, LevelMaxScore) { NewVersionStorage(6, kCompactionStyleLevel); mutable_cf_options_.target_file_size_base = 10000000; - mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); Add(0, 1U, "150", "200", 1000000U); // Level 1 score 1.2 Add(1, 66U, "150", "200", 6000000U); @@ -218,6 +221,9 @@ TEST_F(CompactionPickerTest, LevelMaxScore) { ASSERT_TRUE(compaction.get() != nullptr); ASSERT_EQ(1U, compaction->num_input_files(0)); ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(mutable_cf_options_.target_file_size_base + + mutable_cf_options_.target_file_size_base / 10, + compaction->OutputFilePreallocationSize()); } TEST_F(CompactionPickerTest, NeedsCompactionLevel) { @@ -521,9 +527,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { NewVersionStorage(6, kCompactionStyleLevel); ioptions_.compaction_pri = kMinOverlappingRatio; - mutable_cf_options_.target_file_size_base = 10000000; + mutable_cf_options_.target_file_size_base = 100000000000; mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); Add(2, 6U, "150", "179", 50000000U); Add(2, 7U, "180", "220", 50000000U); @@ -543,6 +550,8 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { ASSERT_EQ(1U, compaction->num_input_files(0)); // Pick file 8 because it overlaps with 0 files on level 3. ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); + // Compaction input size * 1.1 + ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize()); } TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {