andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Restrict L0->L0 compaction according to max_compaction_bytes option (#5329)

Browse Source 
Summary:
Modified FindIntraL0Compaction to stop picking more files if total
amount of compensated bytes would be larger than max_compaction_bytes
option.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5329

Differential Revision: D15435728

Pulled By: ThomasFersch

fbshipit-source-id: d118a6da88d5df8ee20944422ade37cf6b15d60c
This commit is contained in:
Thomas Fersch 2019-05-22 23:38:09 -07:00 committed by Facebook Github Bot
parent 518cd1a62a
commit 3d9d77d900
4 changed files with 89 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
db/compaction_picker.cc
View File

@ -42,19 +42,23 @@ uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files) {
bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
size_t min_files_to_compact,
uint64_t max_compact_bytes_per_del_file,
uint64_t max_compaction_bytes,
CompactionInputFiles* comp_inputs) {
size_t compact_bytes = static_cast<size_t>(level_files[0]->fd.file_size);
uint64_t compensated_compact_bytes = level_files[0]->compensated_file_size;
size_t compact_bytes_per_del_file = port::kMaxSizet;
// compaction range will be [0, span_len).
// Compaction range will be [0, span_len).
size_t span_len;
// pull in files until the amount of compaction work per deleted file begins
// increasing.
// Pull in files until the amount of compaction work per deleted file begins
// increasing or maximum total compaction size is reached.
size_t new_compact_bytes_per_del_file = 0;
for (span_len = 1; span_len < level_files.size(); ++span_len) {
compact_bytes += static_cast<size_t>(level_files[span_len]->fd.file_size);
compensated_compact_bytes += level_files[span_len]->compensated_file_size;
new_compact_bytes_per_del_file = compact_bytes / span_len;
if (level_files[span_len]->being_compacted ||
new_compact_bytes_per_del_file > compact_bytes_per_del_file) {
new_compact_bytes_per_del_file > compact_bytes_per_del_file ||
compensated_compact_bytes > max_compaction_bytes) {
break;
}
compact_bytes_per_del_file = new_compact_bytes_per_del_file;
@ -1627,7 +1631,9 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() {
return false;
}
return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction,
port::kMaxUint64, &start_level_inputs_);
port::kMaxUint64,
mutable_cf_options_.max_compaction_bytes,
&start_level_inputs_);
}
} // namespace

17
db/compaction_picker.h
View File

@ -273,9 +273,26 @@ class NullCompactionPicker : public CompactionPicker {
};
#endif // !ROCKSDB_LITE
// Attempts to find an intra L0 compaction conforming to the given parameters.
//
// @param level_files Metadata for L0 files.
// @param min_files_to_compact Minimum number of files required to
// do the compaction.
// @param max_compact_bytes_per_del_file Maximum average size in bytes per
// file that is going to get deleted by
// the compaction.
// @param max_compaction_bytes Maximum total size in bytes (in terms
// of compensated file size) for files
// to be compacted.
// @param [out] comp_inputs If a compaction was found, will be
// initialized with corresponding input
// files. Cannot be nullptr.
//
// @return true iff compaction was found.
bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
size_t min_files_to_compact,
uint64_t max_compact_bytes_per_del_file,
uint64_t max_compaction_bytes,
CompactionInputFiles* comp_inputs);
CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,

3
db/compaction_picker_fifo.cc
View File

@ -134,7 +134,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
mutable_cf_options
.level0_file_num_compaction_trigger /* min_files_to_compact */
,
max_compact_bytes_per_del_file, &comp_inputs)) {
max_compact_bytes_per_del_file,
mutable_cf_options.max_compaction_bytes, &comp_inputs)) {
Compaction* c = new Compaction(
vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0,
16 * 1024 * 1024 /* output file size limit */,

59
db/compaction_picker_test.cc
View File

@ -1478,6 +1478,65 @@ TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
}
TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesNotHit) {
// Intra L0 compaction triggers only if there are at least
// level0_file_num_compaction_trigger + 2 L0 files.
mutable_cf_options_.level0_file_num_compaction_trigger = 3;
mutable_cf_options_.max_compaction_bytes = 1000000u;
NewVersionStorage(6, kCompactionStyleLevel);
// All 5 L0 files will be picked for intra L0 compaction. The one L1 file
// spans entire L0 key range and is marked as being compacted to avoid
// L0->L1 compaction.
Add(0, 1U, "100", "150", 200000U);
Add(0, 2U, "151", "200", 200000U);
Add(0, 3U, "201", "250", 200000U);
Add(0, 4U, "251", "300", 200000U);
Add(0, 5U, "301", "350", 200000U);
Add(1, 6U, "100", "350", 200000U);
vstorage_->LevelFiles(1)[0]->being_compacted = true;
UpdateVersionStorageInfo();
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_levels());
ASSERT_EQ(5U, compaction->num_input_files(0));
ASSERT_EQ(CompactionReason::kLevelL0FilesNum,
compaction->compaction_reason());
ASSERT_EQ(0U, compaction->output_level());
}
TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesHit) {
// Intra L0 compaction triggers only if there are at least
// level0_file_num_compaction_trigger + 2 L0 files.
mutable_cf_options_.level0_file_num_compaction_trigger = 3;
mutable_cf_options_.max_compaction_bytes = 999999u;
NewVersionStorage(6, kCompactionStyleLevel);
// 4 out of 5 L0 files will be picked for intra L0 compaction due to
// max_compaction_bytes limit (the minimum number of files for triggering
// intra L0 compaction is 4). The one L1 file spans entire L0 key range and
// is marked as being compacted to avoid L0->L1 compaction.
Add(0, 1U, "100", "150", 200000U);
Add(0, 2U, "151", "200", 200000U);
Add(0, 3U, "201", "250", 200000U);
Add(0, 4U, "251", "300", 200000U);
Add(0, 5U, "301", "350", 200000U);
Add(1, 6U, "100", "350", 200000U);
vstorage_->LevelFiles(1)[0]->being_compacted = true;
UpdateVersionStorageInfo();
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
ASSERT_TRUE(compaction.get() != nullptr);
ASSERT_EQ(1U, compaction->num_input_levels());
ASSERT_EQ(4U, compaction->num_input_files(0));
ASSERT_EQ(CompactionReason::kLevelL0FilesNum,
compaction->compaction_reason());
ASSERT_EQ(0U, compaction->output_level());
}
} // namespace rocksdb
int main(int argc, char** argv) {