Report compaction reason in CompactionListener

Summary:
Add CompactionReason to CompactionJobInfo
This will allow users to understand why compaction started which will help options tuning

Test Plan:
added new tests
make check -j64

Reviewers: yhchiang, anthony, kradhakrishnan, sdong, rven

Reviewed By: rven

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D51975
This commit is contained in:
Islam AbdelRahman 2015-12-22 11:37:19 -08:00
parent 8ac7fb8377
commit d005c66faf
7 changed files with 225 additions and 7 deletions

View File

@ -5,6 +5,7 @@
* Change names in CompactionPri and add a new one.
* Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit.
* If options.max_write_buffer_number > 3, writes will be slowed down when writing to the last write buffer to delay a full stop.
* Introduce CompactionJobInfo::compaction_reason, this field include the reason to trigger the compaction.
## 4.3.0 (12/8/2015)
### New Features

View File

@ -147,7 +147,8 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
uint32_t _output_path_id, CompressionType _compression,
std::vector<FileMetaData*> _grandparents,
bool _manual_compaction, double _score,
bool _deletion_compaction)
bool _deletion_compaction,
CompactionReason _compaction_reason)
: start_level_(_inputs[0].level),
output_level_(_output_level),
max_output_file_size_(_target_file_size),
@ -167,8 +168,12 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
score_(_score),
bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
is_manual_compaction_(_manual_compaction) {
is_manual_compaction_(_manual_compaction),
compaction_reason_(_compaction_reason) {
MarkFilesBeingCompacted(true);
if (is_manual_compaction_) {
compaction_reason_ = CompactionReason::kManualCompaction;
}
#ifndef NDEBUG
for (size_t i = 1; i < inputs_.size(); ++i) {

View File

@ -41,7 +41,8 @@ class Compaction {
uint32_t output_path_id, CompressionType compression,
std::vector<FileMetaData*> grandparents,
bool manual_compaction = false, double score = -1,
bool deletion_compaction = false);
bool deletion_compaction = false,
CompactionReason compaction_reason = CompactionReason::kUnknown);
// No copying allowed
Compaction(const Compaction&) = delete;
@ -220,6 +221,8 @@ class Compaction {
Slice GetLargestUserKey() const { return largest_user_key_; }
CompactionReason compaction_reason() { return compaction_reason_; }
private:
// mark (or clear) all files that are being compacted
void MarkFilesBeingCompacted(bool mark_as_compacted);
@ -289,6 +292,9 @@ class Compaction {
// largest user keys in compaction
Slice largest_user_key_;
// Reason for compaction
CompactionReason compaction_reason_;
};
// Utility function

View File

@ -916,6 +916,7 @@ Compaction* LevelCompactionPicker::PickCompaction(
int base_index = -1;
CompactionInputFiles inputs;
double score = 0;
CompactionReason compaction_reason = CompactionReason::kUnknown;
// Find the compactions by size on all levels.
for (int i = 0; i < NumberLevels() - 1; i++) {
@ -928,6 +929,13 @@ Compaction* LevelCompactionPicker::PickCompaction(
&parent_index, &base_index) &&
ExpandWhileOverlapping(cf_name, vstorage, &inputs)) {
// found the compaction!
if (level == 0) {
// L0 score = `num L0 files` / `level0_file_num_compaction_trigger`
compaction_reason = CompactionReason::kLevelL0FilesNum;
} else {
// L1+ score = `Level files size` / `MaxBytesForLevel`
compaction_reason = CompactionReason::kLevelMaxLevelSize;
}
break;
} else {
// didn't find the compaction, clear the inputs
@ -944,6 +952,9 @@ Compaction* LevelCompactionPicker::PickCompaction(
parent_index = base_index = -1;
PickFilesMarkedForCompactionExperimental(cf_name, vstorage, &inputs, &level,
&output_level);
if (!inputs.empty()) {
compaction_reason = CompactionReason::kFilesMarkedForCompaction;
}
}
if (inputs.empty()) {
return nullptr;
@ -994,7 +1005,8 @@ Compaction* LevelCompactionPicker::PickCompaction(
mutable_cf_options.MaxGrandParentOverlapBytes(level),
GetPathId(ioptions_, mutable_cf_options, output_level),
GetCompressionType(ioptions_, output_level, vstorage->base_level()),
std::move(grandparents), is_manual, score);
std::move(grandparents), is_manual, score,
false /* deletion_compaction */, compaction_reason);
// If it's level 0 compaction, make sure we don't execute any other level 0
// compactions in parallel
@ -1595,11 +1607,18 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
file_num_buf);
}
CompactionReason compaction_reason;
if (max_number_of_files_to_compact == UINT_MAX) {
compaction_reason = CompactionReason::kUniversalSortedRunNum;
} else {
compaction_reason = CompactionReason::kUniversalSizeRatio;
}
return new Compaction(
vstorage, mutable_cf_options, std::move(inputs), output_level,
mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
GetCompressionType(ioptions_, start_level, 1, enable_compression),
/* grandparents */ {}, /* is manual */ false, score);
/* grandparents */ {}, /* is manual */ false, score,
false /* deletion_compaction */, compaction_reason);
}
// Look at overall size amplification. If size amplification
@ -1723,7 +1742,9 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1),
/* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
GetCompressionType(ioptions_, vstorage->num_levels() - 1, 1),
/* grandparents */ {}, /* is manual */ false, score);
/* grandparents */ {}, /* is manual */ false, score,
false /* deletion_compaction */,
CompactionReason::kUniversalSizeAmplification);
}
bool FIFOCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage)
@ -1782,7 +1803,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
Compaction* c = new Compaction(
vstorage, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
/* is deletion compaction */ true);
/* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
level0_compactions_in_progress_.insert(c);
return c;
}

View File

@ -1897,6 +1897,7 @@ void DBImpl::NotifyOnCompactionCompleted(
info.output_level = c->output_level();
info.stats = compaction_job_stats;
info.table_properties = c->GetOutputTableProperties();
info.compaction_reason = c->compaction_reason();
for (size_t i = 0; i < c->num_input_levels(); ++i) {
for (const auto fmd : *c->inputs(i)) {
auto fn = TableFileName(db_options_.db_paths, fmd->fd.GetNumber(),

View File

@ -418,6 +418,167 @@ TEST_F(EventListenerTest, DisableBGCompaction) {
ASSERT_GE(listener->slowdown_count, kSlowdownTrigger * 9);
}
class TestCompactionReasonListener : public EventListener {
public:
void OnCompactionCompleted(DB* db, const CompactionJobInfo& ci) override {
std::lock_guard<std::mutex> lock(mutex_);
compaction_reasons_.push_back(ci.compaction_reason);
}
std::vector<CompactionReason> compaction_reasons_;
std::mutex mutex_;
};
TEST_F(EventListenerTest, CompactionReasonLevel) {
Options options;
options.create_if_missing = true;
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
options.level0_file_num_compaction_trigger = 4;
options.compaction_style = kCompactionStyleLevel;
DestroyAndReopen(options);
Random rnd(301);
// Write 4 files in L0
for (int i = 0; i < 4; i++) {
GenerateNewRandomFile(&rnd);
}
dbfull()->TEST_WaitForCompact();
ASSERT_EQ(listener->compaction_reasons_.size(), 1);
ASSERT_EQ(listener->compaction_reasons_[0],
CompactionReason::kLevelL0FilesNum);
DestroyAndReopen(options);
// Write 3 non-overlapping files in L0
for (int k = 1; k <= 30; k++) {
ASSERT_OK(Put(Key(k), Key(k)));
if (k % 10 == 0) {
Flush();
}
}
// Do a trivial move from L0 -> L1
db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
options.max_bytes_for_level_base = 1;
Close();
listener->compaction_reasons_.clear();
Reopen(options);
dbfull()->TEST_WaitForCompact();
ASSERT_GT(listener->compaction_reasons_.size(), 1);
for (auto compaction_reason : listener->compaction_reasons_) {
ASSERT_EQ(compaction_reason, CompactionReason::kLevelMaxLevelSize);
}
options.disable_auto_compactions = true;
Close();
listener->compaction_reasons_.clear();
Reopen(options);
db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_GT(listener->compaction_reasons_.size(), 0);
for (auto compaction_reason : listener->compaction_reasons_) {
ASSERT_EQ(compaction_reason, CompactionReason::kManualCompaction);
}
}
TEST_F(EventListenerTest, CompactionReasonUniversal) {
Options options;
options.create_if_missing = true;
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
options.compaction_style = kCompactionStyleUniversal;
Random rnd(301);
options.level0_file_num_compaction_trigger = 8;
options.compaction_options_universal.max_size_amplification_percent = 100000;
options.compaction_options_universal.size_ratio = 100000;
DestroyAndReopen(options);
listener->compaction_reasons_.clear();
// Write 8 files in L0
for (int i = 0; i < 8; i++) {
GenerateNewRandomFile(&rnd);
}
dbfull()->TEST_WaitForCompact();
ASSERT_GT(listener->compaction_reasons_.size(), 0);
for (auto compaction_reason : listener->compaction_reasons_) {
ASSERT_EQ(compaction_reason, CompactionReason::kUniversalSortedRunNum);
}
options.level0_file_num_compaction_trigger = 8;
options.compaction_options_universal.max_size_amplification_percent = 1;
options.compaction_options_universal.size_ratio = 100000;
DestroyAndReopen(options);
listener->compaction_reasons_.clear();
// Write 8 files in L0
for (int i = 0; i < 8; i++) {
GenerateNewRandomFile(&rnd);
}
dbfull()->TEST_WaitForCompact();
ASSERT_GT(listener->compaction_reasons_.size(), 0);
for (auto compaction_reason : listener->compaction_reasons_) {
ASSERT_EQ(compaction_reason, CompactionReason::kUniversalSizeAmplification);
}
options.disable_auto_compactions = true;
Close();
listener->compaction_reasons_.clear();
Reopen(options);
db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_GT(listener->compaction_reasons_.size(), 0);
for (auto compaction_reason : listener->compaction_reasons_) {
ASSERT_EQ(compaction_reason, CompactionReason::kManualCompaction);
}
}
TEST_F(EventListenerTest, CompactionReasonFIFO) {
Options options;
options.create_if_missing = true;
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
options.level0_file_num_compaction_trigger = 4;
options.compaction_style = kCompactionStyleFIFO;
options.compaction_options_fifo.max_table_files_size = 1;
DestroyAndReopen(options);
Random rnd(301);
// Write 4 files in L0
for (int i = 0; i < 4; i++) {
GenerateNewRandomFile(&rnd);
}
dbfull()->TEST_WaitForCompact();
ASSERT_GT(listener->compaction_reasons_.size(), 0);
for (auto compaction_reason : listener->compaction_reasons_) {
ASSERT_EQ(compaction_reason, CompactionReason::kFIFOMaxSize);
}
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

View File

@ -81,6 +81,26 @@ struct FlushJobInfo {
TableProperties table_properties;
};
enum class CompactionReason {
kUnknown,
// [Level] number of L0 files > level0_file_num_compaction_trigger
kLevelL0FilesNum,
// [Level] total size of level > MaxBytesForLevel()
kLevelMaxLevelSize,
// [Universal] Compacting for size amplification
kUniversalSizeAmplification,
// [Universal] Compacting for size ratio
kUniversalSizeRatio,
// [Universal] number of sorted runs > level0_file_num_compaction_trigger
kUniversalSortedRunNum,
// [FIFO] total size > max_table_files_size
kFIFOMaxSize,
// Manual compaction
kManualCompaction,
// DB::SuggestCompactRange() marked files for compaction
kFilesMarkedForCompaction,
};
struct CompactionJobInfo {
CompactionJobInfo() = default;
explicit CompactionJobInfo(const CompactionJobStats& _stats) :
@ -107,6 +127,9 @@ struct CompactionJobInfo {
// The map is keyed by values from input_files and output_files.
TablePropertiesCollection table_properties;
// Reason to run the compaction
CompactionReason compaction_reason;
// If non-null, this variable stores detailed information
// about this compaction.
CompactionJobStats stats;