DeleteRange compaction statistics

Summary:
- "rocksdb.compaction.key.drop.range_del" - number of keys dropped during compaction due to a range tombstone covering them
- "rocksdb.compaction.range_del.drop.obsolete" - number of range tombstones dropped due to compaction to bottom level and no snapshot saving them
- s/CompactionIteratorStats/CompactionIterationStats/g since this class is no longer specific to CompactionIterator -- it's also updated for range tombstone iteration during compaction
- Move the above class into a separate .h file to avoid circular dependency.
Closes https://github.com/facebook/rocksdb/pull/1520

Differential Revision: D4187179

Pulled By: ajkr

fbshipit-source-id: 10c2103
This commit is contained in:
Andrew Kryczka 2016-11-28 11:44:40 -08:00 committed by Facebook Github Bot
parent 236d4c67e9
commit 7ffb10fc1a
8 changed files with 85 additions and 36 deletions

View File

@ -0,0 +1,29 @@
// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#pragma once
struct CompactionIterationStats {
// Compaction statistics
int64_t num_record_drop_user = 0;
int64_t num_record_drop_hidden = 0;
int64_t num_record_drop_obsolete = 0;
int64_t num_record_drop_range_del = 0;
int64_t num_range_del_drop_obsolete = 0;
uint64_t total_filter_time = 0;
// Input statistics
// TODO(noetzli): The stats are incomplete. They are lacking everything
// consumed by MergeHelper.
uint64_t num_input_records = 0;
uint64_t num_input_deletion_records = 0;
uint64_t num_input_corrupt_records = 0;
uint64_t total_input_raw_key_bytes = 0;
uint64_t total_input_raw_value_bytes = 0;
// Single-Delete diagnostics for exceptional situations
uint64_t num_single_del_fallthru = 0;
uint64_t num_single_del_mismatch = 0;
};

View File

@ -63,6 +63,8 @@ void CompactionIterator::ResetRecordCounts() {
iter_stats_.num_record_drop_user = 0; iter_stats_.num_record_drop_user = 0;
iter_stats_.num_record_drop_hidden = 0; iter_stats_.num_record_drop_hidden = 0;
iter_stats_.num_record_drop_obsolete = 0; iter_stats_.num_record_drop_obsolete = 0;
iter_stats_.num_record_drop_range_del = 0;
iter_stats_.num_range_del_drop_obsolete = 0;
} }
void CompactionIterator::SeekToFirst() { void CompactionIterator::SeekToFirst() {
@ -424,6 +426,8 @@ void CompactionIterator::NextFromInput() {
// 2. different snapshot stripe // 2. different snapshot stripe
bool should_delete = range_del_agg_->ShouldDelete(key_); bool should_delete = range_del_agg_->ShouldDelete(key_);
if (should_delete) { if (should_delete) {
++iter_stats_.num_record_drop_hidden;
++iter_stats_.num_record_drop_range_del;
input_->Next(); input_->Next();
} else { } else {
valid_ = true; valid_ = true;

View File

@ -12,6 +12,7 @@
#include <vector> #include <vector>
#include "db/compaction.h" #include "db/compaction.h"
#include "db/compaction_iteration_stats.h"
#include "db/merge_helper.h" #include "db/merge_helper.h"
#include "db/pinned_iterators_manager.h" #include "db/pinned_iterators_manager.h"
#include "db/range_del_aggregator.h" #include "db/range_del_aggregator.h"
@ -20,27 +21,6 @@
namespace rocksdb { namespace rocksdb {
struct CompactionIteratorStats {
// Compaction statistics
int64_t num_record_drop_user = 0;
int64_t num_record_drop_hidden = 0;
int64_t num_record_drop_obsolete = 0;
uint64_t total_filter_time = 0;
// Input statistics
// TODO(noetzli): The stats are incomplete. They are lacking everything
// consumed by MergeHelper.
uint64_t num_input_records = 0;
uint64_t num_input_deletion_records = 0;
uint64_t num_input_corrupt_records = 0;
uint64_t total_input_raw_key_bytes = 0;
uint64_t total_input_raw_value_bytes = 0;
// Single-Delete diagnostics for exceptional situations
uint64_t num_single_del_fallthru = 0;
uint64_t num_single_del_mismatch = 0;
};
class CompactionIterator { class CompactionIterator {
public: public:
CompactionIterator(InternalIterator* input, const Comparator* cmp, CompactionIterator(InternalIterator* input, const Comparator* cmp,
@ -74,7 +54,7 @@ class CompactionIterator {
const ParsedInternalKey& ikey() const { return ikey_; } const ParsedInternalKey& ikey() const { return ikey_; }
bool Valid() const { return valid_; } bool Valid() const { return valid_; }
const Slice& user_key() const { return current_user_key_; } const Slice& user_key() const { return current_user_key_; }
const CompactionIteratorStats& iter_stats() const { return iter_stats_; } const CompactionIterationStats& iter_stats() const { return iter_stats_; }
private: private:
// Processes the input stream to find the next output // Processes the input stream to find the next output
@ -157,6 +137,6 @@ class CompactionIterator {
// increasing so a later call to the function must be looking for a key that // increasing so a later call to the function must be looking for a key that
// is in or beyond the last file checked during the previous call // is in or beyond the last file checked during the previous call
std::vector<size_t> level_ptrs_; std::vector<size_t> level_ptrs_;
CompactionIteratorStats iter_stats_; CompactionIterationStats iter_stats_;
}; };
} // namespace rocksdb } // namespace rocksdb

View File

@ -768,8 +768,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
sub_compact->ShouldStopBefore( sub_compact->ShouldStopBefore(
key, sub_compact->current_output_file_size) && key, sub_compact->current_output_file_size) &&
sub_compact->builder != nullptr) { sub_compact->builder != nullptr) {
status = FinishCompactionOutputFile(input->status(), sub_compact, CompactionIterationStats range_del_out_stats;
range_del_agg.get()); status =
FinishCompactionOutputFile(input->status(), sub_compact,
range_del_agg.get(), &range_del_out_stats);
RecordDroppedKeys(range_del_out_stats,
&sub_compact->compaction_job_stats);
if (!status.ok()) { if (!status.ok()) {
break; break;
} }
@ -861,8 +865,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
if (c_iter->Valid()) { if (c_iter->Valid()) {
next_key = &c_iter->key(); next_key = &c_iter->key();
} }
CompactionIterationStats range_del_out_stats;
status = FinishCompactionOutputFile(input_status, sub_compact, status = FinishCompactionOutputFile(input_status, sub_compact,
range_del_agg.get(), next_key); range_del_agg.get(),
&range_del_out_stats, next_key);
RecordDroppedKeys(range_del_out_stats,
&sub_compact->compaction_job_stats);
if (sub_compact->outputs.size() == 1) { if (sub_compact->outputs.size() == 1) {
// Use dictionary from first output file for compression of subsequent // Use dictionary from first output file for compression of subsequent
// files. // files.
@ -902,8 +910,11 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
status = OpenCompactionOutputFile(sub_compact); status = OpenCompactionOutputFile(sub_compact);
} }
if (status.ok() && sub_compact->builder != nullptr) { if (status.ok() && sub_compact->builder != nullptr) {
status = FinishCompactionOutputFile(input->status(), sub_compact, CompactionIterationStats range_del_out_stats;
range_del_agg.get()); status =
FinishCompactionOutputFile(input->status(), sub_compact,
range_del_agg.get(), &range_del_out_stats);
RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats);
} }
if (status.ok()) { if (status.ok()) {
status = input->status(); status = input->status();
@ -929,7 +940,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
} }
void CompactionJob::RecordDroppedKeys( void CompactionJob::RecordDroppedKeys(
const CompactionIteratorStats& c_iter_stats, const CompactionIterationStats& c_iter_stats,
CompactionJobStats* compaction_job_stats) { CompactionJobStats* compaction_job_stats) {
if (c_iter_stats.num_record_drop_user > 0) { if (c_iter_stats.num_record_drop_user > 0) {
RecordTick(stats_, COMPACTION_KEY_DROP_USER, RecordTick(stats_, COMPACTION_KEY_DROP_USER,
@ -951,11 +962,20 @@ void CompactionJob::RecordDroppedKeys(
c_iter_stats.num_record_drop_obsolete; c_iter_stats.num_record_drop_obsolete;
} }
} }
if (c_iter_stats.num_record_drop_range_del > 0) {
RecordTick(stats_, COMPACTION_KEY_DROP_RANGE_DEL,
c_iter_stats.num_record_drop_range_del);
}
if (c_iter_stats.num_range_del_drop_obsolete > 0) {
RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE,
c_iter_stats.num_range_del_drop_obsolete);
}
} }
Status CompactionJob::FinishCompactionOutputFile( Status CompactionJob::FinishCompactionOutputFile(
const Status& input_status, SubcompactionState* sub_compact, const Status& input_status, SubcompactionState* sub_compact,
RangeDelAggregator* range_del_agg, RangeDelAggregator* range_del_agg,
CompactionIterationStats* range_del_out_stats,
const Slice* next_table_min_key /* = nullptr */) { const Slice* next_table_min_key /* = nullptr */) {
AutoThreadOperationStageUpdater stage_updater( AutoThreadOperationStageUpdater stage_updater(
ThreadStatus::STAGE_COMPACTION_SYNC_FILE); ThreadStatus::STAGE_COMPACTION_SYNC_FILE);
@ -998,7 +1018,8 @@ Status CompactionJob::FinishCompactionOutputFile(
upper_bound = sub_compact->end; upper_bound = sub_compact->end;
} }
range_del_agg->AddToBuilder(sub_compact->builder.get(), lower_bound, range_del_agg->AddToBuilder(sub_compact->builder.get(), lower_bound,
upper_bound, meta, bottommost_level_); upper_bound, meta, range_del_out_stats,
bottommost_level_);
} }
const uint64_t current_entries = sub_compact->builder->NumEntries(); const uint64_t current_entries = sub_compact->builder->NumEntries();
meta->marked_for_compaction = sub_compact->builder->NeedCompact(); meta->marked_for_compaction = sub_compact->builder->NeedCompact();

View File

@ -96,9 +96,10 @@ class CompactionJob {
// kv-pairs // kv-pairs
void ProcessKeyValueCompaction(SubcompactionState* sub_compact); void ProcessKeyValueCompaction(SubcompactionState* sub_compact);
Status FinishCompactionOutputFile(const Status& input_status, Status FinishCompactionOutputFile(
SubcompactionState* sub_compact, const Status& input_status, SubcompactionState* sub_compact,
RangeDelAggregator* range_del_agg = nullptr, RangeDelAggregator* range_del_agg,
CompactionIterationStats* range_del_out_stats,
const Slice* next_table_min_key = nullptr); const Slice* next_table_min_key = nullptr);
Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options); Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
void RecordCompactionIOStats(); void RecordCompactionIOStats();
@ -106,7 +107,7 @@ class CompactionJob {
void CleanupCompaction(); void CleanupCompaction();
void UpdateCompactionJobStats( void UpdateCompactionJobStats(
const InternalStats::CompactionStats& stats) const; const InternalStats::CompactionStats& stats) const;
void RecordDroppedKeys(const CompactionIteratorStats& c_iter_stats, void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats,
CompactionJobStats* compaction_job_stats = nullptr); CompactionJobStats* compaction_job_stats = nullptr);
void UpdateCompactionStats(); void UpdateCompactionStats();

View File

@ -140,6 +140,7 @@ RangeDelAggregator::TombstoneMap& RangeDelAggregator::GetTombstoneMap(
void RangeDelAggregator::AddToBuilder( void RangeDelAggregator::AddToBuilder(
TableBuilder* builder, const Slice* lower_bound, const Slice* upper_bound, TableBuilder* builder, const Slice* lower_bound, const Slice* upper_bound,
FileMetaData* meta, FileMetaData* meta,
CompactionIterationStats* range_del_out_stats /* = nullptr */,
bool bottommost_level /* = false */) { bool bottommost_level /* = false */) {
if (rep_ == nullptr) { if (rep_ == nullptr) {
return; return;
@ -147,6 +148,10 @@ void RangeDelAggregator::AddToBuilder(
auto stripe_map_iter = rep_->stripe_map_.begin(); auto stripe_map_iter = rep_->stripe_map_.begin();
assert(stripe_map_iter != rep_->stripe_map_.end()); assert(stripe_map_iter != rep_->stripe_map_.end());
if (bottommost_level) { if (bottommost_level) {
range_del_out_stats->num_range_del_drop_obsolete +=
static_cast<int64_t>(stripe_map_iter->second.size());
range_del_out_stats->num_record_drop_obsolete +=
static_cast<int64_t>(stripe_map_iter->second.size());
// For the bottommost level, keys covered by tombstones in the first // For the bottommost level, keys covered by tombstones in the first
// (oldest) stripe have been compacted away, so the tombstones are obsolete. // (oldest) stripe have been compacted away, so the tombstones are obsolete.
++stripe_map_iter; ++stripe_map_iter;

View File

@ -9,6 +9,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "db/compaction_iteration_stats.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/pinned_iterators_manager.h" #include "db/pinned_iterators_manager.h"
#include "db/version_edit.h" #include "db/version_edit.h"
@ -81,6 +82,7 @@ class RangeDelAggregator {
// compaction. // compaction.
void AddToBuilder(TableBuilder* builder, const Slice* lower_bound, void AddToBuilder(TableBuilder* builder, const Slice* lower_bound,
const Slice* upper_bound, FileMetaData* meta, const Slice* upper_bound, FileMetaData* meta,
CompactionIterationStats* range_del_out_stats = nullptr,
bool bottommost_level = false); bool bottommost_level = false);
bool IsEmpty(); bool IsEmpty();

View File

@ -95,12 +95,16 @@ enum Tickers : uint32_t {
/** /**
* COMPACTION_KEY_DROP_* count the reasons for key drop during compaction * COMPACTION_KEY_DROP_* count the reasons for key drop during compaction
* There are 3 reasons currently. * There are 4 reasons currently.
*/ */
COMPACTION_KEY_DROP_NEWER_ENTRY, // key was written with a newer value. COMPACTION_KEY_DROP_NEWER_ENTRY, // key was written with a newer value.
// Also includes keys dropped for range del.
COMPACTION_KEY_DROP_OBSOLETE, // The key is obsolete. COMPACTION_KEY_DROP_OBSOLETE, // The key is obsolete.
COMPACTION_KEY_DROP_RANGE_DEL, // key was covered by a range tombstone.
COMPACTION_KEY_DROP_USER, // user compaction function has dropped the key. COMPACTION_KEY_DROP_USER, // user compaction function has dropped the key.
COMPACTION_RANGE_DEL_DROP_OBSOLETE, // all keys in range were deleted.
// Number of keys written to the database via the Put and Write call's // Number of keys written to the database via the Put and Write call's
NUMBER_KEYS_WRITTEN, NUMBER_KEYS_WRITTEN,
// Number of Keys read, // Number of Keys read,
@ -251,7 +255,10 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
{GET_HIT_L2_AND_UP, "rocksdb.l2andup.hit"}, {GET_HIT_L2_AND_UP, "rocksdb.l2andup.hit"},
{COMPACTION_KEY_DROP_NEWER_ENTRY, "rocksdb.compaction.key.drop.new"}, {COMPACTION_KEY_DROP_NEWER_ENTRY, "rocksdb.compaction.key.drop.new"},
{COMPACTION_KEY_DROP_OBSOLETE, "rocksdb.compaction.key.drop.obsolete"}, {COMPACTION_KEY_DROP_OBSOLETE, "rocksdb.compaction.key.drop.obsolete"},
{COMPACTION_KEY_DROP_RANGE_DEL, "rocksdb.compaction.key.drop.range_del"},
{COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user"}, {COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user"},
{COMPACTION_RANGE_DEL_DROP_OBSOLETE,
"rocksdb.compaction.range_del.drop.obsolete"},
{NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written"}, {NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written"},
{NUMBER_KEYS_READ, "rocksdb.number.keys.read"}, {NUMBER_KEYS_READ, "rocksdb.number.keys.read"},
{NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated"}, {NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated"},