Add Moved(GB) to Compaction IO stats
Summary: Adds counter for bytes moved (files pushed down a level rather than compacted) to compaction IO stats as Moved(GB). From the output removed these infrequently used columns: RW-Amp, Rn(cnt), Rnp1(cnt), Wnp1(cnt), Wnew(cnt). Example old output: Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms) RecordIn RecordDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ L0 0/0 0 0.0 0.0 0.0 0.0 2130.8 2130.8 0.0 0.0 0.0 109.1 0 0 0 0 20002 25068 0.798 28.75 182059 0.16 0 0 L1 142/0 509 1.0 4618.5 2036.5 2582.0 4602.1 2020.2 4.5 2.3 88.5 88.1 24220 701246 1215528 514282 53466 4229 12.643 0.00 0 0.002032745988 300688729 Example new output: Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms) RecordIn RecordDrop -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 7/0 13 1.8 0.0 0.0 0.0 0.6 0.6 0.0 0.0 0.0 14.7 44 353 0.124 0.03 626 0.05 0 0 L1 9/0 16 1.6 0.0 0.0 0.0 0.0 0.0 0.6 0.0 0.0 0.0 0 0 0.000 0.00 0 0.00 0 0 Task ID: # Blame Rev: Test Plan: make check, run db_bench --fillseq --stats_per_interval --stats_interval and look at output Revert Plan: Database Impact: Memcache Impact: Other Notes: EImportant: - begin *PUBLIC* platform impact section - Bugzilla: # - end platform impact - Reviewers: igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D29787
This commit is contained in:
Mark Callaghan
parent
a14b7873ee
commit
32a0a03844
@ -2070,6 +2070,8 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
|
||||
*c->mutable_cf_options());
|
||||
|
||||
VersionStorageInfo::LevelSummaryStorage tmp;
|
||||
c->column_family_data()->internal_stats()->IncBytesMoved(
|
||||
c->level() + 1, f->fd.GetFileSize());
|
||||
LogToBuffer(log_buffer, "[%s] Moved #%" PRIu64 " to level-%d %" PRIu64
|
||||
" bytes %s: %s\n",
|
||||
c->column_family_data()->GetName().c_str(), f->fd.GetNumber(),
|
||||
|
||||
@ -31,18 +31,18 @@ void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
|
||||
buf, len,
|
||||
"\n** Compaction Stats [%s] **\n"
|
||||
"Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) "
|
||||
"Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) "
|
||||
"Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) "
|
||||
"Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) "
|
||||
"Comp(sec) Comp(cnt) Avg(sec) "
|
||||
"Stall(sec) Stall(cnt) Avg(ms) RecordIn RecordDrop\n"
|
||||
"--------------------------------------------------------------------"
|
||||
"--------------------------------------------------------------------"
|
||||
"--------------------------------------------------------------------\n",
|
||||
"----------------------------------------------------------\n",
|
||||
cf_name.c_str());
|
||||
}
|
||||
|
||||
void PrintLevelStats(char* buf, size_t len, const std::string& name,
|
||||
int num_files, int being_compacted, double total_file_size, double score,
|
||||
double rw_amp, double w_amp, double stall_us, uint64_t stalls,
|
||||
double w_amp, double stall_us, uint64_t stalls,
|
||||
const InternalStats::CompactionStats& stats) {
|
||||
uint64_t bytes_read = stats.bytes_readn + stats.bytes_readnp1;
|
||||
uint64_t bytes_new = stats.bytes_written - stats.bytes_readnp1;
|
||||
@ -55,15 +55,11 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
|
||||
"%8.1f " /* Rnp1(GB) */
|
||||
"%9.1f " /* Write(GB) */
|
||||
"%8.1f " /* Wnew(GB) */
|
||||
"%6.1f " /* RW-Amp */
|
||||
"%9.1f " /* Moved(GB) */
|
||||
"%5.1f " /* W-Amp */
|
||||
"%8.1f " /* Rd(MB/s) */
|
||||
"%8.1f " /* Wr(MB/s) */
|
||||
"%8d " /* Rn(cnt) */
|
||||
"%9d " /* Rnp1(cnt) */
|
||||
"%9d " /* Wnp1(cnt) */
|
||||
"%9d " /* Wnew(cnt) */
|
||||
"%10.0f " /* Comp(sec) */
|
||||
"%9.0f " /* Comp(sec) */
|
||||
"%9d " /* Comp(cnt) */
|
||||
"%8.3f " /* Avg(sec) */
|
||||
"%10.2f " /* Stall(sec) */
|
||||
@ -76,10 +72,9 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
|
||||
name.c_str(), num_files, being_compacted, total_file_size / kMB,
|
||||
score, bytes_read / kGB, stats.bytes_readn / kGB,
|
||||
stats.bytes_readnp1 / kGB, stats.bytes_written / kGB,
|
||||
bytes_new / kGB, rw_amp, w_amp, bytes_read / kMB / elapsed,
|
||||
stats.bytes_written / kMB / elapsed, stats.files_in_leveln,
|
||||
stats.files_in_levelnp1, stats.files_out_levelnp1,
|
||||
stats.files_out_levelnp1 - stats.files_in_levelnp1,
|
||||
bytes_new / kGB, stats.bytes_moved / kGB,
|
||||
w_amp, bytes_read / kMB / elapsed,
|
||||
stats.bytes_written / kMB / elapsed,
|
||||
stats.micros / 1000000.0, stats.count,
|
||||
stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count,
|
||||
stall_us / 1000000.0, stalls,
|
||||
@ -428,29 +423,22 @@ void InternalStats::DumpCFStats(std::string* value) {
|
||||
total_slowdown_count_soft += stall_leveln_slowdown_count_soft_[level];
|
||||
total_slowdown_hard += stall_leveln_slowdown_hard_[level];
|
||||
total_slowdown_count_hard += stall_leveln_slowdown_count_hard_[level];
|
||||
int64_t bytes_read = comp_stats_[level].bytes_readn +
|
||||
comp_stats_[level].bytes_readnp1;
|
||||
double rw_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
|
||||
: (comp_stats_[level].bytes_written + bytes_read) /
|
||||
static_cast<double>(comp_stats_[level].bytes_readn);
|
||||
double w_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
|
||||
: comp_stats_[level].bytes_written /
|
||||
static_cast<double>(comp_stats_[level].bytes_readn);
|
||||
PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files,
|
||||
files_being_compacted[level],
|
||||
vstorage->NumLevelBytes(level), compaction_score[level],
|
||||
rw_amp, w_amp, stall_us, stalls, comp_stats_[level]);
|
||||
w_amp, stall_us, stalls, comp_stats_[level]);
|
||||
value->append(buf);
|
||||
}
|
||||
}
|
||||
uint64_t curr_ingest = cf_stats_value_[BYTES_FLUSHED];
|
||||
// Cumulative summary
|
||||
double rw_amp = (stats_sum.bytes_written + stats_sum.bytes_readn +
|
||||
stats_sum.bytes_readnp1) / static_cast<double>(curr_ingest + 1);
|
||||
double w_amp = stats_sum.bytes_written / static_cast<double>(curr_ingest + 1);
|
||||
// Stats summary across levels
|
||||
PrintLevelStats(buf, sizeof(buf), "Sum", total_files,
|
||||
total_files_being_compacted, total_file_size, 0, rw_amp, w_amp,
|
||||
total_files_being_compacted, total_file_size, 0, w_amp,
|
||||
total_stall_us, total_stall_count, stats_sum);
|
||||
value->append(buf);
|
||||
// Interval summary
|
||||
@ -458,12 +446,9 @@ void InternalStats::DumpCFStats(std::string* value) {
|
||||
curr_ingest - cf_stats_snapshot_.ingest_bytes + 1;
|
||||
CompactionStats interval_stats(stats_sum);
|
||||
interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
|
||||
rw_amp = (interval_stats.bytes_written +
|
||||
interval_stats.bytes_readn + interval_stats.bytes_readnp1) /
|
||||
static_cast<double>(interval_ingest);
|
||||
w_amp = interval_stats.bytes_written / static_cast<double>(interval_ingest);
|
||||
PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0,
|
||||
rw_amp, w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
|
||||
w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
|
||||
total_stall_count - cf_stats_snapshot_.stall_count, interval_stats);
|
||||
value->append(buf);
|
||||
|
||||
|
||||
@ -119,6 +119,9 @@ class InternalStats {
|
||||
// Total bytes written during compaction between levels N and N+1
|
||||
uint64_t bytes_written;
|
||||
|
||||
// Total bytes moved to this level
|
||||
uint64_t bytes_moved;
|
||||
|
||||
// Files read from level N during compaction between levels N and N+1
|
||||
int files_in_leveln;
|
||||
|
||||
@ -143,6 +146,7 @@ class InternalStats {
|
||||
bytes_readn(0),
|
||||
bytes_readnp1(0),
|
||||
bytes_written(0),
|
||||
bytes_moved(0),
|
||||
files_in_leveln(0),
|
||||
files_in_levelnp1(0),
|
||||
files_out_levelnp1(0),
|
||||
@ -155,6 +159,7 @@ class InternalStats {
|
||||
bytes_readn(c.bytes_readn),
|
||||
bytes_readnp1(c.bytes_readnp1),
|
||||
bytes_written(c.bytes_written),
|
||||
bytes_moved(c.bytes_moved),
|
||||
files_in_leveln(c.files_in_leveln),
|
||||
files_in_levelnp1(c.files_in_levelnp1),
|
||||
files_out_levelnp1(c.files_out_levelnp1),
|
||||
@ -167,6 +172,7 @@ class InternalStats {
|
||||
this->bytes_readn += c.bytes_readn;
|
||||
this->bytes_readnp1 += c.bytes_readnp1;
|
||||
this->bytes_written += c.bytes_written;
|
||||
this->bytes_moved += c.bytes_moved;
|
||||
this->files_in_leveln += c.files_in_leveln;
|
||||
this->files_in_levelnp1 += c.files_in_levelnp1;
|
||||
this->files_out_levelnp1 += c.files_out_levelnp1;
|
||||
@ -180,6 +186,7 @@ class InternalStats {
|
||||
this->bytes_readn -= c.bytes_readn;
|
||||
this->bytes_readnp1 -= c.bytes_readnp1;
|
||||
this->bytes_written -= c.bytes_written;
|
||||
this->bytes_moved -= c.bytes_moved;
|
||||
this->files_in_leveln -= c.files_in_leveln;
|
||||
this->files_in_levelnp1 -= c.files_in_levelnp1;
|
||||
this->files_out_levelnp1 -= c.files_out_levelnp1;
|
||||
@ -193,6 +200,10 @@ class InternalStats {
|
||||
comp_stats_[level].Add(stats);
|
||||
}
|
||||
|
||||
void IncBytesMoved(int level, uint64_t amount) {
|
||||
comp_stats_[level].bytes_moved += amount;
|
||||
}
|
||||
|
||||
void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {
|
||||
if (soft) {
|
||||
stall_leveln_slowdown_soft_[level] += micros;
|
||||
@ -329,6 +340,7 @@ class InternalStats {
|
||||
uint64_t bytes_readn;
|
||||
uint64_t bytes_readnp1;
|
||||
uint64_t bytes_written;
|
||||
uint64_t bytes_moved;
|
||||
int files_in_leveln;
|
||||
int files_in_levelnp1;
|
||||
int files_out_levelnp1;
|
||||
@ -347,6 +359,8 @@ class InternalStats {
|
||||
|
||||
void AddCompactionStats(int level, const CompactionStats& stats) {}
|
||||
|
||||
void IncBytesMoved(int level, uint64_t amount) {}
|
||||
|
||||
void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {}
|
||||
|
||||
void AddCFStats(InternalCFStatsType type, uint64_t value) {}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user