andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Exporting compaction stats in the form of a map

Browse Source 
Summary:
Currently the compaction stats are printed to stdout. We want to export the compaction stats in a map format so that the upper layer apps (e.g., MySQL) could present
the stats in any format required by the them.
Closes https://github.com/facebook/rocksdb/pull/1477

Differential Revision: D4149836

Pulled By: maysamyabandeh

fbshipit-source-id: b3df19f
This commit is contained in:
Maysam Yabandeh 2016-11-11 20:45:47 -08:00 committed by Facebook Github Bot
parent 672300f47f
commit 361010d447
8 changed files with 422 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
db/db_impl.cc
View File

@ -5292,6 +5292,24 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
return false;
}
bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) {
const DBPropertyInfo* property_info = GetPropertyInfo(property);
value->clear();
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
if (property_info == nullptr) {
return false;
} else if (property_info->handle_map) {
InstrumentedMutexLock l(&mutex_);
return cfd->internal_stats()->GetMapProperty(*property_info, property,
value);
}
// If we reach this point it means that handle_map is not provided for the
// requested property
return false;
}
bool DBImpl::GetIntProperty(ColumnFamilyHandle* column_family,
const Slice& property, uint64_t* value) {
const DBPropertyInfo* property_info = GetPropertyInfo(property);

5
db/db_impl.h
View File

@ -13,6 +13,7 @@
#include <functional>
#include <limits>
#include <list>
#include <map>
#include <queue>
#include <set>
#include <string>
@ -123,6 +124,10 @@ class DBImpl : public DB {
using DB::GetProperty;
virtual bool GetProperty(ColumnFamilyHandle* column_family,
const Slice& property, std::string* value) override;
using DB::GetMapProperty;
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override;
using DB::GetIntProperty;
virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
const Slice& property, uint64_t* value) override;

6
db/db_test.cc
View File

@ -2740,6 +2740,12 @@ class ModelDB : public DB {
const Slice& property, uint64_t* value) override {
return false;
}
using DB::GetMapProperty;
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override {
return false;
}
using DB::GetAggregatedIntProperty;
virtual bool GetAggregatedIntProperty(const Slice& property,
uint64_t* value) override {

25
db/db_test2.cc
View File

@ -2085,8 +2085,26 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
cro.target_level = 2;
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
auto get_stat = [](std::string level_str, LevelStatType type,
std::map<std::string, double> props) {
auto prop_str =
level_str + "." +
InternalStats::compaction_level_stats.at(type).property_name.c_str();
auto prop_item = props.find(prop_str);
return prop_item == props.end() ? 0 : prop_item->second;
};
// Trivial move 2 files to L2
ASSERT_EQ("0,0,2", FilesPerLevel());
// Also test that the stats GetMapProperty API reporting the same result
{
std::map<std::string, double> prop;
ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop));
ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop));
ASSERT_EQ(2, get_stat("L2", LevelStatType::NUM_FILES, prop));
ASSERT_EQ(2, get_stat("Sum", LevelStatType::NUM_FILES, prop));
}
// While the compaction is running, we will create 2 new files that
// can fit in L2, these 2 files will be moved to L2 and overlap with
@ -2113,6 +2131,13 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
// Test that the stats GetMapProperty API reporting 1 file in L2
{
std::map<std::string, double> prop;
ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop));
}
}
TEST_F(DBTest2, ManualCompactionOverlapManualCompaction) {

430
db/internal_stats.cc
View File

@ -25,62 +25,146 @@
namespace rocksdb {
#ifndef ROCKSDB_LITE
const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats =
{
{LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}},
{LevelStatType::COMPACTED_FILES,
LevelStat{"CompactedFiles", "CompactedFiles"}},
{LevelStatType::SIZE_MB, LevelStat{"SizeMB", "Size(MB}"}},
{LevelStatType::SCORE, LevelStat{"Score", "Score"}},
{LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB}"}},
{LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB}"}},
{LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB}"}},
{LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB}"}},
{LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB}"}},
{LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB}"}},
{LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}},
{LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s}"}},
{LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s}"}},
{LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec}"}},
{LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt}"}},
{LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec}"}},
{LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}},
{LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}},
};
namespace {
const double kMB = 1048576.0;
const double kGB = kMB * 1024;
const double kMicrosInSec = 1000000.0;
void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
snprintf(
buf, len,
"\n** Compaction Stats [%s] **\n"
"Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) "
"Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) "
"Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop\n"
"--------------------------------------------------------------------"
"-----------------------------------------------------------"
"--------------------------------------\n",
cf_name.c_str());
int written_size =
snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str());
auto hdr = [](LevelStatType t) {
return InternalStats::compaction_level_stats.at(t).header_name.c_str();
};
int line_size = snprintf(
buf + written_size, len - written_size,
"Level %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
// Note that we skip COMPACTED_FILES and merge it with Files column
hdr(LevelStatType::NUM_FILES), hdr(LevelStatType::SIZE_MB),
hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB),
hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB),
hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB),
hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP),
hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS),
hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_COUNT),
hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN),
hdr(LevelStatType::KEY_DROP));
written_size += line_size;
snprintf(buf + written_size, len - written_size, "%s\n",
std::string(line_size, '-').c_str());
}
void PrintLevelStats(char* buf, size_t len, const std::string& name,
int num_files, int being_compacted, double total_file_size,
double score, double w_amp,
const InternalStats::CompactionStats& stats) {
void PrepareLevelStats(std::map<LevelStatType, double>* level_stats,
int num_files, int being_compacted,
double total_file_size, double score, double w_amp,
const InternalStats::CompactionStats& stats) {
uint64_t bytes_read =
stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
int64_t bytes_new =
stats.bytes_written - stats.bytes_read_output_level;
double elapsed = (stats.micros + 1) / kMicrosInSec;
std::string num_input_records = NumberToHumanString(stats.num_input_records);
std::string num_dropped_records =
NumberToHumanString(stats.num_dropped_records);
(*level_stats)[LevelStatType::NUM_FILES] = num_files;
(*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted;
(*level_stats)[LevelStatType::SIZE_MB] = total_file_size / kMB;
(*level_stats)[LevelStatType::SCORE] = score;
(*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB;
(*level_stats)[LevelStatType::RN_GB] =
stats.bytes_read_non_output_levels / kGB;
(*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB;
(*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB;
(*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB;
(*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB;
(*level_stats)[LevelStatType::WRITE_AMP] = w_amp;
(*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed;
(*level_stats)[LevelStatType::WRITE_MBPS] =
stats.bytes_written / kMB / elapsed;
(*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec;
(*level_stats)[LevelStatType::COMP_COUNT] = stats.count;
(*level_stats)[LevelStatType::AVG_SEC] =
stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count;
(*level_stats)[LevelStatType::KEY_IN] =
static_cast<double>(stats.num_input_records);
(*level_stats)[LevelStatType::KEY_DROP] =
static_cast<double>(stats.num_dropped_records);
}
void PrintLevelStats(char* buf, size_t len, const std::string& name,
const std::map<LevelStatType, double>& stat_value) {
snprintf(buf, len,
"%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */
"%8.1f " /* Read(GB) */
"%7.1f " /* Rn(GB) */
"%8.1f " /* Rnp1(GB) */
"%9.1f " /* Write(GB) */
"%8.1f " /* Wnew(GB) */
"%9.1f " /* Moved(GB) */
"%5.1f " /* W-Amp */
"%8.1f " /* Rd(MB/s) */
"%8.1f " /* Wr(MB/s) */
"%9.0f " /* Comp(sec) */
"%9d " /* Comp(cnt) */
"%8.3f " /* Avg(sec) */
"%7s " /* KeyIn */
"%6s\n", /* KeyDrop */
"%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */
"%8.1f " /* Read(GB) */
"%7.1f " /* Rn(GB) */
"%8.1f " /* Rnp1(GB) */
"%9.1f " /* Write(GB) */
"%8.1f " /* Wnew(GB) */
"%9.1f " /* Moved(GB) */
"%5.1f " /* W-Amp */
"%8.1f " /* Rd(MB/s) */
"%8.1f " /* Wr(MB/s) */
"%9.0f " /* Comp(sec) */
"%9d " /* Comp(cnt) */
"%8.3f " /* Avg(sec) */
"%7s " /* KeyIn */
"%6s\n", /* KeyDrop */
name.c_str(),
num_files, being_compacted, total_file_size / kMB, score,
bytes_read / kGB, stats.bytes_read_non_output_levels / kGB,
stats.bytes_read_output_level / kGB, stats.bytes_written / kGB,
bytes_new / kGB, stats.bytes_moved / kGB, w_amp,
bytes_read / kMB / elapsed, stats.bytes_written / kMB / elapsed,
stats.micros / kMicrosInSec, stats.count,
stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count,
num_input_records.c_str(), num_dropped_records.c_str());
static_cast<int>(stat_value.at(LevelStatType::NUM_FILES)),
static_cast<int>(stat_value.at(LevelStatType::COMPACTED_FILES)),
stat_value.at(LevelStatType::SIZE_MB),
stat_value.at(LevelStatType::SCORE),
stat_value.at(LevelStatType::READ_GB),
stat_value.at(LevelStatType::RN_GB),
stat_value.at(LevelStatType::RNP1_GB),
stat_value.at(LevelStatType::WRITE_GB),
stat_value.at(LevelStatType::W_NEW_GB),
stat_value.at(LevelStatType::MOVED_GB),
stat_value.at(LevelStatType::WRITE_AMP),
stat_value.at(LevelStatType::READ_MBPS),
stat_value.at(LevelStatType::WRITE_MBPS),
stat_value.at(LevelStatType::COMP_SEC),
static_cast<int>(stat_value.at(LevelStatType::COMP_COUNT)),
stat_value.at(LevelStatType::AVG_SEC),
NumberToHumanString(
static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_IN)))
.c_str(),
NumberToHumanString(static_cast<std::int64_t>(
stat_value.at(LevelStatType::KEY_DROP)))
.c_str());
}
void PrintLevelStats(char* buf, size_t len, const std::string& name,
int num_files, int being_compacted, double total_file_size,
double score, double w_amp,
const InternalStats::CompactionStats& stats) {
std::map<LevelStatType, double> level_stats;
PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size,
score, w_amp, stats);
PrintLevelStats(buf, len, name, level_stats);
}
// Assumes that trailing numbers represent an optional argument. This requires
@ -211,73 +295,91 @@ const std::string DB::Properties::kAggregatedTableProperties =
const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
rocksdb_prefix + aggregated_table_properties_at_level;
const std::unordered_map<std::string,
DBPropertyInfo> InternalStats::ppt_name_to_info = {
{DB::Properties::kNumFilesAtLevelPrefix,
{false, &InternalStats::HandleNumFilesAtLevel, nullptr}},
{DB::Properties::kCompressionRatioAtLevelPrefix,
{false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr}},
{DB::Properties::kLevelStats,
{false, &InternalStats::HandleLevelStats, nullptr}},
{DB::Properties::kStats, {false, &InternalStats::HandleStats, nullptr}},
{DB::Properties::kCFStats, {false, &InternalStats::HandleCFStats, nullptr}},
{DB::Properties::kDBStats, {false, &InternalStats::HandleDBStats, nullptr}},
{DB::Properties::kSSTables,
{false, &InternalStats::HandleSsTables, nullptr}},
{DB::Properties::kAggregatedTableProperties,
{false, &InternalStats::HandleAggregatedTableProperties, nullptr}},
{DB::Properties::kAggregatedTablePropertiesAtLevel,
{false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, nullptr}},
{DB::Properties::kNumImmutableMemTable,
{false, nullptr, &InternalStats::HandleNumImmutableMemTable}},
{DB::Properties::kNumImmutableMemTableFlushed,
{false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed}},
{DB::Properties::kMemTableFlushPending,
{false, nullptr, &InternalStats::HandleMemTableFlushPending}},
{DB::Properties::kCompactionPending,
{false, nullptr, &InternalStats::HandleCompactionPending}},
{DB::Properties::kBackgroundErrors,
{false, nullptr, &InternalStats::HandleBackgroundErrors}},
{DB::Properties::kCurSizeActiveMemTable,
{false, nullptr, &InternalStats::HandleCurSizeActiveMemTable}},
{DB::Properties::kCurSizeAllMemTables,
{false, nullptr, &InternalStats::HandleCurSizeAllMemTables}},
{DB::Properties::kSizeAllMemTables,
{false, nullptr, &InternalStats::HandleSizeAllMemTables}},
{DB::Properties::kNumEntriesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable}},
{DB::Properties::kNumEntriesImmMemTables,
{false, nullptr, &InternalStats::HandleNumEntriesImmMemTables}},
{DB::Properties::kNumDeletesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable}},
{DB::Properties::kNumDeletesImmMemTables,
{false, nullptr, &InternalStats::HandleNumDeletesImmMemTables}},
{DB::Properties::kEstimateNumKeys,
{false, nullptr, &InternalStats::HandleEstimateNumKeys}},
{DB::Properties::kEstimateTableReadersMem,
{true, nullptr, &InternalStats::HandleEstimateTableReadersMem}},
{DB::Properties::kIsFileDeletionsEnabled,
{false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled}},
{DB::Properties::kNumSnapshots,
{false, nullptr, &InternalStats::HandleNumSnapshots}},
{DB::Properties::kOldestSnapshotTime,
{false, nullptr, &InternalStats::HandleOldestSnapshotTime}},
{DB::Properties::kNumLiveVersions,
{false, nullptr, &InternalStats::HandleNumLiveVersions}},
{DB::Properties::kCurrentSuperVersionNumber,
{false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber}},
{DB::Properties::kEstimateLiveDataSize,
{true, nullptr, &InternalStats::HandleEstimateLiveDataSize}},
{DB::Properties::kBaseLevel,
{false, nullptr, &InternalStats::HandleBaseLevel}},
{DB::Properties::kTotalSstFilesSize,
{false, nullptr, &InternalStats::HandleTotalSstFilesSize}},
{DB::Properties::kEstimatePendingCompactionBytes,
{false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes}},
{DB::Properties::kNumRunningFlushes,
{false, nullptr, &InternalStats::HandleNumRunningFlushes}},
{DB::Properties::kNumRunningCompactions,
{false, nullptr, &InternalStats::HandleNumRunningCompactions}},
const std::unordered_map<std::string, DBPropertyInfo>
InternalStats::ppt_name_to_info = {
{DB::Properties::kNumFilesAtLevelPrefix,
{false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr}},
{DB::Properties::kCompressionRatioAtLevelPrefix,
{false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr,
nullptr}},
{DB::Properties::kLevelStats,
{false, &InternalStats::HandleLevelStats, nullptr, nullptr}},
{DB::Properties::kStats,
{false, &InternalStats::HandleStats, nullptr, nullptr}},
{DB::Properties::kCFStats,
{false, &InternalStats::HandleCFStats, nullptr,
&InternalStats::HandleCFMapStats}},
{DB::Properties::kDBStats,
{false, &InternalStats::HandleDBStats, nullptr, nullptr}},
{DB::Properties::kSSTables,
{false, &InternalStats::HandleSsTables, nullptr, nullptr}},
{DB::Properties::kAggregatedTableProperties,
{false, &InternalStats::HandleAggregatedTableProperties, nullptr,
nullptr}},
{DB::Properties::kAggregatedTablePropertiesAtLevel,
{false, &InternalStats::HandleAggregatedTablePropertiesAtLevel,
nullptr, nullptr}},
{DB::Properties::kNumImmutableMemTable,
{false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr}},
{DB::Properties::kNumImmutableMemTableFlushed,
{false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed,
nullptr}},
{DB::Properties::kMemTableFlushPending,
{false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr}},
{DB::Properties::kCompactionPending,
{false, nullptr, &InternalStats::HandleCompactionPending, nullptr}},
{DB::Properties::kBackgroundErrors,
{false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr}},
{DB::Properties::kCurSizeActiveMemTable,
{false, nullptr, &InternalStats::HandleCurSizeActiveMemTable,
nullptr}},
{DB::Properties::kCurSizeAllMemTables,
{false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr}},
{DB::Properties::kSizeAllMemTables,
{false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr}},
{DB::Properties::kNumEntriesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable,
nullptr}},
{DB::Properties::kNumEntriesImmMemTables,
{false, nullptr, &InternalStats::HandleNumEntriesImmMemTables,
nullptr}},
{DB::Properties::kNumDeletesActiveMemTable,
{false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable,
nullptr}},
{DB::Properties::kNumDeletesImmMemTables,
{false, nullptr, &InternalStats::HandleNumDeletesImmMemTables,
nullptr}},
{DB::Properties::kEstimateNumKeys,
{false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr}},
{DB::Properties::kEstimateTableReadersMem,
{true, nullptr, &InternalStats::HandleEstimateTableReadersMem,
nullptr}},
{DB::Properties::kIsFileDeletionsEnabled,
{false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled,
nullptr}},
{DB::Properties::kNumSnapshots,
{false, nullptr, &InternalStats::HandleNumSnapshots, nullptr}},
{DB::Properties::kOldestSnapshotTime,
{false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr}},
{DB::Properties::kNumLiveVersions,
{false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr}},
{DB::Properties::kCurrentSuperVersionNumber,
{false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber,
nullptr}},
{DB::Properties::kEstimateLiveDataSize,
{true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr}},
{DB::Properties::kBaseLevel,
{false, nullptr, &InternalStats::HandleBaseLevel, nullptr}},
{DB::Properties::kTotalSstFilesSize,
{false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr}},
{DB::Properties::kEstimatePendingCompactionBytes,
{false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
nullptr}},
{DB::Properties::kNumRunningFlushes,
{false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr}},
{DB::Properties::kNumRunningCompactions,
{false, nullptr, &InternalStats::HandleNumRunningCompactions,
nullptr}},
};
const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
@ -298,6 +400,14 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
return (this->*(property_info.handle_string))(value, arg);
}
bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value) {
assert(value != nullptr);
assert(property_info.handle_map != nullptr);
return (this->*(property_info.handle_map))(value);
}
bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info,
uint64_t* value, DBImpl* db) {
assert(value != nullptr);
@ -370,6 +480,11 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) {
return true;
}
bool InternalStats::HandleCFMapStats(std::map<std::string, double>* cf_stats) {
DumpCFMapStats(cf_stats);
return true;
}
bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
DumpCFStats(value);
return true;
@ -716,7 +831,32 @@ void InternalStats::DumpDBStats(std::string* value) {
db_stats_snapshot_.write_stall_micros = write_stall_micros;
}
void InternalStats::DumpCFStats(std::string* value) {
/**
* Dump Compaction Level stats to a map of stat name to value in double.
* The level in stat name is represented with a prefix "Lx" where "x"
* is the level number. A special level "Sum" represents the sum of a stat
* for all levels.
*/
void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
CompactionStats compaction_stats_sum(0);
std::map<int, std::map<LevelStatType, double>> levels_stats;
DumpCFMapStats(&levels_stats, &compaction_stats_sum);
for (auto const& level_ent : levels_stats) {
auto level_str =
level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first);
for (auto const& stat_ent : level_ent.second) {
auto stat_type = stat_ent.first;
auto key_str =
level_str + "." +
InternalStats::compaction_level_stats.at(stat_type).property_name;
(*cf_stats)[key_str] = stat_ent.second;
}
}
}
int InternalStats::DumpCFMapStats(
std::map<int, std::map<LevelStatType, double>>* levels_stats,
CompactionStats* compaction_stats_sum) {
const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
int num_levels_to_check =
@ -724,7 +864,7 @@ void InternalStats::DumpCFStats(std::string* value) {
? vstorage->num_levels() - 1
: 1;
// Compaction scores are sorted base on its value. Restore them to the
// Compaction scores are sorted based on its value. Restore them to the
// level order
std::vector<double> compaction_score(number_levels_, 0);
for (int i = 0; i < num_levels_to_check; ++i) {
@ -741,12 +881,6 @@ void InternalStats::DumpCFStats(std::string* value) {
}
}
char buf[1000];
// Per-ColumnFamily stats
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
value->append(buf);
CompactionStats stats_sum(0);
int total_files = 0;
int total_files_being_compacted = 0;
double total_file_size = 0;
@ -755,40 +889,67 @@ void InternalStats::DumpCFStats(std::string* value) {
total_files += files;
total_files_being_compacted += files_being_compacted[level];
if (comp_stats_[level].micros > 0 || files > 0) {
stats_sum.Add(comp_stats_[level]);
compaction_stats_sum->Add(comp_stats_[level]);
total_file_size += vstorage->NumLevelBytes(level);
double w_amp =
(comp_stats_[level].bytes_read_non_output_levels == 0) ? 0.0
: static_cast<double>(comp_stats_[level].bytes_written) /
comp_stats_[level].bytes_read_non_output_levels;
PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files,
files_being_compacted[level],
static_cast<double>(vstorage->NumLevelBytes(level)),
compaction_score[level],
w_amp, comp_stats_[level]);
value->append(buf);
(comp_stats_[level].bytes_read_non_output_levels == 0)
? 0.0
: static_cast<double>(comp_stats_[level].bytes_written) /
comp_stats_[level].bytes_read_non_output_levels;
std::map<LevelStatType, double> level_stats;
PrepareLevelStats(&level_stats, files, files_being_compacted[level],
static_cast<double>(vstorage->NumLevelBytes(level)),
compaction_score[level], w_amp, comp_stats_[level]);
(*levels_stats)[level] = level_stats;
}
}
uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
uint64_t curr_ingest = flush_ingest + add_file_ingest;
// Cumulative summary
double w_amp = compaction_stats_sum->bytes_written /
static_cast<double>(curr_ingest + 1);
// Stats summary across levels
std::map<LevelStatType, double> sum_stats;
PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted,
total_file_size, 0, w_amp, *compaction_stats_sum);
(*levels_stats)[-1] = sum_stats; // -1 is for the Sum level
return num_levels_to_check;
}
void InternalStats::DumpCFStats(std::string* value) {
char buf[1000];
// Per-ColumnFamily stats
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
value->append(buf);
// Print stats for each level
std::map<int, std::map<LevelStatType, double>> levels_stats;
CompactionStats compaction_stats_sum(0);
int levels = DumpCFMapStats(&levels_stats, &compaction_stats_sum);
for (int l = 0; l < levels; ++l) {
if (levels_stats.find(l) != levels_stats.end()) {
PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]);
value->append(buf);
}
}
// Print sum of level stats
PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]);
value->append(buf);
uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL];
uint64_t ingest_l0_files_addfile =
cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL];
uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL];
// Cumulative summary
double w_amp = stats_sum.bytes_written / static_cast<double>(curr_ingest + 1);
uint64_t total_stall_count =
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] +
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
cf_stats_count_[MEMTABLE_COMPACTION] + cf_stats_count_[MEMTABLE_SLOWDOWN];
// Stats summary across levels
PrintLevelStats(buf, sizeof(buf), "Sum", total_files,
total_files_being_compacted, total_file_size, 0, w_amp,
stats_sum);
value->append(buf);
// Interval summary
uint64_t interval_flush_ingest =
flush_ingest - cf_stats_snapshot_.ingest_bytes_flush;
@ -796,9 +957,10 @@ void InternalStats::DumpCFStats(std::string* value) {
add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile;
uint64_t interval_ingest =
interval_flush_ingest + interval_add_file_inget + 1;
CompactionStats interval_stats(stats_sum);
CompactionStats interval_stats(compaction_stats_sum);
interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
w_amp = interval_stats.bytes_written / static_cast<double>(interval_ingest);
double w_amp =
interval_stats.bytes_written / static_cast<double>(interval_ingest);
PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats);
value->append(buf);
@ -909,7 +1071,7 @@ void InternalStats::DumpCFStats(std::string* value) {
cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
cf_stats_snapshot_.comp_stats = stats_sum;
cf_stats_snapshot_.comp_stats = compaction_stats_sum;
cf_stats_snapshot_.stall_count = total_stall_count;
}

56
db/internal_stats.h
View File

@ -9,10 +9,11 @@
//
#pragma once
#include "db/version_set.h"
#include <vector>
#include <map>
#include <string>
#include <vector>
#include "db/version_set.h"
class ColumnFamilyData;
@ -41,13 +42,47 @@ struct DBPropertyInfo {
// holding db mutex, which is only supported for int properties.
bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db,
Version* version);
bool (InternalStats::*handle_map)(
std::map<std::string, double>* compaction_stats);
};
extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
#ifndef ROCKSDB_LITE
enum class LevelStatType {
INVALID = 0,
NUM_FILES,
COMPACTED_FILES,
SIZE_MB,
SCORE,
READ_GB,
RN_GB,
RNP1_GB,
WRITE_GB,
W_NEW_GB,
MOVED_GB,
WRITE_AMP,
READ_MBPS,
WRITE_MBPS,
COMP_SEC,
COMP_COUNT,
AVG_SEC,
KEY_IN,
KEY_DROP,
TOTAL // total number of types
};
struct LevelStat {
// This what will be L?.property_name in the flat map returned to the user
std::string property_name;
// This will be what we will print in the header in the cli
std::string header_name;
};
class InternalStats {
public:
static const std::map<LevelStatType, LevelStat> compaction_level_stats;
enum InternalCFStatsType {
LEVEL0_SLOWDOWN_TOTAL,
LEVEL0_SLOWDOWN_WITH_COMPACTION,
@ -221,6 +256,10 @@ class InternalStats {
bool GetStringProperty(const DBPropertyInfo& property_info,
const Slice& property, std::string* value);
bool GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value);
bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
DBImpl* db);
@ -233,6 +272,10 @@ class InternalStats {
private:
void DumpDBStats(std::string* value);
void DumpCFMapStats(std::map<std::string, double>* cf_stats);
int DumpCFMapStats(
std::map<int, std::map<LevelStatType, double>>* level_stats,
CompactionStats* compaction_stats_sum);
void DumpCFStats(std::string* value);
// Per-DB stats
@ -313,6 +356,7 @@ class InternalStats {
bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix);
bool HandleLevelStats(std::string* value, Slice suffix);
bool HandleStats(std::string* value, Slice suffix);
bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
bool HandleCFStats(std::string* value, Slice suffix);
bool HandleDBStats(std::string* value, Slice suffix);
bool HandleSsTables(std::string* value, Slice suffix);
@ -448,6 +492,12 @@ class InternalStats {
return false;
}
bool GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value) {
return false;
}
bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
DBImpl* db) const {
return false;

12
include/rocksdb/db.h
View File

@ -11,6 +11,7 @@
#include <stdint.h>
#include <stdio.h>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
@ -372,6 +373,10 @@ class DB {
// family stats per-level over db's lifetime ("L<n>"), aggregated over
// db's lifetime ("Sum"), and aggregated over the interval since the
// last retrieval ("Int").
// It could also be used to return the stats in the format of the map.
// In this case there will a pair of string to array of double for
// each level as well as for "Sum". "Int" stats will not be affected
// when this form of stats are retrived.
static const std::string kCFStats;
// "rocksdb.dbstats" - returns a multi-line string with general database
@ -511,6 +516,13 @@ class DB {
virtual bool GetProperty(const Slice& property, std::string* value) {
return GetProperty(DefaultColumnFamily(), property, value);
}
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) = 0;
virtual bool GetMapProperty(const Slice& property,
std::map<std::string, double>* value) {
return GetMapProperty(DefaultColumnFamily(), property, value);
}
// Similar to GetProperty(), but only works for a subset of properties whose
// return value is an integer. Return the value by integer. Supported

7
include/rocksdb/utilities/stackable_db.h
View File

@ -3,6 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <map>
#include <string>
#include "rocksdb/db.h"
@ -133,11 +134,17 @@ class StackableDB : public DB {
return db_->ReleaseSnapshot(snapshot);
}
using DB::GetMapProperty;
using DB::GetProperty;
virtual bool GetProperty(ColumnFamilyHandle* column_family,
const Slice& property, std::string* value) override {
return db_->GetProperty(column_family, property, value);
}
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override {
return db_->GetMapProperty(column_family, property, value);
}
using DB::GetIntProperty;
virtual bool GetIntProperty(ColumnFamilyHandle* column_family,