andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Extend property map with compaction stats

Browse Source 
Summary:
This branch extends existing property map which keeps values in doubles to keep values in strings so that it can be used to provide wider range of properties. The immediate need for that is to provide IO stall stats in an easy parseable way to MyRocks which is also part of this branch.
Closes https://github.com/facebook/rocksdb/pull/2794

Differential Revision: D5717676

Pulled By: Tema

fbshipit-source-id: e34ba5b79ba774697f7b97ce1138d8fd55471b8a
This commit is contained in:
Artem Danilov 2017-08-30 15:20:48 -07:00 committed by Facebook Github Bot
parent dc5f29f32f
commit 8a6708f5f2
8 changed files with 73 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
db/db_impl.cc
View File

@ -1689,7 +1689,7 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) {
std::map<std::string, std::string>* value) {
const DBPropertyInfo* property_info = GetPropertyInfo(property);
value->clear();
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();

6
db/db_impl.h
View File

@ -136,9 +136,9 @@ class DBImpl : public DB {
virtual bool GetProperty(ColumnFamilyHandle* column_family,
const Slice& property, std::string* value) override;
using DB::GetMapProperty;
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override;
virtual bool GetMapProperty(
ColumnFamilyHandle* column_family, const Slice& property,
std::map<std::string, std::string>* value) override;
using DB::GetIntProperty;
virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
const Slice& property, uint64_t* value) override;

6
db/db_test.cc
View File

@ -2324,9 +2324,9 @@ class ModelDB : public DB {
return false;
}
using DB::GetMapProperty;
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override {
virtual bool GetMapProperty(
ColumnFamilyHandle* column_family, const Slice& property,
std::map<std::string, std::string>* value) override {
return false;
}
using DB::GetAggregatedIntProperty;

10
db/db_test2.cc
View File

@ -1918,19 +1918,19 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
auto get_stat = [](std::string level_str, LevelStatType type,
std::map<std::string, double> props) {
std::map<std::string, std::string> props) {
auto prop_str =
level_str + "." +
"compaction." + level_str + "." +
InternalStats::compaction_level_stats.at(type).property_name.c_str();
auto prop_item = props.find(prop_str);
return prop_item == props.end() ? 0 : prop_item->second;
return prop_item == props.end() ? 0 : std::stod(prop_item->second);
};
// Trivial move 2 files to L2
ASSERT_EQ("0,0,2", FilesPerLevel());
// Also test that the stats GetMapProperty API reporting the same result
{
std::map<std::string, double> prop;
std::map<std::string, std::string> prop;
ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop));
ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop));
@ -1966,7 +1966,7 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
// Test that the stats GetMapProperty API reporting 1 file in L2
{
std::map<std::string, double> prop;
std::map<std::string, std::string> prop;
ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop));
}

57
db/internal_stats.cc
View File

@ -436,7 +436,7 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value) {
std::map<std::string, std::string>* value) {
assert(value != nullptr);
assert(property_info.handle_map != nullptr);
return (this->*(property_info.handle_map))(value);
@ -514,7 +514,8 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) {
return true;
}
bool InternalStats::HandleCFMapStats(std::map<std::string, double>* cf_stats) {
bool InternalStats::HandleCFMapStats(
std::map<std::string, std::string>* cf_stats) {
DumpCFMapStats(cf_stats);
return true;
}
@ -892,12 +893,15 @@ void InternalStats::DumpDBStats(std::string* value) {
}
/**
* Dump Compaction Level stats to a map of stat name to value in double.
* The level in stat name is represented with a prefix "Lx" where "x"
* is the level number. A special level "Sum" represents the sum of a stat
* for all levels.
* Dump Compaction Level stats to a map of stat name with "compaction." prefix
* to value in double as string. The level in stat name is represented with
* a prefix "Lx" where "x" is the level number. A special level "Sum"
* represents the sum of a stat for all levels.
* The result also contains IO stall counters which keys start with "io_stalls."
* and values represent uint64 encoded as strings.
*/
void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
void InternalStats::DumpCFMapStats(
std::map<std::string, std::string>* cf_stats) {
CompactionStats compaction_stats_sum(0);
std::map<int, std::map<LevelStatType, double>> levels_stats;
DumpCFMapStats(&levels_stats, &compaction_stats_sum);
@ -907,11 +911,13 @@ void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
for (auto const& stat_ent : level_ent.second) {
auto stat_type = stat_ent.first;
auto key_str =
level_str + "." +
"compaction." + level_str + "." +
InternalStats::compaction_level_stats.at(stat_type).property_name;
(*cf_stats)[key_str] = stat_ent.second;
(*cf_stats)[key_str] = std::to_string(stat_ent.second);
}
}
DumpCFMapStatsIOStalls(cf_stats);
}
void InternalStats::DumpCFMapStats(
@ -982,6 +988,39 @@ void InternalStats::DumpCFMapStats(
(*levels_stats)[-1] = sum_stats; // -1 is for the Sum level
}
void InternalStats::DumpCFMapStatsIOStalls(
std::map<std::string, std::string>* cf_stats) {
(*cf_stats)["io_stalls.level0_slowdown"] =
std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL]);
(*cf_stats)["io_stalls.level0_slowdown_with_compaction"] =
std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION]);
(*cf_stats)["io_stalls.level0_numfiles"] =
std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_TOTAL]);
(*cf_stats)["io_stalls.level0_numfiles_with_compaction"] =
std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION]);
(*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] =
std::to_string(cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT]);
(*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] =
std::to_string(cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT]);
(*cf_stats)["io_stalls.memtable_compaction"] =
std::to_string(cf_stats_count_[MEMTABLE_COMPACTION]);
(*cf_stats)["io_stalls.memtable_slowdown"] =
std::to_string(cf_stats_count_[MEMTABLE_SLOWDOWN]);
uint64_t total_stop =
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
cf_stats_count_[MEMTABLE_COMPACTION];
uint64_t total_slowdown =
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] +
cf_stats_count_[MEMTABLE_SLOWDOWN];
(*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop);
(*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown);
}
void InternalStats::DumpCFStats(std::string* value) {
DumpCFStatsNoFileHistogram(value);
DumpCFFileHistogram(value);

14
db/internal_stats.h
View File

@ -42,8 +42,9 @@ struct DBPropertyInfo {
// holding db mutex, which is only supported for int properties.
bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db,
Version* version);
bool (InternalStats::*handle_map)(
std::map<std::string, double>* compaction_stats);
// @param props Map of general properties to populate
bool (InternalStats::*handle_map)(std::map<std::string, std::string>* props);
};
extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
@ -298,7 +299,7 @@ class InternalStats {
bool GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value);
std::map<std::string, std::string>* value);
bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
DBImpl* db);
@ -312,10 +313,11 @@ class InternalStats {
private:
void DumpDBStats(std::string* value);
void DumpCFMapStats(std::map<std::string, double>* cf_stats);
void DumpCFMapStats(std::map<std::string, std::string>* cf_stats);
void DumpCFMapStats(
std::map<int, std::map<LevelStatType, double>>* level_stats,
CompactionStats* compaction_stats_sum);
void DumpCFMapStatsIOStalls(std::map<std::string, std::string>* cf_stats);
void DumpCFStats(std::string* value);
void DumpCFStatsNoFileHistogram(std::string* value);
void DumpCFFileHistogram(std::string* value);
@ -424,7 +426,7 @@ class InternalStats {
bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix);
bool HandleLevelStats(std::string* value, Slice suffix);
bool HandleStats(std::string* value, Slice suffix);
bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
bool HandleCFMapStats(std::map<std::string, std::string>* compaction_stats);
bool HandleCFStats(std::string* value, Slice suffix);
bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix);
bool HandleCFFileHistogram(std::string* value, Slice suffix);
@ -569,7 +571,7 @@ class InternalStats {
bool GetMapProperty(const DBPropertyInfo& property_info,
const Slice& property,
std::map<std::string, double>* value) {
std::map<std::string, std::string>* value) {
return false;
}

4
include/rocksdb/db.h
View File

@ -596,9 +596,9 @@ class DB {
}
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) = 0;
std::map<std::string, std::string>* value) = 0;
virtual bool GetMapProperty(const Slice& property,
std::map<std::string, double>* value) {
std::map<std::string, std::string>* value) {
return GetMapProperty(DefaultColumnFamily(), property, value);
}

6
include/rocksdb/utilities/stackable_db.h
View File

@ -160,9 +160,9 @@ class StackableDB : public DB {
const Slice& property, std::string* value) override {
return db_->GetProperty(column_family, property, value);
}
virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
const Slice& property,
std::map<std::string, double>* value) override {
virtual bool GetMapProperty(
ColumnFamilyHandle* column_family, const Slice& property,
std::map<std::string, std::string>* value) override {
return db_->GetMapProperty(column_family, property, value);
}