From 8a6708f5f2208b328ade511c94b817c480077137 Mon Sep 17 00:00:00 2001 From: Artem Danilov Date: Wed, 30 Aug 2017 15:20:48 -0700 Subject: [PATCH] Extend property map with compaction stats Summary: This branch extends existing property map which keeps values in doubles to keep values in strings so that it can be used to provide wider range of properties. The immediate need for that is to provide IO stall stats in an easy parseable way to MyRocks which is also part of this branch. Closes https://github.com/facebook/rocksdb/pull/2794 Differential Revision: D5717676 Pulled By: Tema fbshipit-source-id: e34ba5b79ba774697f7b97ce1138d8fd55471b8a --- db/db_impl.cc | 2 +- db/db_impl.h | 6 +-- db/db_test.cc | 6 +-- db/db_test2.cc | 10 ++--- db/internal_stats.cc | 57 ++++++++++++++++++++---- db/internal_stats.h | 14 +++--- include/rocksdb/db.h | 4 +- include/rocksdb/utilities/stackable_db.h | 6 +-- 8 files changed, 73 insertions(+), 32 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 4aba14c60..6b410b83a 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1689,7 +1689,7 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family, bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family, const Slice& property, - std::map* value) { + std::map* value) { const DBPropertyInfo* property_info = GetPropertyInfo(property); value->clear(); auto cfd = reinterpret_cast(column_family)->cfd(); diff --git a/db/db_impl.h b/db/db_impl.h index 39c1d6103..e44b81260 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -136,9 +136,9 @@ class DBImpl : public DB { virtual bool GetProperty(ColumnFamilyHandle* column_family, const Slice& property, std::string* value) override; using DB::GetMapProperty; - virtual bool GetMapProperty(ColumnFamilyHandle* column_family, - const Slice& property, - std::map* value) override; + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override; using DB::GetIntProperty; virtual bool GetIntProperty(ColumnFamilyHandle* column_family, const Slice& property, uint64_t* value) override; diff --git a/db/db_test.cc b/db/db_test.cc index fddb7aea6..5ef5e8833 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2324,9 +2324,9 @@ class ModelDB : public DB { return false; } using DB::GetMapProperty; - virtual bool GetMapProperty(ColumnFamilyHandle* column_family, - const Slice& property, - std::map* value) override { + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override { return false; } using DB::GetAggregatedIntProperty; diff --git a/db/db_test2.cc b/db/db_test2.cc index 8f00d20e7..02923cebc 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -1918,19 +1918,19 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); auto get_stat = [](std::string level_str, LevelStatType type, - std::map props) { + std::map props) { auto prop_str = - level_str + "." + + "compaction." + level_str + "." + InternalStats::compaction_level_stats.at(type).property_name.c_str(); auto prop_item = props.find(prop_str); - return prop_item == props.end() ? 0 : prop_item->second; + return prop_item == props.end() ? 0 : std::stod(prop_item->second); }; // Trivial move 2 files to L2 ASSERT_EQ("0,0,2", FilesPerLevel()); // Also test that the stats GetMapProperty API reporting the same result { - std::map prop; + std::map prop; ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop)); ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop)); @@ -1966,7 +1966,7 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { // Test that the stats GetMapProperty API reporting 1 file in L2 { - std::map prop; + std::map prop; ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop)); } diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 54723ea91..463f67b8e 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -436,7 +436,7 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info, bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info, const Slice& property, - std::map* value) { + std::map* value) { assert(value != nullptr); assert(property_info.handle_map != nullptr); return (this->*(property_info.handle_map))(value); @@ -514,7 +514,8 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) { return true; } -bool InternalStats::HandleCFMapStats(std::map* cf_stats) { +bool InternalStats::HandleCFMapStats( + std::map* cf_stats) { DumpCFMapStats(cf_stats); return true; } @@ -892,12 +893,15 @@ void InternalStats::DumpDBStats(std::string* value) { } /** - * Dump Compaction Level stats to a map of stat name to value in double. - * The level in stat name is represented with a prefix "Lx" where "x" - * is the level number. A special level "Sum" represents the sum of a stat - * for all levels. + * Dump Compaction Level stats to a map of stat name with "compaction." prefix + * to value in double as string. The level in stat name is represented with + * a prefix "Lx" where "x" is the level number. A special level "Sum" + * represents the sum of a stat for all levels. + * The result also contains IO stall counters which keys start with "io_stalls." + * and values represent uint64 encoded as strings. */ -void InternalStats::DumpCFMapStats(std::map* cf_stats) { +void InternalStats::DumpCFMapStats( + std::map* cf_stats) { CompactionStats compaction_stats_sum(0); std::map> levels_stats; DumpCFMapStats(&levels_stats, &compaction_stats_sum); @@ -907,11 +911,13 @@ void InternalStats::DumpCFMapStats(std::map* cf_stats) { for (auto const& stat_ent : level_ent.second) { auto stat_type = stat_ent.first; auto key_str = - level_str + "." + + "compaction." + level_str + "." + InternalStats::compaction_level_stats.at(stat_type).property_name; - (*cf_stats)[key_str] = stat_ent.second; + (*cf_stats)[key_str] = std::to_string(stat_ent.second); } } + + DumpCFMapStatsIOStalls(cf_stats); } void InternalStats::DumpCFMapStats( @@ -982,6 +988,39 @@ void InternalStats::DumpCFMapStats( (*levels_stats)[-1] = sum_stats; // -1 is for the Sum level } +void InternalStats::DumpCFMapStatsIOStalls( + std::map* cf_stats) { + (*cf_stats)["io_stalls.level0_slowdown"] = + std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL]); + (*cf_stats)["io_stalls.level0_slowdown_with_compaction"] = + std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION]); + (*cf_stats)["io_stalls.level0_numfiles"] = + std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_TOTAL]); + (*cf_stats)["io_stalls.level0_numfiles_with_compaction"] = + std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION]); + (*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] = + std::to_string(cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT]); + (*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] = + std::to_string(cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT]); + (*cf_stats)["io_stalls.memtable_compaction"] = + std::to_string(cf_stats_count_[MEMTABLE_COMPACTION]); + (*cf_stats)["io_stalls.memtable_slowdown"] = + std::to_string(cf_stats_count_[MEMTABLE_SLOWDOWN]); + + uint64_t total_stop = + cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] + + cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] + + cf_stats_count_[MEMTABLE_COMPACTION]; + + uint64_t total_slowdown = + cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] + + cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] + + cf_stats_count_[MEMTABLE_SLOWDOWN]; + + (*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop); + (*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown); +} + void InternalStats::DumpCFStats(std::string* value) { DumpCFStatsNoFileHistogram(value); DumpCFFileHistogram(value); diff --git a/db/internal_stats.h b/db/internal_stats.h index 1dd393f73..66b9a9149 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -42,8 +42,9 @@ struct DBPropertyInfo { // holding db mutex, which is only supported for int properties. bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db, Version* version); - bool (InternalStats::*handle_map)( - std::map* compaction_stats); + + // @param props Map of general properties to populate + bool (InternalStats::*handle_map)(std::map* props); }; extern const DBPropertyInfo* GetPropertyInfo(const Slice& property); @@ -298,7 +299,7 @@ class InternalStats { bool GetMapProperty(const DBPropertyInfo& property_info, const Slice& property, - std::map* value); + std::map* value); bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, DBImpl* db); @@ -312,10 +313,11 @@ class InternalStats { private: void DumpDBStats(std::string* value); - void DumpCFMapStats(std::map* cf_stats); + void DumpCFMapStats(std::map* cf_stats); void DumpCFMapStats( std::map>* level_stats, CompactionStats* compaction_stats_sum); + void DumpCFMapStatsIOStalls(std::map* cf_stats); void DumpCFStats(std::string* value); void DumpCFStatsNoFileHistogram(std::string* value); void DumpCFFileHistogram(std::string* value); @@ -424,7 +426,7 @@ class InternalStats { bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix); bool HandleLevelStats(std::string* value, Slice suffix); bool HandleStats(std::string* value, Slice suffix); - bool HandleCFMapStats(std::map* compaction_stats); + bool HandleCFMapStats(std::map* compaction_stats); bool HandleCFStats(std::string* value, Slice suffix); bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix); bool HandleCFFileHistogram(std::string* value, Slice suffix); @@ -569,7 +571,7 @@ class InternalStats { bool GetMapProperty(const DBPropertyInfo& property_info, const Slice& property, - std::map* value) { + std::map* value) { return false; } diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 078c24b4f..32392c2f7 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -596,9 +596,9 @@ class DB { } virtual bool GetMapProperty(ColumnFamilyHandle* column_family, const Slice& property, - std::map* value) = 0; + std::map* value) = 0; virtual bool GetMapProperty(const Slice& property, - std::map* value) { + std::map* value) { return GetMapProperty(DefaultColumnFamily(), property, value); } diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index 991de90aa..74e74b3fa 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -160,9 +160,9 @@ class StackableDB : public DB { const Slice& property, std::string* value) override { return db_->GetProperty(column_family, property, value); } - virtual bool GetMapProperty(ColumnFamilyHandle* column_family, - const Slice& property, - std::map* value) override { + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override { return db_->GetMapProperty(column_family, property, value); }