diff --git a/db/db_impl.cc b/db/db_impl.cc index 4aba14c60..6b410b83a 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1689,7 +1689,7 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family, bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family, const Slice& property, - std::map* value) { + std::map* value) { const DBPropertyInfo* property_info = GetPropertyInfo(property); value->clear(); auto cfd = reinterpret_cast(column_family)->cfd(); diff --git a/db/db_impl.h b/db/db_impl.h index 39c1d6103..e44b81260 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -136,9 +136,9 @@ class DBImpl : public DB { virtual bool GetProperty(ColumnFamilyHandle* column_family, const Slice& property, std::string* value) override; using DB::GetMapProperty; - virtual bool GetMapProperty(ColumnFamilyHandle* column_family, - const Slice& property, - std::map* value) override; + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override; using DB::GetIntProperty; virtual bool GetIntProperty(ColumnFamilyHandle* column_family, const Slice& property, uint64_t* value) override; diff --git a/db/db_test.cc b/db/db_test.cc index fddb7aea6..5ef5e8833 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2324,9 +2324,9 @@ class ModelDB : public DB { return false; } using DB::GetMapProperty; - virtual bool GetMapProperty(ColumnFamilyHandle* column_family, - const Slice& property, - std::map* value) override { + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override { return false; } using DB::GetAggregatedIntProperty; diff --git a/db/db_test2.cc b/db/db_test2.cc index 8f00d20e7..02923cebc 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -1918,19 +1918,19 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); auto get_stat = [](std::string level_str, LevelStatType type, - std::map props) { + std::map props) { auto prop_str = - level_str + "." + + "compaction." + level_str + "." + InternalStats::compaction_level_stats.at(type).property_name.c_str(); auto prop_item = props.find(prop_str); - return prop_item == props.end() ? 0 : prop_item->second; + return prop_item == props.end() ? 0 : std::stod(prop_item->second); }; // Trivial move 2 files to L2 ASSERT_EQ("0,0,2", FilesPerLevel()); // Also test that the stats GetMapProperty API reporting the same result { - std::map prop; + std::map prop; ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop)); ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop)); @@ -1966,7 +1966,7 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { // Test that the stats GetMapProperty API reporting 1 file in L2 { - std::map prop; + std::map prop; ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop)); } diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 54723ea91..463f67b8e 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -436,7 +436,7 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info, bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info, const Slice& property, - std::map* value) { + std::map* value) { assert(value != nullptr); assert(property_info.handle_map != nullptr); return (this->*(property_info.handle_map))(value); @@ -514,7 +514,8 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) { return true; } -bool InternalStats::HandleCFMapStats(std::map* cf_stats) { +bool InternalStats::HandleCFMapStats( + std::map* cf_stats) { DumpCFMapStats(cf_stats); return true; } @@ -892,12 +893,15 @@ void InternalStats::DumpDBStats(std::string* value) { } /** - * Dump Compaction Level stats to a map of stat name to value in double. - * The level in stat name is represented with a prefix "Lx" where "x" - * is the level number. A special level "Sum" represents the sum of a stat - * for all levels. + * Dump Compaction Level stats to a map of stat name with "compaction." prefix + * to value in double as string. The level in stat name is represented with + * a prefix "Lx" where "x" is the level number. A special level "Sum" + * represents the sum of a stat for all levels. + * The result also contains IO stall counters which keys start with "io_stalls." + * and values represent uint64 encoded as strings. */ -void InternalStats::DumpCFMapStats(std::map* cf_stats) { +void InternalStats::DumpCFMapStats( + std::map* cf_stats) { CompactionStats compaction_stats_sum(0); std::map> levels_stats; DumpCFMapStats(&levels_stats, &compaction_stats_sum); @@ -907,11 +911,13 @@ void InternalStats::DumpCFMapStats(std::map* cf_stats) { for (auto const& stat_ent : level_ent.second) { auto stat_type = stat_ent.first; auto key_str = - level_str + "." + + "compaction." + level_str + "." + InternalStats::compaction_level_stats.at(stat_type).property_name; - (*cf_stats)[key_str] = stat_ent.second; + (*cf_stats)[key_str] = std::to_string(stat_ent.second); } } + + DumpCFMapStatsIOStalls(cf_stats); } void InternalStats::DumpCFMapStats( @@ -982,6 +988,39 @@ void InternalStats::DumpCFMapStats( (*levels_stats)[-1] = sum_stats; // -1 is for the Sum level } +void InternalStats::DumpCFMapStatsIOStalls( + std::map* cf_stats) { + (*cf_stats)["io_stalls.level0_slowdown"] = + std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL]); + (*cf_stats)["io_stalls.level0_slowdown_with_compaction"] = + std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION]); + (*cf_stats)["io_stalls.level0_numfiles"] = + std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_TOTAL]); + (*cf_stats)["io_stalls.level0_numfiles_with_compaction"] = + std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION]); + (*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] = + std::to_string(cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT]); + (*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] = + std::to_string(cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT]); + (*cf_stats)["io_stalls.memtable_compaction"] = + std::to_string(cf_stats_count_[MEMTABLE_COMPACTION]); + (*cf_stats)["io_stalls.memtable_slowdown"] = + std::to_string(cf_stats_count_[MEMTABLE_SLOWDOWN]); + + uint64_t total_stop = + cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] + + cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] + + cf_stats_count_[MEMTABLE_COMPACTION]; + + uint64_t total_slowdown = + cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] + + cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] + + cf_stats_count_[MEMTABLE_SLOWDOWN]; + + (*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop); + (*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown); +} + void InternalStats::DumpCFStats(std::string* value) { DumpCFStatsNoFileHistogram(value); DumpCFFileHistogram(value); diff --git a/db/internal_stats.h b/db/internal_stats.h index 1dd393f73..66b9a9149 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -42,8 +42,9 @@ struct DBPropertyInfo { // holding db mutex, which is only supported for int properties. bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db, Version* version); - bool (InternalStats::*handle_map)( - std::map* compaction_stats); + + // @param props Map of general properties to populate + bool (InternalStats::*handle_map)(std::map* props); }; extern const DBPropertyInfo* GetPropertyInfo(const Slice& property); @@ -298,7 +299,7 @@ class InternalStats { bool GetMapProperty(const DBPropertyInfo& property_info, const Slice& property, - std::map* value); + std::map* value); bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, DBImpl* db); @@ -312,10 +313,11 @@ class InternalStats { private: void DumpDBStats(std::string* value); - void DumpCFMapStats(std::map* cf_stats); + void DumpCFMapStats(std::map* cf_stats); void DumpCFMapStats( std::map>* level_stats, CompactionStats* compaction_stats_sum); + void DumpCFMapStatsIOStalls(std::map* cf_stats); void DumpCFStats(std::string* value); void DumpCFStatsNoFileHistogram(std::string* value); void DumpCFFileHistogram(std::string* value); @@ -424,7 +426,7 @@ class InternalStats { bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix); bool HandleLevelStats(std::string* value, Slice suffix); bool HandleStats(std::string* value, Slice suffix); - bool HandleCFMapStats(std::map* compaction_stats); + bool HandleCFMapStats(std::map* compaction_stats); bool HandleCFStats(std::string* value, Slice suffix); bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix); bool HandleCFFileHistogram(std::string* value, Slice suffix); @@ -569,7 +571,7 @@ class InternalStats { bool GetMapProperty(const DBPropertyInfo& property_info, const Slice& property, - std::map* value) { + std::map* value) { return false; } diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 078c24b4f..32392c2f7 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -596,9 +596,9 @@ class DB { } virtual bool GetMapProperty(ColumnFamilyHandle* column_family, const Slice& property, - std::map* value) = 0; + std::map* value) = 0; virtual bool GetMapProperty(const Slice& property, - std::map* value) { + std::map* value) { return GetMapProperty(DefaultColumnFamily(), property, value); } diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index 991de90aa..74e74b3fa 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -160,9 +160,9 @@ class StackableDB : public DB { const Slice& property, std::string* value) override { return db_->GetProperty(column_family, property, value); } - virtual bool GetMapProperty(ColumnFamilyHandle* column_family, - const Slice& property, - std::map* value) override { + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override { return db_->GetMapProperty(column_family, property, value); }