Extend property map with compaction stats

Summary:
This branch extends existing property map which keeps values in doubles to keep values in strings so that it can be used to provide wider range of properties. The immediate need for that is to provide IO stall stats in an easy parseable way to MyRocks which is also part of this branch.
Closes https://github.com/facebook/rocksdb/pull/2794

Differential Revision: D5717676

Pulled By: Tema

fbshipit-source-id: e34ba5b79ba774697f7b97ce1138d8fd55471b8a

db/db_impl.cc

@@ -1689,7 +1689,7 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
 
 bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
                             const Slice& property,
-                            std::map<std::string, double>* value) {
+                            std::map<std::string, std::string>* value) {
   const DBPropertyInfo* property_info = GetPropertyInfo(property);
   value->clear();
   auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();

db/db_impl.h

@@ -136,9 +136,9 @@ class DBImpl : public DB {
   virtual bool GetProperty(ColumnFamilyHandle* column_family,
                            const Slice& property, std::string* value) override;
   using DB::GetMapProperty;
-  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+  virtual bool GetMapProperty(
-                              const Slice& property,
+      ColumnFamilyHandle* column_family, const Slice& property,
-                              std::map<std::string, double>* value) override;
+      std::map<std::string, std::string>* value) override;
   using DB::GetIntProperty;
   virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                               const Slice& property, uint64_t* value) override;

db/db_test.cc

@@ -2324,9 +2324,9 @@ class ModelDB : public DB {
     return false;
   }
   using DB::GetMapProperty;
-  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+  virtual bool GetMapProperty(
-                              const Slice& property,
+      ColumnFamilyHandle* column_family, const Slice& property,
-                              std::map<std::string, double>* value) override {
+      std::map<std::string, std::string>* value) override {
     return false;
   }
   using DB::GetAggregatedIntProperty;

db/db_test2.cc

@@ -1918,19 +1918,19 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
 
   auto get_stat = [](std::string level_str, LevelStatType type,
-                     std::map<std::string, double> props) {
+                     std::map<std::string, std::string> props) {
     auto prop_str =
-        level_str + "." +
+        "compaction." + level_str + "." +
         InternalStats::compaction_level_stats.at(type).property_name.c_str();
     auto prop_item = props.find(prop_str);
-    return prop_item == props.end() ? 0 : prop_item->second;
+    return prop_item == props.end() ? 0 : std::stod(prop_item->second);
   };
 
   // Trivial move 2 files to L2
   ASSERT_EQ("0,0,2", FilesPerLevel());
   // Also test that the stats GetMapProperty API reporting the same result
   {
-    std::map<std::string, double> prop;
+    std::map<std::string, std::string> prop;
     ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
     ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop));
     ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop));
@@ -1966,7 +1966,7 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
 
   // Test that the stats GetMapProperty API reporting 1 file in L2
   {
-    std::map<std::string, double> prop;
+    std::map<std::string, std::string> prop;
     ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
     ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop));
   }

db/internal_stats.cc

@@ -436,7 +436,7 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
 
 bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
                                    const Slice& property,
-                                   std::map<std::string, double>* value) {
+                                   std::map<std::string, std::string>* value) {
   assert(value != nullptr);
   assert(property_info.handle_map != nullptr);
   return (this->*(property_info.handle_map))(value);
@@ -514,7 +514,8 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) {
   return true;
 }
 
-bool InternalStats::HandleCFMapStats(std::map<std::string, double>* cf_stats) {
+bool InternalStats::HandleCFMapStats(
+    std::map<std::string, std::string>* cf_stats) {
   DumpCFMapStats(cf_stats);
   return true;
 }
@@ -892,12 +893,15 @@ void InternalStats::DumpDBStats(std::string* value) {
 }
 
 /**
- * Dump Compaction Level stats to a map of stat name to value in double.
+ * Dump Compaction Level stats to a map of stat name with "compaction." prefix
- * The level in stat name is represented with a prefix "Lx" where "x"
+ * to value in double as string. The level in stat name is represented with
- * is the level number. A special level "Sum" represents the sum of a stat
+ * a prefix "Lx" where "x" is the level number. A special level "Sum"
- * for all levels.
+ * represents the sum of a stat for all levels.
+ * The result also contains IO stall counters which keys start with "io_stalls."
+ * and values represent uint64 encoded as strings.
  */
-void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
+void InternalStats::DumpCFMapStats(
+        std::map<std::string, std::string>* cf_stats) {
   CompactionStats compaction_stats_sum(0);
   std::map<int, std::map<LevelStatType, double>> levels_stats;
   DumpCFMapStats(&levels_stats, &compaction_stats_sum);
@@ -907,11 +911,13 @@ void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
     for (auto const& stat_ent : level_ent.second) {
       auto stat_type = stat_ent.first;
       auto key_str =
-          level_str + "." +
+          "compaction." + level_str + "." +
           InternalStats::compaction_level_stats.at(stat_type).property_name;
-      (*cf_stats)[key_str] = stat_ent.second;
+      (*cf_stats)[key_str] = std::to_string(stat_ent.second);
     }
   }
+
+  DumpCFMapStatsIOStalls(cf_stats);
 }
 
 void InternalStats::DumpCFMapStats(
@@ -982,6 +988,39 @@ void InternalStats::DumpCFMapStats(
   (*levels_stats)[-1] = sum_stats;  //  -1 is for the Sum level
 }
 
+void InternalStats::DumpCFMapStatsIOStalls(
+    std::map<std::string, std::string>* cf_stats) {
+  (*cf_stats)["io_stalls.level0_slowdown"] =
+      std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL]);
+  (*cf_stats)["io_stalls.level0_slowdown_with_compaction"] =
+      std::to_string(cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION]);
+  (*cf_stats)["io_stalls.level0_numfiles"] =
+      std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_TOTAL]);
+  (*cf_stats)["io_stalls.level0_numfiles_with_compaction"] =
+      std::to_string(cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION]);
+  (*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] =
+      std::to_string(cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT]);
+  (*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] =
+      std::to_string(cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT]);
+  (*cf_stats)["io_stalls.memtable_compaction"] =
+      std::to_string(cf_stats_count_[MEMTABLE_COMPACTION]);
+  (*cf_stats)["io_stalls.memtable_slowdown"] =
+      std::to_string(cf_stats_count_[MEMTABLE_SLOWDOWN]);
+
+  uint64_t total_stop =
+      cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
+      cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
+      cf_stats_count_[MEMTABLE_COMPACTION];
+
+  uint64_t total_slowdown =
+      cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
+      cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] +
+      cf_stats_count_[MEMTABLE_SLOWDOWN];
+
+  (*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop);
+  (*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown);
+}
+
 void InternalStats::DumpCFStats(std::string* value) {
   DumpCFStatsNoFileHistogram(value);
   DumpCFFileHistogram(value);

db/internal_stats.h

@@ -42,8 +42,9 @@ struct DBPropertyInfo {
   //      holding db mutex, which is only supported for int properties.
   bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db,
                                     Version* version);
-  bool (InternalStats::*handle_map)(
+
-      std::map<std::string, double>* compaction_stats);
+  // @param props Map of general properties to populate
+  bool (InternalStats::*handle_map)(std::map<std::string, std::string>* props);
 };
 
 extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
@@ -298,7 +299,7 @@ class InternalStats {
 
   bool GetMapProperty(const DBPropertyInfo& property_info,
                       const Slice& property,
-                      std::map<std::string, double>* value);
+                      std::map<std::string, std::string>* value);
 
   bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
                       DBImpl* db);
@@ -312,10 +313,11 @@ class InternalStats {
 
  private:
   void DumpDBStats(std::string* value);
-  void DumpCFMapStats(std::map<std::string, double>* cf_stats);
+  void DumpCFMapStats(std::map<std::string, std::string>* cf_stats);
   void DumpCFMapStats(
       std::map<int, std::map<LevelStatType, double>>* level_stats,
       CompactionStats* compaction_stats_sum);
+  void DumpCFMapStatsIOStalls(std::map<std::string, std::string>* cf_stats);
   void DumpCFStats(std::string* value);
   void DumpCFStatsNoFileHistogram(std::string* value);
   void DumpCFFileHistogram(std::string* value);
@@ -424,7 +426,7 @@ class InternalStats {
   bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix);
   bool HandleLevelStats(std::string* value, Slice suffix);
   bool HandleStats(std::string* value, Slice suffix);
-  bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
+  bool HandleCFMapStats(std::map<std::string, std::string>* compaction_stats);
   bool HandleCFStats(std::string* value, Slice suffix);
   bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix);
   bool HandleCFFileHistogram(std::string* value, Slice suffix);
@@ -569,7 +571,7 @@ class InternalStats {
 
   bool GetMapProperty(const DBPropertyInfo& property_info,
                       const Slice& property,
-                      std::map<std::string, double>* value) {
+                      std::map<std::string, std::string>* value) {
     return false;
   }
 

include/rocksdb/db.h

@@ -596,9 +596,9 @@ class DB {
   }
   virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
                               const Slice& property,
-                              std::map<std::string, double>* value) = 0;
+                              std::map<std::string, std::string>* value) = 0;
   virtual bool GetMapProperty(const Slice& property,
-                              std::map<std::string, double>* value) {
+                              std::map<std::string, std::string>* value) {
     return GetMapProperty(DefaultColumnFamily(), property, value);
   }
 

include/rocksdb/utilities/stackable_db.h

@@ -160,9 +160,9 @@ class StackableDB : public DB {
                            const Slice& property, std::string* value) override {
     return db_->GetProperty(column_family, property, value);
   }
-  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+  virtual bool GetMapProperty(
-                              const Slice& property,
+      ColumnFamilyHandle* column_family, const Slice& property,
-                              std::map<std::string, double>* value) override {
+      std::map<std::string, std::string>* value) override {
     return db_->GetMapProperty(column_family, property, value);
   }