diff --git a/db/db_impl.cc b/db/db_impl.cc index b3ddda295..f13dfeb97 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -5292,6 +5292,24 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family, return false; } +bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family, + const Slice& property, + std::map* value) { + const DBPropertyInfo* property_info = GetPropertyInfo(property); + value->clear(); + auto cfd = reinterpret_cast(column_family)->cfd(); + if (property_info == nullptr) { + return false; + } else if (property_info->handle_map) { + InstrumentedMutexLock l(&mutex_); + return cfd->internal_stats()->GetMapProperty(*property_info, property, + value); + } + // If we reach this point it means that handle_map is not provided for the + // requested property + return false; +} + bool DBImpl::GetIntProperty(ColumnFamilyHandle* column_family, const Slice& property, uint64_t* value) { const DBPropertyInfo* property_info = GetPropertyInfo(property); diff --git a/db/db_impl.h b/db/db_impl.h index cbf9a292d..6a1e91133 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -123,6 +124,10 @@ class DBImpl : public DB { using DB::GetProperty; virtual bool GetProperty(ColumnFamilyHandle* column_family, const Slice& property, std::string* value) override; + using DB::GetMapProperty; + virtual bool GetMapProperty(ColumnFamilyHandle* column_family, + const Slice& property, + std::map* value) override; using DB::GetIntProperty; virtual bool GetIntProperty(ColumnFamilyHandle* column_family, const Slice& property, uint64_t* value) override; diff --git a/db/db_test.cc b/db/db_test.cc index d3a82881a..f82dff501 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2740,6 +2740,12 @@ class ModelDB : public DB { const Slice& property, uint64_t* value) override { return false; } + using DB::GetMapProperty; + virtual bool GetMapProperty(ColumnFamilyHandle* column_family, + const Slice& property, + std::map* value) override { + return false; + } using DB::GetAggregatedIntProperty; virtual bool GetAggregatedIntProperty(const Slice& property, uint64_t* value) override { diff --git a/db/db_test2.cc b/db/db_test2.cc index c72837eff..85e605791 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -2085,8 +2085,26 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { cro.target_level = 2; ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + auto get_stat = [](std::string level_str, LevelStatType type, + std::map props) { + auto prop_str = + level_str + "." + + InternalStats::compaction_level_stats.at(type).property_name.c_str(); + auto prop_item = props.find(prop_str); + return prop_item == props.end() ? 0 : prop_item->second; + }; + // Trivial move 2 files to L2 ASSERT_EQ("0,0,2", FilesPerLevel()); + // Also test that the stats GetMapProperty API reporting the same result + { + std::map prop; + ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); + ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop)); + ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop)); + ASSERT_EQ(2, get_stat("L2", LevelStatType::NUM_FILES, prop)); + ASSERT_EQ(2, get_stat("Sum", LevelStatType::NUM_FILES, prop)); + } // While the compaction is running, we will create 2 new files that // can fit in L2, these 2 files will be moved to L2 and overlap with @@ -2113,6 +2131,13 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); rocksdb::SyncPoint::GetInstance()->DisableProcessing(); + + // Test that the stats GetMapProperty API reporting 1 file in L2 + { + std::map prop; + ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); + ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop)); + } } TEST_F(DBTest2, ManualCompactionOverlapManualCompaction) { diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 358b58e84..122e8c9b6 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -25,62 +25,146 @@ namespace rocksdb { #ifndef ROCKSDB_LITE + +const std::map InternalStats::compaction_level_stats = + { + {LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}}, + {LevelStatType::COMPACTED_FILES, + LevelStat{"CompactedFiles", "CompactedFiles"}}, + {LevelStatType::SIZE_MB, LevelStat{"SizeMB", "Size(MB}"}}, + {LevelStatType::SCORE, LevelStat{"Score", "Score"}}, + {LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB}"}}, + {LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB}"}}, + {LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB}"}}, + {LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB}"}}, + {LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB}"}}, + {LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB}"}}, + {LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}}, + {LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s}"}}, + {LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s}"}}, + {LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec}"}}, + {LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt}"}}, + {LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec}"}}, + {LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}}, + {LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}}, +}; + namespace { const double kMB = 1048576.0; const double kGB = kMB * 1024; const double kMicrosInSec = 1000000.0; void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) { - snprintf( - buf, len, - "\n** Compaction Stats [%s] **\n" - "Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) " - "Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) " - "Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop\n" - "--------------------------------------------------------------------" - "-----------------------------------------------------------" - "--------------------------------------\n", - cf_name.c_str()); + int written_size = + snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str()); + auto hdr = [](LevelStatType t) { + return InternalStats::compaction_level_stats.at(t).header_name.c_str(); + }; + int line_size = snprintf( + buf + written_size, len - written_size, + "Level %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n", + // Note that we skip COMPACTED_FILES and merge it with Files column + hdr(LevelStatType::NUM_FILES), hdr(LevelStatType::SIZE_MB), + hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB), + hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB), + hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB), + hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP), + hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS), + hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_COUNT), + hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN), + hdr(LevelStatType::KEY_DROP)); + + written_size += line_size; + snprintf(buf + written_size, len - written_size, "%s\n", + std::string(line_size, '-').c_str()); } -void PrintLevelStats(char* buf, size_t len, const std::string& name, - int num_files, int being_compacted, double total_file_size, - double score, double w_amp, - const InternalStats::CompactionStats& stats) { +void PrepareLevelStats(std::map* level_stats, + int num_files, int being_compacted, + double total_file_size, double score, double w_amp, + const InternalStats::CompactionStats& stats) { uint64_t bytes_read = stats.bytes_read_non_output_levels + stats.bytes_read_output_level; int64_t bytes_new = stats.bytes_written - stats.bytes_read_output_level; double elapsed = (stats.micros + 1) / kMicrosInSec; - std::string num_input_records = NumberToHumanString(stats.num_input_records); - std::string num_dropped_records = - NumberToHumanString(stats.num_dropped_records); + (*level_stats)[LevelStatType::NUM_FILES] = num_files; + (*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted; + (*level_stats)[LevelStatType::SIZE_MB] = total_file_size / kMB; + (*level_stats)[LevelStatType::SCORE] = score; + (*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB; + (*level_stats)[LevelStatType::RN_GB] = + stats.bytes_read_non_output_levels / kGB; + (*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB; + (*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB; + (*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB; + (*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB; + (*level_stats)[LevelStatType::WRITE_AMP] = w_amp; + (*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed; + (*level_stats)[LevelStatType::WRITE_MBPS] = + stats.bytes_written / kMB / elapsed; + (*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec; + (*level_stats)[LevelStatType::COMP_COUNT] = stats.count; + (*level_stats)[LevelStatType::AVG_SEC] = + stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count; + (*level_stats)[LevelStatType::KEY_IN] = + static_cast(stats.num_input_records); + (*level_stats)[LevelStatType::KEY_DROP] = + static_cast(stats.num_dropped_records); +} + +void PrintLevelStats(char* buf, size_t len, const std::string& name, + const std::map& stat_value) { snprintf(buf, len, - "%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */ - "%8.1f " /* Read(GB) */ - "%7.1f " /* Rn(GB) */ - "%8.1f " /* Rnp1(GB) */ - "%9.1f " /* Write(GB) */ - "%8.1f " /* Wnew(GB) */ - "%9.1f " /* Moved(GB) */ - "%5.1f " /* W-Amp */ - "%8.1f " /* Rd(MB/s) */ - "%8.1f " /* Wr(MB/s) */ - "%9.0f " /* Comp(sec) */ - "%9d " /* Comp(cnt) */ - "%8.3f " /* Avg(sec) */ - "%7s " /* KeyIn */ - "%6s\n", /* KeyDrop */ + "%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */ + "%8.1f " /* Read(GB) */ + "%7.1f " /* Rn(GB) */ + "%8.1f " /* Rnp1(GB) */ + "%9.1f " /* Write(GB) */ + "%8.1f " /* Wnew(GB) */ + "%9.1f " /* Moved(GB) */ + "%5.1f " /* W-Amp */ + "%8.1f " /* Rd(MB/s) */ + "%8.1f " /* Wr(MB/s) */ + "%9.0f " /* Comp(sec) */ + "%9d " /* Comp(cnt) */ + "%8.3f " /* Avg(sec) */ + "%7s " /* KeyIn */ + "%6s\n", /* KeyDrop */ name.c_str(), - num_files, being_compacted, total_file_size / kMB, score, - bytes_read / kGB, stats.bytes_read_non_output_levels / kGB, - stats.bytes_read_output_level / kGB, stats.bytes_written / kGB, - bytes_new / kGB, stats.bytes_moved / kGB, w_amp, - bytes_read / kMB / elapsed, stats.bytes_written / kMB / elapsed, - stats.micros / kMicrosInSec, stats.count, - stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count, - num_input_records.c_str(), num_dropped_records.c_str()); + static_cast(stat_value.at(LevelStatType::NUM_FILES)), + static_cast(stat_value.at(LevelStatType::COMPACTED_FILES)), + stat_value.at(LevelStatType::SIZE_MB), + stat_value.at(LevelStatType::SCORE), + stat_value.at(LevelStatType::READ_GB), + stat_value.at(LevelStatType::RN_GB), + stat_value.at(LevelStatType::RNP1_GB), + stat_value.at(LevelStatType::WRITE_GB), + stat_value.at(LevelStatType::W_NEW_GB), + stat_value.at(LevelStatType::MOVED_GB), + stat_value.at(LevelStatType::WRITE_AMP), + stat_value.at(LevelStatType::READ_MBPS), + stat_value.at(LevelStatType::WRITE_MBPS), + stat_value.at(LevelStatType::COMP_SEC), + static_cast(stat_value.at(LevelStatType::COMP_COUNT)), + stat_value.at(LevelStatType::AVG_SEC), + NumberToHumanString( + static_cast(stat_value.at(LevelStatType::KEY_IN))) + .c_str(), + NumberToHumanString(static_cast( + stat_value.at(LevelStatType::KEY_DROP))) + .c_str()); +} + +void PrintLevelStats(char* buf, size_t len, const std::string& name, + int num_files, int being_compacted, double total_file_size, + double score, double w_amp, + const InternalStats::CompactionStats& stats) { + std::map level_stats; + PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size, + score, w_amp, stats); + PrintLevelStats(buf, len, name, level_stats); } // Assumes that trailing numbers represent an optional argument. This requires @@ -211,73 +295,91 @@ const std::string DB::Properties::kAggregatedTableProperties = const std::string DB::Properties::kAggregatedTablePropertiesAtLevel = rocksdb_prefix + aggregated_table_properties_at_level; -const std::unordered_map InternalStats::ppt_name_to_info = { - {DB::Properties::kNumFilesAtLevelPrefix, - {false, &InternalStats::HandleNumFilesAtLevel, nullptr}}, - {DB::Properties::kCompressionRatioAtLevelPrefix, - {false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr}}, - {DB::Properties::kLevelStats, - {false, &InternalStats::HandleLevelStats, nullptr}}, - {DB::Properties::kStats, {false, &InternalStats::HandleStats, nullptr}}, - {DB::Properties::kCFStats, {false, &InternalStats::HandleCFStats, nullptr}}, - {DB::Properties::kDBStats, {false, &InternalStats::HandleDBStats, nullptr}}, - {DB::Properties::kSSTables, - {false, &InternalStats::HandleSsTables, nullptr}}, - {DB::Properties::kAggregatedTableProperties, - {false, &InternalStats::HandleAggregatedTableProperties, nullptr}}, - {DB::Properties::kAggregatedTablePropertiesAtLevel, - {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, nullptr}}, - {DB::Properties::kNumImmutableMemTable, - {false, nullptr, &InternalStats::HandleNumImmutableMemTable}}, - {DB::Properties::kNumImmutableMemTableFlushed, - {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed}}, - {DB::Properties::kMemTableFlushPending, - {false, nullptr, &InternalStats::HandleMemTableFlushPending}}, - {DB::Properties::kCompactionPending, - {false, nullptr, &InternalStats::HandleCompactionPending}}, - {DB::Properties::kBackgroundErrors, - {false, nullptr, &InternalStats::HandleBackgroundErrors}}, - {DB::Properties::kCurSizeActiveMemTable, - {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable}}, - {DB::Properties::kCurSizeAllMemTables, - {false, nullptr, &InternalStats::HandleCurSizeAllMemTables}}, - {DB::Properties::kSizeAllMemTables, - {false, nullptr, &InternalStats::HandleSizeAllMemTables}}, - {DB::Properties::kNumEntriesActiveMemTable, - {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable}}, - {DB::Properties::kNumEntriesImmMemTables, - {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables}}, - {DB::Properties::kNumDeletesActiveMemTable, - {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable}}, - {DB::Properties::kNumDeletesImmMemTables, - {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables}}, - {DB::Properties::kEstimateNumKeys, - {false, nullptr, &InternalStats::HandleEstimateNumKeys}}, - {DB::Properties::kEstimateTableReadersMem, - {true, nullptr, &InternalStats::HandleEstimateTableReadersMem}}, - {DB::Properties::kIsFileDeletionsEnabled, - {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled}}, - {DB::Properties::kNumSnapshots, - {false, nullptr, &InternalStats::HandleNumSnapshots}}, - {DB::Properties::kOldestSnapshotTime, - {false, nullptr, &InternalStats::HandleOldestSnapshotTime}}, - {DB::Properties::kNumLiveVersions, - {false, nullptr, &InternalStats::HandleNumLiveVersions}}, - {DB::Properties::kCurrentSuperVersionNumber, - {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber}}, - {DB::Properties::kEstimateLiveDataSize, - {true, nullptr, &InternalStats::HandleEstimateLiveDataSize}}, - {DB::Properties::kBaseLevel, - {false, nullptr, &InternalStats::HandleBaseLevel}}, - {DB::Properties::kTotalSstFilesSize, - {false, nullptr, &InternalStats::HandleTotalSstFilesSize}}, - {DB::Properties::kEstimatePendingCompactionBytes, - {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes}}, - {DB::Properties::kNumRunningFlushes, - {false, nullptr, &InternalStats::HandleNumRunningFlushes}}, - {DB::Properties::kNumRunningCompactions, - {false, nullptr, &InternalStats::HandleNumRunningCompactions}}, +const std::unordered_map + InternalStats::ppt_name_to_info = { + {DB::Properties::kNumFilesAtLevelPrefix, + {false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr}}, + {DB::Properties::kCompressionRatioAtLevelPrefix, + {false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr, + nullptr}}, + {DB::Properties::kLevelStats, + {false, &InternalStats::HandleLevelStats, nullptr, nullptr}}, + {DB::Properties::kStats, + {false, &InternalStats::HandleStats, nullptr, nullptr}}, + {DB::Properties::kCFStats, + {false, &InternalStats::HandleCFStats, nullptr, + &InternalStats::HandleCFMapStats}}, + {DB::Properties::kDBStats, + {false, &InternalStats::HandleDBStats, nullptr, nullptr}}, + {DB::Properties::kSSTables, + {false, &InternalStats::HandleSsTables, nullptr, nullptr}}, + {DB::Properties::kAggregatedTableProperties, + {false, &InternalStats::HandleAggregatedTableProperties, nullptr, + nullptr}}, + {DB::Properties::kAggregatedTablePropertiesAtLevel, + {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, + nullptr, nullptr}}, + {DB::Properties::kNumImmutableMemTable, + {false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr}}, + {DB::Properties::kNumImmutableMemTableFlushed, + {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed, + nullptr}}, + {DB::Properties::kMemTableFlushPending, + {false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr}}, + {DB::Properties::kCompactionPending, + {false, nullptr, &InternalStats::HandleCompactionPending, nullptr}}, + {DB::Properties::kBackgroundErrors, + {false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr}}, + {DB::Properties::kCurSizeActiveMemTable, + {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable, + nullptr}}, + {DB::Properties::kCurSizeAllMemTables, + {false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr}}, + {DB::Properties::kSizeAllMemTables, + {false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr}}, + {DB::Properties::kNumEntriesActiveMemTable, + {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable, + nullptr}}, + {DB::Properties::kNumEntriesImmMemTables, + {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables, + nullptr}}, + {DB::Properties::kNumDeletesActiveMemTable, + {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable, + nullptr}}, + {DB::Properties::kNumDeletesImmMemTables, + {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables, + nullptr}}, + {DB::Properties::kEstimateNumKeys, + {false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr}}, + {DB::Properties::kEstimateTableReadersMem, + {true, nullptr, &InternalStats::HandleEstimateTableReadersMem, + nullptr}}, + {DB::Properties::kIsFileDeletionsEnabled, + {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled, + nullptr}}, + {DB::Properties::kNumSnapshots, + {false, nullptr, &InternalStats::HandleNumSnapshots, nullptr}}, + {DB::Properties::kOldestSnapshotTime, + {false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr}}, + {DB::Properties::kNumLiveVersions, + {false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr}}, + {DB::Properties::kCurrentSuperVersionNumber, + {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber, + nullptr}}, + {DB::Properties::kEstimateLiveDataSize, + {true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr}}, + {DB::Properties::kBaseLevel, + {false, nullptr, &InternalStats::HandleBaseLevel, nullptr}}, + {DB::Properties::kTotalSstFilesSize, + {false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr}}, + {DB::Properties::kEstimatePendingCompactionBytes, + {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes, + nullptr}}, + {DB::Properties::kNumRunningFlushes, + {false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr}}, + {DB::Properties::kNumRunningCompactions, + {false, nullptr, &InternalStats::HandleNumRunningCompactions, + nullptr}}, }; const DBPropertyInfo* GetPropertyInfo(const Slice& property) { @@ -298,6 +400,14 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info, return (this->*(property_info.handle_string))(value, arg); } +bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info, + const Slice& property, + std::map* value) { + assert(value != nullptr); + assert(property_info.handle_map != nullptr); + return (this->*(property_info.handle_map))(value); +} + bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, DBImpl* db) { assert(value != nullptr); @@ -370,6 +480,11 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) { return true; } +bool InternalStats::HandleCFMapStats(std::map* cf_stats) { + DumpCFMapStats(cf_stats); + return true; +} + bool InternalStats::HandleCFStats(std::string* value, Slice suffix) { DumpCFStats(value); return true; @@ -716,7 +831,32 @@ void InternalStats::DumpDBStats(std::string* value) { db_stats_snapshot_.write_stall_micros = write_stall_micros; } -void InternalStats::DumpCFStats(std::string* value) { +/** + * Dump Compaction Level stats to a map of stat name to value in double. + * The level in stat name is represented with a prefix "Lx" where "x" + * is the level number. A special level "Sum" represents the sum of a stat + * for all levels. + */ +void InternalStats::DumpCFMapStats(std::map* cf_stats) { + CompactionStats compaction_stats_sum(0); + std::map> levels_stats; + DumpCFMapStats(&levels_stats, &compaction_stats_sum); + for (auto const& level_ent : levels_stats) { + auto level_str = + level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first); + for (auto const& stat_ent : level_ent.second) { + auto stat_type = stat_ent.first; + auto key_str = + level_str + "." + + InternalStats::compaction_level_stats.at(stat_type).property_name; + (*cf_stats)[key_str] = stat_ent.second; + } + } +} + +int InternalStats::DumpCFMapStats( + std::map>* levels_stats, + CompactionStats* compaction_stats_sum) { const VersionStorageInfo* vstorage = cfd_->current()->storage_info(); int num_levels_to_check = @@ -724,7 +864,7 @@ void InternalStats::DumpCFStats(std::string* value) { ? vstorage->num_levels() - 1 : 1; - // Compaction scores are sorted base on its value. Restore them to the + // Compaction scores are sorted based on its value. Restore them to the // level order std::vector compaction_score(number_levels_, 0); for (int i = 0; i < num_levels_to_check; ++i) { @@ -741,12 +881,6 @@ void InternalStats::DumpCFStats(std::string* value) { } } - char buf[1000]; - // Per-ColumnFamily stats - PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName()); - value->append(buf); - - CompactionStats stats_sum(0); int total_files = 0; int total_files_being_compacted = 0; double total_file_size = 0; @@ -755,40 +889,67 @@ void InternalStats::DumpCFStats(std::string* value) { total_files += files; total_files_being_compacted += files_being_compacted[level]; if (comp_stats_[level].micros > 0 || files > 0) { - stats_sum.Add(comp_stats_[level]); + compaction_stats_sum->Add(comp_stats_[level]); total_file_size += vstorage->NumLevelBytes(level); double w_amp = - (comp_stats_[level].bytes_read_non_output_levels == 0) ? 0.0 - : static_cast(comp_stats_[level].bytes_written) / - comp_stats_[level].bytes_read_non_output_levels; - PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files, - files_being_compacted[level], - static_cast(vstorage->NumLevelBytes(level)), - compaction_score[level], - w_amp, comp_stats_[level]); - value->append(buf); + (comp_stats_[level].bytes_read_non_output_levels == 0) + ? 0.0 + : static_cast(comp_stats_[level].bytes_written) / + comp_stats_[level].bytes_read_non_output_levels; + std::map level_stats; + PrepareLevelStats(&level_stats, files, files_being_compacted[level], + static_cast(vstorage->NumLevelBytes(level)), + compaction_score[level], w_amp, comp_stats_[level]); + (*levels_stats)[level] = level_stats; } } uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED]; uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE]; uint64_t curr_ingest = flush_ingest + add_file_ingest; + // Cumulative summary + double w_amp = compaction_stats_sum->bytes_written / + static_cast(curr_ingest + 1); + // Stats summary across levels + std::map sum_stats; + PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted, + total_file_size, 0, w_amp, *compaction_stats_sum); + (*levels_stats)[-1] = sum_stats; // -1 is for the Sum level + return num_levels_to_check; +} + +void InternalStats::DumpCFStats(std::string* value) { + char buf[1000]; + // Per-ColumnFamily stats + PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName()); + value->append(buf); + + // Print stats for each level + std::map> levels_stats; + CompactionStats compaction_stats_sum(0); + int levels = DumpCFMapStats(&levels_stats, &compaction_stats_sum); + for (int l = 0; l < levels; ++l) { + if (levels_stats.find(l) != levels_stats.end()) { + PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]); + value->append(buf); + } + } + // Print sum of level stats + PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]); + value->append(buf); + + uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED]; + uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE]; uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL]; uint64_t ingest_l0_files_addfile = cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL]; uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL]; // Cumulative summary - double w_amp = stats_sum.bytes_written / static_cast(curr_ingest + 1); uint64_t total_stall_count = cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] + cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] + cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] + cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] + cf_stats_count_[MEMTABLE_COMPACTION] + cf_stats_count_[MEMTABLE_SLOWDOWN]; - // Stats summary across levels - PrintLevelStats(buf, sizeof(buf), "Sum", total_files, - total_files_being_compacted, total_file_size, 0, w_amp, - stats_sum); - value->append(buf); // Interval summary uint64_t interval_flush_ingest = flush_ingest - cf_stats_snapshot_.ingest_bytes_flush; @@ -796,9 +957,10 @@ void InternalStats::DumpCFStats(std::string* value) { add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile; uint64_t interval_ingest = interval_flush_ingest + interval_add_file_inget + 1; - CompactionStats interval_stats(stats_sum); + CompactionStats interval_stats(compaction_stats_sum); interval_stats.Subtract(cf_stats_snapshot_.comp_stats); - w_amp = interval_stats.bytes_written / static_cast(interval_ingest); + double w_amp = + interval_stats.bytes_written / static_cast(interval_ingest); PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats); value->append(buf); @@ -909,7 +1071,7 @@ void InternalStats::DumpCFStats(std::string* value) { cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile; cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile; cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile; - cf_stats_snapshot_.comp_stats = stats_sum; + cf_stats_snapshot_.comp_stats = compaction_stats_sum; cf_stats_snapshot_.stall_count = total_stall_count; } diff --git a/db/internal_stats.h b/db/internal_stats.h index d6b1ed6cf..e6c2365c2 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -9,10 +9,11 @@ // #pragma once -#include "db/version_set.h" - -#include +#include #include +#include + +#include "db/version_set.h" class ColumnFamilyData; @@ -41,13 +42,47 @@ struct DBPropertyInfo { // holding db mutex, which is only supported for int properties. bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db, Version* version); + bool (InternalStats::*handle_map)( + std::map* compaction_stats); }; extern const DBPropertyInfo* GetPropertyInfo(const Slice& property); #ifndef ROCKSDB_LITE +enum class LevelStatType { + INVALID = 0, + NUM_FILES, + COMPACTED_FILES, + SIZE_MB, + SCORE, + READ_GB, + RN_GB, + RNP1_GB, + WRITE_GB, + W_NEW_GB, + MOVED_GB, + WRITE_AMP, + READ_MBPS, + WRITE_MBPS, + COMP_SEC, + COMP_COUNT, + AVG_SEC, + KEY_IN, + KEY_DROP, + TOTAL // total number of types +}; + +struct LevelStat { + // This what will be L?.property_name in the flat map returned to the user + std::string property_name; + // This will be what we will print in the header in the cli + std::string header_name; +}; + class InternalStats { public: + static const std::map compaction_level_stats; + enum InternalCFStatsType { LEVEL0_SLOWDOWN_TOTAL, LEVEL0_SLOWDOWN_WITH_COMPACTION, @@ -221,6 +256,10 @@ class InternalStats { bool GetStringProperty(const DBPropertyInfo& property_info, const Slice& property, std::string* value); + bool GetMapProperty(const DBPropertyInfo& property_info, + const Slice& property, + std::map* value); + bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, DBImpl* db); @@ -233,6 +272,10 @@ class InternalStats { private: void DumpDBStats(std::string* value); + void DumpCFMapStats(std::map* cf_stats); + int DumpCFMapStats( + std::map>* level_stats, + CompactionStats* compaction_stats_sum); void DumpCFStats(std::string* value); // Per-DB stats @@ -313,6 +356,7 @@ class InternalStats { bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix); bool HandleLevelStats(std::string* value, Slice suffix); bool HandleStats(std::string* value, Slice suffix); + bool HandleCFMapStats(std::map* compaction_stats); bool HandleCFStats(std::string* value, Slice suffix); bool HandleDBStats(std::string* value, Slice suffix); bool HandleSsTables(std::string* value, Slice suffix); @@ -448,6 +492,12 @@ class InternalStats { return false; } + bool GetMapProperty(const DBPropertyInfo& property_info, + const Slice& property, + std::map* value) { + return false; + } + bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, DBImpl* db) const { return false; diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 283265909..f3c2924ba 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -372,6 +373,10 @@ class DB { // family stats per-level over db's lifetime ("L"), aggregated over // db's lifetime ("Sum"), and aggregated over the interval since the // last retrieval ("Int"). + // It could also be used to return the stats in the format of the map. + // In this case there will a pair of string to array of double for + // each level as well as for "Sum". "Int" stats will not be affected + // when this form of stats are retrived. static const std::string kCFStats; // "rocksdb.dbstats" - returns a multi-line string with general database @@ -511,6 +516,13 @@ class DB { virtual bool GetProperty(const Slice& property, std::string* value) { return GetProperty(DefaultColumnFamily(), property, value); } + virtual bool GetMapProperty(ColumnFamilyHandle* column_family, + const Slice& property, + std::map* value) = 0; + virtual bool GetMapProperty(const Slice& property, + std::map* value) { + return GetMapProperty(DefaultColumnFamily(), property, value); + } // Similar to GetProperty(), but only works for a subset of properties whose // return value is an integer. Return the value by integer. Supported diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index d223a7615..a712d1848 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -3,6 +3,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once +#include #include #include "rocksdb/db.h" @@ -133,11 +134,17 @@ class StackableDB : public DB { return db_->ReleaseSnapshot(snapshot); } + using DB::GetMapProperty; using DB::GetProperty; virtual bool GetProperty(ColumnFamilyHandle* column_family, const Slice& property, std::string* value) override { return db_->GetProperty(column_family, property, value); } + virtual bool GetMapProperty(ColumnFamilyHandle* column_family, + const Slice& property, + std::map* value) override { + return db_->GetMapProperty(column_family, property, value); + } using DB::GetIntProperty; virtual bool GetIntProperty(ColumnFamilyHandle* column_family,