diff --git a/db/db_impl.cc b/db/db_impl.cc index 5c3462e77..62d0092fc 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -432,12 +432,16 @@ void DBImpl::MaybeDumpStats() { std::string stats; { InstrumentedMutexLock l(&mutex_); - for (auto cfd : *versions_->GetColumnFamilySet()) { - cfd->internal_stats()->GetStringProperty( - *cf_property_info, DB::Properties::kCFStats, &stats); - } default_cf_internal_stats_->GetStringProperty( *db_property_info, DB::Properties::kDBStats, &stats); + for (auto cfd : *versions_->GetColumnFamilySet()) { + cfd->internal_stats()->GetStringProperty( + *cf_property_info, DB::Properties::kCFStatsNoFileHistogram, &stats); + } + for (auto cfd : *versions_->GetColumnFamilySet()) { + cfd->internal_stats()->GetStringProperty( + *cf_property_info, DB::Properties::kCFFileHistogram, &stats); + } } ROCKS_LOG_WARN(immutable_db_options_.info_log, "------- DUMPING STATS -------"); diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc index c38422615..ebc5f96e0 100644 --- a/db/db_properties_test.cc +++ b/db/db_properties_test.cc @@ -342,7 +342,7 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) { BlockBasedTableOptions table_options; table_options.no_block_cache = true; - DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); int key_index = 0; Random rnd(301); for (int num = 0; num < 8; num++) { @@ -359,25 +359,26 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) { for (int key = 0; key < key_index; key++) { Get(Key(key)); } - ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop)); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop)); ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); // Reopen and issue Get(). See thee latency tracked - Reopen(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); dbfull()->TEST_WaitForCompact(); for (int key = 0; key < key_index; key++) { Get(Key(key)); } - ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop)); + ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(), + "rocksdb.cf-file-histogram", &prop)); ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); // Reopen and issue iterating. See thee latency tracked - Reopen(options); - ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop)); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop)); ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); @@ -386,22 +387,41 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) { for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) { } } - ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop)); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop)); ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + // CF 1 should show no histogram. + ASSERT_TRUE( + dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop)); + ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + // put something and read it back , CF 1 should show histogram. + Put(1, "foo", "bar"); + Flush(1); + dbfull()->TEST_WaitForCompact(); + ASSERT_EQ("bar", Get(1, "foo")); + + ASSERT_TRUE( + dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop)); + ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + // options.max_open_files preloads table readers. options.max_open_files = -1; - Reopen(options); - ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop)); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(), + "rocksdb.cf-file-histogram", &prop)); ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); for (int key = 0; key < key_index; key++) { Get(Key(key)); } - ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop)); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop)); ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); diff --git a/db/internal_stats.cc b/db/internal_stats.cc index f4abbad77..5de370e18 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -195,6 +195,9 @@ static const std::string compression_ratio_at_level_prefix = static const std::string allstats = "stats"; static const std::string sstables = "sstables"; static const std::string cfstats = "cfstats"; +static const std::string cfstats_no_file_histogram = + "cfstats-no-file-histogram"; +static const std::string cf_file_histogram = "cf-file-histogram"; static const std::string dbstats = "dbstats"; static const std::string levelstats = "levelstats"; static const std::string num_immutable_mem_table = "num-immutable-mem-table"; @@ -248,6 +251,10 @@ const std::string DB::Properties::kCompressionRatioAtLevelPrefix = const std::string DB::Properties::kStats = rocksdb_prefix + allstats; const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables; const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats; +const std::string DB::Properties::kCFStatsNoFileHistogram = + rocksdb_prefix + cfstats_no_file_histogram; +const std::string DB::Properties::kCFFileHistogram = + rocksdb_prefix + cf_file_histogram; const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats; const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats; const std::string DB::Properties::kNumImmutableMemTable = @@ -324,6 +331,11 @@ const std::unordered_map {DB::Properties::kCFStats, {false, &InternalStats::HandleCFStats, nullptr, &InternalStats::HandleCFMapStats}}, + {DB::Properties::kCFStatsNoFileHistogram, + {false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr, + nullptr}}, + {DB::Properties::kCFFileHistogram, + {false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr}}, {DB::Properties::kDBStats, {false, &InternalStats::HandleDBStats, nullptr, nullptr}}, {DB::Properties::kSSTables, @@ -512,6 +524,17 @@ bool InternalStats::HandleCFStats(std::string* value, Slice suffix) { return true; } +bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value, + Slice suffix) { + DumpCFStatsNoFileHistogram(value); + return true; +} + +bool InternalStats::HandleCFFileHistogram(std::string* value, Slice suffix) { + DumpCFFileHistogram(value); + return true; +} + bool InternalStats::HandleDBStats(std::string* value, Slice suffix) { DumpDBStats(value); return true; @@ -855,16 +878,6 @@ void InternalStats::DumpDBStats(std::string* value) { 10000.0 / std::max(interval_seconds_up, 0.001)); value->append(buf); - for (int level = 0; level < number_levels_; level++) { - if (!file_read_latency_[level].Empty()) { - char buf2[5000]; - snprintf(buf2, sizeof(buf2), - "** Level %d read latency histogram (micros):\n%s\n", level, - file_read_latency_[level].ToString().c_str()); - value->append(buf2); - } - } - db_stats_snapshot_.seconds_up = seconds_up; db_stats_snapshot_.ingest_bytes = user_bytes_written; db_stats_snapshot_.write_other = write_other; @@ -968,6 +981,11 @@ void InternalStats::DumpCFMapStats( } void InternalStats::DumpCFStats(std::string* value) { + DumpCFStatsNoFileHistogram(value); + DumpCFFileHistogram(value); +} + +void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) { char buf[2000]; // Per-ColumnFamily stats PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName()); @@ -1126,6 +1144,23 @@ void InternalStats::DumpCFStats(std::string* value) { cf_stats_snapshot_.stall_count = total_stall_count; } +void InternalStats::DumpCFFileHistogram(std::string* value) { + char buf[2000]; + snprintf(buf, sizeof(buf), + "\n** File Read Latency Histogram By Level [%s] **\n", + cfd_->GetName().c_str()); + value->append(buf); + + for (int level = 0; level < number_levels_; level++) { + if (!file_read_latency_[level].Empty()) { + char buf2[5000]; + snprintf(buf2, sizeof(buf2), + "** Level %d read latency histogram (micros):\n%s\n", level, + file_read_latency_[level].ToString().c_str()); + value->append(buf2); + } + } +} #else diff --git a/db/internal_stats.h b/db/internal_stats.h index cfd275517..584184437 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -277,6 +277,8 @@ class InternalStats { std::map>* level_stats, CompactionStats* compaction_stats_sum); void DumpCFStats(std::string* value); + void DumpCFStatsNoFileHistogram(std::string* value); + void DumpCFFileHistogram(std::string* value); // Per-DB stats std::atomic db_stats_[INTERNAL_DB_STATS_ENUM_MAX]; @@ -358,6 +360,8 @@ class InternalStats { bool HandleStats(std::string* value, Slice suffix); bool HandleCFMapStats(std::map* compaction_stats); bool HandleCFStats(std::string* value, Slice suffix); + bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix); + bool HandleCFFileHistogram(std::string* value, Slice suffix); bool HandleDBStats(std::string* value, Slice suffix); bool HandleSsTables(std::string* value, Slice suffix); bool HandleAggregatedTableProperties(std::string* value, Slice suffix); diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 8407e0ea5..723e4dea3 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -400,15 +400,24 @@ class DB { // SST files. static const std::string kSSTables; - // "rocksdb.cfstats" - returns a multi-line string with general column - // family stats per-level over db's lifetime ("L"), aggregated over - // db's lifetime ("Sum"), and aggregated over the interval since the - // last retrieval ("Int"). + // "rocksdb.cfstats" - Both of "rocksdb.cfstats-no-file-histogram" and + // "rocksdb.cf-file-histogram" together. See below for description + // of the two. + static const std::string kCFStats; + + // "rocksdb.cfstats-no-file-histogram" - returns a multi-line string with + // general columm family stats per-level over db's lifetime ("L"), + // aggregated over db's lifetime ("Sum"), and aggregated over the + // interval since the last retrieval ("Int"). // It could also be used to return the stats in the format of the map. // In this case there will a pair of string to array of double for // each level as well as for "Sum". "Int" stats will not be affected // when this form of stats are retrived. - static const std::string kCFStats; + static const std::string kCFStatsNoFileHistogram; + + // "rocksdb.cf-file-histogram" - print out how many file reads to every + // level, as well as the histogram of latency of single requests. + static const std::string kCFFileHistogram; // "rocksdb.dbstats" - returns a multi-line string with general database // stats, both cumulative (over the db's lifetime) and interval (since