andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

File level histogram should be printed per CF, not per DB

Browse Source 
Summary:
Currently level histogram is only printed out for DB stats and for default CF. This is confusing. Change to print for every CF instead.
Closes https://github.com/facebook/rocksdb/pull/2126

Differential Revision: D4865373

Pulled By: siying

fbshipit-source-id: 1c853e0ac66e00120ee931cabc9daf69ccc2d577
This commit is contained in:
Siying Dong 2017-04-11 08:38:22 -07:00 committed by Facebook Github Bot
parent 9300ef5455
commit 8f47a97512
5 changed files with 101 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
db/db_impl.cc
View File

@ -432,12 +432,16 @@ void DBImpl::MaybeDumpStats() {
std::string stats;
{
InstrumentedMutexLock l(&mutex_);
for (auto cfd : *versions_->GetColumnFamilySet()) {
cfd->internal_stats()->GetStringProperty(
*cf_property_info, DB::Properties::kCFStats, &stats);
}
default_cf_internal_stats_->GetStringProperty(
*db_property_info, DB::Properties::kDBStats, &stats);
for (auto cfd : *versions_->GetColumnFamilySet()) {
cfd->internal_stats()->GetStringProperty(
*cf_property_info, DB::Properties::kCFStatsNoFileHistogram, &stats);
}
for (auto cfd : *versions_->GetColumnFamilySet()) {
cfd->internal_stats()->GetStringProperty(
*cf_property_info, DB::Properties::kCFFileHistogram, &stats);
}
}
ROCKS_LOG_WARN(immutable_db_options_.info_log,
"------- DUMPING STATS -------");

40
db/db_properties_test.cc
View File

@ -342,7 +342,7 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
BlockBasedTableOptions table_options;
table_options.no_block_cache = true;
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
int key_index = 0;
Random rnd(301);
for (int num = 0; num < 8; num++) {
@ -359,25 +359,26 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
for (int key = 0; key < key_index; key++) {
Get(Key(key));
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// Reopen and issue Get(). See thee latency tracked
Reopen(options);
ReopenWithColumnFamilies({"default", "pikachu"}, options);
dbfull()->TEST_WaitForCompact();
for (int key = 0; key < key_index; key++) {
Get(Key(key));
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
"rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// Reopen and issue iterating. See thee latency tracked
Reopen(options);
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
@ -386,22 +387,41 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
}
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// CF 1 should show no histogram.
ASSERT_TRUE(
dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// put something and read it back , CF 1 should show histogram.
Put(1, "foo", "bar");
Flush(1);
dbfull()->TEST_WaitForCompact();
ASSERT_EQ("bar", Get(1, "foo"));
ASSERT_TRUE(
dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
// options.max_open_files preloads table readers.
options.max_open_files = -1;
Reopen(options);
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
"rocksdb.cf-file-histogram", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
for (int key = 0; key < key_index; key++) {
Get(Key(key));
}
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));

55
db/internal_stats.cc
View File

@ -195,6 +195,9 @@ static const std::string compression_ratio_at_level_prefix =
static const std::string allstats = "stats";
static const std::string sstables = "sstables";
static const std::string cfstats = "cfstats";
static const std::string cfstats_no_file_histogram =
"cfstats-no-file-histogram";
static const std::string cf_file_histogram = "cf-file-histogram";
static const std::string dbstats = "dbstats";
static const std::string levelstats = "levelstats";
static const std::string num_immutable_mem_table = "num-immutable-mem-table";
@ -248,6 +251,10 @@ const std::string DB::Properties::kCompressionRatioAtLevelPrefix =
const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
const std::string DB::Properties::kCFStatsNoFileHistogram =
rocksdb_prefix + cfstats_no_file_histogram;
const std::string DB::Properties::kCFFileHistogram =
rocksdb_prefix + cf_file_histogram;
const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
const std::string DB::Properties::kNumImmutableMemTable =
@ -324,6 +331,11 @@ const std::unordered_map<std::string, DBPropertyInfo>
{DB::Properties::kCFStats,
{false, &InternalStats::HandleCFStats, nullptr,
&InternalStats::HandleCFMapStats}},
{DB::Properties::kCFStatsNoFileHistogram,
{false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr,
nullptr}},
{DB::Properties::kCFFileHistogram,
{false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr}},
{DB::Properties::kDBStats,
{false, &InternalStats::HandleDBStats, nullptr, nullptr}},
{DB::Properties::kSSTables,
@ -512,6 +524,17 @@ bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
return true;
}
bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
Slice suffix) {
DumpCFStatsNoFileHistogram(value);
return true;
}
bool InternalStats::HandleCFFileHistogram(std::string* value, Slice suffix) {
DumpCFFileHistogram(value);
return true;
}
bool InternalStats::HandleDBStats(std::string* value, Slice suffix) {
DumpDBStats(value);
return true;
@ -855,16 +878,6 @@ void InternalStats::DumpDBStats(std::string* value) {
10000.0 / std::max(interval_seconds_up, 0.001));
value->append(buf);
for (int level = 0; level < number_levels_; level++) {
if (!file_read_latency_[level].Empty()) {
char buf2[5000];
snprintf(buf2, sizeof(buf2),
"** Level %d read latency histogram (micros):\n%s\n", level,
file_read_latency_[level].ToString().c_str());
value->append(buf2);
}
}
db_stats_snapshot_.seconds_up = seconds_up;
db_stats_snapshot_.ingest_bytes = user_bytes_written;
db_stats_snapshot_.write_other = write_other;
@ -968,6 +981,11 @@ void InternalStats::DumpCFMapStats(
}
void InternalStats::DumpCFStats(std::string* value) {
DumpCFStatsNoFileHistogram(value);
DumpCFFileHistogram(value);
}
void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
char buf[2000];
// Per-ColumnFamily stats
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
@ -1126,6 +1144,23 @@ void InternalStats::DumpCFStats(std::string* value) {
cf_stats_snapshot_.stall_count = total_stall_count;
}
void InternalStats::DumpCFFileHistogram(std::string* value) {
char buf[2000];
snprintf(buf, sizeof(buf),
"\n** File Read Latency Histogram By Level [%s] **\n",
cfd_->GetName().c_str());
value->append(buf);
for (int level = 0; level < number_levels_; level++) {
if (!file_read_latency_[level].Empty()) {
char buf2[5000];
snprintf(buf2, sizeof(buf2),
"** Level %d read latency histogram (micros):\n%s\n", level,
file_read_latency_[level].ToString().c_str());
value->append(buf2);
}
}
}
#else

4
db/internal_stats.h
View File

@ -277,6 +277,8 @@ class InternalStats {
std::map<int, std::map<LevelStatType, double>>* level_stats,
CompactionStats* compaction_stats_sum);
void DumpCFStats(std::string* value);
void DumpCFStatsNoFileHistogram(std::string* value);
void DumpCFFileHistogram(std::string* value);
// Per-DB stats
std::atomic<uint64_t> db_stats_[INTERNAL_DB_STATS_ENUM_MAX];
@ -358,6 +360,8 @@ class InternalStats {
bool HandleStats(std::string* value, Slice suffix);
bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
bool HandleCFStats(std::string* value, Slice suffix);
bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix);
bool HandleCFFileHistogram(std::string* value, Slice suffix);
bool HandleDBStats(std::string* value, Slice suffix);
bool HandleSsTables(std::string* value, Slice suffix);
bool HandleAggregatedTableProperties(std::string* value, Slice suffix);

19
include/rocksdb/db.h
View File

@ -400,15 +400,24 @@ class DB {
// SST files.
static const std::string kSSTables;
// "rocksdb.cfstats" - returns a multi-line string with general column
// family stats per-level over db's lifetime ("L<n>"), aggregated over
// db's lifetime ("Sum"), and aggregated over the interval since the
// last retrieval ("Int").
// "rocksdb.cfstats" - Both of "rocksdb.cfstats-no-file-histogram" and
// "rocksdb.cf-file-histogram" together. See below for description
// of the two.
static const std::string kCFStats;
// "rocksdb.cfstats-no-file-histogram" - returns a multi-line string with
// general columm family stats per-level over db's lifetime ("L<n>"),
// aggregated over db's lifetime ("Sum"), and aggregated over the
// interval since the last retrieval ("Int").
// It could also be used to return the stats in the format of the map.
// In this case there will a pair of string to array of double for
// each level as well as for "Sum". "Int" stats will not be affected
// when this form of stats are retrived.
static const std::string kCFStats;
static const std::string kCFStatsNoFileHistogram;
// "rocksdb.cf-file-histogram" - print out how many file reads to every
// level, as well as the histogram of latency of single requests.
static const std::string kCFFileHistogram;
// "rocksdb.dbstats" - returns a multi-line string with general database
// stats, both cumulative (over the db's lifetime) and interval (since