Metrics: record compaction drop's and bloom filter effectiveness

Summary: Record BloomFliter hits and drop off reasons during compaction.

Test Plan: Unit tests work.

Reviewers: dhruba, heyongqiang

Reviewed By: dhruba

Differential Revision: https://reviews.facebook.net/D6591

db/db_bench.cc

@@ -575,12 +575,19 @@ class Benchmark {
   void PrintStatistics() {
     if (FLAGS_statistics) {
       fprintf(stdout, "File opened:%ld closed:%ld errors:%ld\n"
-          "Block Cache Hit Count:%ld Block Cache Miss Count:%ld\n",
-              dbstats->getNumFileOpens(),
-              dbstats->getNumFileCloses(),
-              dbstats->getNumFileErrors(),
+          "Block Cache Hit Count:%ld Block Cache Miss Count:%ld\n"
+          "Bloom Filter Useful: %ld \n"
+          "Compaction key_drop_newer_entry: %ld key_drop_obsolete: %ld "
+          "Compaction key_drop_user: %ld",
+          dbstats->getNumFileOpens(),
+          dbstats->getNumFileCloses(),
+          dbstats->getNumFileErrors(),
           dbstats->getTickerCount(BLOCK_CACHE_HIT),
-          dbstats->getTickerCount(BLOCK_CACHE_MISS));
+          dbstats->getTickerCount(BLOCK_CACHE_MISS),
+          dbstats->getTickerCount(BLOOM_FILTER_USEFUL),
+          dbstats->getTickerCount(COMPACTION_KEY_DROP_NEWER_ENTRY),
+          dbstats->getTickerCount(COMPACTION_KEY_DROP_OBSOLETE),
+          dbstats->getTickerCount(COMPACTION_KEY_DROP_USER));
     }
   }
 

db/db_impl.cc

@@ -23,6 +23,7 @@
 #include "db/write_batch_internal.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
+#include "leveldb/statistics.h"
 #include "leveldb/status.h"
 #include "leveldb/table.h"
 #include "leveldb/table_builder.h"
@@ -1143,6 +1144,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
       if (last_sequence_for_key <= compact->smallest_snapshot) {
         // Hidden by an newer entry for same user key
         drop = true;    // (A)
+        RecordTick(options_.statistics, COMPACTION_KEY_DROP_NEWER_ENTRY);
       } else if (ikey.type == kTypeDeletion &&
                  ikey.sequence <= compact->smallest_snapshot &&
                  compact->compaction->IsBaseLevelForKey(ikey.user_key)) {
@@ -1154,6 +1156,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
         //     few iterations of this loop (by rule (A) above).
         // Therefore this deletion marker is obsolete and can be dropped.
         drop = true;
+        RecordTick(options_.statistics, COMPACTION_KEY_DROP_OBSOLETE);
       } else if (options_.CompactionFilter != NULL &&
                  ikey.type != kTypeDeletion &&
                  ikey.sequence < compact->smallest_snapshot) {
@@ -1164,6 +1167,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
         drop = options_.CompactionFilter(compact->compaction->level(),
                          ikey.user_key, value, &compaction_filter_value);
 
+        if (drop) {
+          RecordTick(options_.statistics, COMPACTION_KEY_DROP_USER);
+        }
         // If the application wants to change the value, then do so here.
         if (compaction_filter_value != NULL) {
           value = *compaction_filter_value;

include/leveldb/statistics.h

@@ -16,7 +16,15 @@ namespace leveldb {
 enum Tickers {
   BLOCK_CACHE_MISS = 0,
   BLOCK_CACHE_HIT = 1,
-  TICKER_ENUM_MAX = 2,
+  BLOOM_FILTER_USEFUL = 2, // no. of times bloom filter has avoided file reads.
+  /**
+   * COMPACTION_KEY_DROP_* count the reasons for key drop during compaction
+   * There are 3 reasons currently.
+   */
+  COMPACTION_KEY_DROP_NEWER_ENTRY = 3, // key was written with a newer value.
+  COMPACTION_KEY_DROP_OBSOLETE = 4, // The key is obsolete.
+  COMPACTION_KEY_DROP_USER = 5, // user compaction function has dropped the key.
+  TICKER_ENUM_MAX = 6,
 };
 
 

table/table.cc

@@ -243,6 +243,7 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k,
         handle.DecodeFrom(&handle_value).ok() &&
         !filter->KeyMayMatch(handle.offset(), k)) {
       // Not found
+      RecordTick(rep_->options.statistics, BLOOM_FILTER_USEFUL);
     } else {
       bool didIO = false;
       Iterator* block_iter = BlockReader(this, options, iiter->value(),