prefetch bloom filter data block for L0 files

Summary: as title

Test Plan:
db_bench
the initial result is very promising. I will post results of complete
runs

Reviewers: dhruba, haobo, sdong, igor

Reviewed By: sdong

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D18867

db/version_set.cc

@@ -567,6 +567,16 @@ void Version::Get(const ReadOptions& options,
       continue;
     }
 
+    // Prefetch table data to avoid cache miss if possible
+    if (level == 0) {
+      for (int i = 0; i < num_files; ++i) {
+        auto* r = files_[0][i]->table_reader;
+        if (r) {
+          r->Prepare(ikey);
+        }
+      }
+    }
+
     // Get the list of files to search in this level
     FileMetaData* const* files = &files_[level][0];
 
@@ -607,6 +617,7 @@ void Version::Get(const ReadOptions& options,
 
     for (int32_t i = start_index; i < num_files;) {
       FileMetaData* f = files[i];
+
       // Check if key is within a file's range. If search left bound and right
       // bound point to the same find, we are sure key falls in range.
       assert(level == 0 || i == start_index ||

db/version_set.h

@@ -92,6 +92,7 @@ class Version {
     FileMetaData* seek_file;
     int seek_file_level;
   };
+
   void Get(const ReadOptions&, const LookupKey& key, std::string* val,
            Status* status, MergeContext* merge_context, GetStats* stats,
            bool* value_found = nullptr);

port/port_posix.h

@@ -482,6 +482,8 @@ inline bool LZ4HC_Compress(const CompressionOptions &opts, const char* input,
 
 #define CACHE_LINE_SIZE 64U
 
+#define PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality)
+
 } // namespace port
 } // namespace rocksdb
 

table/plain_table_reader.cc

@@ -610,6 +610,13 @@ Status PlainTableReader::Next(uint32_t* offset, ParsedInternalKey* key,
   return Status::OK();
 }
 
+void PlainTableReader::Prepare(const Slice& target) {
+  if (enable_bloom_) {
+    uint32_t prefix_hash = GetSliceHash(GetPrefix(target));
+    bloom_.Prefetch(prefix_hash);
+  }
+}
+
 Status PlainTableReader::Get(const ReadOptions& ro, const Slice& target,
                              void* arg,
                              bool (*saver)(void*, const ParsedInternalKey&,

table/plain_table_reader.h

@@ -57,6 +57,8 @@ class PlainTableReader: public TableReader {
 
   Iterator* NewIterator(const ReadOptions&, Arena* arena = nullptr) override;
 
+  void Prepare(const Slice& target);
+
   Status Get(const ReadOptions&, const Slice& key, void* arg,
              bool (*result_handler)(void* arg, const ParsedInternalKey& k,
                                     const Slice& v, bool),

table/table_reader.h

@@ -49,6 +49,9 @@ class TableReader {
 
   virtual std::shared_ptr<const TableProperties> GetTableProperties() const = 0;
 
+  // Prepare work that can be done before the real Get()
+  virtual void Prepare(const Slice& target) {}
+
   // Calls (*result_handler)(handle_context, ...) repeatedly, starting with
   // the entry found after a call to Seek(key), until result_handler returns
   // false, where k is the actual internal key for a row found and v as the

util/dynamic_bloom.h

@@ -5,12 +5,12 @@
 
 #pragma once
 
+#include <util/arena.h>
+#include <port/port_posix.h>
+
 #include <atomic>
 #include <memory>
 
-#include "port/port.h"
-#include <util/arena.h>
-
 namespace rocksdb {
 
 class Slice;
@@ -53,6 +53,8 @@ class DynamicBloom {
   // Multithreaded access to this function is OK
   bool MayContainHash(uint32_t hash) const;
 
+  void Prefetch(uint32_t h);
+
  private:
   uint32_t kTotalBits;
   uint32_t kNumBlocks;
@@ -71,6 +73,13 @@ inline bool DynamicBloom::MayContain(const Slice& key) const {
   return (MayContainHash(hash_func_(key)));
 }
 
+inline void DynamicBloom::Prefetch(uint32_t h) {
+  if (kNumBlocks != 0) {
+    uint32_t b = ((h >> 11 | (h << 21)) % kNumBlocks) * (CACHE_LINE_SIZE * 8);
+    PREFETCH(&(data_[b]), 0, 3);
+  }
+}
+
 inline bool DynamicBloom::MayContainHash(uint32_t h) const {
   assert(kNumBlocks > 0 || kTotalBits > 0);
   const uint32_t delta = (h >> 17) | (h << 15);  // Rotate right 17 bits