Merge branch 'master' into columnfamilies

Conflicts:
	db/version_set.cc
	db/version_set_reduce_num_levels.cc
	util/ldb_cmd.cc

Makefile

@@ -136,13 +136,12 @@ endif  # PLATFORM_SHARED_EXT
 all: $(LIBRARY) $(PROGRAMS)
 
 .PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests \
-	release tags valgrind_check whitebox_crash_test format
+	release tags valgrind_check whitebox_crash_test format shared_lib
 
 # Will also generate shared libraries. 
 release:
 	$(MAKE) clean
 	OPT=-DNDEBUG $(MAKE) all -j32
-	OPT=-DNDEBUG $(MAKE) $(SHARED) -j32
 
 coverage:
 	$(MAKE) clean
@@ -202,6 +201,8 @@ tags:
 format:
 	build_tools/format-diff.sh
 
+shared_lib: $(SHARED)
+
 # ---------------------------------------------------------------------------
 # 	Unit tests and tools
 # ---------------------------------------------------------------------------

db/compaction_picker.cc

@@ -22,6 +22,21 @@ uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
   return sum;
 }
 
+// Multiple two operands. If they overflow, return op1.
+uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
+  if (op1 == 0) {
+    return 0;
+  }
+  if (op2 <= 0) {
+    return op1;
+  }
+  uint64_t casted_op2 = (uint64_t) op2;
+  if (std::numeric_limits<uint64_t>::max() / op1 < casted_op2) {
+    return op1;
+  }
+  return op1 * casted_op2;
+}
+
 }  // anonymous namespace
 
 CompactionPicker::CompactionPicker(const Options* options,
@@ -30,15 +45,7 @@ CompactionPicker::CompactionPicker(const Options* options,
       options_(options),
       num_levels_(options->num_levels),
       icmp_(icmp) {
-  Init();
-}
 
-void CompactionPicker::ReduceNumberOfLevels(int new_levels) {
-  num_levels_ = new_levels;
-  Init();
-}
-
-void CompactionPicker::Init() {
   max_file_size_.reset(new uint64_t[NumberLevels()]);
   level_max_bytes_.reset(new uint64_t[NumberLevels()]);
   int target_file_size_multiplier = options_->target_file_size_multiplier;
@@ -48,10 +55,11 @@ void CompactionPicker::Init() {
       max_file_size_[i] = ULLONG_MAX;
       level_max_bytes_[i] = options_->max_bytes_for_level_base;
     } else if (i > 1) {
-      max_file_size_[i] = max_file_size_[i - 1] * target_file_size_multiplier;
+      max_file_size_[i] = MultiplyCheckOverflow(max_file_size_[i - 1],
-      level_max_bytes_[i] =
+                                                target_file_size_multiplier);
-          level_max_bytes_[i - 1] * max_bytes_multiplier *
+      level_max_bytes_[i] = MultiplyCheckOverflow(
-          options_->max_bytes_for_level_multiplier_additional[i - 1];
+          MultiplyCheckOverflow(level_max_bytes_[i - 1], max_bytes_multiplier),
+          options_->max_bytes_for_level_multiplier_additional[i - 1]);
     } else {
       max_file_size_[i] = options_->target_file_size_base;
       level_max_bytes_[i] = options_->max_bytes_for_level_base;

db/compaction_picker.h

@@ -27,9 +27,6 @@ class CompactionPicker {
   CompactionPicker(const Options* options, const InternalKeyComparator* icmp);
   virtual ~CompactionPicker();
 
-  // See VersionSet::ReduceNumberOfLevels()
-  void ReduceNumberOfLevels(int new_levels);
-
   // Pick level and inputs for a new compaction.
   // Returns nullptr if there is no compaction to be done.
   // Otherwise returns a pointer to a heap-allocated object that
@@ -120,8 +117,6 @@ class CompactionPicker {
 
   const Options* const options_;
  private:
-  void Init();
-
   int num_levels_;
 
   const InternalKeyComparator* const icmp_;

db/db_impl.cc

@@ -267,6 +267,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
       bg_cv_(&mutex_),
       mem_rep_factory_(options_.memtable_factory.get()),
       mem_(new MemTable(internal_comparator_, options_)),
+      imm_(options_.min_write_buffer_number_to_merge),
       logfile_number_(0),
       super_version_(nullptr),
       super_version_number_(0),
@@ -363,7 +364,7 @@ DBImpl::~DBImpl() {
     delete mem_->Unref();
   }
 
-  imm_.UnrefAll(&to_delete);
+  imm_.current()->Unref(&to_delete);
   for (MemTable* m: to_delete) {
     delete m;
   }
@@ -511,7 +512,7 @@ bool DBImpl::SuperVersion::Unref() {
 
 void DBImpl::SuperVersion::Cleanup() {
   assert(refs.load(std::memory_order_relaxed) == 0);
-  imm.UnrefAll(&to_delete);
+  imm->Unref(&to_delete);
   MemTable* m = mem->Unref();
   if (m != nullptr) {
     to_delete.push_back(m);
@@ -519,13 +520,13 @@ void DBImpl::SuperVersion::Cleanup() {
   current->Unref();
 }
 
-void DBImpl::SuperVersion::Init(MemTable* new_mem, const MemTableList& new_imm,
+void DBImpl::SuperVersion::Init(MemTable* new_mem, MemTableListVersion* new_imm,
                                 Version* new_current) {
   mem = new_mem;
   imm = new_imm;
   current = new_current;
   mem->Ref();
-  imm.RefAll();
+  imm->Ref();
   current->Ref();
   refs.store(1, std::memory_order_relaxed);
 }
@@ -1226,7 +1227,7 @@ Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress,
   mutex_.AssertHeld();
   assert(imm_.size() != 0);
 
-  if (!imm_.IsFlushPending(options_.min_write_buffer_number_to_merge)) {
+  if (!imm_.IsFlushPending()) {
     Log(options_.info_log, "FlushMemTableToOutputFile already in progress");
     Status s = Status::IOError("FlushMemTableToOutputFile already in progress");
     return s;
@@ -1767,8 +1768,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
   } else if (shutting_down_.Acquire_Load()) {
     // DB is being deleted; no more background compactions
   } else {
-    bool is_flush_pending =
+    bool is_flush_pending = imm_.IsFlushPending();
-      imm_.IsFlushPending(options_.min_write_buffer_number_to_merge);
     if (is_flush_pending &&
         (bg_flush_scheduled_ < options_.max_background_flushes)) {
       // memtable flush needed
@@ -1803,8 +1803,7 @@ void DBImpl::BGWorkCompaction(void* db) {
 Status DBImpl::BackgroundFlush(bool* madeProgress,
                                DeletionState& deletion_state) {
   Status stat;
-  while (stat.ok() &&
+  while (stat.ok() && imm_.IsFlushPending()) {
-         imm_.IsFlushPending(options_.min_write_buffer_number_to_merge)) {
     Log(options_.info_log,
         "BackgroundCallFlush doing FlushMemTableToOutputFile, flush slots available %d",
         options_.max_background_flushes - bg_flush_scheduled_);
@@ -1924,7 +1923,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
   mutex_.AssertHeld();
 
   // TODO: remove memtable flush from formal compaction
-  while (imm_.IsFlushPending(options_.min_write_buffer_number_to_merge)) {
+  while (imm_.IsFlushPending()) {
     Log(options_.info_log,
         "BackgroundCompaction doing FlushMemTableToOutputFile, compaction slots "
         "available %d",
@@ -2330,7 +2329,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact,
       const uint64_t imm_start = env_->NowMicros();
       LogFlush(options_.info_log);
       mutex_.Lock();
-      if (imm_.IsFlushPending(options_.min_write_buffer_number_to_merge)) {
+      if (imm_.IsFlushPending()) {
         FlushMemTableToOutputFile(nullptr, deletion_state);
         bg_cv_.SignalAll();  // Wakeup MakeRoomForWrite() if necessary
       }
@@ -2663,8 +2662,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact,
 namespace {
 struct IterState {
   port::Mutex* mu;
-  Version* version;
+  Version* version = nullptr;
-  std::vector<MemTable*> mem; // includes both mem_ and imm_
+  MemTable* mem = nullptr;
+  MemTableListVersion* imm = nullptr;
   DBImpl *db;
 };
 
@@ -2673,15 +2673,16 @@ static void CleanupIteratorState(void* arg1, void* arg2) {
   DBImpl::DeletionState deletion_state(state->db->GetOptions().
                                        max_write_buffer_number);
   state->mu->Lock();
-  for (unsigned int i = 0; i < state->mem.size(); i++) {
+  MemTable* m = state->mem->Unref();
-    MemTable* m = state->mem[i]->Unref();
   if (m != nullptr) {
     deletion_state.memtables_to_free.push_back(m);
   }
-  }
   if (state->version) {  // not set for memtable-only iterator
     state->version->Unref();
   }
+  if (state->imm) {  // not set for memtable-only iterator
+    state->imm->Unref(&deletion_state.memtables_to_free);
+  }
   // fast path FindObsoleteFiles
   state->db->FindObsoleteFiles(deletion_state, false, true);
   state->mu->Unlock();
@@ -2695,7 +2696,7 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
                                       SequenceNumber* latest_snapshot) {
   IterState* cleanup = new IterState;
   MemTable* mutable_mem;
-  std::vector<MemTable*> immutables;
+  MemTableListVersion* immutable_mems;
   Version* version;
 
   // Collect together all needed child iterators for mem
@@ -2704,27 +2705,22 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
   mem_->Ref();
   mutable_mem = mem_;
   // Collect together all needed child iterators for imm_
-  imm_.GetMemTables(&immutables);
+  immutable_mems = imm_.current();
-  for (unsigned int i = 0; i < immutables.size(); i++) {
+  immutable_mems->Ref();
-    immutables[i]->Ref();
-  }
   versions_->current()->Ref();
   version = versions_->current();
   mutex_.Unlock();
 
-  std::vector<Iterator*> list;
+  std::vector<Iterator*> iterator_list;
-  list.push_back(mutable_mem->NewIterator(options));
+  iterator_list.push_back(mutable_mem->NewIterator(options));
-  cleanup->mem.push_back(mutable_mem);
+  cleanup->mem = mutable_mem;
-
+  cleanup->imm = immutable_mems;
   // Collect all needed child iterators for immutable memtables
-  for (MemTable* m : immutables) {
+  immutable_mems->AddIterators(options, &iterator_list);
-    list.push_back(m->NewIterator(options));
-    cleanup->mem.push_back(m);
-  }
   // Collect iterators for files in L0 - Ln
-  version->AddIterators(options, storage_options_, &list);
+  version->AddIterators(options, storage_options_, &iterator_list);
-  Iterator* internal_iter =
+  Iterator* internal_iter = NewMergingIterator(
-      NewMergingIterator(&internal_comparator_, &list[0], list.size());
+      &internal_comparator_, &iterator_list[0], iterator_list.size());
   cleanup->version = version;
   cleanup->mu = &mutex_;
   cleanup->db = this;
@@ -2743,19 +2739,15 @@ std::pair<Iterator*, Iterator*> DBImpl::GetTailingIteratorPair(
     uint64_t* superversion_number) {
 
   MemTable* mutable_mem;
-  std::vector<MemTable*> immutables;
+  MemTableListVersion* immutable_mems;
   Version* version;
 
-  immutables.reserve(options_.max_write_buffer_number);
-
   // get all child iterators and bump their refcounts under lock
   mutex_.Lock();
   mutable_mem = mem_;
   mutable_mem->Ref();
-  imm_.GetMemTables(&immutables);
+  immutable_mems = imm_.current();
-  for (size_t i = 0; i < immutables.size(); ++i) {
+  immutable_mems->Ref();
-    immutables[i]->Ref();
-  }
   version = versions_->current();
   version->Ref();
   if (superversion_number != nullptr) {
@@ -2765,7 +2757,7 @@ std::pair<Iterator*, Iterator*> DBImpl::GetTailingIteratorPair(
 
   Iterator* mutable_iter = mutable_mem->NewIterator(options);
   IterState* mutable_cleanup = new IterState();
-  mutable_cleanup->mem.push_back(mutable_mem);
+  mutable_cleanup->mem = mutable_mem;
   mutable_cleanup->db = this;
   mutable_cleanup->mu = &mutex_;
   mutable_iter->RegisterCleanup(CleanupIteratorState, mutable_cleanup, nullptr);
@@ -2777,10 +2769,8 @@ std::pair<Iterator*, Iterator*> DBImpl::GetTailingIteratorPair(
   Iterator* immutable_iter;
   IterState* immutable_cleanup = new IterState();
   std::vector<Iterator*> list;
-  for (MemTable* m : immutables) {
+  immutable_mems->AddIterators(options, &list);
-    list.push_back(m->NewIterator(options));
+  immutable_cleanup->imm = immutable_mems;
-    immutable_cleanup->mem.push_back(m);
-  }
   version->AddIterators(options, storage_options_, &list);
   immutable_cleanup->version = version;
   immutable_cleanup->db = this;
@@ -2837,7 +2827,7 @@ void DBImpl::InstallSuperVersion(DeletionState& deletion_state) {
 DBImpl::SuperVersion* DBImpl::InstallSuperVersion(
     SuperVersion* new_superversion) {
   mutex_.AssertHeld();
-  new_superversion->Init(mem_, imm_, versions_->current());
+  new_superversion->Init(mem_, imm_.current(), versions_->current());
   SuperVersion* old_superversion = super_version_;
   super_version_ = new_superversion;
   ++super_version_number_;
@@ -2880,7 +2870,7 @@ Status DBImpl::GetImpl(const ReadOptions& options,
   if (get_version->mem->Get(lkey, value, &s, merge_context, options_)) {
     // Done
     RecordTick(options_.statistics.get(), MEMTABLE_HIT);
-  } else if (get_version->imm.Get(lkey, value, &s, merge_context, options_)) {
+  } else if (get_version->imm->Get(lkey, value, &s, merge_context, options_)) {
     // Done
     RecordTick(options_.statistics.get(), MEMTABLE_HIT);
   } else {
@@ -2936,10 +2926,10 @@ std::vector<Status> DBImpl::MultiGet(
   }
 
   MemTable* mem = mem_;
-  MemTableList imm = imm_;
+  MemTableListVersion* imm = imm_.current();
   Version* current = versions_->current();
   mem->Ref();
-  imm.RefAll();
+  imm->Ref();
   current->Ref();
 
   // Unlock while reading from files and memtables
@@ -2971,7 +2961,7 @@ std::vector<Status> DBImpl::MultiGet(
     LookupKey lkey(keys[i], snapshot);
     if (mem->Get(lkey, value, &s, merge_context, options_)) {
       // Done
-    } else if (imm.Get(lkey, value, &s, merge_context, options_)) {
+    } else if (imm->Get(lkey, value, &s, merge_context, options_)) {
       // Done
     } else {
       current->Get(options, lkey, value, &s, &merge_context, &stats, options_);
@@ -2990,7 +2980,7 @@ std::vector<Status> DBImpl::MultiGet(
     MaybeScheduleFlushOrCompaction();
   }
   MemTable* m = mem->Unref();
-  imm.UnrefAll(&to_delete);
+  imm->Unref(&to_delete);
   current->Unref();
   mutex_.Unlock();
 

db/db_impl.h

@@ -177,10 +177,10 @@ class DBImpl : public DB {
   // holds references to memtable, all immutable memtables and version
   struct SuperVersion {
     MemTable* mem;
-    MemTableList imm;
+    MemTableListVersion* imm;
     Version* current;
     std::atomic<uint32_t> refs;
-    // We need to_delete because during Cleanup(), imm.UnrefAll() returns
+    // We need to_delete because during Cleanup(), imm->Unref() returns
     // all memtables that we need to free through this vector. We then
     // delete all those memtables outside of mutex, during destruction
     std::vector<MemTable*> to_delete;
@@ -198,7 +198,7 @@ class DBImpl : public DB {
     // that needs to be deleted in to_delete vector. Unrefing those
     // objects needs to be done in the mutex
     void Cleanup();
-    void Init(MemTable* new_mem, const MemTableList& new_imm,
+    void Init(MemTable* new_mem, MemTableListVersion* new_imm,
               Version* new_current);
   };
 

db/db_test.cc

@@ -17,6 +17,7 @@
 #include "db/filename.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
+#include "table/block_based_table_factory.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/env.h"
@@ -735,6 +736,9 @@ TEST(DBTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
   options.filter_policy = filter_policy.get();
   options.create_if_missing = true;
   options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
   DestroyAndReopen(&options);
 
   ASSERT_OK(db_->Put(WriteOptions(), "key", "val"));

db/memtablelist.cc

@@ -16,41 +16,85 @@ namespace rocksdb {
 
 class InternalKeyComparator;
 class Mutex;
-class MemTableListIterator;
 class VersionSet;
 
-using std::list;
+MemTableListVersion::MemTableListVersion(MemTableListVersion* old) {
-
+  if (old != nullptr) {
-// Increase reference count on all underling memtables
+    memlist_ = old->memlist_;
-void MemTableList::RefAll() {
+    size_ = old->size_;
-  for (auto &memtable : memlist_) {
+    for (auto& m : memlist_) {
-    memtable->Ref();
+      m->Ref();
+    }
   }
 }
 
-// Drop reference count on all underling memtables. If the
+void MemTableListVersion::Ref() { ++refs_; }
-// refcount of an underlying memtable drops to zero, then
+
-// return it in to_delete vector.
+void MemTableListVersion::Unref(std::vector<MemTable*>* to_delete) {
-void MemTableList::UnrefAll(std::vector<MemTable*>* to_delete) {
+  --refs_;
-  for (auto &memtable : memlist_) {
+  if (refs_ == 0) {
-    MemTable* m = memtable->Unref();
+    // if to_delete is equal to nullptr it means we're confident
-    if (m != nullptr) {
+    // that refs_ will not be zero
-      to_delete->push_back(m);
+    assert(to_delete != nullptr);
+    for (const auto& m : memlist_) {
+      MemTable* x = m->Unref();
+      if (x != nullptr) {
+        to_delete->push_back(x);
       }
     }
+    delete this;
   }
+}
+
+int MemTableListVersion::size() const { return size_; }
 
 // Returns the total number of memtables in the list
-int MemTableList::size() {
+int MemTableList::size() const {
-  assert(num_flush_not_started_ <= size_);
+  assert(num_flush_not_started_ <= current_->size_);
-  return size_;
+  return current_->size_;
+}
+
+// Search all the memtables starting from the most recent one.
+// Return the most recent value found, if any.
+// Operands stores the list of merge operations to apply, so far.
+bool MemTableListVersion::Get(const LookupKey& key, std::string* value,
+                              Status* s, MergeContext& merge_context,
+                              const Options& options) {
+  for (auto& memtable : memlist_) {
+    if (memtable->Get(key, value, s, merge_context, options)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+void MemTableListVersion::AddIterators(const ReadOptions& options,
+                                       std::vector<Iterator*>* iterator_list) {
+  for (auto& m : memlist_) {
+    iterator_list->push_back(m->NewIterator(options));
+  }
+}
+
+void MemTableListVersion::Add(MemTable* m) {
+  assert(refs_ == 1);  // only when refs_ == 1 is MemTableListVersion mutable
+  m->Ref();
+  memlist_.push_front(m);
+  ++size_;
+}
+
+void MemTableListVersion::Remove(MemTable* m) {
+  assert(refs_ == 1);  // only when refs_ == 1 is MemTableListVersion mutable
+  MemTable* x __attribute__((unused)) = m->Unref();
+  assert(x == nullptr);  // it still needs to be alive!
+  memlist_.remove(m);
+  --size_;
 }
 
 // Returns true if there is at least one memtable on which flush has
 // not yet started.
-bool MemTableList::IsFlushPending(int min_write_buffer_number_to_merge) {
+bool MemTableList::IsFlushPending() {
   if ((flush_requested_ && num_flush_not_started_ >= 1) ||
-      (num_flush_not_started_ >= min_write_buffer_number_to_merge)) {
+      (num_flush_not_started_ >= min_write_buffer_number_to_merge_)) {
     assert(imm_flush_needed.NoBarrier_Load() != nullptr);
     return true;
   }
@@ -59,7 +103,8 @@ bool MemTableList::IsFlushPending(int min_write_buffer_number_to_merge) {
 
 // Returns the memtables that need to be flushed.
 void MemTableList::PickMemtablesToFlush(std::vector<MemTable*>* ret) {
-  for (auto it = memlist_.rbegin(); it != memlist_.rend(); it++) {
+  const auto& memlist = current_->memlist_;
+  for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
     MemTable* m = *it;
     if (!m->flush_in_progress_) {
       assert(!m->flush_completed_);
@@ -122,8 +167,8 @@ Status MemTableList::InstallMemtableFlushResults(
   // scan all memtables from the earliest, and commit those
   // (in that order) that have finished flushing. Memetables
   // are always committed in the order that they were created.
-  while (!memlist_.empty() && s.ok()) {
+  while (!current_->memlist_.empty() && s.ok()) {
-    MemTable* m = memlist_.back(); // get the last element
+    MemTable* m = current_->memlist_.back();  // get the last element
     if (!m->flush_completed_) {
       break;
     }
@@ -135,6 +180,10 @@ Status MemTableList::InstallMemtableFlushResults(
     // this can release and reacquire the mutex.
     s = vset->LogAndApply(&m->edit_, mu);
 
+    // we will be changing the version in the next code path,
+    // so we better create a new one, since versions are immutable
+    InstallNewVersion();
+
     // All the later memtables that have the same filenum
     // are part of the same batch. They can be committed now.
     uint64_t mem_id = 1;  // how many memtables has been flushed.
@@ -144,7 +193,7 @@ Status MemTableList::InstallMemtableFlushResults(
             "Level-0 commit table #%lu: memtable #%lu done",
             (unsigned long)m->file_number_,
             (unsigned long)mem_id);
-        memlist_.remove(m);
+        current_->Remove(m);
         assert(m->file_number_ > 0);
 
         // pending_outputs can be cleared only after the newly created file
@@ -155,7 +204,6 @@ Status MemTableList::InstallMemtableFlushResults(
         if (m->Unref() != nullptr) {
           to_delete->push_back(m);
         }
-        size_--;
       } else {
         //commit failed. setup state so that we can flush again.
         Log(info_log,
@@ -172,7 +220,7 @@ Status MemTableList::InstallMemtableFlushResults(
         s = Status::IOError("Unable to commit flushed memtable");
       }
       ++mem_id;
-    } while (!memlist_.empty() && (m = memlist_.back()) &&
+    } while (!current_->memlist_.empty() && (m = current_->memlist_.back()) &&
              m->file_number_ == file_number);
   }
   commit_in_progress_ = false;
@@ -181,9 +229,9 @@ Status MemTableList::InstallMemtableFlushResults(
 
 // New memtables are inserted at the front of the list.
 void MemTableList::Add(MemTable* m) {
-  assert(size_ >= num_flush_not_started_);
+  assert(current_->size_ >= num_flush_not_started_);
-  size_++;
+  InstallNewVersion();
-  memlist_.push_front(m);
+  current_->Add(m);
   m->MarkImmutable();
   num_flush_not_started_++;
   if (num_flush_not_started_ == 1) {
@@ -194,28 +242,20 @@ void MemTableList::Add(MemTable* m) {
 // Returns an estimate of the number of bytes of data in use.
 size_t MemTableList::ApproximateMemoryUsage() {
   size_t size = 0;
-  for (auto &memtable : memlist_) {
+  for (auto& memtable : current_->memlist_) {
     size += memtable->ApproximateMemoryUsage();
   }
   return size;
 }
 
-// Search all the memtables starting from the most recent one.
+void MemTableList::InstallNewVersion() {
-// Return the most recent value found, if any.
+  if (current_->refs_ == 1) {
-// Operands stores the list of merge operations to apply, so far.
+    // we're the only one using the version, just keep using it
-bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s,
+  } else {
-                       MergeContext& merge_context, const Options& options) {
+    // somebody else holds the current version, we need to create new one
-  for (auto &memtable : memlist_) {
+    MemTableListVersion* version = current_;
-    if (memtable->Get(key, value, s, merge_context, options)) {
+    current_ = new MemTableListVersion(current_);
-      return true;
+    version->Unref();
-    }
-  }
-  return false;
-}
-
-void MemTableList::GetMemTables(std::vector<MemTable*>* output) {
-  for (auto &memtable : memlist_) {
-    output->push_back(memtable);
   }
 }
 

db/memtablelist.h

@@ -7,8 +7,10 @@
 #pragma once
 #include <string>
 #include <list>
-#include <deque>
+#include <vector>
 #include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "rocksdb/iterator.h"
 #include "db/dbformat.h"
 #include "db/skiplist.h"
 #include "memtable.h"
@@ -17,44 +19,71 @@ namespace rocksdb {
 
 class InternalKeyComparator;
 class Mutex;
-class MemTableListIterator;
 
-//
+// keeps a list of immutable memtables in a vector. the list is immutable
+// if refcount is bigger than one. It is used as a state for Get() and
+// Iterator code paths
+class MemTableListVersion {
+ public:
+  explicit MemTableListVersion(MemTableListVersion* old = nullptr);
+
+  void Ref();
+  void Unref(std::vector<MemTable*>* to_delete = nullptr);
+
+  int size() const;
+
+  // Search all the memtables starting from the most recent one.
+  // Return the most recent value found, if any.
+  bool Get(const LookupKey& key, std::string* value, Status* s,
+           MergeContext& merge_context, const Options& options);
+
+  void AddIterators(const ReadOptions& options,
+                    std::vector<Iterator*>* iterator_list);
+
+  // REQUIRE: m is mutable memtable
+  void Add(MemTable* m);
+  // REQUIRE: m is mutable memtable
+  void Remove(MemTable* m);
+
+ private:
+  friend class MemTableList;
+  std::list<MemTable*> memlist_;
+  int size_ = 0;
+  int refs_ = 1;
+};
+
 // This class stores references to all the immutable memtables.
 // The memtables are flushed to L0 as soon as possible and in
 // any order. If there are more than one immutable memtable, their
 // flushes can occur concurrently.  However, they are 'committed'
 // to the manifest in FIFO order to maintain correctness and
 // recoverability from a crash.
-//
 class MemTableList {
  public:
   // A list of memtables.
-  MemTableList() : size_(0), num_flush_not_started_(0),
+  explicit MemTableList(int min_write_buffer_number_to_merge)
+      : min_write_buffer_number_to_merge_(min_write_buffer_number_to_merge),
+        current_(new MemTableListVersion()),
+        num_flush_not_started_(0),
         commit_in_progress_(false),
         flush_requested_(false) {
     imm_flush_needed.Release_Store(nullptr);
+    current_->Ref();
   }
-  ~MemTableList() {};
+  ~MemTableList() {}
+
+  MemTableListVersion* current() { return current_; }
 
   // so that backgrund threads can detect non-nullptr pointer to
   // determine whether this is anything more to start flushing.
   port::AtomicPointer imm_flush_needed;
 
-  // Increase reference count on all underling memtables
-  void RefAll();
-
-  // Drop reference count on all underling memtables. If the refcount
-  // on an underlying memtable drops to zero, then return it in
-  // to_delete vector.
-  void UnrefAll(std::vector<MemTable*>* to_delete);
-
   // Returns the total number of memtables in the list
-  int size();
+  int size() const;
 
   // Returns true if there is at least one memtable on which flush has
   // not yet started.
-  bool IsFlushPending(int min_write_buffer_number_to_merge);
+  bool IsFlushPending();
 
   // Returns the earliest memtables that needs to be flushed. The returned
   // memtables are guaranteed to be in the ascending order of created time.
@@ -75,14 +104,6 @@ class MemTableList {
   // Returns an estimate of the number of bytes of data in use.
   size_t ApproximateMemoryUsage();
 
-  // Search all the memtables starting from the most recent one.
-  // Return the most recent value found, if any.
-  bool Get(const LookupKey& key, std::string* value, Status* s,
-           MergeContext& merge_context, const Options& options);
-
-  // Returns the list of underlying memtables.
-  void GetMemTables(std::vector<MemTable*>* list);
-
   // Request a flush of all existing memtables to storage
   void FlushRequested() { flush_requested_ = true; }
 
@@ -91,8 +112,12 @@ class MemTableList {
   // void operator=(const MemTableList&);
 
  private:
-  std::list<MemTable*> memlist_;
+  // DB mutex held
-  int size_;
+  void InstallNewVersion();
+
+  int min_write_buffer_number_to_merge_;
+
+  MemTableListVersion* current_;
 
   // the number of elements that still need flushing
   int num_flush_not_started_;

db/version_set.cc

@@ -1874,6 +1874,77 @@ Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
   return s;
 }
 
+Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
+                                        const Options* options,
+                                        const EnvOptions& storage_options,
+                                        int new_levels) {
+  if (new_levels <= 1) {
+    return Status::InvalidArgument(
+        "Number of levels needs to be bigger than 1");
+  }
+
+  const InternalKeyComparator cmp(options->comparator);
+  TableCache tc(dbname, options, storage_options, 10);
+  VersionSet versions(dbname, options, storage_options, &tc, &cmp);
+  Status status;
+
+  std::vector<ColumnFamilyDescriptor> dummy;
+  dummy.push_back(ColumnFamilyDescriptor());
+  status = versions.Recover(dummy);
+  if (!status.ok()) {
+    return status;
+  }
+
+  Version* current_version = versions.current();
+  int current_levels = current_version->NumberLevels();
+
+  if (current_levels <= new_levels) {
+    return Status::OK();
+  }
+
+  // Make sure there are file only on one level from
+  // (new_levels-1) to (current_levels-1)
+  int first_nonempty_level = -1;
+  int first_nonempty_level_filenum = 0;
+  for (int i = new_levels - 1; i < current_levels; i++) {
+    int file_num = current_version->NumLevelFiles(i);
+    if (file_num != 0) {
+      if (first_nonempty_level < 0) {
+        first_nonempty_level = i;
+        first_nonempty_level_filenum = file_num;
+      } else {
+        char msg[255];
+        snprintf(msg, sizeof(msg),
+                 "Found at least two levels containing files: "
+                 "[%d:%d],[%d:%d].\n",
+                 first_nonempty_level, first_nonempty_level_filenum, i,
+                 file_num);
+        return Status::InvalidArgument(msg);
+      }
+    }
+  }
+
+  std::vector<FileMetaData*>* old_files_list = current_version->files_;
+  std::vector<FileMetaData*>* new_files_list =
+      new std::vector<FileMetaData*>[new_levels];
+  for (int i = 0; i < new_levels - 1; i++) {
+    new_files_list[i] = old_files_list[i];
+  }
+
+  if (first_nonempty_level > 0) {
+    new_files_list[new_levels - 1] = old_files_list[first_nonempty_level];
+  }
+
+  delete[] current_version->files_;
+  current_version->files_ = new_files_list;
+  current_version->num_levels_ = new_levels;
+
+  VersionEdit ve;
+  port::Mutex dummy_mutex;
+  MutexLock l(&dummy_mutex);
+  return versions.LogAndApply(&ve, &dummy_mutex, true);
+}
+
 Status VersionSet::DumpManifest(Options& options, std::string& dscname,
                                 bool verbose, bool hex) {
   // Open the specified manifest file.

db/version_set.h

@@ -302,10 +302,16 @@ class VersionSet {
   // Try to reduce the number of levels. This call is valid when
   // only one level from the new max level to the old
   // max level containing files.
+  // The call is static, since number of levels is immutable during
+  // the lifetime of a RocksDB instance. It reduces number of levels
+  // in a DB by applying changes to manifest.
   // For example, a db currently has 7 levels [0-6], and a call to
   // to reduce to 5 [0-4] can only be executed when only one level
   // among [4-6] contains files.
-  Status ReduceNumberOfLevels(int new_levels, port::Mutex* mu);
+  static Status ReduceNumberOfLevels(const std::string& dbname,
+                                     const Options* options,
+                                     const EnvOptions& storage_options,
+                                     int new_levels);
 
   // Return the current version.
   Version* current() const {

db/version_set_reduce_num_levels.cc

@@ -1,78 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2012 Facebook. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "db/version_set.h"
-
-#include <algorithm>
-#include <stdio.h>
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "util/logging.h"
-
-namespace rocksdb {
-
-Status VersionSet::ReduceNumberOfLevels(int new_levels, port::Mutex* mu) {
-
-  if(new_levels <= 1) {
-    return Status::InvalidArgument(
-        "Number of levels needs to be bigger than 1");
-  }
-
-  // TODO this only works for default column family now
-  Version* current_version = column_family_set_->GetDefault()->current;
-  int current_levels = current_version->NumberLevels();
-
-  if (current_levels <= new_levels) {
-    return Status::OK();
-  }
-
-  // Make sure there are file only on one level from
-  // (new_levels-1) to (current_levels-1)
-  int first_nonempty_level = -1;
-  int first_nonempty_level_filenum = 0;
-  for (int i = new_levels - 1; i < current_levels; i++) {
-    int file_num = current_version->NumLevelFiles(i);
-    if (file_num != 0) {
-      if (first_nonempty_level < 0) {
-        first_nonempty_level = i;
-        first_nonempty_level_filenum = file_num;
-      } else {
-        char msg[255];
-        sprintf(msg, "Found at least two levels containing files: "
-            "[%d:%d],[%d:%d].\n",
-            first_nonempty_level, first_nonempty_level_filenum, i, file_num);
-        return Status::InvalidArgument(msg);
-      }
-    }
-  }
-
-  Status st;
-  std::vector<FileMetaData*>*  old_files_list = current_version->files_;
-  std::vector<FileMetaData*>* new_files_list =
-      new std::vector<FileMetaData*>[new_levels];
-  for (int i = 0; i < new_levels - 1; i++) {
-    new_files_list[i] = old_files_list[i];
-  }
-
-  if (first_nonempty_level > 0) {
-    new_files_list[new_levels - 1] = old_files_list[first_nonempty_level];
-  }
-
-  delete[] current_version->files_;
-  current_version->files_ = new_files_list;
-  current_version->num_levels_ = new_levels;
-
-  num_levels_ = new_levels;
-  compaction_picker_->ReduceNumberOfLevels(new_levels);
-  VersionEdit ve;
-  st = LogAndApply(&ve, mu, true);
-  return st;
-}
-
-}

include/rocksdb/cache.h

@@ -104,6 +104,15 @@ class Cache {
   // returns the maximum configured capacity of the cache
   virtual size_t GetCapacity() = 0;
 
+  // Call this on shutdown if you want to speed it up. Cache will disown
+  // any underlying data and will not free it on delete. This call will leak
+  // memory - call this only if you're shutting down the process.
+  // Any attempts of using cache after this call will fail terribly.
+  // Always delete the DB object before calling this method!
+  virtual void DisownData() {
+    // default implementation is noop
+  };
+
  private:
   void LRU_Remove(Handle* e);
   void LRU_Append(Handle* e);

table/block_based_table_factory.cc

@@ -22,8 +22,8 @@ Status BlockBasedTableFactory::GetTableReader(
     const Options& options, const EnvOptions& soptions,
     unique_ptr<RandomAccessFile>&& file, uint64_t file_size,
     unique_ptr<TableReader>* table_reader) const {
-  return BlockBasedTable::Open(options, soptions, std::move(file), file_size,
+  return BlockBasedTable::Open(options, soptions, table_options_,
-                               table_reader);
+                               std::move(file), file_size, table_reader);
 }
 
 TableBuilder* BlockBasedTableFactory::GetTableBuilder(

table/block_based_table_factory.h

@@ -14,6 +14,7 @@
 #include "rocksdb/flush_block_policy.h"
 #include "rocksdb/options.h"
 #include "rocksdb/table.h"
+#include "table/block_based_table_options.h"
 
 namespace rocksdb {
 
@@ -31,39 +32,24 @@ class BlockBasedTableBuilder;
 
 class BlockBasedTableFactory: public TableFactory {
  public:
-  struct TableOptions {
+  BlockBasedTableFactory() : BlockBasedTableFactory(BlockBasedTableOptions()) {}
-    // @flush_block_policy_factory creates the instances of flush block policy.
+  explicit BlockBasedTableFactory(const BlockBasedTableOptions& table_options)
-    // which provides a configurable way to determine when to flush a block in
+      : table_options_(table_options) {}
-    // the block based tables.  If not set, table builder will use the default
-    // block flush policy, which cut blocks by block size (please refer to
-    // `FlushBlockBySizePolicy`).
-    std::shared_ptr<FlushBlockPolicyFactory> flush_block_policy_factory;
-  };
 
-  BlockBasedTableFactory() : BlockBasedTableFactory(TableOptions()) { }
+  ~BlockBasedTableFactory() {}
-  BlockBasedTableFactory(const TableOptions& table_options): 
-      table_options_(table_options) { 
-  }
 
-  ~BlockBasedTableFactory() {
+  const char* Name() const override { return "BlockBasedTable"; }
-  }
-
-  const char* Name() const override {
-    return "BlockBasedTable";
-  }
 
   Status GetTableReader(const Options& options, const EnvOptions& soptions,
-                        unique_ptr<RandomAccessFile> && file,
+                        unique_ptr<RandomAccessFile>&& file, uint64_t file_size,
-                        uint64_t file_size,
                         unique_ptr<TableReader>* table_reader) const override;
 
   TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
-                                CompressionType compression_type) const
+                                CompressionType compression_type)
-                                    override;
+      const override;
 
  private:
-  TableOptions table_options_;
+  BlockBasedTableOptions table_options_;
 };
 
-
 }  // namespace rocksdb

table/block_based_table_options.h

@@ -0,0 +1,31 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#pragma once
+#include <memory>
+
+namespace rocksdb {
+
+class FlushBlockPolicyFactory;
+
+struct BlockBasedTableOptions {
+  // @flush_block_policy_factory creates the instances of flush block policy.
+  // which provides a configurable way to determine when to flush a block in
+  // the block based tables.  If not set, table builder will use the default
+  // block flush policy, which cut blocks by block size (please refer to
+  // `FlushBlockBySizePolicy`).
+  std::shared_ptr<FlushBlockPolicyFactory> flush_block_policy_factory;
+
+  // TODO(kailiu) Temporarily disable this feature by making the default value
+  // to be false. Also in master branch, this file is non-public so no user
+  // will be able to change the value of `cache_index_and_filter_blocks`.
+  //
+  // Indicating if we'd put index/filter blocks to the block cache.
+  // If not specified, each "table reader" object will pre-load index/filter
+  // block during table initialization.
+  bool cache_index_and_filter_blocks = false;
+};
+
+}  // namespace rocksdb

table/block_based_table_reader.cc

@@ -26,6 +26,7 @@
 #include "util/coding.h"
 #include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
+#include "table/block_based_table_options.h"
 
 namespace rocksdb {
 
@@ -45,9 +46,9 @@ struct BlockBasedTable::Rep {
   Status status;
   unique_ptr<RandomAccessFile> file;
   char cache_key_prefix[kMaxCacheKeyPrefixSize];
-  size_t cache_key_prefix_size;
+  size_t cache_key_prefix_size = 0;
   char compressed_cache_key_prefix[kMaxCacheKeyPrefixSize];
-  size_t compressed_cache_key_prefix_size;
+  size_t compressed_cache_key_prefix_size = 0;
 
   // Handle to metaindex_block: saved from footer
   BlockHandle metaindex_handle;
@@ -220,20 +221,21 @@ Cache::Handle* GetFromBlockCache(
 
 } // end of anonymous namespace
 
-Status BlockBasedTable::Open(const Options& options,
+Status BlockBasedTable::Open(const Options& options, const EnvOptions& soptions,
-                             const EnvOptions& soptions,
+                             const BlockBasedTableOptions& table_options,
                              unique_ptr<RandomAccessFile>&& file,
-                             uint64_t size,
+                             uint64_t file_size,
                              unique_ptr<TableReader>* table_reader) {
   table_reader->reset();
-  if (size < Footer::kEncodedLength) {
+
+  if (file_size < Footer::kEncodedLength) {
     return Status::InvalidArgument("file is too short to be an sstable");
   }
 
   char footer_space[Footer::kEncodedLength];
   Slice footer_input;
-  Status s = file->Read(size - Footer::kEncodedLength, Footer::kEncodedLength,
+  Status s = file->Read(file_size - Footer::kEncodedLength,
-                        &footer_input, footer_space);
+                        Footer::kEncodedLength, &footer_input, footer_space);
   if (!s.ok()) return s;
 
   // Check that we actually read the whole footer from the file. It may be
@@ -277,11 +279,21 @@ Status BlockBasedTable::Open(const Options& options,
     }
   }
 
-  // Initialize index/filter blocks. If block cache is not specified,
+  // Will use block cache for index/filter blocks access?
-  // these blocks will be kept in member variables in Rep, which will
+  if (options.block_cache && table_options.cache_index_and_filter_blocks) {
-  // reside in the memory as long as this table object is alive; otherwise
+    // Call IndexBlockReader() to implicitly add index to the block_cache
-  // they will be added to block cache.
+    unique_ptr<Iterator> iter(new_table->IndexBlockReader(ReadOptions()));
-  if (!options.block_cache) {
+    s = iter->status();
+
+    if (s.ok()) {
+      // Call GetFilter() to implicitly add filter to the block_cache
+      auto filter_entry = new_table->GetFilter();
+      filter_entry.Release(options.block_cache.get());
+    }
+  } else {
+    // If we don't use block cache for index/filter blocks access, we'll
+    // pre-load these blocks, which will kept in member variables in Rep
+    // and with a same life-time as this table object.
     Block* index_block = nullptr;
     // TODO: we never really verify check sum for index block
     s = ReadBlockFromFile(
@@ -309,18 +321,7 @@ Status BlockBasedTable::Open(const Options& options,
     } else {
       delete index_block;
     }
-  } else {
-    // Call IndexBlockReader() to implicitly add index to the block_cache
-    unique_ptr<Iterator> iter(
-        new_table->IndexBlockReader(ReadOptions())
-    );
-    s = iter->status();
 
-    if (s.ok()) {
-      // Call GetFilter() to implicitly add filter to the block_cache
-      auto filter_entry = new_table->GetFilter();
-      filter_entry.Release(options.block_cache.get());
-    }
   }
 
   if (s.ok()) {
@@ -836,7 +837,6 @@ BlockBasedTable::GetFilter(bool no_io) const {
 // Get the iterator from the index block.
 Iterator* BlockBasedTable::IndexBlockReader(const ReadOptions& options) const {
   if (rep_->index_block) {
-    assert (!rep_->options.block_cache);
     return rep_->index_block->NewIterator(rep_->options.comparator);
   }
 

table/block_based_table_reader.h

@@ -29,6 +29,7 @@ struct ReadOptions;
 class TableCache;
 class TableReader;
 class FilterBlockReader;
+struct BlockBasedTableOptions;
 
 using std::unique_ptr;
 
@@ -50,10 +51,9 @@ class BlockBasedTable : public TableReader {
   // to nullptr and returns a non-ok status.
   //
   // *file must remain live while this Table is in use.
-  static Status Open(const Options& options,
+  static Status Open(const Options& db_options, const EnvOptions& env_options,
-                     const EnvOptions& soptions,
+                     const BlockBasedTableOptions& table_options,
-                     unique_ptr<RandomAccessFile>&& file,
+                     unique_ptr<RandomAccessFile>&& file, uint64_t file_size,
-                     uint64_t file_size,
                      unique_ptr<TableReader>* table_reader);
 
   bool PrefixMayMatch(const Slice& internal_prefix) override;

table/table_test.cc

@@ -243,13 +243,12 @@ class BlockConstructor: public Constructor {
 
 class BlockBasedTableConstructor: public Constructor {
  public:
-  explicit BlockBasedTableConstructor(
+  explicit BlockBasedTableConstructor(const Comparator* cmp)
-      const Comparator* cmp)
+      : Constructor(cmp) {}
-      : Constructor(cmp) {
-  }
   ~BlockBasedTableConstructor() {
     Reset();
   }
+
   virtual Status FinishImpl(const Options& options, const KVMap& data) {
     Reset();
     sink_.reset(new StringSink());
@@ -277,7 +276,6 @@ class BlockBasedTableConstructor: public Constructor {
     // Open the table
     uniq_id_ = cur_uniq_id_++;
     source_.reset(new StringSource(sink_->contents(), uniq_id_));
-    unique_ptr<TableFactory> table_factory;
     return options.table_factory->GetTableReader(options, soptions,
                                                  std::move(source_),
                                                  sink_->contents().size(),
@@ -979,6 +977,11 @@ TEST(TableTest, BlockCacheTest) {
   options.create_if_missing = true;
   options.statistics = CreateDBStatistics();
   options.block_cache = NewLRUCache(1024);
+
+  // Enable the cache for index/filter blocks
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
   std::vector<std::string> keys;
   KVMap kvmap;
 
@@ -1292,7 +1295,6 @@ TEST(MemTableTest, Simple) {
   delete memtable->Unref();
 }
 
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

util/cache.cc

@@ -409,6 +409,9 @@ class ShardedLRUCache : public Cache {
   virtual size_t GetCapacity() {
     return capacity_;
   }
+  virtual void DisownData() {
+    shard_ = nullptr;
+  }
 };
 
 }  // end anonymous namespace

util/ldb_cmd.cc

@@ -1072,25 +1072,7 @@ void ReduceDBLevelsCommand::DoCommand() {
   CloseDB();
 
   EnvOptions soptions;
-  TableCache tc(db_path_, &opt, soptions, 10);
+  st = VersionSet::ReduceNumberOfLevels(db_path_, &opt, soptions, new_levels_);
-  const InternalKeyComparator cmp(opt.comparator);
-  VersionSet versions(db_path_, &opt, soptions, &tc, &cmp);
-  std::vector<ColumnFamilyDescriptor> dummy;
-  dummy.push_back(ColumnFamilyDescriptor());
-  // We rely the VersionSet::Recover to tell us the internal data structures
-  // in the db. And the Recover() should never do any change (like LogAndApply)
-  // to the manifest file.
-  st = versions.Recover(dummy);
-  if (!st.ok()) {
-    exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString());
-    return;
-  }
-
-  port::Mutex mu;
-  mu.Lock();
-  st = versions.ReduceNumberOfLevels(new_levels_, &mu);
-  mu.Unlock();
-
   if (!st.ok()) {
     exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString());
     return;