Retire memenv https://github.com/facebook/rocksdb/pull/2082

Summary:
This is a manual commit of this PR:
Retire InMemoryEnv in favor of MockEnv #2082
With MockEnv doing the same yet being more mature, InMemoryEnv is redundant.

Reviewed By: IslamAbdelRahman

Differential Revision: D5162323

fbshipit-source-id: 59fd0082a891dc99cc531e4da9d68bf891eae3f5

CMakeLists.txt

@@ -381,7 +381,7 @@ set(SOURCES
         env/env.cc
         env/env_chroot.cc
         env/env_hdfs.cc
-        env/memenv.cc
+        env/mock_env.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
@@ -745,7 +745,6 @@ endforeach(sourcefile ${BENCHMARKS})
 # and linked to tests. Add test only code that is not #ifdefed for Release here.
 set(TESTUTIL_SOURCE
     db/db_test_util.cc
-    env/mock_env.cc
     monitoring/thread_status_updater_debug.cc
     table/mock_table.cc
     util/fault_injection_test_env.cc

TARGETS

@@ -46,7 +46,7 @@ rocksdb_preprocessor_flags = [
 
 cpp_library(
     name = "rocksdb_lib",
-    auto_headers = AutoHeaders.RECURSIVE_GLOB,
+    headers = AutoHeaders.RECURSIVE_GLOB,
     srcs = [
       "cache/clock_cache.cc",
       "cache/lru_cache.cc",
@@ -108,7 +108,7 @@ cpp_library(
       "env/env_hdfs.cc",
       "env/env_posix.cc",
       "env/io_posix.cc",
-      "env/memenv.cc",
+      "env/mock_env.cc",
       "memtable/hash_cuckoo_rep.cc",
       "memtable/hash_linklist_rep.cc",
       "memtable/hash_skiplist_rep.cc",
@@ -201,7 +201,6 @@ cpp_library(
       "utilities/blob_db/blob_db.cc",
       "utilities/blob_db/blob_db_impl.cc",
       "utilities/blob_db/blob_db_options_impl.cc",
-      "utilities/blob_db/blob_dump_tool.cc",
       "utilities/blob_db/blob_file.cc",
       "utilities/blob_db/blob_log_reader.cc",
       "utilities/blob_db/blob_log_writer.cc",
@@ -260,9 +259,8 @@ cpp_library(
 
 cpp_library(
     name = "rocksdb_test_lib",
-    auto_headers = AutoHeaders.RECURSIVE_GLOB,
+    headers = AutoHeaders.RECURSIVE_GLOB,
     srcs = [
-      "env/mock_env.cc",
       "table/mock_table.cc",
       "util/fault_injection_test_env.cc",
       "util/testharness.cc",
@@ -280,7 +278,7 @@ cpp_library(
 
 cpp_library(
     name = "rocksdb_tools_lib",
-    auto_headers = AutoHeaders.RECURSIVE_GLOB,
+    headers = AutoHeaders.RECURSIVE_GLOB,
     srcs = [
       "tools/db_bench_tool.cc",
       "util/testutil.cc",
@@ -293,7 +291,7 @@ cpp_library(
 
 cpp_library(
     name = "env_basic_test_lib",
-    auto_headers = AutoHeaders.RECURSIVE_GLOB,
+    headers = AutoHeaders.RECURSIVE_GLOB,
     srcs = ["env/env_basic_test.cc"],
     deps = [":rocksdb_test_lib"],
     preprocessor_flags = rocksdb_preprocessor_flags,
@@ -334,6 +332,7 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
  ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
  ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
  ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
+ ['db_write_test', 'db/db_write_test.cc', 'serial'],
  ['write_batch_with_index_test',
   'utilities/write_batch_with_index/write_batch_with_index_test.cc',
   'serial'],
@@ -458,9 +457,8 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
  ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
  ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
  ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
- ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
+ ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
- ['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'],
+
- ['db_write_test', 'db/db_write_test.cc', 'serial']]
 
 # Generate a test rule for each entry in ROCKS_TESTS
 for test_cfg in ROCKS_TESTS:
@@ -489,12 +487,10 @@ custom_unittest(
     name = "make_rocksdbjavastatic",
     type = "simple",
     command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
-    deps=[],
 )
 
 custom_unittest(
     name = "make_rocksdb_lite_release",
     type = "simple",
     command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
-    deps=[],
 )

env/memenv.cc

@@ -1,451 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <string.h>
-
-#include <algorithm>
-#include <map>
-#include <string>
-#include <vector>
-
-#include "rocksdb/env.h"
-#include "rocksdb/status.h"
-#include "port/port.h"
-#include "util/mutexlock.h"
-
-namespace rocksdb {
-
-#ifndef ROCKSDB_LITE
-
-namespace {
-
-std::string NormalizeFileName(const std::string fname) {
-  if (fname.find("//") == std::string::npos) {
-    return fname;
-  }
-  std::string out_name = "";
-  bool is_slash = false;
-  for (char c : fname) {
-    if (c == '/' && is_slash) {
-      continue;
-    }
-    out_name.append(1, c);
-    if (c == '/') {
-      is_slash = true;
-    } else {
-      is_slash = false;
-    }
-  }
-  return out_name;
-}
-
-class FileState {
- public:
-  // FileStates are reference counted. The initial reference count is zero
-  // and the caller must call Ref() at least once.
-  FileState() : refs_(0), size_(0) {}
-
-  // Increase the reference count.
-  void Ref() {
-    MutexLock lock(&refs_mutex_);
-    ++refs_;
-  }
-
-  // Decrease the reference count. Delete if this is the last reference.
-  void Unref() {
-    bool do_delete = false;
-
-    {
-      MutexLock lock(&refs_mutex_);
-      --refs_;
-      assert(refs_ >= 0);
-      if (refs_ <= 0) {
-        do_delete = true;
-      }
-    }
-
-    if (do_delete) {
-      delete this;
-    }
-  }
-
-  uint64_t Size() const { return size_; }
-
-  Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
-    const uint64_t available = size_ - std::min(size_, offset);
-    if (n > available) {
-      n = available;
-    }
-    if (n == 0) {
-      *result = Slice();
-      return Status::OK();
-    }
-
-    size_t block = offset / kBlockSize;
-    size_t block_offset = offset % kBlockSize;
-
-    if (n <= kBlockSize - block_offset) {
-      // The requested bytes are all in the first block.
-      *result = Slice(blocks_[block] + block_offset, n);
-      return Status::OK();
-    }
-
-    size_t bytes_to_copy = n;
-    char* dst = scratch;
-
-    while (bytes_to_copy > 0) {
-      size_t avail = kBlockSize - block_offset;
-      if (avail > bytes_to_copy) {
-        avail = bytes_to_copy;
-      }
-      memcpy(dst, blocks_[block] + block_offset, avail);
-
-      bytes_to_copy -= avail;
-      dst += avail;
-      block++;
-      block_offset = 0;
-    }
-
-    *result = Slice(scratch, n);
-    return Status::OK();
-  }
-
-  Status Append(const Slice& data) {
-    const char* src = data.data();
-    size_t src_len = data.size();
-
-    while (src_len > 0) {
-      size_t avail;
-      size_t offset = size_ % kBlockSize;
-
-      if (offset != 0) {
-        // There is some room in the last block.
-        avail = kBlockSize - offset;
-      } else {
-        // No room in the last block; push new one.
-        blocks_.push_back(new char[kBlockSize]);
-        avail = kBlockSize;
-      }
-
-      if (avail > src_len) {
-        avail = src_len;
-      }
-      memcpy(blocks_.back() + offset, src, avail);
-      src_len -= avail;
-      src += avail;
-      size_ += avail;
-    }
-
-    return Status::OK();
-  }
-
- private:
-  // Private since only Unref() should be used to delete it.
-  ~FileState() {
-    for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
-         ++i) {
-      delete [] *i;
-    }
-  }
-
-  // No copying allowed.
-  FileState(const FileState&);
-  void operator=(const FileState&);
-
-  port::Mutex refs_mutex_;
-  int refs_;  // Protected by refs_mutex_;
-
-  // The following fields are not protected by any mutex. They are only mutable
-  // while the file is being written, and concurrent access is not allowed
-  // to writable files.
-  std::vector<char*> blocks_;
-  uint64_t size_;
-
-  enum { kBlockSize = 8 * 1024 };
-};
-
-class SequentialFileImpl : public SequentialFile {
- public:
-  explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
-    file_->Ref();
-  }
-
-  ~SequentialFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Read(size_t n, Slice* result, char* scratch) override {
-    Status s = file_->Read(pos_, n, result, scratch);
-    if (s.ok()) {
-      pos_ += result->size();
-    }
-    return s;
-  }
-
-  virtual Status Skip(uint64_t n) override {
-    if (pos_ > file_->Size()) {
-      return Status::IOError("pos_ > file_->Size()");
-    }
-    const size_t available = file_->Size() - pos_;
-    if (n > available) {
-      n = available;
-    }
-    pos_ += n;
-    return Status::OK();
-  }
-
- private:
-  FileState* file_;
-  size_t pos_;
-};
-
-class RandomAccessFileImpl : public RandomAccessFile {
- public:
-  explicit RandomAccessFileImpl(FileState* file) : file_(file) {
-    file_->Ref();
-  }
-
-  ~RandomAccessFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const override {
-    return file_->Read(offset, n, result, scratch);
-  }
-
- private:
-  FileState* file_;
-};
-
-class WritableFileImpl : public WritableFile {
- public:
-  explicit WritableFileImpl(FileState* file) : file_(file) { file_->Ref(); }
-
-  ~WritableFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Append(const Slice& data) override {
-    return file_->Append(data);
-  }
-  virtual Status Truncate(uint64_t size) override {
-    return Status::OK();
-  }
-  virtual Status Close() override { return Status::OK(); }
-  virtual Status Flush() override { return Status::OK(); }
-  virtual Status Sync() override { return Status::OK(); }
-
- private:
-  FileState* file_;
-};
-
-class InMemoryDirectory : public Directory {
- public:
-  virtual Status Fsync() override { return Status::OK(); }
-};
-
-class InMemoryEnv : public EnvWrapper {
- public:
-  explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
-
-  virtual ~InMemoryEnv() {
-    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
-      i->second->Unref();
-    }
-  }
-
-  // Partial implementation of the Env interface.
-  virtual Status NewSequentialFile(const std::string& fname,
-                                   unique_ptr<SequentialFile>* result,
-                                   const EnvOptions& soptions) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(fname) == file_map_.end()) {
-      *result = NULL;
-      return Status::IOError(fname, "File not found");
-    }
-
-    result->reset(new SequentialFileImpl(file_map_[nfname]));
-    return Status::OK();
-  }
-
-  virtual Status NewRandomAccessFile(const std::string& fname,
-                                     unique_ptr<RandomAccessFile>* result,
-                                     const EnvOptions& soptions) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) == file_map_.end()) {
-      *result = NULL;
-      return Status::IOError(fname, "File not found");
-    }
-
-    result->reset(new RandomAccessFileImpl(file_map_[nfname]));
-    return Status::OK();
-  }
-
-  virtual Status NewWritableFile(const std::string& fname,
-                                 unique_ptr<WritableFile>* result,
-                                 const EnvOptions& soptions) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) != file_map_.end()) {
-      DeleteFileInternal(nfname);
-    }
-
-    FileState* file = new FileState();
-    file->Ref();
-    file_map_[nfname] = file;
-
-    result->reset(new WritableFileImpl(file));
-    return Status::OK();
-  }
-
-  virtual Status NewDirectory(const std::string& name,
-                              unique_ptr<Directory>* result) override {
-    result->reset(new InMemoryDirectory());
-    return Status::OK();
-  }
-
-  virtual Status FileExists(const std::string& fname) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) != file_map_.end()) {
-      return Status::OK();
-    } else {
-      return Status::NotFound();
-    }
-  }
-
-  virtual Status GetChildren(const std::string& dir,
-                             std::vector<std::string>* result) override {
-    MutexLock lock(&mutex_);
-    result->clear();
-
-    bool found_dir = false;
-    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
-      const std::string& filename = i->first;
-
-      if (dir == filename) {
-        found_dir = true;
-      } else if (filename.size() >= dir.size() + 1 &&
-                 filename[dir.size()] == '/' &&
-                 Slice(filename).starts_with(Slice(dir))) {
-        found_dir = true;
-        result->push_back(filename.substr(dir.size() + 1));
-      }
-    }
-
-    return found_dir ? Status::OK() : Status::NotFound();
-  }
-
-  void DeleteFileInternal(const std::string& fname) {
-    if (file_map_.find(fname) == file_map_.end()) {
-      return;
-    }
-
-    file_map_[fname]->Unref();
-    file_map_.erase(fname);
-  }
-
-  virtual Status DeleteFile(const std::string& fname) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) == file_map_.end()) {
-      return Status::IOError(fname, "File not found");
-    }
-
-    DeleteFileInternal(nfname);
-    return Status::OK();
-  }
-
-  virtual Status CreateDir(const std::string& dirname) override {
-    auto ndirname = NormalizeFileName(dirname);
-    if (file_map_.find(ndirname) == file_map_.end()) {
-      FileState* file = new FileState();
-      file->Ref();
-      file_map_[ndirname] = file;
-    } else {
-      return Status::IOError();
-    }
-    return Status::OK();
-  }
-
-  virtual Status CreateDirIfMissing(const std::string& dirname) override {
-    CreateDir(dirname);
-    return Status::OK();
-  }
-
-  virtual Status DeleteDir(const std::string& dirname) override {
-    return DeleteFile(dirname);
-  }
-
-  virtual Status GetFileSize(const std::string& fname,
-                             uint64_t* file_size) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-
-    if (file_map_.find(nfname) == file_map_.end()) {
-      return Status::IOError(fname, "File not found");
-    }
-
-    *file_size = file_map_[nfname]->Size();
-    return Status::OK();
-  }
-
-  virtual Status GetFileModificationTime(const std::string& fname,
-                                         uint64_t* time) override {
-    return Status::NotSupported("getFileMTime", "Not supported in MemEnv");
-  }
-
-  virtual Status RenameFile(const std::string& src,
-                            const std::string& dest) override {
-    std::string nsrc = NormalizeFileName(src);
-    std::string ndest = NormalizeFileName(dest);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nsrc) == file_map_.end()) {
-      return Status::IOError(src, "File not found");
-    }
-
-    DeleteFileInternal(dest);
-    file_map_[ndest] = file_map_[nsrc];
-    file_map_.erase(nsrc);
-    return Status::OK();
-  }
-
-  virtual Status LockFile(const std::string& fname, FileLock** lock) override {
-    *lock = new FileLock;
-    return Status::OK();
-  }
-
-  virtual Status UnlockFile(FileLock* lock) override {
-    delete lock;
-    return Status::OK();
-  }
-
-  virtual Status GetTestDirectory(std::string* path) override {
-    *path = "/test";
-    return Status::OK();
-  }
-
- private:
-  // Map from filenames to FileState objects, representing a simple file system.
-  typedef std::map<std::string, FileState*> FileSystem;
-  port::Mutex mutex_;
-  FileSystem file_map_;  // Protected by mutex_.
-};
-
-}  // namespace
-
-Env* NewMemEnv(Env* base_env) {
-  return new InMemoryEnv(base_env);
-}
-
-#else  // ROCKSDB_LITE
-
-Env* NewMemEnv(Env* base_env) { return nullptr; }
-
-#endif  // !ROCKSDB_LITE
-
-}  // namespace rocksdb

env/mock_env.cc

@@ -782,4 +782,14 @@ void MockEnv::FakeSleepForMicroseconds(int64_t micros) {
   fake_sleep_micros_.fetch_add(micros);
 }
 
+#ifndef ROCKSDB_LITE
+// This is to maintain the behavior before swithcing from InMemoryEnv to MockEnv
+Env* NewMemEnv(Env* base_env) { return new MockEnv(base_env); }
+
+#else  // ROCKSDB_LITE
+
+Env* NewMemEnv(Env* base_env) { return nullptr; }
+
+#endif  // !ROCKSDB_LITE
+
 }  // namespace rocksdb

src.mk

@@ -60,7 +60,7 @@ LIB_SOURCES =                                                   \
   env/env_hdfs.cc                                               \
   env/env_posix.cc                                              \
   env/io_posix.cc                                               \
-  env/memenv.cc                                                 \
+  env/mock_env.cc                                              \
   memtable/hash_cuckoo_rep.cc                                   \
   memtable/hash_linklist_rep.cc                                 \
   memtable/hash_skiplist_rep.cc                                 \
@@ -206,7 +206,6 @@ TOOL_LIB_SOURCES = \
   utilities/blob_db/blob_dump_tool.cc                            \
 
 MOCK_LIB_SOURCES = \
-  env/mock_env.cc \
   table/mock_table.cc \
   util/fault_injection_test_env.cc