From 824e6463413c3d3d76188e172ef6c8002c461c50 Mon Sep 17 00:00:00 2001
From: Yoshinori Matsunobu <yoshinori@fb.com>
Date: Thu, 2 Apr 2015 18:14:49 -0700
Subject: [PATCH] Adding another NewFlashcacheAwareEnv function to support
 pre-opened fd

Summary:
There are some cases when flachcache file descriptor was
already allocated (i.e. fb-MySQL). Then NewFlashcacheAwareEnv returns an
error at open() because fd was already assigned. This diff adds another
function to instantiate FlashcacheAwareEnv, with pre-allocated fd cachedev_fd.

Test Plan: Tested with MyRocks using this function, then worked

Reviewers: sdong, igor

Reviewed By: igor

Subscribers: dhruba, MarkCallaghan, rven

Differential Revision: https://reviews.facebook.net/D36447
---
 db/db_bench.cc                         | 12 +++++++++++-
 include/rocksdb/utilities/flashcache.h |  5 +++--
 utilities/flashcache/flashcache.cc     | 11 +----------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/db/db_bench.cc b/db/db_bench.cc
index 6e1679578..5fed923c7 100644
--- a/db/db_bench.cc
+++ b/db/db_bench.cc
@@ -24,6 +24,7 @@ int main() {
 #include <numaif.h>
 #endif
 
+#include <fcntl.h>
 #include <inttypes.h>
 #include <cstddef>
 #include <sys/types.h>
@@ -1189,6 +1190,7 @@ class Benchmark {
   int64_t readwrites_;
   int64_t merge_keys_;
   bool report_file_operations_;
+  int cachedev_fd_;
 
   bool SanityCheck() {
     if (FLAGS_compression_ratio > 1) {
@@ -1456,6 +1458,9 @@ class Benchmark {
       // this will leak, but we're shutting down so nobody cares
       cache_->DisownData();
     }
+    if (FLAGS_disable_flashcache_for_background_threads && cachedev_fd_ != -1) {
+      close(cachedev_fd_);
+    }
   }
 
   Slice AllocateKey(std::unique_ptr<const char[]>* key_guard) {
@@ -2026,8 +2031,13 @@ class Benchmark {
       FLAGS_env->LowerThreadPoolIOPriority(Env::HIGH);
     }
     if (FLAGS_disable_flashcache_for_background_threads) {
+      cachedev_fd_ = open(FLAGS_flashcache_dev.c_str(), O_RDONLY);
+      if (cachedev_fd_ < 0) {
+        fprintf(stderr, "Open flash device failed\n");
+        exit(1);
+      }
       flashcache_aware_env_ =
-          std::move(NewFlashcacheAwareEnv(FLAGS_env, FLAGS_flashcache_dev));
+          std::move(NewFlashcacheAwareEnv(FLAGS_env, cachedev_fd_));
       if (flashcache_aware_env_.get() == nullptr) {
         fprintf(stderr, "Failed to open flashcahce device at %s\n",
                 FLAGS_flashcache_dev.c_str());
diff --git a/include/rocksdb/utilities/flashcache.h b/include/rocksdb/utilities/flashcache.h
index c9bd4fd26..7bb760924 100644
--- a/include/rocksdb/utilities/flashcache.h
+++ b/include/rocksdb/utilities/flashcache.h
@@ -17,8 +17,9 @@ namespace rocksdb {
 // reads. Reads from compaction thread don't need to be cached because they are
 // going to be soon made obsolete (due to nature of compaction)
 // Usually you would pass Env::Default() as base.
-// flashcache_dev is a path to the flashcache device
+// cachedev_fd is a file descriptor of the flashcache device. Caller has to
+// open flashcache device before calling this API.
 extern std::unique_ptr<Env> NewFlashcacheAwareEnv(
-    Env* base, const std::string& flashcache_dev);
+    Env* base, const int cachedev_fd);
 
 }  // namespace rocksdb
diff --git a/utilities/flashcache/flashcache.cc b/utilities/flashcache/flashcache.cc
index 75435cca0..059e6efe2 100644
--- a/utilities/flashcache/flashcache.cc
+++ b/utilities/flashcache/flashcache.cc
@@ -29,7 +29,6 @@ class FlashcacheAwareEnv : public EnvWrapper {
     pid_t pid = getpid();
     /* cleanup previous whitelistings */
     if (ioctl(cachedev_fd_, FLASHCACHEDELALLWHITELIST, &pid) < 0) {
-      close(cachedev_fd_);
       cachedev_fd_ = -1;
       fprintf(stderr, "ioctl del-all-whitelist for flashcache failed\n");
       return;
@@ -46,7 +45,6 @@ class FlashcacheAwareEnv : public EnvWrapper {
       if (ioctl(cachedev_fd_, FLASHCACHEDELWHITELIST, &pid) < 0) {
         fprintf(stderr, "ioctl del-whitelist for flashcache failed\n");
       }
-      close(cachedev_fd_);
     }
   }
 
@@ -103,14 +101,7 @@ class FlashcacheAwareEnv : public EnvWrapper {
 };
 
 std::unique_ptr<Env> NewFlashcacheAwareEnv(Env* base,
-                                           const std::string& flashcache_dev) {
-  // Cachedev should remain open or ioctl will be lost
-  int cachedev_fd = open(flashcache_dev.c_str(), O_RDONLY);
-  if (cachedev_fd < 0) {
-    fprintf(stderr, "Open flash device failed\n");
-    return nullptr;
-  }
-
+                                           const int cachedev_fd) {
   std::unique_ptr<Env> ret(new FlashcacheAwareEnv(base, cachedev_fd));
   return std::move(ret);
 }