SimulatedHybridFileSystem to simulate HDD behavior more accurately (#9259)

Summary:
SimulatedHybridFileSystem now takes a more thorough simualtion of an HDD:
1. cover writes too, not just read
2. Latency and throughput is now simulated as seek + read time, using a rate limiter
This implementation can be modified to simulate full HDD behavior, which is not yet done.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9259

Test Plan: Run db_bench and observe the desired behavior.

Reviewed By: jay-zhuang

Differential Revision: D32903039

fbshipit-source-id: a83f5d72143e114d5e75edf39d647bf0b71978e1

tools/simulated_hybrid_file_system.cc

@@ -16,8 +16,28 @@
 namespace ROCKSDB_NAMESPACE {
 
 const int kLatencyAddedPerRequestUs = 15000;
-const int64_t kRequestPerSec = 100;
+const int64_t kUsPerSec = 1000000;
-const int64_t kDummyBytesPerRequest = 1024 * 1024;
+const int64_t kDummyBytesPerUs = 1024;
+
+namespace {
+// From bytes to read/write, calculate service time needed by an HDD.
+// This is used to simulate latency from HDD.
+int CalculateServeTimeUs(size_t bytes) {
+  return 12200 + static_cast<int>(static_cast<double>(bytes) * 0.005215);
+}
+
+// There is a bug in rater limiter that would crash with small requests
+// Hack to get it around.
+void RateLimiterRequest(RateLimiter* rater_limiter, int64_t amount) {
+  int64_t left = amount * kDummyBytesPerUs;
+  const int64_t kMaxToRequest = kDummyBytesPerUs * kUsPerSec / 1024;
+  while (left > 0) {
+    int64_t to_request = std::min(kMaxToRequest, left);
+    rater_limiter->Request(to_request, Env::IOPriority::IO_LOW, nullptr);
+    left -= to_request;
+  }
+}
+}  // namespace
 
 // The metadata file format: each line is a full filename of a file which is
 // warm
@@ -27,7 +47,7 @@ SimulatedHybridFileSystem::SimulatedHybridFileSystem(
     : FileSystemWrapper(base),
       // Limit to 100 requests per second.
       rate_limiter_(NewGenericRateLimiter(
-          kDummyBytesPerRequest * kRequestPerSec /* rate_bytes_per_sec */,
+          kDummyBytesPerUs * kUsPerSec /* rate_bytes_per_sec */,
           1000 /* refill_period_us */)),
       metadata_file_name_(metadata_file_name),
       name_("SimulatedHybridFileSystem: " + std::string(target()->Name())) {
@@ -91,10 +111,14 @@ IOStatus SimulatedHybridFileSystem::NewWritableFile(
     std::unique_ptr<FSWritableFile>* result, IODebugContext* dbg) {
   if (file_opts.temperature == Temperature::kWarm) {
     const std::lock_guard<std::mutex> lock(mutex_);
-    fprintf(stderr, "warm file %s\n", fname.c_str());
     warm_file_set_.insert(fname);
   }
-  return target()->NewWritableFile(fname, file_opts, result, dbg);
+
+  IOStatus s = target()->NewWritableFile(fname, file_opts, result, dbg);
+  if (file_opts.temperature == Temperature::kWarm) {
+    result->reset(new SimulatedWritableFile(std::move(*result), rate_limiter_));
+  }
+  return s;
 }
 
 IOStatus SimulatedHybridFileSystem::DeleteFile(const std::string& fname,
@@ -112,7 +136,7 @@ IOStatus SimulatedHybridRaf::Read(uint64_t offset, size_t n,
                                   char* scratch, IODebugContext* dbg) const {
   if (temperature_ == Temperature::kWarm) {
     Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs);
-    RequestRateLimit(1);
+    RequestRateLimit(n);
   }
   return target()->Read(offset, n, options, result, scratch, dbg);
 }
@@ -121,7 +145,9 @@ IOStatus SimulatedHybridRaf::MultiRead(FSReadRequest* reqs, size_t num_reqs,
                                        const IOOptions& options,
                                        IODebugContext* dbg) {
   if (temperature_ == Temperature::kWarm) {
-    RequestRateLimit(static_cast<int64_t>(num_reqs));
+    for (size_t i = 0; i < num_reqs; i++) {
+      RequestRateLimit(reqs[i].len);
+    }
     Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs *
                                          static_cast<int>(num_reqs));
   }
@@ -132,22 +158,74 @@ IOStatus SimulatedHybridRaf::Prefetch(uint64_t offset, size_t n,
                                       const IOOptions& options,
                                       IODebugContext* dbg) {
   if (temperature_ == Temperature::kWarm) {
-    RequestRateLimit(1);
+    RequestRateLimit(n);
     Env::Default()->SleepForMicroseconds(kLatencyAddedPerRequestUs);
   }
   return target()->Prefetch(offset, n, options, dbg);
 }
 
-void SimulatedHybridRaf::RequestRateLimit(int64_t num_requests) const {
+void SimulatedHybridRaf::RequestRateLimit(int64_t bytes) const {
-  int64_t left = num_requests * kDummyBytesPerRequest;
+  RateLimiterRequest(rate_limiter_.get(), CalculateServeTimeUs(bytes));
-  const int64_t kMaxToRequest = kDummyBytesPerRequest / 100;
-  while (left > 0) {
-    int64_t to_request = std::min(kMaxToRequest, left);
-    rate_limiter_->Request(to_request, Env::IOPriority::IO_LOW, nullptr);
-    left -= to_request;
-  }
 }
 
+void SimulatedWritableFile::RequestRateLimit(int64_t bytes) const {
+  RateLimiterRequest(rate_limiter_.get(), CalculateServeTimeUs(bytes));
+}
+
+IOStatus SimulatedWritableFile::Append(const Slice& data, const IOOptions& ioo,
+                                       IODebugContext* idc) {
+  if (use_direct_io()) {
+    RequestRateLimit(data.size());
+  } else {
+    unsynced_bytes += data.size();
+  }
+  return target()->Append(data, ioo, idc);
+}
+
+IOStatus SimulatedWritableFile::Append(
+    const Slice& data, const IOOptions& options,
+    const DataVerificationInfo& verification_info, IODebugContext* dbg) {
+  if (use_direct_io()) {
+    RequestRateLimit(data.size());
+  } else {
+    unsynced_bytes += data.size();
+  }
+  return target()->Append(data, options, verification_info, dbg);
+}
+
+IOStatus SimulatedWritableFile::PositionedAppend(const Slice& data,
+                                                 uint64_t offset,
+                                                 const IOOptions& options,
+                                                 IODebugContext* dbg) {
+  if (use_direct_io()) {
+    RequestRateLimit(data.size());
+  } else {
+    // This might be overcalculated, but it's probably OK.
+    unsynced_bytes += data.size();
+  }
+  return target()->PositionedAppend(data, offset, options, dbg);
+}
+IOStatus SimulatedWritableFile::PositionedAppend(
+    const Slice& data, uint64_t offset, const IOOptions& options,
+    const DataVerificationInfo& verification_info, IODebugContext* dbg) {
+  if (use_direct_io()) {
+    RequestRateLimit(data.size());
+  } else {
+    // This might be overcalculated, but it's probably OK.
+    unsynced_bytes += data.size();
+  }
+  return target()->PositionedAppend(data, offset, options, verification_info,
+                                    dbg);
+}
+
+IOStatus SimulatedWritableFile::Sync(const IOOptions& options,
+                                     IODebugContext* dbg) {
+  if (unsynced_bytes > 0) {
+    RequestRateLimit(unsynced_bytes);
+    unsynced_bytes = 0;
+  }
+  return target()->Sync(options, dbg);
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 #endif  // ROCKSDB_LITE

tools/simulated_hybrid_file_system.h

@@ -86,6 +86,35 @@ class SimulatedHybridRaf : public FSRandomAccessFileOwnerWrapper {
 
   void RequestRateLimit(int64_t num_requests) const;
 };
+
+class SimulatedWritableFile : public FSWritableFileWrapper {
+ public:
+  SimulatedWritableFile(std::unique_ptr<FSWritableFile>&& t,
+                        std::shared_ptr<RateLimiter> rate_limiter)
+      : FSWritableFileWrapper(t.get()),
+        file_guard_(std::move(t)),
+        rate_limiter_(rate_limiter) {}
+  IOStatus Append(const Slice& data, const IOOptions&,
+                  IODebugContext*) override;
+  IOStatus Append(const Slice& data, const IOOptions& options,
+                  const DataVerificationInfo& verification_info,
+                  IODebugContext* dbg) override;
+  IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override;
+  IOStatus PositionedAppend(const Slice& data, uint64_t offset,
+                            const IOOptions& options,
+                            IODebugContext* dbg) override;
+  IOStatus PositionedAppend(const Slice& data, uint64_t offset,
+                            const IOOptions& options,
+                            const DataVerificationInfo& verification_info,
+                            IODebugContext* dbg) override;
+
+ private:
+  std::unique_ptr<FSWritableFile> file_guard_;
+  std::shared_ptr<RateLimiter> rate_limiter_;
+  size_t unsynced_bytes = 0;
+
+  void RequestRateLimit(int64_t num_requests) const;
+};
 }  // namespace ROCKSDB_NAMESPACE
 
 #endif  // ROCKSDB_LITE