Add IO Tracer Parser (#7333)

Summary:
Implement a parsing tool io_tracer_parser that takes IO trace file (binary file) with command line argument --io_trace_file and output file with --output_file and dumps the IO trace records in outputfile in human readable form.

Also added unit test cases that generates IO trace records and calls io_tracer_parse to parse those records.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7333

Test Plan:
make check -j64,
 Add unit test cases.

Reviewed By: anand1976

Differential Revision: D23772360

Pulled By: akankshamahajan15

fbshipit-source-id: 9c20519c189362e6663352d08863326f3e496271

CMakeLists.txt

@@ -737,6 +737,7 @@ set(SOURCES
         test_util/transaction_test_util.cc
         tools/block_cache_analyzer/block_cache_trace_analyzer.cc
         tools/dump/db_dump_tool.cc
+        tools/io_tracer_parser_tool.cc
         tools/ldb_cmd.cc
         tools/ldb_tool.cc
         tools/sst_dump_tool.cc
@@ -1147,6 +1148,7 @@ if(WITH_TESTS)
         test_util/testutil_test.cc
         trace_replay/io_tracer_test.cc
         tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
+        tools/io_tracer_parser_test.cc
         tools/ldb_cmd_test.cc
         tools/reduce_levels_test.cc
         tools/sst_dump_test.cc

Makefile

@@ -1897,6 +1897,12 @@ io_tracer_test: $(OBJ_DIR)/trace_replay/io_tracer_test.o $(OBJ_DIR)/trace_replay
 prefetch_test: $(OBJ_DIR)/file/prefetch_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+io_tracer_parser_test: $(OBJ_DIR)/tools/io_tracer_parser_test.o $(OBJ_DIR)/tools/io_tracer_parser_tool.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
+io_tracer_parser: $(OBJ_DIR)/tools/io_tracer_parser.o $(TOOLS_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 PREFIX ?= /usr/local

TARGETS

@@ -316,6 +316,7 @@ cpp_library(
         "test_util/sync_point_impl.cc",
         "test_util/transaction_test_util.cc",
         "tools/dump/db_dump_tool.cc",
+        "tools/io_tracer_parser_tool.cc",
         "tools/ldb_cmd.cc",
         "tools/ldb_tool.cc",
         "tools/sst_dump_tool.cc",
@@ -1213,6 +1214,13 @@ ROCKS_TESTS = [
         [],
         [],
     ],
+    [
+        "io_tracer_parser_test",
+        "tools/io_tracer_parser_test.cc",
+        "serial",
+        [],
+        [],
+    ],
     [
         "io_tracer_test",
         "trace_replay/io_tracer_test.cc",

db/db_impl/db_impl.cc

@@ -3035,6 +3035,7 @@ FileSystem* DBImpl::GetFileSystem() const {
 
 Status DBImpl::StartIOTrace(Env* env, const TraceOptions& trace_options,
                             std::unique_ptr<TraceWriter>&& trace_writer) {
+  assert(trace_writer != nullptr);
   return io_tracer_->StartIOTrace(env, trace_options, std::move(trace_writer));
 }
 

env/file_system_tracer.cc

@@ -271,7 +271,7 @@ uint64_t FSWritableFileTracingWrapper::GetFileSize(const IOOptions& options,
   uint64_t file_size = target()->GetFileSize(options, dbg);
   uint64_t elapsed = timer.ElapsedNanos();
   IOTraceRecord io_record(env_->NowNanos(), TraceType::kIOFileNameAndFileSize,
-                          "GetFileSize", elapsed, "" /* file_name */,
+                          "GetFileSize", elapsed, "OK", "" /* file_name */,
                           file_size);
   io_tracer_->WriteIOOp(io_record);
   return file_size;

src.mk

@@ -283,6 +283,7 @@ LIB_SOURCES_C =
 endif
 
 TOOL_LIB_SOURCES =                                              \
+  tools/io_tracer_parser_tool.cc                                \
   tools/ldb_cmd.cc                                              \
   tools/ldb_tool.cc                                             \
   tools/sst_dump_tool.cc                                        \
@@ -330,11 +331,13 @@ TOOLS_MAIN_SOURCES =                                                    \
   tools/db_repl_stress.cc                                               \
   tools/db_sanity_test.cc                                               \
   tools/ldb.cc                                                          \
+  tools/io_tracer_parser.cc                                             \
   tools/sst_dump.cc                                                     \
   tools/write_stress.cc                                                 \
   tools/dump/rocksdb_dump.cc                                            \
   tools/dump/rocksdb_undump.cc                                          \
   tools/trace_analyzer.cc                                               \
+  tools/io_tracer_parser_tool.cc                                        \
 
 BENCH_MAIN_SOURCES =                                                    \
   cache/cache_bench.cc                                                  \
@@ -466,6 +469,7 @@ TEST_MAIN_SOURCES =                                                     \
   table/block_fetcher_test.cc                                           \
   test_util/testutil_test.cc                                            \
   tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc         \
+  tools/io_tracer_parser_test.cc                                        \
   tools/ldb_cmd_test.cc                                                 \
   tools/reduce_levels_test.cc                                           \
   tools/sst_dump_test.cc                                                \

tools/io_tracer_parser.cc

@@ -0,0 +1,25 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+#ifndef ROCKSDB_LITE
+#ifndef GFLAGS
+#include <cstdio>
+int main() {
+  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+  return 1;
+}
+#else  // GFLAGS
+#include "tools/io_tracer_parser_tool.h"
+int main(int argc, char** argv) {
+  return ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv);
+}
+#endif  // GFLAGS
+#else   // ROCKSDB_LITE
+#include <stdio.h>
+int main(int /*argc*/, char** /*argv*/) {
+  fprintf(stderr, "Not supported in lite mode.\n");
+  return 1;
+}
+#endif  // ROCKSDB_LITE

tools/io_tracer_parser_test.cc

@@ -0,0 +1,187 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+
+#ifndef ROCKSDB_LITE
+#ifndef GFLAGS
+#include <cstdio>
+int main() {
+  fprintf(stderr, "Please install gflags to run io_tracer_parser_test\n");
+  return 1;
+}
+#else
+
+#include <string>
+#include <vector>
+
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+#include "tools/io_tracer_parser_tool.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+namespace {
+const int kMaxArgCount = 100;
+const size_t kArgBufferSize = 100000;
+}  // namespace
+
+class IOTracerParserTest : public testing::Test {
+ public:
+  IOTracerParserTest() {
+    test_path_ = test::PerThreadDBPath("io_tracer_parser_test");
+    env_ = ROCKSDB_NAMESPACE::Env::Default();
+    EXPECT_OK(env_->CreateDirIfMissing(test_path_));
+    trace_file_path_ = test_path_ + "/io_trace_file";
+    dbname_ = test_path_ + "/db";
+    Options options;
+    options.create_if_missing = true;
+    EXPECT_OK(DB::Open(options, dbname_, &db_));
+  }
+
+  ~IOTracerParserTest() {
+    if (env_->FileExists(trace_file_path_).ok()) {
+      EXPECT_OK(env_->DeleteFile(trace_file_path_));
+    }
+    if (db_ != nullptr) {
+      Options options;
+      options.env = env_;
+      delete db_;
+      db_ = nullptr;
+      EXPECT_OK(DestroyDB(dbname_, options));
+    }
+    EXPECT_OK(env_->DeleteDir(test_path_));
+  }
+
+  void GenerateIOTrace() {
+    WriteOptions write_opt;
+    TraceOptions trace_opt;
+    std::unique_ptr<TraceWriter> trace_writer;
+
+    ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_,
+                                 &trace_writer));
+
+    ASSERT_OK(db_->StartIOTrace(env_, trace_opt, std::move(trace_writer)));
+
+    for (int i = 0; i < 10; i++) {
+      ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i),
+                         "value_" + std::to_string(i)));
+      ASSERT_OK(db_->Flush(FlushOptions()));
+    }
+
+    ASSERT_OK(db_->EndIOTrace());
+    ASSERT_OK(env_->FileExists(trace_file_path_));
+  }
+
+  void RunIOTracerParserTool() {
+    std::vector<std::string> params = {"./io_tracer_parser",
+                                       "-io_trace_file=" + trace_file_path_};
+
+    char arg_buffer[kArgBufferSize];
+    char* argv[kMaxArgCount];
+    int argc = 0;
+    int cursor = 0;
+    for (const auto& arg : params) {
+      ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize);
+      ASSERT_LE(argc + 1, kMaxArgCount);
+
+      snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str());
+
+      argv[argc++] = arg_buffer + cursor;
+      cursor += static_cast<int>(arg.size()) + 1;
+    }
+    ASSERT_EQ(0, ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv));
+  }
+
+  DB* db_;
+  Env* env_;
+  EnvOptions env_options_;
+  std::string trace_file_path_;
+  std::string output_file_;
+  std::string test_path_;
+  std::string dbname_;
+};
+
+TEST_F(IOTracerParserTest, InvalidArguments) {
+  {
+    std::vector<std::string> params = {"./io_tracer_parser"};
+    char arg_buffer[kArgBufferSize];
+    char* argv[kMaxArgCount];
+    int argc = 0;
+    int cursor = 0;
+    for (const auto& arg : params) {
+      ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize);
+      ASSERT_LE(argc + 1, kMaxArgCount);
+
+      snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str());
+
+      argv[argc++] = arg_buffer + cursor;
+      cursor += static_cast<int>(arg.size()) + 1;
+    }
+    ASSERT_EQ(1, ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv));
+  }
+}
+
+TEST_F(IOTracerParserTest, DumpAndParseIOTraceRecords) {
+  GenerateIOTrace();
+  RunIOTracerParserTool();
+}
+
+TEST_F(IOTracerParserTest, NoRecordingAfterEndIOTrace) {
+  uint64_t file_size = 0;
+  // Generate IO trace records and parse them.
+  {
+    GenerateIOTrace();
+    RunIOTracerParserTool();
+    ASSERT_OK(env_->GetFileSize(trace_file_path_, &file_size));
+  }
+  // Once DB::EndIOTrace is invoked in GenerateIOTrace(), no new records should
+  // be appended.
+  {
+    WriteOptions write_opt;
+    for (int i = 10; i < 20; i++) {
+      ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i),
+                         "value_" + std::to_string(i)));
+      ASSERT_OK(db_->Flush(FlushOptions()));
+    }
+  }
+
+  uint64_t new_file_size = 0;
+  ASSERT_OK(env_->GetFileSize(trace_file_path_, &new_file_size));
+  ASSERT_EQ(file_size, new_file_size);
+}
+
+TEST_F(IOTracerParserTest, NoRecordingBeforeStartIOTrace) {
+  {
+    WriteOptions write_opt;
+    for (int i = 10; i < 20; i++) {
+      ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i),
+                         "value_" + std::to_string(i)));
+      ASSERT_OK(db_->Flush(FlushOptions()));
+    }
+    // IO trace file doesn't exist
+    ASSERT_NOK(env_->FileExists(trace_file_path_));
+  }
+  // Generate IO trace records and parse them.
+  {
+    GenerateIOTrace();
+    RunIOTracerParserTool();
+  }
+}
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+#endif  // GFLAGS
+#else
+#include <stdio.h>
+int main(int /*argc*/, char** /*argv*/) {
+  fprintf(stderr, "io_tracer_parser_test is not supported in ROCKSDB_LITE\n");
+  return 0;
+}
+#endif  // ROCKSDB_LITE

tools/io_tracer_parser_tool.cc

@@ -0,0 +1,120 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//    This source code is licensed under both the GPLv2 (found in the
+//    COPYING file in the root directory) and Apache 2.0 License
+//    (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+#ifdef GFLAGS
+#include "tools/io_tracer_parser_tool.h"
+
+#include <cinttypes>
+#include <cstdio>
+#include <iomanip>
+#include <memory>
+#include <sstream>
+
+#include "port/lang.h"
+#include "trace_replay/io_tracer.h"
+#include "util/gflags_compat.h"
+
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+
+DEFINE_string(io_trace_file, "", "The IO trace file path.");
+
+namespace ROCKSDB_NAMESPACE {
+
+IOTraceRecordParser::IOTraceRecordParser(const std::string& input_file)
+    : input_file_(input_file) {}
+
+void IOTraceRecordParser::PrintHumanReadableHeader(
+    const IOTraceHeader& header) {
+  std::stringstream ss;
+  ss << "Start Time: " << header.start_time
+     << "\nRocksDB Major Version: " << header.rocksdb_major_version
+     << "\nRocksDB Minor Version: " << header.rocksdb_minor_version << "\n";
+  fprintf(stdout, "%s", ss.str().c_str());
+}
+
+void IOTraceRecordParser::PrintHumanReadableIOTraceRecord(
+    const IOTraceRecord& record) {
+  std::stringstream ss;
+  ss << "Access Time : " << std::setw(17) << std::left
+     << record.access_timestamp << ", File Operation: " << std::setw(18)
+     << std::left << record.file_operation.c_str()
+     << ", Latency: " << std::setw(9) << std::left << record.latency
+     << ", IO Status: " << record.io_status.c_str();
+
+  switch (record.trace_type) {
+    case TraceType::kIOGeneral:
+      break;
+    case TraceType::kIOFileNameAndFileSize:
+      ss << ", File Size: " << record.file_size;
+      FALLTHROUGH_INTENDED;
+    case TraceType::kIOFileName: {
+      if (!record.file_name.empty()) {
+        ss << ", File Name: " << record.file_name.c_str();
+      }
+      break;
+    }
+    case TraceType::kIOLenAndOffset:
+      ss << ", Offset: " << record.offset;
+      FALLTHROUGH_INTENDED;
+    case TraceType::kIOLen:
+      ss << ", Length: " << record.len;
+      break;
+    default:
+      assert(false);
+  }
+  ss << "\n";
+  fprintf(stdout, "%s", ss.str().c_str());
+}
+
+int IOTraceRecordParser::ReadIOTraceRecords() {
+  Status status;
+  Env* env(Env::Default());
+  std::unique_ptr<TraceReader> trace_reader;
+  std::unique_ptr<IOTraceReader> io_trace_reader;
+
+  status = NewFileTraceReader(env, EnvOptions(), input_file_, &trace_reader);
+  if (!status.ok()) {
+    fprintf(stderr, "%s: %s\n", input_file_.c_str(), status.ToString().c_str());
+    return 1;
+  }
+  io_trace_reader.reset(new IOTraceReader(std::move(trace_reader)));
+
+  // Read the header and dump it in a file.
+  IOTraceHeader header;
+  status = io_trace_reader->ReadHeader(&header);
+  if (!status.ok()) {
+    fprintf(stderr, "%s: %s\n", input_file_.c_str(), status.ToString().c_str());
+    return 1;
+  }
+  PrintHumanReadableHeader(header);
+
+  // Read the records one by one and print them in human readable format.
+  while (status.ok()) {
+    IOTraceRecord record;
+    status = io_trace_reader->ReadIOOp(&record);
+    if (!status.ok()) {
+      break;
+    }
+    PrintHumanReadableIOTraceRecord(record);
+  }
+  return 0;
+}
+
+int io_tracer_parser(int argc, char** argv) {
+  ParseCommandLineFlags(&argc, &argv, true);
+
+  if (FLAGS_io_trace_file.empty()) {
+    fprintf(stderr, "IO Trace file path is empty\n");
+    return 1;
+  }
+
+  IOTraceRecordParser io_tracer_parser(FLAGS_io_trace_file);
+  return io_tracer_parser.ReadIOTraceRecords();
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+#endif  // GFLAGS
+#endif  // ROCKSDB_LITE

tools/io_tracer_parser_tool.h

@@ -0,0 +1,40 @@
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+#pragma once
+
+#include <memory>
+
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+struct IOTraceHeader;
+struct IOTraceRecord;
+
+// IOTraceRecordParser class reads the IO trace file (in binary format) and
+// dumps the human readable records in output_file_.
+class IOTraceRecordParser {
+ public:
+  explicit IOTraceRecordParser(const std::string& input_file);
+
+  // ReadIOTraceRecords reads the binary trace file records one by one and
+  // invoke PrintHumanReadableIOTraceRecord to dump the records in output_file_.
+  int ReadIOTraceRecords();
+
+ private:
+  void PrintHumanReadableHeader(const IOTraceHeader& header);
+  void PrintHumanReadableIOTraceRecord(const IOTraceRecord& record);
+
+  // Binary file that contains IO trace records.
+  std::string input_file_;
+};
+
+int io_tracer_parser(int argc, char** argv);
+
+}  // namespace ROCKSDB_NAMESPACE
+#endif  // ROCKSDB_LITE

trace_replay/io_tracer.h

@@ -128,14 +128,35 @@ class IOTracer {
   IOTracer(IOTracer&&) = delete;
   IOTracer& operator=(IOTracer&&) = delete;
 
+  // no_sanitize is added for tracing_enabled. writer_ is protected under mutex
+  // so even if user call Start/EndIOTrace and tracing_enabled is not updated in
+  // the meanwhile, WriteIOOp will anyways check the writer_ protected under
+  // mutex and ignore the operation if writer_is null. So its ok if
+  // tracing_enabled shows non updated value.
+
+#if defined(__clang__)
+#if defined(__has_feature) && __has_feature(thread_sanitizer)
+#define TSAN_SUPPRESSION __attribute__((no_sanitize("thread")))
+#endif  // __has_feature(thread_sanitizer)
+#else   // __clang__
+#ifdef __SANITIZE_THREAD__
+#define TSAN_SUPPRESSION __attribute__((no_sanitize("thread")))
+#endif  // __SANITIZE_THREAD__
+#endif  // __clang__
+
+#ifndef TSAN_SUPPRESSION
+#define TSAN_SUPPRESSION
+#endif  // TSAN_SUPPRESSION
+
   // Start writing IO operations to the trace_writer.
-  Status StartIOTrace(Env* env, const TraceOptions& trace_options,
+  TSAN_SUPPRESSION Status
-                      std::unique_ptr<TraceWriter>&& trace_writer);
+  StartIOTrace(Env* env, const TraceOptions& trace_options,
+               std::unique_ptr<TraceWriter>&& trace_writer);
 
   // Stop writing IO operations to the trace_writer.
-  void EndIOTrace();
+  TSAN_SUPPRESSION void EndIOTrace();
 
-  bool is_tracing_enabled() const { return tracing_enabled; }
+  TSAN_SUPPRESSION bool is_tracing_enabled() const { return tracing_enabled; }
 
   Status WriteIOOp(const IOTraceRecord& record);