andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add IO Tracer Parser (#7333)

Browse Source 
Summary:
Implement a parsing tool io_tracer_parser that takes IO trace file (binary file) with command line argument --io_trace_file and output file with --output_file and dumps the IO trace records in outputfile in human readable form.

Also added unit test cases that generates IO trace records and calls io_tracer_parse to parse those records.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7333

Test Plan:
make check -j64,
 Add unit test cases.

Reviewed By: anand1976

Differential Revision: D23772360

Pulled By: akankshamahajan15

fbshipit-source-id: 9c20519c189362e6663352d08863326f3e496271
This commit is contained in:
Akanksha Mahajan 2020-09-23 15:49:11 -07:00 committed by Facebook GitHub Bot
parent 31d1cea4f3
commit 98ac6b646a
11 changed files with 419 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -737,6 +737,7 @@ set(SOURCES
test_util/transaction_test_util.cc
tools/block_cache_analyzer/block_cache_trace_analyzer.cc
tools/dump/db_dump_tool.cc
tools/io_tracer_parser_tool.cc
tools/ldb_cmd.cc
tools/ldb_tool.cc
tools/sst_dump_tool.cc
@ -1147,6 +1148,7 @@ if(WITH_TESTS)
test_util/testutil_test.cc
trace_replay/io_tracer_test.cc
tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
tools/io_tracer_parser_test.cc
tools/ldb_cmd_test.cc
tools/reduce_levels_test.cc
tools/sst_dump_test.cc

6
Makefile
View File

@ -1897,6 +1897,12 @@ io_tracer_test: $(OBJ_DIR)/trace_replay/io_tracer_test.o $(OBJ_DIR)/trace_replay
prefetch_test: $(OBJ_DIR)/file/prefetch_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
io_tracer_parser_test: $(OBJ_DIR)/tools/io_tracer_parser_test.o $(OBJ_DIR)/tools/io_tracer_parser_tool.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
io_tracer_parser: $(OBJ_DIR)/tools/io_tracer_parser.o $(TOOLS_LIBRARY) $(LIBRARY)
$(AM_LINK)
#-------------------------------------------------
# make install related stuff
PREFIX ?= /usr/local

8
TARGETS
View File

@ -316,6 +316,7 @@ cpp_library(
"test_util/sync_point_impl.cc",
"test_util/transaction_test_util.cc",
"tools/dump/db_dump_tool.cc",
"tools/io_tracer_parser_tool.cc",
"tools/ldb_cmd.cc",
"tools/ldb_tool.cc",
"tools/sst_dump_tool.cc",
@ -1213,6 +1214,13 @@ ROCKS_TESTS = [
[],
[],
],
[
"io_tracer_parser_test",
"tools/io_tracer_parser_test.cc",
"serial",
[],
[],
],
[
"io_tracer_test",
"trace_replay/io_tracer_test.cc",

1
db/db_impl/db_impl.cc
View File

@ -3035,6 +3035,7 @@ FileSystem* DBImpl::GetFileSystem() const {
Status DBImpl::StartIOTrace(Env* env, const TraceOptions& trace_options,
std::unique_ptr<TraceWriter>&& trace_writer) {
assert(trace_writer != nullptr);
return io_tracer_->StartIOTrace(env, trace_options, std::move(trace_writer));
}

2
env/file_system_tracer.cc vendored
View File

@ -271,7 +271,7 @@ uint64_t FSWritableFileTracingWrapper::GetFileSize(const IOOptions& options,
uint64_t file_size = target()->GetFileSize(options, dbg);
uint64_t elapsed = timer.ElapsedNanos();
IOTraceRecord io_record(env_->NowNanos(), TraceType::kIOFileNameAndFileSize,
"GetFileSize", elapsed, "" /* file_name */,
"GetFileSize", elapsed, "OK", "" /* file_name */,
file_size);
io_tracer_->WriteIOOp(io_record);
return file_size;

4
src.mk
View File

@ -283,6 +283,7 @@ LIB_SOURCES_C =
endif
TOOL_LIB_SOURCES = \
tools/io_tracer_parser_tool.cc \
tools/ldb_cmd.cc \
tools/ldb_tool.cc \
tools/sst_dump_tool.cc \
@ -330,11 +331,13 @@ TOOLS_MAIN_SOURCES = \
tools/db_repl_stress.cc \
tools/db_sanity_test.cc \
tools/ldb.cc \
tools/io_tracer_parser.cc \
tools/sst_dump.cc \
tools/write_stress.cc \
tools/dump/rocksdb_dump.cc \
tools/dump/rocksdb_undump.cc \
tools/trace_analyzer.cc \
tools/io_tracer_parser_tool.cc \
BENCH_MAIN_SOURCES = \
cache/cache_bench.cc \
@ -466,6 +469,7 @@ TEST_MAIN_SOURCES = \
table/block_fetcher_test.cc \
test_util/testutil_test.cc \
tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc \
tools/io_tracer_parser_test.cc \
tools/ldb_cmd_test.cc \
tools/reduce_levels_test.cc \
tools/sst_dump_test.cc \

25
tools/io_tracer_parser.cc Normal file
View File

@ -0,0 +1,25 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#ifndef GFLAGS
#include <cstdio>
int main() {
fprintf(stderr, "Please install gflags to run rocksdb tools\n");
return 1;
}
#else // GFLAGS
#include "tools/io_tracer_parser_tool.h"
int main(int argc, char** argv) {
return ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv);
}
#endif // GFLAGS
#else // ROCKSDB_LITE
#include <stdio.h>
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "Not supported in lite mode.\n");
return 1;
}
#endif // ROCKSDB_LITE

187
tools/io_tracer_parser_test.cc Normal file
View File

@ -0,0 +1,187 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#ifndef GFLAGS
#include <cstdio>
int main() {
fprintf(stderr, "Please install gflags to run io_tracer_parser_test\n");
return 1;
}
#else
#include <string>
#include <vector>
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#include "tools/io_tracer_parser_tool.h"
namespace ROCKSDB_NAMESPACE {
namespace {
const int kMaxArgCount = 100;
const size_t kArgBufferSize = 100000;
} // namespace
class IOTracerParserTest : public testing::Test {
public:
IOTracerParserTest() {
test_path_ = test::PerThreadDBPath("io_tracer_parser_test");
env_ = ROCKSDB_NAMESPACE::Env::Default();
EXPECT_OK(env_->CreateDirIfMissing(test_path_));
trace_file_path_ = test_path_ + "/io_trace_file";
dbname_ = test_path_ + "/db";
Options options;
options.create_if_missing = true;
EXPECT_OK(DB::Open(options, dbname_, &db_));
}
~IOTracerParserTest() {
if (env_->FileExists(trace_file_path_).ok()) {
EXPECT_OK(env_->DeleteFile(trace_file_path_));
}
if (db_ != nullptr) {
Options options;
options.env = env_;
delete db_;
db_ = nullptr;
EXPECT_OK(DestroyDB(dbname_, options));
}
EXPECT_OK(env_->DeleteDir(test_path_));
}
void GenerateIOTrace() {
WriteOptions write_opt;
TraceOptions trace_opt;
std::unique_ptr<TraceWriter> trace_writer;
ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_,
&trace_writer));
ASSERT_OK(db_->StartIOTrace(env_, trace_opt, std::move(trace_writer)));
for (int i = 0; i < 10; i++) {
ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i),
"value_" + std::to_string(i)));
ASSERT_OK(db_->Flush(FlushOptions()));
}
ASSERT_OK(db_->EndIOTrace());
ASSERT_OK(env_->FileExists(trace_file_path_));
}
void RunIOTracerParserTool() {
std::vector<std::string> params = {"./io_tracer_parser",
"-io_trace_file=" + trace_file_path_};
char arg_buffer[kArgBufferSize];
char* argv[kMaxArgCount];
int argc = 0;
int cursor = 0;
for (const auto& arg : params) {
ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize);
ASSERT_LE(argc + 1, kMaxArgCount);
snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str());
argv[argc++] = arg_buffer + cursor;
cursor += static_cast<int>(arg.size()) + 1;
}
ASSERT_EQ(0, ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv));
}
DB* db_;
Env* env_;
EnvOptions env_options_;
std::string trace_file_path_;
std::string output_file_;
std::string test_path_;
std::string dbname_;
};
TEST_F(IOTracerParserTest, InvalidArguments) {
{
std::vector<std::string> params = {"./io_tracer_parser"};
char arg_buffer[kArgBufferSize];
char* argv[kMaxArgCount];
int argc = 0;
int cursor = 0;
for (const auto& arg : params) {
ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize);
ASSERT_LE(argc + 1, kMaxArgCount);
snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str());
argv[argc++] = arg_buffer + cursor;
cursor += static_cast<int>(arg.size()) + 1;
}
ASSERT_EQ(1, ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv));
}
}
TEST_F(IOTracerParserTest, DumpAndParseIOTraceRecords) {
GenerateIOTrace();
RunIOTracerParserTool();
}
TEST_F(IOTracerParserTest, NoRecordingAfterEndIOTrace) {
uint64_t file_size = 0;
// Generate IO trace records and parse them.
{
GenerateIOTrace();
RunIOTracerParserTool();
ASSERT_OK(env_->GetFileSize(trace_file_path_, &file_size));
}
// Once DB::EndIOTrace is invoked in GenerateIOTrace(), no new records should
// be appended.
{
WriteOptions write_opt;
for (int i = 10; i < 20; i++) {
ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i),
"value_" + std::to_string(i)));
ASSERT_OK(db_->Flush(FlushOptions()));
}
}
uint64_t new_file_size = 0;
ASSERT_OK(env_->GetFileSize(trace_file_path_, &new_file_size));
ASSERT_EQ(file_size, new_file_size);
}
TEST_F(IOTracerParserTest, NoRecordingBeforeStartIOTrace) {
{
WriteOptions write_opt;
for (int i = 10; i < 20; i++) {
ASSERT_OK(db_->Put(write_opt, "key_" + std::to_string(i),
"value_" + std::to_string(i)));
ASSERT_OK(db_->Flush(FlushOptions()));
}
// IO trace file doesn't exist
ASSERT_NOK(env_->FileExists(trace_file_path_));
}
// Generate IO trace records and parse them.
{
GenerateIOTrace();
RunIOTracerParserTool();
}
}
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#endif // GFLAGS
#else
#include <stdio.h>
int main(int /*argc*/, char** /*argv*/) {
fprintf(stderr, "io_tracer_parser_test is not supported in ROCKSDB_LITE\n");
return 0;
}
#endif // ROCKSDB_LITE

120
tools/io_tracer_parser_tool.cc Normal file
View File

@ -0,0 +1,120 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#ifdef GFLAGS
#include "tools/io_tracer_parser_tool.h"
#include <cinttypes>
#include <cstdio>
#include <iomanip>
#include <memory>
#include <sstream>
#include "port/lang.h"
#include "trace_replay/io_tracer.h"
#include "util/gflags_compat.h"
using GFLAGS_NAMESPACE::ParseCommandLineFlags;
DEFINE_string(io_trace_file, "", "The IO trace file path.");
namespace ROCKSDB_NAMESPACE {
IOTraceRecordParser::IOTraceRecordParser(const std::string& input_file)
: input_file_(input_file) {}
void IOTraceRecordParser::PrintHumanReadableHeader(
const IOTraceHeader& header) {
std::stringstream ss;
ss << "Start Time: " << header.start_time
<< "\nRocksDB Major Version: " << header.rocksdb_major_version
<< "\nRocksDB Minor Version: " << header.rocksdb_minor_version << "\n";
fprintf(stdout, "%s", ss.str().c_str());
}
void IOTraceRecordParser::PrintHumanReadableIOTraceRecord(
const IOTraceRecord& record) {
std::stringstream ss;
ss << "Access Time : " << std::setw(17) << std::left
<< record.access_timestamp << ", File Operation: " << std::setw(18)
<< std::left << record.file_operation.c_str()
<< ", Latency: " << std::setw(9) << std::left << record.latency
<< ", IO Status: " << record.io_status.c_str();
switch (record.trace_type) {
case TraceType::kIOGeneral:
break;
case TraceType::kIOFileNameAndFileSize:
ss << ", File Size: " << record.file_size;
FALLTHROUGH_INTENDED;
case TraceType::kIOFileName: {
if (!record.file_name.empty()) {
ss << ", File Name: " << record.file_name.c_str();
}
break;
}
case TraceType::kIOLenAndOffset:
ss << ", Offset: " << record.offset;
FALLTHROUGH_INTENDED;
case TraceType::kIOLen:
ss << ", Length: " << record.len;
break;
default:
assert(false);
}
ss << "\n";
fprintf(stdout, "%s", ss.str().c_str());
}
int IOTraceRecordParser::ReadIOTraceRecords() {
Status status;
Env* env(Env::Default());
std::unique_ptr<TraceReader> trace_reader;
std::unique_ptr<IOTraceReader> io_trace_reader;
status = NewFileTraceReader(env, EnvOptions(), input_file_, &trace_reader);
if (!status.ok()) {
fprintf(stderr, "%s: %s\n", input_file_.c_str(), status.ToString().c_str());
return 1;
}
io_trace_reader.reset(new IOTraceReader(std::move(trace_reader)));
// Read the header and dump it in a file.
IOTraceHeader header;
status = io_trace_reader->ReadHeader(&header);
if (!status.ok()) {
fprintf(stderr, "%s: %s\n", input_file_.c_str(), status.ToString().c_str());
return 1;
}
PrintHumanReadableHeader(header);
// Read the records one by one and print them in human readable format.
while (status.ok()) {
IOTraceRecord record;
status = io_trace_reader->ReadIOOp(&record);
if (!status.ok()) {
break;
}
PrintHumanReadableIOTraceRecord(record);
}
return 0;
}
int io_tracer_parser(int argc, char** argv) {
ParseCommandLineFlags(&argc, &argv, true);
if (FLAGS_io_trace_file.empty()) {
fprintf(stderr, "IO Trace file path is empty\n");
return 1;
}
IOTraceRecordParser io_tracer_parser(FLAGS_io_trace_file);
return io_tracer_parser.ReadIOTraceRecords();
}
} // namespace ROCKSDB_NAMESPACE
#endif // GFLAGS
#endif // ROCKSDB_LITE

40
tools/io_tracer_parser_tool.h Normal file
View File

@ -0,0 +1,40 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#pragma once
#include <memory>
#include "rocksdb/env.h"
#include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE {
struct IOTraceHeader;
struct IOTraceRecord;
// IOTraceRecordParser class reads the IO trace file (in binary format) and
// dumps the human readable records in output_file_.
class IOTraceRecordParser {
public:
explicit IOTraceRecordParser(const std::string& input_file);
// ReadIOTraceRecords reads the binary trace file records one by one and
// invoke PrintHumanReadableIOTraceRecord to dump the records in output_file_.
int ReadIOTraceRecords();
private:
void PrintHumanReadableHeader(const IOTraceHeader& header);
void PrintHumanReadableIOTraceRecord(const IOTraceRecord& record);
// Binary file that contains IO trace records.
std::string input_file_;
};
int io_tracer_parser(int argc, char** argv);
} // namespace ROCKSDB_NAMESPACE
#endif // ROCKSDB_LITE

29
trace_replay/io_tracer.h
View File

@ -128,14 +128,35 @@ class IOTracer {
IOTracer(IOTracer&&) = delete;
IOTracer& operator=(IOTracer&&) = delete;
// no_sanitize is added for tracing_enabled. writer_ is protected under mutex
// so even if user call Start/EndIOTrace and tracing_enabled is not updated in
// the meanwhile, WriteIOOp will anyways check the writer_ protected under
// mutex and ignore the operation if writer_is null. So its ok if
// tracing_enabled shows non updated value.
#if defined(__clang__)
#if defined(__has_feature) && __has_feature(thread_sanitizer)
#define TSAN_SUPPRESSION __attribute__((no_sanitize("thread")))
#endif // __has_feature(thread_sanitizer)
#else // __clang__
#ifdef __SANITIZE_THREAD__
#define TSAN_SUPPRESSION __attribute__((no_sanitize("thread")))
#endif // __SANITIZE_THREAD__
#endif // __clang__
#ifndef TSAN_SUPPRESSION
#define TSAN_SUPPRESSION
#endif // TSAN_SUPPRESSION
// Start writing IO operations to the trace_writer.
Status StartIOTrace(Env* env, const TraceOptions& trace_options,
std::unique_ptr<TraceWriter>&& trace_writer);
TSAN_SUPPRESSION Status
StartIOTrace(Env* env, const TraceOptions& trace_options,
std::unique_ptr<TraceWriter>&& trace_writer);
// Stop writing IO operations to the trace_writer.
void EndIOTrace();
TSAN_SUPPRESSION void EndIOTrace();
bool is_tracing_enabled() const { return tracing_enabled; }
TSAN_SUPPRESSION bool is_tracing_enabled() const { return tracing_enabled; }
Status WriteIOOp(const IOTraceRecord& record);