From fdc1cb43a64a1a523740eedf3ab0a733d0878743 Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Thu, 17 Oct 2019 19:35:22 -0700 Subject: [PATCH] Support decoding blob indexes in sst_dump (#5926) Summary: The patch adds a new command line parameter --decode_blob_index to sst_dump. If this switch is specified, sst_dump prints blob indexes in a human readable format, printing the blob file number, offset, size, and expiration (if applicable) for blob references, and the blob value (and expiration) for inlined blobs. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5926 Test Plan: Used db_bench's BlobDB mode to generate SST files containing blob references with and without expiration, as well as inlined blobs with and without expiration (note: the latter are stored as plain values), and confirmed sst_dump correctly prints all four types of records. Differential Revision: D17939077 Pulled By: ltamasi fbshipit-source-id: edc5f58fee94ba35f6699c6a042d5758f5b3963d --- db/blob_index.h | 20 ++++++++++++++++++++ tools/ldb_cmd.cc | 4 +++- tools/sst_dump_tool.cc | 31 ++++++++++++++++++++++++++----- tools/sst_dump_tool_imp.h | 4 +++- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/db/blob_index.h b/db/blob_index.h index 4acb4ddef..e1d41e274 100644 --- a/db/blob_index.h +++ b/db/blob_index.h @@ -5,6 +5,9 @@ #pragma once #ifndef ROCKSDB_LITE +#include +#include + #include "rocksdb/options.h" #include "util/coding.h" #include "util/string_util.h" @@ -108,6 +111,23 @@ class BlobIndex { return Status::OK(); } + std::string DebugString(bool output_hex) { + std::ostringstream oss; + + if (IsInlined()) { + oss << "[inlined blob] value:" << value_.ToString(output_hex); + } else { + oss << "[blob ref] file:" << file_number_ << " offset:" << offset_ + << " size:" << size_; + } + + if (HasTTL()) { + oss << " exp:" << expiration_; + } + + return oss.str(); + } + static void EncodeInlinedTTL(std::string* dst, uint64_t expiration, const Slice& value) { assert(dst != nullptr); diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index a47b3d4b8..9ff9bb0e9 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -2945,7 +2945,9 @@ void DumpSstFile(Options options, std::string filename, bool output_hex, return; } // no verification - rocksdb::SstFileDumper dumper(options, filename, false, output_hex); + // TODO: add support for decoding blob indexes in ldb as well + rocksdb::SstFileDumper dumper(options, filename, /* verify_checksum */ false, + output_hex, /* decode_blob_index */ false); Status st = dumper.ReadSequential(true, std::numeric_limits::max(), false, // has_from from_key, false, // has_to diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index fa4a431e8..9c7de34cd 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -15,6 +15,7 @@ #include #include +#include "db/blob_index.h" #include "db/memtable.h" #include "db/write_batch_internal.h" #include "options/cf_options.h" @@ -42,11 +43,12 @@ namespace rocksdb { SstFileDumper::SstFileDumper(const Options& options, const std::string& file_path, bool verify_checksum, - bool output_hex) + bool output_hex, bool decode_blob_index) : file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum), output_hex_(output_hex), + decode_blob_index_(decode_blob_index), options_(options), ioptions_(options_), moptions_(ColumnFamilyOptions(options_)), @@ -379,9 +381,22 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, } if (print_kv) { - fprintf(stdout, "%s => %s\n", - ikey.DebugString(output_hex_).c_str(), - value.ToString(output_hex_).c_str()); + if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) { + fprintf(stdout, "%s => %s\n", ikey.DebugString(output_hex_).c_str(), + value.ToString(output_hex_).c_str()); + } else { + BlobIndex blob_index; + + const Status s = blob_index.DecodeFrom(value); + if (!s.ok()) { + fprintf(stderr, "%s => error decoding blob index\n", + ikey.DebugString(output_hex_).c_str()); + continue; + } + + fprintf(stdout, "%s => %s\n", ikey.DebugString(output_hex_).c_str(), + blob_index.DebugString(output_hex_).c_str()); + } } } @@ -425,6 +440,9 @@ void print_help() { --output_hex Can be combined with scan command to print the keys and values in Hex + --decode_blob_index + Decode blob indexes and print them in a human-readable format during scans. + --from= Key to start reading from when executing check|scan @@ -475,6 +493,7 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { uint64_t n; bool verify_checksum = false; bool output_hex = false; + bool decode_blob_index = false; bool input_key_hex = false; bool has_from = false; bool has_to = false; @@ -499,6 +518,8 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { dir_or_file = argv[i] + 7; } else if (strcmp(argv[i], "--output_hex") == 0) { output_hex = true; + } else if (strcmp(argv[i], "--decode_blob_index") == 0) { + decode_blob_index = true; } else if (strcmp(argv[i], "--input_key_hex") == 0) { input_key_hex = true; } else if (sscanf(argv[i], "--read_num=%lu%c", (unsigned long*)&n, &junk) == @@ -638,7 +659,7 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) { } rocksdb::SstFileDumper dumper(options, filename, verify_checksum, - output_hex); + output_hex, decode_blob_index); if (!dumper.getStatus().ok()) { fprintf(stderr, "%s: %s\n", filename.c_str(), dumper.getStatus().ToString().c_str()); diff --git a/tools/sst_dump_tool_imp.h b/tools/sst_dump_tool_imp.h index f8d544352..5226502ed 100644 --- a/tools/sst_dump_tool_imp.h +++ b/tools/sst_dump_tool_imp.h @@ -18,7 +18,8 @@ namespace rocksdb { class SstFileDumper { public: explicit SstFileDumper(const Options& options, const std::string& file_name, - bool verify_checksum, bool output_hex); + bool verify_checksum, bool output_hex, + bool decode_blob_index); Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from, const std::string& from_key, bool has_to, @@ -64,6 +65,7 @@ class SstFileDumper { uint64_t read_num_; bool verify_checksum_; bool output_hex_; + bool decode_blob_index_; EnvOptions soptions_; // options_ and internal_comparator_ will also be used in