Support decoding blob indexes in sst_dump (#5926)

Summary:
The patch adds a new command line parameter --decode_blob_index to sst_dump.
If this switch is specified, sst_dump prints blob indexes in a human readable format,
printing the blob file number, offset, size, and expiration (if applicable) for blob
references, and the blob value (and expiration) for inlined blobs.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5926

Test Plan:
Used db_bench's BlobDB mode to generate SST files containing blob references with
and without expiration, as well as inlined blobs with and without expiration (note: the
latter are stored as plain values), and confirmed sst_dump correctly prints all four types
of records.

Differential Revision: D17939077

Pulled By: ltamasi

fbshipit-source-id: edc5f58fee94ba35f6699c6a042d5758f5b3963d

db/blob_index.h

@@ -5,6 +5,9 @@
 #pragma once
 #ifndef ROCKSDB_LITE
 
+#include <sstream>
+#include <string>
+
 #include "rocksdb/options.h"
 #include "util/coding.h"
 #include "util/string_util.h"
@@ -108,6 +111,23 @@ class BlobIndex {
     return Status::OK();
   }
 
+  std::string DebugString(bool output_hex) {
+    std::ostringstream oss;
+
+    if (IsInlined()) {
+      oss << "[inlined blob] value:" << value_.ToString(output_hex);
+    } else {
+      oss << "[blob ref] file:" << file_number_ << " offset:" << offset_
+          << " size:" << size_;
+    }
+
+    if (HasTTL()) {
+      oss << " exp:" << expiration_;
+    }
+
+    return oss.str();
+  }
+
   static void EncodeInlinedTTL(std::string* dst, uint64_t expiration,
                                const Slice& value) {
     assert(dst != nullptr);

tools/ldb_cmd.cc

@@ -2945,7 +2945,9 @@ void DumpSstFile(Options options, std::string filename, bool output_hex,
     return;
   }
   // no verification
-  rocksdb::SstFileDumper dumper(options, filename, false, output_hex);
+  // TODO: add support for decoding blob indexes in ldb as well
+  rocksdb::SstFileDumper dumper(options, filename, /* verify_checksum */ false,
+                                output_hex, /* decode_blob_index */ false);
   Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
                                     false,            // has_from
                                     from_key, false,  // has_to

tools/sst_dump_tool.cc

@@ -15,6 +15,7 @@
 #include <sstream>
 #include <vector>
 
+#include "db/blob_index.h"
 #include "db/memtable.h"
 #include "db/write_batch_internal.h"
 #include "options/cf_options.h"
@@ -42,11 +43,12 @@ namespace rocksdb {
 
 SstFileDumper::SstFileDumper(const Options& options,
                              const std::string& file_path, bool verify_checksum,
-                             bool output_hex)
+                             bool output_hex, bool decode_blob_index)
     : file_name_(file_path),
       read_num_(0),
       verify_checksum_(verify_checksum),
       output_hex_(output_hex),
+      decode_blob_index_(decode_blob_index),
       options_(options),
       ioptions_(options_),
       moptions_(ColumnFamilyOptions(options_)),
@@ -379,9 +381,22 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num,
     }
 
     if (print_kv) {
-      fprintf(stdout, "%s => %s\n",
+      if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) {
-          ikey.DebugString(output_hex_).c_str(),
+        fprintf(stdout, "%s => %s\n", ikey.DebugString(output_hex_).c_str(),
-          value.ToString(output_hex_).c_str());
+                value.ToString(output_hex_).c_str());
+      } else {
+        BlobIndex blob_index;
+
+        const Status s = blob_index.DecodeFrom(value);
+        if (!s.ok()) {
+          fprintf(stderr, "%s => error decoding blob index\n",
+                  ikey.DebugString(output_hex_).c_str());
+          continue;
+        }
+
+        fprintf(stdout, "%s => %s\n", ikey.DebugString(output_hex_).c_str(),
+                blob_index.DebugString(output_hex_).c_str());
+      }
     }
   }
 
@@ -425,6 +440,9 @@ void print_help() {
     --output_hex
       Can be combined with scan command to print the keys and values in Hex
 
+    --decode_blob_index
+      Decode blob indexes and print them in a human-readable format during scans.
+
     --from=<user_key>
       Key to start reading from when executing check|scan
 
@@ -475,6 +493,7 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) {
   uint64_t n;
   bool verify_checksum = false;
   bool output_hex = false;
+  bool decode_blob_index = false;
   bool input_key_hex = false;
   bool has_from = false;
   bool has_to = false;
@@ -499,6 +518,8 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) {
       dir_or_file = argv[i] + 7;
     } else if (strcmp(argv[i], "--output_hex") == 0) {
       output_hex = true;
+    } else if (strcmp(argv[i], "--decode_blob_index") == 0) {
+      decode_blob_index = true;
     } else if (strcmp(argv[i], "--input_key_hex") == 0) {
       input_key_hex = true;
     } else if (sscanf(argv[i], "--read_num=%lu%c", (unsigned long*)&n, &junk) ==
@@ -638,7 +659,7 @@ int SSTDumpTool::Run(int argc, char** argv, Options options) {
     }
 
     rocksdb::SstFileDumper dumper(options, filename, verify_checksum,
-                                  output_hex);
+                                  output_hex, decode_blob_index);
     if (!dumper.getStatus().ok()) {
       fprintf(stderr, "%s: %s\n", filename.c_str(),
               dumper.getStatus().ToString().c_str());

tools/sst_dump_tool_imp.h

@@ -18,7 +18,8 @@ namespace rocksdb {
 class SstFileDumper {
  public:
   explicit SstFileDumper(const Options& options, const std::string& file_name,
-                         bool verify_checksum, bool output_hex);
+                         bool verify_checksum, bool output_hex,
+                         bool decode_blob_index);
 
   Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from,
                         const std::string& from_key, bool has_to,
@@ -64,6 +65,7 @@ class SstFileDumper {
   uint64_t read_num_;
   bool verify_checksum_;
   bool output_hex_;
+  bool decode_blob_index_;
   EnvOptions soptions_;
 
   // options_ and internal_comparator_ will also be used in