andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Blob DB: blob_dump to show uncompressed values

Browse Source 
Summary:
Make blob_dump tool able to show uncompressed values if the blob file is compressed. Also show total compressed vs. raw size at the end if --show_summary is provided.
Closes https://github.com/facebook/rocksdb/pull/3633

Differential Revision: D7348926

Pulled By: yiwu-arbug

fbshipit-source-id: ca709cb4ed5cf6a550ff2987df8033df81516f8e
This commit is contained in:
Yi Wu 2018-04-05 11:00:52 -07:00 committed by Facebook Github Bot
parent c827b2dc2a
commit 36a9f22931
3 changed files with 99 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
tools/blob_dump.cc
View File

@ -27,9 +27,13 @@ int main(int argc, char** argv) {
{"file", required_argument, nullptr, 'f'},
{"show_key", optional_argument, nullptr, 'k'},
{"show_blob", optional_argument, nullptr, 'b'},
{"show_uncompressed_blob", optional_argument, nullptr, 'r'},
{"show_summary", optional_argument, nullptr, 's'},
};
DisplayType show_key = DisplayType::kRaw;
DisplayType show_blob = DisplayType::kNone;
DisplayType show_uncompressed_blob = DisplayType::kNone;
bool show_summary = false;
std::string file;
while (true) {
int c = getopt_long(argc, argv, "hk::b::f:", options, nullptr);
@ -42,7 +46,9 @@ int main(int argc, char** argv) {
fprintf(stdout,
"Usage: blob_dump --file=filename "
"[--show_key[=none|raw|hex|detail]] "
"[--show_blob[=none|raw|hex|detail]]\n");
"[--show_blob[=none|raw|hex|detail]] "
"[--show_uncompressed_blob[=none|raw|hex|detail]] "
"[--show_summary]\n");
return 0;
case 'f':
file = optarg;
@ -64,16 +70,31 @@ int main(int argc, char** argv) {
}
show_blob = display_types.at(arg_str);
} else {
show_blob = DisplayType::kDetail;
show_blob = DisplayType::kHex;
}
break;
case 'r':
if (optarg) {
if (display_types.count(arg_str) == 0) {
fprintf(stderr, "Unrecognized blob display type.\n");
return -1;
}
show_uncompressed_blob = display_types.at(arg_str);
} else {
show_uncompressed_blob = DisplayType::kHex;
}
break;
case 's':
show_summary = true;
break;
default:
fprintf(stderr, "Unrecognized option.\n");
return -1;
}
}
BlobDumpTool tool;
Status s = tool.Run(file, show_key, show_blob);
Status s =
tool.Run(file, show_key, show_blob, show_uncompressed_blob, show_summary);
if (!s.ok()) {
fprintf(stderr, "Failed: %s\n", s.ToString().c_str());
return -1;

69
utilities/blob_db/blob_dump_tool.cc
View File

@ -17,6 +17,7 @@
#include "port/port.h"
#include "rocksdb/convenience.h"
#include "rocksdb/env.h"
#include "table/format.h"
#include "util/coding.h"
#include "util/string_util.h"
@ -27,7 +28,9 @@ BlobDumpTool::BlobDumpTool()
: reader_(nullptr), buffer_(nullptr), buffer_size_(0) {}
Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
DisplayType show_blob) {
DisplayType show_blob,
DisplayType show_uncompressed_blob,
bool show_summary) {
Status s;
Env* env = Env::Default();
s = env->FileExists(filename);
@ -50,7 +53,8 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
reader_.reset(new RandomAccessFileReader(std::move(file), filename));
uint64_t offset = 0;
uint64_t footer_offset = 0;
s = DumpBlobLogHeader(&offset);
CompressionType compression = kNoCompression;
s = DumpBlobLogHeader(&offset, &compression);
if (!s.ok()) {
return s;
}
@ -58,14 +62,30 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
if (!s.ok()) {
return s;
}
if (show_key != DisplayType::kNone) {
uint64_t total_records = 0;
uint64_t total_key_size = 0;
uint64_t total_blob_size = 0;
uint64_t total_uncompressed_blob_size = 0;
if (show_key != DisplayType::kNone || show_summary) {
while (offset < footer_offset) {
s = DumpRecord(show_key, show_blob, &offset);
s = DumpRecord(show_key, show_blob, show_uncompressed_blob, show_summary,
compression, &offset, &total_records, &total_key_size,
&total_blob_size, &total_uncompressed_blob_size);
if (!s.ok()) {
return s;
break;
}
}
}
if (show_summary) {
fprintf(stdout, "Summary:\n");
fprintf(stdout, " total records: %" PRIu64 "\n", total_records);
fprintf(stdout, " total key size: %" PRIu64 "\n", total_key_size);
fprintf(stdout, " total blob size: %" PRIu64 "\n", total_blob_size);
if (compression != kNoCompression) {
fprintf(stdout, " total raw blob size: %" PRIu64 "\n",
total_uncompressed_blob_size);
}
}
return s;
}
@ -89,7 +109,8 @@ Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) {
return s;
}
Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset,
CompressionType* compression) {
Slice slice;
Status s = Read(0, BlobLogHeader::kSize, &slice);
if (!s.ok()) {
@ -114,6 +135,7 @@ Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
fprintf(stdout, " Expiration range : %s\n",
GetString(header.expiration_range).c_str());
*offset = BlobLogHeader::kSize;
*compression = header.compression;
return s;
}
@ -136,7 +158,7 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
BlobLogFooter footer;
s = footer.DecodeFrom(slice);
if (!s.ok()) {
return s;
return no_footer();
}
fprintf(stdout, "Blob log footer:\n");
fprintf(stdout, " Blob count : %" PRIu64 "\n", footer.blob_count);
@ -146,9 +168,16 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
}
Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
uint64_t* offset) {
DisplayType show_uncompressed_blob,
bool show_summary, CompressionType compression,
uint64_t* offset, uint64_t* total_records,
uint64_t* total_key_size,
uint64_t* total_blob_size,
uint64_t* total_uncompressed_blob_size) {
if (show_key != DisplayType::kNone) {
fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
*offset, *offset);
}
Slice slice;
Status s = Read(*offset, BlobLogRecord::kHeaderSize, &slice);
if (!s.ok()) {
@ -161,14 +190,30 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
}
uint64_t key_size = record.key_size;
uint64_t value_size = record.value_size;
if (show_key != DisplayType::kNone) {
fprintf(stdout, " key size : %" PRIu64 "\n", key_size);
fprintf(stdout, " value size : %" PRIu64 "\n", value_size);
fprintf(stdout, " expiration : %" PRIu64 "\n", record.expiration);
}
*offset += BlobLogRecord::kHeaderSize;
s = Read(*offset, key_size + value_size, &slice);
if (!s.ok()) {
return s;
}
// Decompress value
std::string uncompressed_value;
if (compression != kNoCompression &&
(show_uncompressed_blob != DisplayType::kNone || show_summary)) {
BlockContents contents;
s = UncompressBlockContentsForCompressionType(
slice.data() + key_size, value_size, &contents,
2 /*compress_format_version*/, Slice(), compression,
ImmutableCFOptions(Options()));
if (!s.ok()) {
return s;
}
uncompressed_value = contents.data.ToString();
}
if (show_key != DisplayType::kNone) {
fprintf(stdout, " key : ");
DumpSlice(Slice(slice.data(), key_size), show_key);
@ -176,8 +221,16 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
fprintf(stdout, " blob : ");
DumpSlice(Slice(slice.data() + key_size, value_size), show_blob);
}
if (show_uncompressed_blob != DisplayType::kNone) {
fprintf(stdout, " raw blob : ");
DumpSlice(Slice(uncompressed_value), show_uncompressed_blob);
}
}
*offset += key_size + value_size;
*total_records += 1;
*total_key_size += key_size;
*total_blob_size += value_size;
*total_uncompressed_blob_size += uncompressed_value.size();
return s;
}

13
utilities/blob_db/blob_dump_tool.h
View File

@ -27,8 +27,9 @@ class BlobDumpTool {
BlobDumpTool();
Status Run(const std::string& filename, DisplayType key_type,
DisplayType blob_type);
Status Run(const std::string& filename, DisplayType show_key,
DisplayType show_blob, DisplayType show_uncompressed_blob,
bool show_summary);
private:
std::unique_ptr<RandomAccessFileReader> reader_;
@ -36,10 +37,14 @@ class BlobDumpTool {
size_t buffer_size_;
Status Read(uint64_t offset, size_t size, Slice* result);
Status DumpBlobLogHeader(uint64_t* offset);
Status DumpBlobLogHeader(uint64_t* offset, CompressionType* compression);
Status DumpBlobLogFooter(uint64_t file_size, uint64_t* footer_offset);
Status DumpRecord(DisplayType show_key, DisplayType show_blob,
uint64_t* offset);
DisplayType show_uncompressed_blob, bool show_summary,
CompressionType compression, uint64_t* offset,
uint64_t* total_records, uint64_t* total_key_size,
uint64_t* total_blob_size,
uint64_t* total_uncompressed_blob_size);
void DumpSlice(const Slice s, DisplayType type);
template <class T>