andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added JSON manifest dump option to ldb command

Browse Source 
Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.

Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```

**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```

Reviewers: sdong, anthony, yhchiang, igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D41727
This commit is contained in:
Ari Ekmekji 2015-07-17 10:07:40 -07:00
parent a96fcd09b7
commit 74c755c552
7 changed files with 115 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
db/version_edit.cc
View File

@ -11,6 +11,7 @@
#include "db/version_set.h"
#include "util/coding.h"
#include "util/event_logger.h"
#include "rocksdb/slice.h"
namespace rocksdb {
@ -359,7 +360,7 @@ std::string VersionEdit::DebugString(bool hex_key) const {
AppendNumberTo(&r, prev_log_number_);
}
if (has_next_file_number_) {
r.append("\n NextFile: ");
r.append("\n NextFileNumber: ");
AppendNumberTo(&r, next_file_number_);
}
if (has_last_sequence_) {
@ -404,4 +405,75 @@ std::string VersionEdit::DebugString(bool hex_key) const {
return r;
}
std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
JSONWriter jw;
jw << "EditNumber" << edit_num;
if (has_comparator_) {
jw << "Comparator" << comparator_;
}
if (has_log_number_) {
jw << "LogNumber" << log_number_;
}
if (has_prev_log_number_) {
jw << "PrevLogNumber" << prev_log_number_;
}
if (has_next_file_number_) {
jw << "NextFileNumber" << next_file_number_;
}
if (has_last_sequence_) {
jw << "LastSeq" << last_sequence_;
}
if (!deleted_files_.empty()) {
jw << "DeletedFiles";
jw.StartArray();
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
iter != deleted_files_.end();
++iter) {
jw.StartArrayedObject();
jw << "Level" << iter->first;
jw << "FileNumber" << iter->second;
jw.EndArrayedObject();
}
jw.EndArray();
}
if (!new_files_.empty()) {
jw << "AddedFiles";
jw.StartArray();
for (size_t i = 0; i < new_files_.size(); i++) {
jw.StartArrayedObject();
jw << "Level" << new_files_[i].first;
const FileMetaData& f = new_files_[i].second;
jw << "FileNumber" << f.fd.GetNumber();
jw << "FileSize" << f.fd.GetFileSize();
jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
jw << "LargestIKey" << f.largest.DebugString(hex_key);
jw.EndArrayedObject();
}
jw.EndArray();
}
jw << "ColumnFamily" << column_family_;
if (is_column_family_add_) {
jw << "ColumnFamilyAdd" << column_family_name_;
}
if (is_column_family_drop_) {
jw << "ColumnFamilyDrop" << column_family_name_;
}
if (has_max_column_family_) {
jw << "MaxColumnFamily" << max_column_family_;
}
jw.EndObject();
return jw.Get();
}
} // namespace rocksdb

1
db/version_edit.h
View File

@ -227,6 +227,7 @@ class VersionEdit {
}
std::string DebugString(bool hex_key = false) const;
std::string DebugJSON(int edit_num, bool hex_key = false) const;
private:
friend class VersionSet;

9
db/version_set.cc
View File

@ -2572,7 +2572,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
}
Status VersionSet::DumpManifest(Options& options, std::string& dscname,
bool verbose, bool hex) {
bool verbose, bool hex, bool json) {
// Open the specified manifest file.
unique_ptr<SequentialFile> file;
Status s = options.env->NewSequentialFile(dscname, &file, env_options_);
@ -2613,9 +2613,10 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
}
// Write out each individual edit
if (verbose) {
printf("*************************Edit[%d] = %s\n",
count, edit.DebugString(hex).c_str());
if (verbose && !json) {
printf("%s\n", edit.DebugString(hex).c_str());
} else if (json) {
printf("%s\n", edit.DebugJSON(count, hex).c_str());
}
count++;

2
db/version_set.h
View File

@ -572,7 +572,7 @@ class VersionSet {
// printf contents (for debugging)
Status DumpManifest(Options& options, std::string& manifestFileName,
bool verbose, bool hex = false);
bool verbose, bool hex = false, bool json = false);
#endif // ROCKSDB_LITE

25
util/event_logger.h
View File

@ -15,11 +15,11 @@
namespace rocksdb {
// JSONWriter doesn't support objects in arrays yet. There wasn't a need for
// that.
class JSONWriter {
public:
JSONWriter() : state_(kExpectKey), first_element_(true) { stream_ << "{"; }
JSONWriter() : state_(kExpectKey), first_element_(true), in_array_(false) {
stream_ << "{";
}
void AddKey(const std::string& key) {
assert(state_ == kExpectKey);
@ -59,6 +59,7 @@ class JSONWriter {
void StartArray() {
assert(state_ == kExpectValue);
state_ = kInArray;
in_array_ = true;
stream_ << "[";
first_element_ = true;
}
@ -66,6 +67,7 @@ class JSONWriter {
void EndArray() {
assert(state_ == kInArray);
state_ = kExpectKey;
in_array_ = false;
stream_ << "]";
first_element_ = false;
}
@ -83,6 +85,21 @@ class JSONWriter {
first_element_ = false;
}
void StartArrayedObject() {
assert(state_ == kInArray && in_array_);
state_ = kExpectValue;
if (!first_element_) {
stream_ << ", ";
}
StartObject();
}
void EndArrayedObject() {
assert(in_array_);
EndObject();
state_ = kInArray;
}
std::string Get() const { return stream_.str(); }
JSONWriter& operator<<(const char* val) {
@ -110,9 +127,11 @@ class JSONWriter {
kExpectKey,
kExpectValue,
kInArray,
kInArrayedObject,
};
JSONWriterState state_;
bool first_element_;
bool in_array_;
std::ostringstream stream_;
};

17
util/ldb_cmd.cc
View File

@ -527,7 +527,7 @@ void DBLoaderCommand::DoCommand() {
namespace {
void DumpManifestFile(std::string file, bool verbose, bool hex) {
void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) {
Options options;
EnvOptions sopt;
std::string dbname("dummy");
@ -540,7 +540,7 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) {
WriteController wc(options.delayed_write_rate);
WriteBuffer wb(options.db_write_buffer_size);
VersionSet versions(dbname, &options, sopt, tc.get(), &wb, &wc);
Status s = versions.DumpManifest(options, file, verbose, hex);
Status s = versions.DumpManifest(options, file, verbose, hex, json);
if (!s.ok()) {
printf("Error in processing file %s %s\n", file.c_str(),
s.ToString().c_str());
@ -550,12 +550,14 @@ void DumpManifestFile(std::string file, bool verbose, bool hex) {
} // namespace
const string ManifestDumpCommand::ARG_VERBOSE = "verbose";
const string ManifestDumpCommand::ARG_PATH = "path";
const string ManifestDumpCommand::ARG_JSON = "json";
const string ManifestDumpCommand::ARG_PATH = "path";
void ManifestDumpCommand::Help(string& ret) {
ret.append(" ");
ret.append(ManifestDumpCommand::Name());
ret.append(" [--" + ARG_VERBOSE + "]");
ret.append(" [--" + ARG_JSON + "]");
ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
ret.append("\n");
}
@ -563,11 +565,13 @@ void ManifestDumpCommand::Help(string& ret) {
ManifestDumpCommand::ManifestDumpCommand(const vector<string>& params,
const map<string, string>& options, const vector<string>& flags) :
LDBCommand(options, flags, false,
BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX})),
BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
verbose_(false),
json_(false),
path_("")
{
verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
json_ = IsFlagPresent(flags, ARG_JSON);
map<string, string>::const_iterator itr = options.find(ARG_PATH);
if (itr != options.end()) {
@ -623,7 +627,8 @@ void ManifestDumpCommand::DoCommand() {
printf("Processing Manifest file %s\n", manifestfile.c_str());
}
DumpManifestFile(manifestfile, verbose_, is_key_hex_);
DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_);
if (verbose_) {
printf("Processing Manifest file %s done\n", manifestfile.c_str());
}
@ -2025,7 +2030,7 @@ void DBFileDumperCommand::DoCommand() {
manifest_filename.resize(manifest_filename.size() - 1);
string manifest_filepath = db_->GetName() + "/" + manifest_filename;
std::cout << manifest_filepath << std::endl;
DumpManifestFile(manifest_filepath, false, false);
DumpManifestFile(manifest_filepath, false, false, false);
std::cout << std::endl;
std::cout << "SST Files" << std::endl;

2
util/ldb_cmd.h
View File

@ -507,9 +507,11 @@ public:
private:
bool verbose_;
bool json_;
string path_;
static const string ARG_VERBOSE;
static const string ARG_JSON;
static const string ARG_PATH;
};