Merge remote-tracking branch 'origin' into performance
This commit is contained in:
commit
f3967a5132
@ -239,7 +239,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string VersionEdit::DebugString() const {
|
std::string VersionEdit::DebugString(bool hex_key) const {
|
||||||
std::string r;
|
std::string r;
|
||||||
r.append("VersionEdit {");
|
r.append("VersionEdit {");
|
||||||
if (has_comparator_) {
|
if (has_comparator_) {
|
||||||
@ -266,7 +266,7 @@ std::string VersionEdit::DebugString() const {
|
|||||||
r.append("\n CompactPointer: ");
|
r.append("\n CompactPointer: ");
|
||||||
AppendNumberTo(&r, compact_pointers_[i].first);
|
AppendNumberTo(&r, compact_pointers_[i].first);
|
||||||
r.append(" ");
|
r.append(" ");
|
||||||
r.append(compact_pointers_[i].second.DebugString());
|
r.append(compact_pointers_[i].second.DebugString(hex_key));
|
||||||
}
|
}
|
||||||
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
||||||
iter != deleted_files_.end();
|
iter != deleted_files_.end();
|
||||||
@ -285,9 +285,9 @@ std::string VersionEdit::DebugString() const {
|
|||||||
r.append(" ");
|
r.append(" ");
|
||||||
AppendNumberTo(&r, f.file_size);
|
AppendNumberTo(&r, f.file_size);
|
||||||
r.append(" ");
|
r.append(" ");
|
||||||
r.append(f.smallest.DebugString());
|
r.append(f.smallest.DebugString(hex_key));
|
||||||
r.append(" .. ");
|
r.append(" .. ");
|
||||||
r.append(f.largest.DebugString());
|
r.append(f.largest.DebugString(hex_key));
|
||||||
}
|
}
|
||||||
r.append("\n}\n");
|
r.append("\n}\n");
|
||||||
return r;
|
return r;
|
||||||
|
@ -96,7 +96,7 @@ class VersionEdit {
|
|||||||
void EncodeTo(std::string* dst) const;
|
void EncodeTo(std::string* dst) const;
|
||||||
Status DecodeFrom(const Slice& src);
|
Status DecodeFrom(const Slice& src);
|
||||||
|
|
||||||
std::string DebugString() const;
|
std::string DebugString(bool hex_key = false) const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
friend class VersionSet;
|
friend class VersionSet;
|
||||||
|
@ -1512,7 +1512,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
|
|||||||
// Write out each individual edit
|
// Write out each individual edit
|
||||||
if (verbose) {
|
if (verbose) {
|
||||||
printf("*************************Edit[%d] = %s\n",
|
printf("*************************Edit[%d] = %s\n",
|
||||||
count, edit.DebugString().c_str());
|
count, edit.DebugString(hex).c_str());
|
||||||
}
|
}
|
||||||
count++;
|
count++;
|
||||||
|
|
||||||
|
@ -14,6 +14,7 @@
|
|||||||
#include "table/block.h"
|
#include "table/block.h"
|
||||||
#include "table/block_builder.h"
|
#include "table/block_builder.h"
|
||||||
#include "table/format.h"
|
#include "table/format.h"
|
||||||
|
#include "util/ldb_cmd.h"
|
||||||
#include "util/random.h"
|
#include "util/random.h"
|
||||||
#include "util/testharness.h"
|
#include "util/testharness.h"
|
||||||
#include "util/testutil.h"
|
#include "util/testutil.h"
|
||||||
@ -22,10 +23,16 @@ namespace leveldb {
|
|||||||
|
|
||||||
class SstFileReader {
|
class SstFileReader {
|
||||||
public:
|
public:
|
||||||
explicit SstFileReader(std::string file_name,
|
explicit SstFileReader(const std::string& file_name,
|
||||||
bool verify_checksum = false,
|
bool verify_checksum,
|
||||||
bool output_hex = false);
|
bool output_hex);
|
||||||
Status ReadSequential(bool print_kv, uint64_t read_num = -1);
|
|
||||||
|
Status ReadSequential(bool print_kv,
|
||||||
|
uint64_t read_num,
|
||||||
|
bool has_from,
|
||||||
|
const std::string& from_key,
|
||||||
|
bool has_to,
|
||||||
|
const std::string& to_key);
|
||||||
|
|
||||||
uint64_t GetReadNumber() { return read_num_; }
|
uint64_t GetReadNumber() { return read_num_; }
|
||||||
|
|
||||||
@ -37,17 +44,25 @@ private:
|
|||||||
EnvOptions soptions_;
|
EnvOptions soptions_;
|
||||||
};
|
};
|
||||||
|
|
||||||
SstFileReader::SstFileReader(std::string file_path,
|
SstFileReader::SstFileReader(const std::string& file_path,
|
||||||
bool verify_checksum,
|
bool verify_checksum,
|
||||||
bool output_hex)
|
bool output_hex)
|
||||||
:file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum),
|
:file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum),
|
||||||
output_hex_(output_hex) {
|
output_hex_(output_hex) {
|
||||||
|
std::cout << "Process " << file_path << "\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
Status SstFileReader::ReadSequential(bool print_kv, uint64_t read_num)
|
Status SstFileReader::ReadSequential(bool print_kv,
|
||||||
|
uint64_t read_num,
|
||||||
|
bool has_from,
|
||||||
|
const std::string& from_key,
|
||||||
|
bool has_to,
|
||||||
|
const std::string& to_key)
|
||||||
{
|
{
|
||||||
unique_ptr<Table> table;
|
unique_ptr<Table> table;
|
||||||
|
InternalKeyComparator internal_comparator_(BytewiseComparator());
|
||||||
Options table_options;
|
Options table_options;
|
||||||
|
table_options.comparator = &internal_comparator_;
|
||||||
unique_ptr<RandomAccessFile> file;
|
unique_ptr<RandomAccessFile> file;
|
||||||
Status s = table_options.env->NewRandomAccessFile(file_name_, &file,
|
Status s = table_options.env->NewRandomAccessFile(file_name_, &file,
|
||||||
soptions_);
|
soptions_);
|
||||||
@ -63,17 +78,38 @@ Status SstFileReader::ReadSequential(bool print_kv, uint64_t read_num)
|
|||||||
|
|
||||||
Iterator* iter = table->NewIterator(ReadOptions(verify_checksum_, false));
|
Iterator* iter = table->NewIterator(ReadOptions(verify_checksum_, false));
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
if (has_from) {
|
||||||
|
InternalKey ikey(from_key, kMaxSequenceNumber, kValueTypeForSeek);
|
||||||
|
iter->Seek(ikey.Encode());
|
||||||
|
} else {
|
||||||
|
iter->SeekToFirst();
|
||||||
|
}
|
||||||
|
for (; iter->Valid(); iter->Next()) {
|
||||||
Slice key = iter->key();
|
Slice key = iter->key();
|
||||||
Slice value = iter->value();
|
Slice value = iter->value();
|
||||||
++i;
|
++i;
|
||||||
if (read_num > 0 && i > read_num)
|
if (read_num > 0 && i > read_num)
|
||||||
break;
|
break;
|
||||||
if (print_kv) {
|
|
||||||
fprintf(stdout, "%s ==> %s\n",
|
ParsedInternalKey ikey;
|
||||||
key.ToString(output_hex_).c_str(),
|
if (!ParseInternalKey(key, &ikey)) {
|
||||||
value.ToString(output_hex_).c_str());
|
std::cerr << "Internal Key ["
|
||||||
|
<< key.ToString(true /* in hex*/)
|
||||||
|
<< "] parse error!\n";
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If end marker was specified, we stop before it
|
||||||
|
if (has_to && BytewiseComparator()->Compare(ikey.user_key, to_key) >= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (print_kv) {
|
||||||
|
std::cout << ikey.DebugString(output_hex_)
|
||||||
|
<< " => "
|
||||||
|
<< value.ToString(output_hex_) << "\n";
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
read_num_ += i;
|
read_num_ += i;
|
||||||
@ -90,9 +126,29 @@ static void print_help() {
|
|||||||
"sst_dump [--command=check|scan] [--verify_checksum] "
|
"sst_dump [--command=check|scan] [--verify_checksum] "
|
||||||
"--file=data_dir_OR_sst_file"
|
"--file=data_dir_OR_sst_file"
|
||||||
" [--output_hex]"
|
" [--output_hex]"
|
||||||
|
" [--input_key_hex]"
|
||||||
|
" [--from=<user_key>]"
|
||||||
|
" [--to=<user_key>]"
|
||||||
" [--read_num=NUM]\n");
|
" [--read_num=NUM]\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string HexToString(const string& str) {
|
||||||
|
string parsed;
|
||||||
|
if (str[0] != '0' || str[1] != 'x') {
|
||||||
|
fprintf(stderr, "Invalid hex input %s. Must start with 0x\n",
|
||||||
|
str.c_str());
|
||||||
|
throw "Invalid hex input";
|
||||||
|
}
|
||||||
|
|
||||||
|
for (unsigned int i = 2; i < str.length();) {
|
||||||
|
int c;
|
||||||
|
sscanf(str.c_str() + i, "%2X", &c);
|
||||||
|
parsed.push_back(c);
|
||||||
|
i += 2;
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char** argv) {
|
int main(int argc, char** argv) {
|
||||||
|
|
||||||
const char* dir_or_file = nullptr;
|
const char* dir_or_file = nullptr;
|
||||||
@ -103,24 +159,47 @@ int main(int argc, char** argv) {
|
|||||||
uint64_t n;
|
uint64_t n;
|
||||||
bool verify_checksum = false;
|
bool verify_checksum = false;
|
||||||
bool output_hex = false;
|
bool output_hex = false;
|
||||||
|
bool input_key_hex = false;
|
||||||
|
bool has_from = false;
|
||||||
|
bool has_to = false;
|
||||||
|
std::string from_key;
|
||||||
|
std::string to_key;
|
||||||
for (int i = 1; i < argc; i++)
|
for (int i = 1; i < argc; i++)
|
||||||
{
|
{
|
||||||
if (strncmp(argv[i], "--file=", 7) == 0) {
|
if (strncmp(argv[i], "--file=", 7) == 0) {
|
||||||
dir_or_file = argv[i] + 7;
|
dir_or_file = argv[i] + 7;
|
||||||
} else if (strcmp(argv[i], "--output_hex") == 0) {
|
} else if (strcmp(argv[i], "--output_hex") == 0) {
|
||||||
output_hex = true;
|
output_hex = true;
|
||||||
|
} else if (strcmp(argv[i], "--input_key_hex") == 0) {
|
||||||
|
input_key_hex = true;
|
||||||
} else if (sscanf(argv[i], "--read_num=%ld%c", &n, &junk) == 1) {
|
} else if (sscanf(argv[i], "--read_num=%ld%c", &n, &junk) == 1) {
|
||||||
read_num = n;
|
read_num = n;
|
||||||
} else if (strcmp(argv[i], "--verify_checksum") == 0) {
|
} else if (strcmp(argv[i], "--verify_checksum") == 0) {
|
||||||
verify_checksum = true;
|
verify_checksum = true;
|
||||||
} else if (strncmp(argv[i], "--command=", 10) == 0) {
|
} else if (strncmp(argv[i], "--command=", 10) == 0) {
|
||||||
command = argv[i] + 10;
|
command = argv[i] + 10;
|
||||||
|
} else if (strncmp(argv[i], "--from=", 7) == 0) {
|
||||||
|
from_key = argv[i] + 7;
|
||||||
|
has_from = true;
|
||||||
|
} else if (strncmp(argv[i], "--to=", 5) == 0) {
|
||||||
|
to_key = argv[i] + 5;
|
||||||
|
has_to = true;
|
||||||
}else {
|
}else {
|
||||||
print_help();
|
print_help();
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if (input_key_hex) {
|
||||||
|
if (has_from) {
|
||||||
|
from_key = HexToString(from_key);
|
||||||
|
}
|
||||||
|
if (has_to) {
|
||||||
|
to_key = HexToString(to_key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if(dir_or_file == nullptr) {
|
if(dir_or_file == nullptr) {
|
||||||
print_help();
|
print_help();
|
||||||
exit(1);
|
exit(1);
|
||||||
@ -136,6 +215,9 @@ int main(int argc, char** argv) {
|
|||||||
dir = false;
|
dir = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::cout << "from [" << leveldb::Slice(from_key).ToString(true)
|
||||||
|
<< "] to [" << leveldb::Slice(to_key).ToString(true) << "]\n";
|
||||||
|
|
||||||
uint64_t total_read = 0;
|
uint64_t total_read = 0;
|
||||||
for (size_t i = 0; i < filenames.size(); i++) {
|
for (size_t i = 0; i < filenames.size(); i++) {
|
||||||
std::string filename = filenames.at(i);
|
std::string filename = filenames.at(i);
|
||||||
@ -145,7 +227,7 @@ int main(int argc, char** argv) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if(dir) {
|
if(dir) {
|
||||||
filename = std::string(dir_or_file) + "//" + filename;
|
filename = std::string(dir_or_file) + "/" + filename;
|
||||||
}
|
}
|
||||||
leveldb::SstFileReader reader(filename, verify_checksum,
|
leveldb::SstFileReader reader(filename, verify_checksum,
|
||||||
output_hex);
|
output_hex);
|
||||||
@ -153,7 +235,9 @@ int main(int argc, char** argv) {
|
|||||||
// scan all files in give file path.
|
// scan all files in give file path.
|
||||||
if (command == "" || command == "scan" || command == "check") {
|
if (command == "" || command == "scan" || command == "check") {
|
||||||
st = reader.ReadSequential(command != "check",
|
st = reader.ReadSequential(command != "check",
|
||||||
read_num > 0 ? (read_num - total_read) : read_num);
|
read_num > 0 ? (read_num - total_read) :
|
||||||
|
read_num,
|
||||||
|
has_from, from_key, has_to, to_key);
|
||||||
if (!st.ok()) {
|
if (!st.ok()) {
|
||||||
fprintf(stderr, "%s: %s\n", filename.c_str(),
|
fprintf(stderr, "%s: %s\n", filename.c_str(),
|
||||||
st.ToString().c_str());
|
st.ToString().c_str());
|
||||||
|
@ -578,6 +578,7 @@ void PrintBucketCounts(const vector<uint64_t>& bucket_counts, int ttl_start,
|
|||||||
|
|
||||||
const string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
|
const string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
|
||||||
const string InternalDumpCommand::ARG_STATS = "stats";
|
const string InternalDumpCommand::ARG_STATS = "stats";
|
||||||
|
const string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex";
|
||||||
|
|
||||||
InternalDumpCommand::InternalDumpCommand(const vector<string>& params,
|
InternalDumpCommand::InternalDumpCommand(const vector<string>& params,
|
||||||
const map<string, string>& options,
|
const map<string, string>& options,
|
||||||
@ -585,12 +586,14 @@ InternalDumpCommand::InternalDumpCommand(const vector<string>& params,
|
|||||||
LDBCommand(options, flags, true,
|
LDBCommand(options, flags, true,
|
||||||
BuildCmdLineOptions({ ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
|
BuildCmdLineOptions({ ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
|
||||||
ARG_FROM, ARG_TO, ARG_MAX_KEYS,
|
ARG_FROM, ARG_TO, ARG_MAX_KEYS,
|
||||||
ARG_COUNT_ONLY, ARG_STATS})),
|
ARG_COUNT_ONLY, ARG_STATS,
|
||||||
|
ARG_INPUT_KEY_HEX})),
|
||||||
has_from_(false),
|
has_from_(false),
|
||||||
has_to_(false),
|
has_to_(false),
|
||||||
max_keys_(-1),
|
max_keys_(-1),
|
||||||
count_only_(false),
|
count_only_(false),
|
||||||
print_stats_(false) {
|
print_stats_(false),
|
||||||
|
is_input_key_hex_(false) {
|
||||||
|
|
||||||
has_from_ = ParseStringOption(options, ARG_FROM, &from_);
|
has_from_ = ParseStringOption(options, ARG_FROM, &from_);
|
||||||
has_to_ = ParseStringOption(options, ARG_TO, &to_);
|
has_to_ = ParseStringOption(options, ARG_TO, &to_);
|
||||||
@ -599,8 +602,9 @@ InternalDumpCommand::InternalDumpCommand(const vector<string>& params,
|
|||||||
|
|
||||||
print_stats_ = IsFlagPresent(flags, ARG_STATS);
|
print_stats_ = IsFlagPresent(flags, ARG_STATS);
|
||||||
count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
|
count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
|
||||||
|
is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX);
|
||||||
|
|
||||||
if (is_key_hex_) {
|
if (is_input_key_hex_) {
|
||||||
if (has_from_) {
|
if (has_from_) {
|
||||||
from_ = HexToString(from_);
|
from_ = HexToString(from_);
|
||||||
}
|
}
|
||||||
@ -614,6 +618,7 @@ void InternalDumpCommand::Help(string& ret) {
|
|||||||
ret.append(" ");
|
ret.append(" ");
|
||||||
ret.append(InternalDumpCommand::Name());
|
ret.append(InternalDumpCommand::Name());
|
||||||
ret.append(HelpRangeCmdArgs());
|
ret.append(HelpRangeCmdArgs());
|
||||||
|
ret.append(" [--" + ARG_INPUT_KEY_HEX + "]");
|
||||||
ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
|
ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
|
||||||
ret.append(" [--" + ARG_COUNT_ONLY + "]");
|
ret.append(" [--" + ARG_COUNT_ONLY + "]");
|
||||||
ret.append(" [--" + ARG_STATS + "]");
|
ret.append(" [--" + ARG_STATS + "]");
|
||||||
@ -674,7 +679,7 @@ void InternalDumpCommand::DoCommand() {
|
|||||||
if (!count_only_) {
|
if (!count_only_) {
|
||||||
string key = ikey.DebugString(is_key_hex_);
|
string key = ikey.DebugString(is_key_hex_);
|
||||||
string value = iter->value().ToString(is_value_hex_);
|
string value = iter->value().ToString(is_value_hex_);
|
||||||
fprintf(stdout, "%s => %s\n", key.data(), value.data());
|
std::cout << key << " => " << value << "\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
// Terminate if maximum number of keys have been dumped
|
// Terminate if maximum number of keys have been dumped
|
||||||
|
@ -424,9 +424,11 @@ private:
|
|||||||
int max_keys_;
|
int max_keys_;
|
||||||
bool count_only_;
|
bool count_only_;
|
||||||
bool print_stats_;
|
bool print_stats_;
|
||||||
|
bool is_input_key_hex_;
|
||||||
|
|
||||||
static const string ARG_COUNT_ONLY;
|
static const string ARG_COUNT_ONLY;
|
||||||
static const string ARG_STATS;
|
static const string ARG_STATS;
|
||||||
|
static const string ARG_INPUT_KEY_HEX;
|
||||||
};
|
};
|
||||||
|
|
||||||
class DBLoaderCommand: public LDBCommand {
|
class DBLoaderCommand: public LDBCommand {
|
||||||
|
Loading…
Reference in New Issue
Block a user