Commandline tool to compace LevelDB databases.
Summary: A simple CLI which calles DB->CompactRange() Can take String key's as range. Test Plan: Inserted data into a table. Waited for a minute, used compact tool on it. File modification time's changed so Compact did something on the files. Existing unit tests work. Reviewers: heyongqiang, dhruba Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D5697
This commit is contained in:
Abhishek Kona
parent
a321d5be9e
commit
fec81318b0
@ -209,7 +209,7 @@ class RandomGenerator {
|
||||
pos_ += len;
|
||||
return Slice(data_.data() + pos_ - len, len);
|
||||
}
|
||||
};
|
||||
};
|
||||
static Slice TrimSpace(Slice s) {
|
||||
int start = 0;
|
||||
while (start < s.size() && isspace(s[start])) {
|
||||
@ -515,9 +515,9 @@ class Benchmark {
|
||||
|
||||
public:
|
||||
Benchmark()
|
||||
: cache_(FLAGS_cache_size >= 0 ?
|
||||
(FLAGS_cache_numshardbits >= 1 ?
|
||||
NewLRUCache(FLAGS_cache_size, FLAGS_cache_numshardbits) :
|
||||
: cache_(FLAGS_cache_size >= 0 ?
|
||||
(FLAGS_cache_numshardbits >= 1 ?
|
||||
NewLRUCache(FLAGS_cache_size, FLAGS_cache_numshardbits) :
|
||||
NewLRUCache(FLAGS_cache_size)) : NULL),
|
||||
filter_policy_(FLAGS_bloom_bits >= 0
|
||||
? NewBloomFilterPolicy(FLAGS_bloom_bits)
|
||||
@ -528,9 +528,9 @@ class Benchmark {
|
||||
entries_per_batch_(1),
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
writes_(FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes),
|
||||
readwrites_((FLAGS_writes < 0 && FLAGS_reads < 0)? FLAGS_num :
|
||||
readwrites_((FLAGS_writes < 0 && FLAGS_reads < 0)? FLAGS_num :
|
||||
((FLAGS_writes > FLAGS_reads) ? FLAGS_writes : FLAGS_reads)
|
||||
),
|
||||
),
|
||||
heap_counter_(0) {
|
||||
std::vector<std::string> files;
|
||||
FLAGS_env->GetChildren(FLAGS_db, &files);
|
||||
@ -1048,7 +1048,7 @@ class Benchmark {
|
||||
|
||||
//
|
||||
// This is diffferent from ReadWhileWriting because it does not use
|
||||
// an extra thread.
|
||||
// an extra thread.
|
||||
//
|
||||
void ReadRandomWriteRandom(ThreadState* thread) {
|
||||
ReadOptions options(FLAGS_verify_checksum, true);
|
||||
@ -1090,7 +1090,7 @@ class Benchmark {
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
char msg[100];
|
||||
snprintf(msg, sizeof(msg), "( reads:%ld writes:%ld total:%ld )",
|
||||
snprintf(msg, sizeof(msg), "( reads:%ld writes:%ld total:%ld )",
|
||||
reads_done, writes_done, readwrites_);
|
||||
thread->stats.AddMessage(msg);
|
||||
}
|
||||
|
||||
202
tools/ldb.cc
202
tools/ldb.cc
@ -7,9 +7,11 @@
|
||||
#include <sstream>
|
||||
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/slice.h"
|
||||
|
||||
std::string HexToString(std::string str) {
|
||||
std::string HexToString(const std::string& str) {
|
||||
std::string parsed;
|
||||
for (int i = 0; i < str.length(); ) {
|
||||
int c;
|
||||
@ -20,73 +22,58 @@ std::string HexToString(std::string str) {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
static void print_help() {
|
||||
|
||||
static void print_usage() {
|
||||
fprintf(stderr,
|
||||
"db_dump "
|
||||
"--start=[START_KEY] "
|
||||
"--end=[END_KEY] "
|
||||
"--max_keys=[NUM] "
|
||||
"--hex "
|
||||
"--count_only "
|
||||
"--stats "
|
||||
"[PATH]\n");
|
||||
"ldb [compact|dump] "
|
||||
"--db-path=database_path "
|
||||
"[--from=START KEY] "
|
||||
"[--to=END KEY ] "
|
||||
"[--max_keys=[NUM] (only for dump)] "
|
||||
"[--hex ] "
|
||||
"[--count_only (only for dump)] "
|
||||
"[--stats (only for dump) ] \n");
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
std::string db_path;
|
||||
std::string start;
|
||||
std::string end;
|
||||
uint64_t max_keys = -1;
|
||||
bool print_stats = false;
|
||||
bool count_only = false;
|
||||
uint64_t count = 0;
|
||||
bool hex = false;
|
||||
|
||||
// Parse command line args
|
||||
char junk;
|
||||
uint64_t n;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (strncmp(argv[i], "--start=", 8) == 0) {
|
||||
start= argv[i] + 8;
|
||||
} else if (strncmp(argv[i], "--end=", 6) == 0) {
|
||||
end = argv[i] + 6;
|
||||
} else if (sscanf(argv[i], "--max_keys=%ld%c", &n, &junk) == 1) {
|
||||
max_keys = n;
|
||||
} else if (strncmp(argv[i], "--stats", 7) == 0) {
|
||||
print_stats = true;
|
||||
} else if (strncmp(argv[i], "--count_only", 12) == 0) {
|
||||
count_only = true;
|
||||
} else if (strncmp(argv[i], "--hex", 5) == 0) {
|
||||
hex = true;
|
||||
} else if (i == (argc - 1)) {
|
||||
db_path = argv[i];
|
||||
} else {
|
||||
print_help();
|
||||
exit(1);
|
||||
}
|
||||
static void safe_open_db(const std::string& dbname, leveldb::DB** db) {
|
||||
leveldb::Options options;
|
||||
options.create_if_missing = false;
|
||||
leveldb::Status status = leveldb::DB::Open(options, dbname, db);
|
||||
|
||||
if(!status.ok()) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Could not open db at %s\nERROR: %s",
|
||||
dbname.data(),
|
||||
status.ToString().data()
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void dump_db(
|
||||
const std::string& db_path,
|
||||
std::string& start,
|
||||
std::string& end,
|
||||
int64_t max_keys,
|
||||
const bool hex,
|
||||
const bool print_stats,
|
||||
const bool count_only
|
||||
) {
|
||||
// Parse command line args
|
||||
uint64_t count = 0;
|
||||
|
||||
|
||||
if (hex) {
|
||||
start = HexToString(start);
|
||||
end = HexToString(end);
|
||||
}
|
||||
|
||||
if (db_path.empty()) {
|
||||
print_help();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Open DB
|
||||
leveldb::Options options;
|
||||
leveldb::DB *db;
|
||||
leveldb::Status status = leveldb::DB::Open(options, db_path, &db);
|
||||
safe_open_db(db_path, &db);
|
||||
|
||||
if (!status.ok()) {
|
||||
fprintf(stderr, "%s\n", status.ToString().c_str());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Print DB stats if desired
|
||||
if (print_stats) {
|
||||
std::string stats;
|
||||
if (db->GetProperty("leveldb.stats", &stats)) {
|
||||
@ -96,7 +83,7 @@ int main(int argc, char** argv) {
|
||||
|
||||
// Setup key iterator
|
||||
leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions());
|
||||
status = iter->status();
|
||||
leveldb::Status status = iter->status();
|
||||
if (!status.ok()) {
|
||||
fprintf(stderr, "%s\n", status.ToString().c_str());
|
||||
delete db;
|
||||
@ -140,7 +127,108 @@ int main(int argc, char** argv) {
|
||||
// Clean up
|
||||
delete iter;
|
||||
delete db;
|
||||
}
|
||||
|
||||
static void compact(
|
||||
const std::string dbname,
|
||||
std::string from,
|
||||
std::string to,
|
||||
const bool hex
|
||||
) {
|
||||
|
||||
leveldb::DB* db;
|
||||
safe_open_db(dbname, &db);
|
||||
|
||||
if(hex) {
|
||||
from = HexToString(from);
|
||||
to = HexToString(to);
|
||||
}
|
||||
|
||||
leveldb::Slice* begin = from.empty() ? NULL : new leveldb::Slice(from);
|
||||
leveldb::Slice* end = to.empty() ? NULL : new leveldb::Slice(to);
|
||||
db->CompactRange(begin, end);
|
||||
delete db;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
enum {
|
||||
DUMP, COMPACT
|
||||
} command;
|
||||
|
||||
if (argc < 2) {
|
||||
print_usage();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
size_t n;
|
||||
|
||||
const std::string dbnameKey = "--db-path=";
|
||||
const std::string toKey = "--to=";
|
||||
const std::string fromKey = "--from=";
|
||||
std::string dbname;
|
||||
bool dbnameFound = false;
|
||||
std::string from;
|
||||
std::string to;
|
||||
int64_t temp;
|
||||
int64_t max_keys = -1;
|
||||
|
||||
bool print_stats = false;
|
||||
bool count_only = false;
|
||||
bool hex = false;
|
||||
char junk;
|
||||
|
||||
|
||||
std::string commandString = argv[1];
|
||||
if (commandString == "dump") {
|
||||
command = DUMP;
|
||||
} else if (commandString == "compact") {
|
||||
command = COMPACT;
|
||||
} else {
|
||||
print_usage();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
for (int i = 2; i < argc; i++) {
|
||||
std::string param(argv[i]);
|
||||
if ((n = param.find(dbnameKey)) != std::string::npos) {
|
||||
dbname = param.substr(dbnameKey.size());
|
||||
dbnameFound = true;
|
||||
} else if ((n = param.find(fromKey)) != std::string::npos) {
|
||||
from = param.substr(fromKey.size());
|
||||
} else if ((n = param.find(toKey)) != std::string::npos) {
|
||||
to = param.substr(toKey.size());
|
||||
} else if (sscanf(argv[i], "--max_keys=%ld%c", &temp, &junk) == 1) {
|
||||
max_keys = temp;
|
||||
} else if (strncmp(argv[i], "--stats", 7) == 0) {
|
||||
print_stats = true;
|
||||
} else if (strncmp(argv[i], "--count_only", 12) == 0) {
|
||||
count_only = true;
|
||||
} else if (strncmp(argv[i], "--hex", 5) == 0) {
|
||||
hex = true;
|
||||
} else {
|
||||
print_usage();
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dbnameFound || dbname.empty()) {
|
||||
fprintf(stderr, "DB path required. See help\n");
|
||||
print_usage();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
switch(command) {
|
||||
case DUMP:
|
||||
dump_db(dbname, from, to, max_keys, hex, print_stats, count_only);
|
||||
break;
|
||||
case COMPACT:
|
||||
compact(dbname, from, to, hex);
|
||||
break;
|
||||
default:
|
||||
print_usage();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user