From 11ce6a060e0e4d742c75985ed1405a41f9a814b0 Mon Sep 17 00:00:00 2001 From: Dilip Antony Joseph Date: Fri, 11 Jan 2013 11:09:23 -0800 Subject: [PATCH] Enhanced ldb to support data access commands Summary: Added put/get/scan/batchput/delete/approxsize Test Plan: Added pyunit script to test the newly added commands Reviewers: chip, leveldb Reviewed By: chip CC: zshao, emayanke Differential Revision: https://reviews.facebook.net/D7947 --- Makefile | 5 +- tools/ldb.cc | 107 ++-- tools/ldb_test.py | 308 ++++++++++ tools/reduce_levels_test.cc | 12 +- util/ldb_cmd.cc | 1070 +++++++++++++++++++++++---------- util/ldb_cmd.h | 580 +++++++++++------- util/ldb_cmd_execute_result.h | 74 +++ 7 files changed, 1573 insertions(+), 583 deletions(-) create mode 100644 tools/ldb_test.py create mode 100644 util/ldb_cmd_execute_result.h diff --git a/Makefile b/Makefile index 4436aa0d6..86beb05e8 100644 --- a/Makefile +++ b/Makefile @@ -104,9 +104,12 @@ release: make clean OPT=-DNDEBUG make -j32 -check: all $(PROGRAMS) $(TESTS) $(TOOLS) +check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done +ldb_tests: all $(PROGRAMS) $(TOOLS) + python tools/ldb_test.py + clean: -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk -rm -rf ios-x86/* ios-arm/* diff --git a/tools/ldb.cc b/tools/ldb.cc index 7585e75d3..56e3d9670 100644 --- a/tools/ldb.cc +++ b/tools/ldb.cc @@ -10,36 +10,53 @@ class LDBCommandRunner { public: static void PrintHelp(const char* exec_name) { - std::string ret; - ret.append("--- compact ----:\n"); - ret.append(exec_name); - ret.append(" compact "); - Compactor::Help(ret); + string ret; - ret.append("\n--- dump ----:\n"); - ret.append(exec_name); - ret.append(" dump "); - DBDumper::Help(ret); + ret.append("ldb - LevelDB Tool"); + ret.append("\n\n"); + ret.append("All commands MUST specify --" + LDBCommand::ARG_DB + + "=\n"); + ret.append("\n"); + ret.append("The following optional parameters control if keys/values are " + "input/output as hex or as plain strings:\n"); + ret.append(" --" + LDBCommand::ARG_KEY_HEX + + " : Keys are input/output as hex\n"); + ret.append(" --" + LDBCommand::ARG_VALUE_HEX + + " : Values are input/output as hex\n"); + ret.append(" --" + LDBCommand::ARG_HEX + + " : Both keys and values are input/output as hex\n"); + ret.append("\n"); - ret.append("\n--- load ----:\n"); - ret.append(exec_name); - ret.append(" load "); - DBLoader::Help(ret); + ret.append("The following optional parameters control the database " + "internals:\n"); + ret.append(" --" + LDBCommand::ARG_BLOOM_BITS + "=\n"); + ret.append(" --" + LDBCommand::ARG_COMPRESSION_TYPE + + "=\n"); + ret.append(" --" + LDBCommand::ARG_BLOCK_SIZE + + "=\n"); + ret.append(" --" + LDBCommand::ARG_AUTO_COMPACTION + "=\n"); + ret.append(" --" + LDBCommand::ARG_WRITE_BUFFER_SIZE + + "=\n"); + ret.append(" --" + LDBCommand::ARG_FILE_SIZE + "=\n"); - ret.append("\n--- query ----:\n"); - ret.append(exec_name); - ret.append(" query "); - DBQuerier::Help(ret); + ret.append("\n\n"); + ret.append("Data Access Commands:\n"); + PutCommand::Help(ret); + GetCommand::Help(ret); + BatchPutCommand::Help(ret); + ScanCommand::Help(ret); + DeleteCommand::Help(ret); + DBQuerierCommand::Help(ret); + ApproxSizeCommand::Help(ret); - ret.append("\n---reduce_levels ----:\n"); - ret.append(exec_name); - ret.append(" reduce_levels "); - ReduceDBLevels::Help(ret); + ret.append("\n\n"); + ret.append("Admin Commands:\n"); + WALDumperCommand::Help(ret); + CompactorCommand::Help(ret); + ReduceDBLevelsCommand::Help(ret); + DBDumperCommand::Help(ret); + DBLoaderCommand::Help(ret); - ret.append("\n---dump_wal----:\n"); - ret.append(exec_name); - ret.append(" dump_wal "); - WALDumper::Help(ret); fprintf(stderr, "%s\n", ret.c_str()); } @@ -48,38 +65,15 @@ public: PrintHelp(argv[0]); exit(1); } - const char* cmd = argv[1]; - std::string db_name; - std::vector args; - for (int i = 2; i < argc; i++) { - if (strncmp(argv[i], "--db=", strlen("--db=")) == 0) { - db_name = argv[i] + strlen("--db="); - } else { - args.push_back(argv[i]); - } + + LDBCommand* cmdObj = LDBCommand::InitFromCmdLineArgs(argc, argv); + if (cmdObj == NULL) { + fprintf(stderr, "Unknown command\n"); + PrintHelp(argv[0]); + exit(1); } - LDBCommand* cmdObj = NULL; - if (strcmp(cmd, "compact") == 0) { - // run compactor - cmdObj = new Compactor(db_name, args); - } else if (strcmp(cmd, "dump") == 0) { - // run dump - cmdObj = new DBDumper(db_name, args); - } else if (strcmp(cmd, "load") == 0) { - // run loader - cmdObj = new DBLoader(db_name, args); - } else if (strcmp(cmd, "query") == 0) { - // run querier - cmdObj = new DBQuerier(db_name, args); - } else if (strcmp(cmd, "reduce_levels") == 0) { - // reduce db levels - cmdObj = new ReduceDBLevels(db_name, args); - } else if (strcmp(cmd, "dump_wal") == 0) { - cmdObj = new WALDumper(args); - } else { - fprintf(stderr, "Unknown command: %s\n", cmd); - PrintHelp(argv[0]); + if (!cmdObj->ValidateCmdLineOptions()) { exit(1); } @@ -87,7 +81,10 @@ public: LDBCommandExecuteResult ret = cmdObj->GetExecuteState(); fprintf(stderr, "%s\n", ret.ToString().c_str()); delete cmdObj; + + exit(ret.IsFailed()); } + }; } diff --git a/tools/ldb_test.py b/tools/ldb_test.py new file mode 100644 index 000000000..142a5a890 --- /dev/null +++ b/tools/ldb_test.py @@ -0,0 +1,308 @@ +import os +import os.path +import shutil +import subprocess +import time +import unittest +import tempfile + +def my_check_output(*popenargs, **kwargs): + """ + If we had python 2.7, we should simply use subprocess.check_output. + This is a stop-gap solution for python 2.6 + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise Exception("Exit code is not 0. It is %d. Command: %s" % + (retcode, cmd)) + return output + + +class LDBTestCase(unittest.TestCase): + def setUp(self): + self.TMP_DIR = tempfile.mkdtemp(prefix="ldb_test_") + self.DB_NAME = "testdb" + + def tearDown(self): + assert(self.TMP_DIR.strip() != "/" + and self.TMP_DIR.strip() != "/tmp" + and self.TMP_DIR.strip() != "/tmp/") #Just some paranoia + + shutil.rmtree(self.TMP_DIR) + + def dbParam(self, dbName): + return "--db=%s" % os.path.join(self.TMP_DIR, dbName) + + def assertRunOKFull(self, params, expectedOutput): + """ + All command-line params must be specified. + Allows full flexibility in testing; for example: missing db param. + + """ + + output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" % + params, shell=True) + self.assertEquals(output.strip(), expectedOutput.strip()); + + def assertRunFAILFull(self, params): + """ + All command-line params must be specified. + Allows full flexibility in testing; for example: missing db param. + + """ + try: + my_check_output("./ldb %s |grep -v \"Created bg thread\"" % params, + shell=True) + except Exception, e: + return + self.fail( + "Exception should have been raised for command with params: %s" % + params) + + def assertRunOK(self, params, expectedOutput): + """ + Uses the default test db. + + """ + self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params), + expectedOutput) + + def assertRunFAIL(self, params): + """ + Uses the default test db. + """ + self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params)) + + def testSimpleStringPutGet(self): + self.assertRunFAIL("put x1 y1") + self.assertRunOK("put --create_if_missing x1 y1", "OK") + self.assertRunOK("get x1", "y1") + self.assertRunFAIL("get x2") + + self.assertRunOK("put x2 y2", "OK") + self.assertRunOK("get x1", "y1") + self.assertRunOK("get x2", "y2") + self.assertRunFAIL("get x3") + + self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2") + self.assertRunOK("put x3 y3", "OK") + + self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2\nx3 : y3") + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3") + self.assertRunOK("scan --from=x", "x1 : y1\nx2 : y2\nx3 : y3") + + self.assertRunOK("scan --to=x2", "x1 : y1") + self.assertRunOK("scan --from=x1 --to=z --max_keys=1", "x1 : y1") + self.assertRunOK("scan --from=x1 --to=z --max_keys=2", + "x1 : y1\nx2 : y2") + + self.assertRunOK("scan --from=x1 --to=z --max_keys=3", + "x1 : y1\nx2 : y2\nx3 : y3") + self.assertRunOK("scan --from=x1 --to=z --max_keys=4", + "x1 : y1\nx2 : y2\nx3 : y3") + self.assertRunOK("scan --from=x1 --to=x2", "x1 : y1") + self.assertRunOK("scan --from=x2 --to=x4", "x2 : y2\nx3 : y3") + self.assertRunFAIL("scan --from=x4 --to=z") # No results => FAIL + self.assertRunFAIL("scan --from=x1 --to=z --max_keys=foo") + + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3") + + self.assertRunOK("delete x1", "OK") + self.assertRunOK("scan", "x2 : y2\nx3 : y3") + + self.assertRunOK("delete NonExistentKey", "OK") + # It is wierd that GET and SCAN raise exception for + # non-existent key, while delete does not + + def dumpDb(self, params, dumpFile): + return 0 == os.system("./ldb dump %s > %s" % (params, dumpFile)) + + def loadDb(self, params, dumpFile): + return 0 == os.system("cat %s | ./ldb load %s" % (dumpFile, params)) + + def testStringBatchPut(self): + self.assertRunOK("batchput x1 y1 --create_if_missing", "OK") + self.assertRunOK("scan", "x1 : y1") + self.assertRunOK("batchput x2 y2 x3 y3 \"x4 abc\" \"y4 xyz\"", "OK") + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz") + self.assertRunFAIL("batchput") + self.assertRunFAIL("batchput k1") + self.assertRunFAIL("batchput k1 v1 k2") + + + def testHexPutGet(self): + self.assertRunOK("put a1 b1 --create_if_missing", "OK") + self.assertRunOK("scan", "a1 : b1") + self.assertRunOK("scan --hex", "0x6131 : 0x6231") + self.assertRunFAIL("put --hex 6132 6232") + self.assertRunOK("put --hex 0x6132 0x6232", "OK") + self.assertRunOK("scan --hex", "0x6131 : 0x6231\n0x6132 : 0x6232") + self.assertRunOK("scan", "a1 : b1\na2 : b2") + self.assertRunOK("get a1", "b1") + self.assertRunOK("get --hex 0x6131", "0x6231") + self.assertRunOK("get a2", "b2") + self.assertRunOK("get --hex 0x6132", "0x6232") + self.assertRunOK("get --key_hex 0x6132", "b2") + self.assertRunOK("get --key_hex --value_hex 0x6132", "0x6232") + self.assertRunOK("get --value_hex a2", "0x6232") + self.assertRunOK("scan --key_hex --value_hex", + "0x6131 : 0x6231\n0x6132 : 0x6232") + self.assertRunOK("scan --hex --from=0x6131 --to=0x6133", + "0x6131 : 0x6231\n0x6132 : 0x6232") + self.assertRunOK("scan --hex --from=0x6131 --to=0x6132", + "0x6131 : 0x6231") + self.assertRunOK("scan --key_hex", "0x6131 : b1\n0x6132 : b2") + self.assertRunOK("scan --value_hex", "a1 : 0x6231\na2 : 0x6232") + self.assertRunOK("batchput --hex 0x6133 0x6233 0x6134 0x6234", "OK") + self.assertRunOK("scan", "a1 : b1\na2 : b2\na3 : b3\na4 : b4") + self.assertRunOK("delete --hex 0x6133", "OK") + self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4") + + + def testInvalidCmdLines(self): + # db not specified + self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing") + # No param called he + self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing") + # max_keys is not applicable for put + self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing") + # hex has invalid boolean value + self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing") + + + def testDumpLoad(self): + self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", + "OK") + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME) + + # Dump and load without any additional params specified + dumpFilePath = os.path.join(self.TMP_DIR, "dump1") + loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump1") + self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath)) + self.assertTrue(self.loadDb( + "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)) + self.assertRunOKFull("scan --db=%s" % loadedDbPath, + "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + # Dump and load in hex + dumpFilePath = os.path.join(self.TMP_DIR, "dump2") + loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump2") + self.assertTrue(self.dumpDb("--db=%s --hex" % origDbPath, dumpFilePath)) + self.assertTrue(self.loadDb( + "--db=%s --hex --create_if_missing" % loadedDbPath, dumpFilePath)) + self.assertRunOKFull("scan --db=%s" % loadedDbPath, + "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + # Dump only a portion of the key range + dumpFilePath = os.path.join(self.TMP_DIR, "dump3") + loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump3") + self.assertTrue(self.dumpDb( + "--db=%s --from=x1 --to=x3" % origDbPath, dumpFilePath)) + self.assertTrue(self.loadDb( + "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)) + self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2") + + # Dump upto max_keys rows + dumpFilePath = os.path.join(self.TMP_DIR, "dump4") + loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump4") + self.assertTrue(self.dumpDb( + "--db=%s --max_keys=3" % origDbPath, dumpFilePath)) + self.assertTrue(self.loadDb( + "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)) + self.assertRunOKFull("scan --db=%s" % loadedDbPath, + "x1 : y1\nx2 : y2\nx3 : y3") + + # Load into an existing db, create_if_missing is not specified + self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath)) + self.assertTrue(self.loadDb("--db=%s" % loadedDbPath, dumpFilePath)) + self.assertRunOKFull("scan --db=%s" % loadedDbPath, + "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + # Dump and load with WAL disabled + dumpFilePath = os.path.join(self.TMP_DIR, "dump5") + loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump5") + self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath)) + self.assertTrue(self.loadDb( + "--db=%s --disable_wal --create_if_missing" % loadedDbPath, + dumpFilePath)) + self.assertRunOKFull("scan --db=%s" % loadedDbPath, + "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + # Dump and load with lots of extra params specified + extraParams = " ".join(["--bloom_bits=14", "--compression_type=bzip2", + "--block_size=1024", "--auto_compaction=true", + "--write_buffer_size=4194304", + "--file_size=2097152"]) + dumpFilePath = os.path.join(self.TMP_DIR, "dump6") + loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump6") + self.assertTrue(self.dumpDb( + "--db=%s %s" % (origDbPath, extraParams), dumpFilePath)) + self.assertTrue(self.loadDb( + "--db=%s %s --create_if_missing" % (loadedDbPath, extraParams), + dumpFilePath)) + self.assertRunOKFull("scan --db=%s" % loadedDbPath, + "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + # Dump with count_only + dumpFilePath = os.path.join(self.TMP_DIR, "dump7") + loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump7") + self.assertTrue(self.dumpDb( + "--db=%s --count_only" % origDbPath, dumpFilePath)) + self.assertTrue(self.loadDb( + "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)) + # DB should have atleast one value for scan to work + self.assertRunOKFull("put --db=%s k1 v1" % loadedDbPath, "OK") + self.assertRunOKFull("scan --db=%s" % loadedDbPath, "k1 : v1") + + # Dump command fails because of typo in params + dumpFilePath = os.path.join(self.TMP_DIR, "dump8") + self.assertFalse(self.dumpDb( + "--db=%s --create_if_missin" % origDbPath, dumpFilePath)) + + + def testMiscAdminTask(self): + # These tests need to be improved; for example with asserts about + # whether compaction or level reduction actually took place. + self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", + "OK") + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME) + + self.assertTrue(0 == os.system("./ldb compact --db=%s" % origDbPath)) + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + self.assertTrue(0 == os.system( + "./ldb reduce_levels --db=%s --new_levels=2" % origDbPath)) + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + self.assertTrue(0 == os.system( + "./ldb reduce_levels --db=%s --new_levels=3" % origDbPath)) + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + self.assertTrue(0 == os.system( + "./ldb compact --db=%s --from=x1 --to=x3" % origDbPath)) + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + self.assertTrue(0 == os.system( + "./ldb compact --db=%s --hex --from=0x6131 --to=0x6134" % + origDbPath)) + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + #TODO(dilip): Not sure what should be passed to WAL.Currently corrupted. + self.assertTrue(0 == os.system( + "./ldb dump_wal --db=%s --walfile=%s --header" % ( + origDbPath, os.path.join(origDbPath, "LOG")))) + self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + + +if __name__ == "__main__": + unittest.main() + diff --git a/tools/reduce_levels_test.cc b/tools/reduce_levels_test.cc index f7e9377c2..1e48e296e 100644 --- a/tools/reduce_levels_test.cc +++ b/tools/reduce_levels_test.cc @@ -84,11 +84,13 @@ Status ReduceLevelTest::OpenDB(bool create_if_missing, int num_levels, } bool ReduceLevelTest::ReduceLevels(int target_level) { - std::vector args = leveldb::ReduceDBLevels::PrepareArgs( - target_level, false); - ReduceDBLevels level_reducer(dbname_, args); - level_reducer.Run(); - return level_reducer.GetExecuteState().IsSucceed(); + std::vector args = leveldb::ReduceDBLevelsCommand::PrepareArgs( + dbname_, target_level, false); + LDBCommand* level_reducer = LDBCommand::InitFromCmdLineArgs(args); + level_reducer->Run(); + bool is_succeed = level_reducer->GetExecuteState().IsSucceed(); + delete level_reducer; + return is_succeed; } TEST(ReduceLevelTest, Last_Level) { diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index 4fda2da4a..e60ea60ac 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include +#include #include "leveldb/write_batch.h" #include "db/dbformat.h" @@ -11,129 +13,281 @@ namespace leveldb { -const char* LDBCommand::BLOOM_ARG = "--bloom_bits="; -const char* LDBCommand::COMPRESSION_TYPE_ARG = "--compression_type="; -const char* LDBCommand::BLOCK_SIZE = "--block_size="; -const char* LDBCommand::AUTO_COMPACTION = "--auto_compaction="; -const char* LDBCommand::WRITE_BUFFER_SIZE_ARG = "--write_buffer_size="; -const char* LDBCommand::FILE_SIZE_ARG = "--file_size="; +const string LDBCommand::ARG_DB = "db"; +const string LDBCommand::ARG_HEX = "hex"; +const string LDBCommand::ARG_KEY_HEX = "key_hex"; +const string LDBCommand::ARG_VALUE_HEX = "value_hex"; +const string LDBCommand::ARG_FROM = "from"; +const string LDBCommand::ARG_TO = "to"; +const string LDBCommand::ARG_MAX_KEYS = "max_keys"; +const string LDBCommand::ARG_BLOOM_BITS = "bloom_bits"; +const string LDBCommand::ARG_COMPRESSION_TYPE = "compression_type"; +const string LDBCommand::ARG_BLOCK_SIZE = "block_size"; +const string LDBCommand::ARG_AUTO_COMPACTION = "auto_compaction"; +const string LDBCommand::ARG_WRITE_BUFFER_SIZE = "write_buffer_size"; +const string LDBCommand::ARG_FILE_SIZE = "file_size"; +const string LDBCommand::ARG_CREATE_IF_MISSING = "create_if_missing"; + const char* LDBCommand::DELIM = " ==> "; -void LDBCommand::parse_open_args(std::vector& args) { - std::vector rest_of_args; - for (unsigned int i = 0; i < args.size(); i++) { - std::string& arg = args.at(i); - if (arg.find(BLOOM_ARG) == 0 - || arg.find(COMPRESSION_TYPE_ARG) == 0 - || arg.find(BLOCK_SIZE) == 0 - || arg.find(AUTO_COMPACTION) == 0 - || arg.find(WRITE_BUFFER_SIZE_ARG) == 0 - || arg.find(FILE_SIZE_ARG) == 0) { - open_args_.push_back(arg); +LDBCommand* LDBCommand::InitFromCmdLineArgs(int argc, char** argv) { + vector args; + for (int i = 1; i < argc; i++) { + args.push_back(argv[i]); + } + return InitFromCmdLineArgs(args); +} + +/** + * Parse the command-line arguments and create the appropriate LDBCommand2 + * instance. + * The command line arguments must be in the following format: + * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] .. + * COMMAND ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] .. + * This is similar to the command line format used by HBaseClientTool. + * Command name is not included in args. + * Returns NULL if the command-line cannot be parsed. + */ +LDBCommand* LDBCommand::InitFromCmdLineArgs(const vector& args) { + // --x=y command line arguments are added as x->y map entries. + map options; + + // Command-line arguments of the form --hex end up in this array as hex + vector flags; + + // Everything other than options and flags. Represents commands + // and their parameters. For eg: put key1 value1 go into this vector. + vector cmdTokens; + + const string OPTION_PREFIX = "--"; + + for (vector::const_iterator itr = args.begin(); + itr != args.end(); itr++) { + string arg = *itr; + if (boost::starts_with(arg, OPTION_PREFIX)){ + vector splits; + boost::split(splits, arg, boost::is_any_of("=")); + if (splits.size() == 2) { + string optionKey = splits[0].substr(OPTION_PREFIX.size()); + options[optionKey] = splits[1]; + } else { + string optionKey = splits[0].substr(OPTION_PREFIX.size()); + flags.push_back(optionKey); + } } else { - rest_of_args.push_back(arg); + cmdTokens.push_back(string(arg)); } } - swap(args, rest_of_args); + + if (cmdTokens.size() < 1) { + fprintf(stderr, "Command not specified!"); + return NULL; + } + + string cmd = cmdTokens[0]; + vector cmdParams(cmdTokens.begin()+1, cmdTokens.end()); + + if (cmd == GetCommand::Name()) { + return new GetCommand(cmdParams, options, flags); + } else if (cmd == PutCommand::Name()) { + return new PutCommand(cmdParams, options, flags); + } else if (cmd == BatchPutCommand::Name()) { + return new BatchPutCommand(cmdParams, options, flags); + } else if (cmd == ScanCommand::Name()) { + return new ScanCommand(cmdParams, options, flags); + } else if (cmd == DeleteCommand::Name()) { + return new DeleteCommand(cmdParams, options, flags); + } else if (cmd == ApproxSizeCommand::Name()) { + return new ApproxSizeCommand(cmdParams, options, flags); + } else if (cmd == DBQuerierCommand::Name()) { + return new DBQuerierCommand(cmdParams, options, flags); + } else if (cmd == CompactorCommand::Name()) { + return new CompactorCommand(cmdParams, options, flags); + } else if (cmd == WALDumperCommand::Name()) { + return new WALDumperCommand(cmdParams, options, flags); + } else if (cmd == ReduceDBLevelsCommand::Name()) { + return new ReduceDBLevelsCommand(cmdParams, options, flags); + } else if (cmd == DBDumperCommand::Name()) { + return new DBDumperCommand(cmdParams, options, flags); + } else if (cmd == DBLoaderCommand::Name()) { + return new DBLoaderCommand(cmdParams, options, flags); + } + + return NULL; +} + +/** + * Parses the specific integer option and fills in the value. + * Returns true if the option is found. + * Returns false if the option is not found or if there is an error parsing the + * value. If there is an error, the specified exec_state is also + * updated. + */ +bool LDBCommand::ParseIntOption(const map& options, + string option, int& value, LDBCommandExecuteResult& exec_state) { + + map::const_iterator itr = options_.find(option); + if (itr != options_.end()) { + try { + value = boost::lexical_cast(itr->second); + return true; + } catch( const boost::bad_lexical_cast & ) { + exec_state = LDBCommandExecuteResult::FAILED(option + + " has an invalid value."); + } + } + return false; } leveldb::Options LDBCommand::PrepareOptionsForOpenDB() { + leveldb::Options opt; opt.create_if_missing = false; - for (unsigned int i = 0; i < open_args_.size(); i++) { - std::string& arg = open_args_.at(i); - if (arg.find(BLOOM_ARG) == 0) { - std::string bits_string = arg.substr(strlen(BLOOM_ARG)); - int bits = atoi(bits_string.c_str()); - if (bits == 0) { - // Badly-formatted bits. - exec_state_ = LDBCommandExecuteResult::FAILED( - std::string("Badly-formatted bits: ") + bits_string); - } + + map::const_iterator itr; + + int bits; + if (ParseIntOption(options_, ARG_BLOOM_BITS, bits, exec_state_)) { + if (bits > 0) { opt.filter_policy = leveldb::NewBloomFilterPolicy(bits); - } else if (arg.find(BLOCK_SIZE) == 0) { - std::string block_size_string = arg.substr(strlen(BLOCK_SIZE)); - int block_size = atoi(block_size_string.c_str()); - if (block_size == 0) { - // Badly-formatted bits. - exec_state_ = LDBCommandExecuteResult::FAILED( - std::string("Badly-formatted block size: ") + block_size_string); - } + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_BLOOM_BITS + + " must be > 0."); + } + } + + int block_size; + if (ParseIntOption(options_, ARG_BLOCK_SIZE, block_size, exec_state_)) { + if (block_size > 0) { opt.block_size = block_size; - } else if (arg.find(AUTO_COMPACTION) == 0) { - std::string value = arg.substr(strlen(AUTO_COMPACTION)); - if (value == "false") { - opt.disable_auto_compactions = true; - } else if (value == "true") { - opt.disable_auto_compactions = false; - } else { - // Unknown compression. - exec_state_ = LDBCommandExecuteResult::FAILED( - "Unknown auto_compaction value: " + value); - } - } else if (arg.find(COMPRESSION_TYPE_ARG) == 0) { - std::string comp = arg.substr(strlen(COMPRESSION_TYPE_ARG)); - if (comp == "no") { - opt.compression = leveldb::kNoCompression; - } else if (comp == "snappy") { - opt.compression = leveldb::kSnappyCompression; - } else if (comp == "zlib") { - opt.compression = leveldb::kZlibCompression; - } else if (comp == "bzip2") { - opt.compression = leveldb::kBZip2Compression; - } else { - // Unknown compression. - exec_state_ = LDBCommandExecuteResult::FAILED( - "Unknown compression level: " + comp); - } - } else if (arg.find(WRITE_BUFFER_SIZE_ARG) == 0) { - std::string write_buffer_str = arg.substr(strlen(WRITE_BUFFER_SIZE_ARG)); - int write_buffer_size = atoi(write_buffer_str.c_str()); - if (write_buffer_size == 0) { - exec_state_ = LDBCommandExecuteResult::FAILED( - std::string("Badly-formatted buffer size: ") + write_buffer_str); - } + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_BLOCK_SIZE + + " must be > 0."); + } + } + + itr = options_.find(ARG_AUTO_COMPACTION); + if (itr != options_.end()) { + opt.disable_auto_compactions = ! StringToBool(itr->second); + } + + itr = options_.find(ARG_COMPRESSION_TYPE); + if (itr != options_.end()) { + string comp = itr->second; + if (comp == "no") { + opt.compression = leveldb::kNoCompression; + } else if (comp == "snappy") { + opt.compression = leveldb::kSnappyCompression; + } else if (comp == "zlib") { + opt.compression = leveldb::kZlibCompression; + } else if (comp == "bzip2") { + opt.compression = leveldb::kBZip2Compression; + } else { + // Unknown compression. + exec_state_ = LDBCommandExecuteResult::FAILED( + "Unknown compression level: " + comp); + } + } + + int write_buffer_size; + if (ParseIntOption(options_, ARG_WRITE_BUFFER_SIZE, write_buffer_size, + exec_state_)) { + if (write_buffer_size > 0) { opt.write_buffer_size = write_buffer_size; - } else if (arg.find(FILE_SIZE_ARG) == 0) { - std::string file_size_str = arg.substr(strlen(FILE_SIZE_ARG)); - int file_size = atoi(file_size_str.c_str()); - if (file_size == 0) { - exec_state_ = LDBCommandExecuteResult::FAILED( - std::string("Badly-formatted file size: ") + file_size_str); - } + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_WRITE_BUFFER_SIZE + + " must be > 0."); + } + } + + int file_size; + if (ParseIntOption(options_, ARG_FILE_SIZE, file_size, exec_state_)) { + if (file_size > 0) { opt.target_file_size_base = file_size; } else { - exec_state_ = LDBCommandExecuteResult::FAILED( - "Unknown option: " + arg); + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_FILE_SIZE + + " must be > 0."); } } return opt; } +bool LDBCommand::ParseKeyValue(const string& line, string* key, string* value, + bool is_key_hex, bool is_value_hex) { + size_t pos = line.find(DELIM); + if (pos != std::string::npos) { + *key = line.substr(0, pos); + *value = line.substr(pos + strlen(DELIM)); + if (is_key_hex) { + *key = HexToString(*key); + } + if (is_value_hex) { + *value = HexToString(*value); + } + return true; + } else { + return false; + } +} -const char* LDBCommand::FROM_ARG = "--from="; -const char* LDBCommand::END_ARG = "--to="; -const char* LDBCommand::HEX_ARG = "--hex"; +/** + * Make sure that ONLY the command-line options and flags expected by this + * command are specified on the command-line. Extraneous options are usually + * the result of user error. + * Returns true if all checks pass. Else returns false, and prints an + * appropriate error msg to stderr. + */ +bool LDBCommand::ValidateCmdLineOptions() { -Compactor::Compactor(std::string& db_name, std::vector& args) : - LDBCommand(db_name, args), null_from_(true), null_to_(true), hex_(false) { - for (unsigned int i = 0; i < args.size(); i++) { - std::string& arg = args.at(i); - if (arg.find(FROM_ARG) == 0) { - null_from_ = false; - from_ = arg.substr(strlen(FROM_ARG)); - } else if (arg.find(END_ARG) == 0) { - null_to_ = false; - to_ = arg.substr(strlen(END_ARG)); - } else if (arg == HEX_ARG) { - hex_ = true; - } else { - exec_state_ = LDBCommandExecuteResult::FAILED("Unknown argument." + arg); + for (map::const_iterator itr = options_.begin(); + itr != options_.end(); itr++) { + if (std::find(valid_cmd_line_options_.begin(), + valid_cmd_line_options_.end(), itr->first) == + valid_cmd_line_options_.end()) { + fprintf(stderr, "Invalid command-line option %s\n", itr->first.c_str()); + return false; } } - if (hex_) { + for (vector::const_iterator itr = flags_.begin(); + itr != flags_.end(); itr++) { + if (std::find(valid_cmd_line_options_.begin(), + valid_cmd_line_options_.end(), *itr) == + valid_cmd_line_options_.end()) { + fprintf(stderr, "Invalid command-line flag %s\n", itr->c_str()); + return false; + } + } + + if (options_.find(ARG_DB) == options_.end()) { + fprintf(stderr, "%s must be specified\n", ARG_DB.c_str()); + return false; + } + + return true; +} + +CompactorCommand::CompactorCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_FROM, ARG_TO, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX})), + null_from_(true), null_to_(true) { + + map::const_iterator itr = options.find(ARG_FROM); + if (itr != options.end()) { + null_from_ = false; + from_ = itr->second; + } + + itr = options.find(ARG_TO); + if (itr != options.end()) { + null_to_ = false; + to_ = itr->second; + } + + if (is_key_hex_) { if (!null_from_) { from_ = HexToString(from_); } @@ -143,14 +297,14 @@ Compactor::Compactor(std::string& db_name, std::vector& args) : } } -void Compactor::Help(std::string& ret) { - LDBCommand::Help(ret); - ret.append("[--from=START KEY] "); - ret.append("[--to=START KEY] "); - ret.append("[--hex] "); +void CompactorCommand::Help(string& ret) { + ret.append(" "); + ret.append(CompactorCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append("\n"); } -void Compactor::DoCommand() { +void CompactorCommand::DoCommand() { leveldb::Slice* begin = NULL; leveldb::Slice* end = NULL; @@ -168,46 +322,35 @@ void Compactor::DoCommand() { delete end; } -const char* DBLoader::HEX_INPUT_ARG = "--input_hex"; -const char* DBLoader::CREATE_IF_MISSING_ARG = "--create_if_missing"; -const char* DBLoader::DISABLE_WAL_ARG = "--disable_wal"; +const string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal"; -DBLoader::DBLoader(std::string& db_name, std::vector& args) : - LDBCommand(db_name, args), - hex_input_(false), +DBLoaderCommand::DBLoaderCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + ARG_FROM, ARG_TO, ARG_CREATE_IF_MISSING, + ARG_DISABLE_WAL})), create_if_missing_(false) { - for (unsigned int i = 0; i < args.size(); i++) { - std::string& arg = args.at(i); - if (arg == HEX_INPUT_ARG) { - hex_input_ = true; - } else if (arg == CREATE_IF_MISSING_ARG) { - create_if_missing_ = true; - } else if (arg == DISABLE_WAL_ARG) { - disable_wal_ = true; - } else { - exec_state_ = LDBCommandExecuteResult::FAILED("Unknown argument:" + arg); - } - } + + create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING); + disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL); } -void DBLoader::Help(std::string& ret) { - LDBCommand::Help(ret); - ret.append("["); - ret.append(HEX_INPUT_ARG); - ret.append("] ["); - ret.append(CREATE_IF_MISSING_ARG); - ret.append("] ["); - ret.append(DISABLE_WAL_ARG); - ret.append("]"); +void DBLoaderCommand::Help(string& ret) { + ret.append(" "); + ret.append(DBLoaderCommand::Name()); + ret.append(" [--" + ARG_CREATE_IF_MISSING + "]"); + ret.append(" [--" + ARG_DISABLE_WAL + "]"); + ret.append("\n"); } -leveldb::Options DBLoader::PrepareOptionsForOpenDB() { +leveldb::Options DBLoaderCommand::PrepareOptionsForOpenDB() { leveldb::Options opt = LDBCommand::PrepareOptionsForOpenDB(); opt.create_if_missing = create_if_missing_; return opt; } -void DBLoader::DoCommand() { +void DBLoaderCommand::DoCommand() { if (!db_) { return; } @@ -218,11 +361,11 @@ void DBLoader::DoCommand() { } int bad_lines = 0; - std::string line; + string line; while (std::getline(std::cin, line, '\n')) { - std::string key; - std::string value; - if (ParseKeyValue(line, &key, &value, hex_input_)) { + string key; + string value; + if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) { db_->Put(write_options, Slice(key), Slice(value)); } else if (0 == line.find("Keys in range:")) { // ignore this line @@ -232,50 +375,53 @@ void DBLoader::DoCommand() { bad_lines ++; } } - + if (bad_lines > 0) { std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl; } } -const char* DBDumper::MAX_KEYS_ARG = "--max_keys="; -const char* DBDumper::COUNT_ONLY_ARG = "--count_only"; -const char* DBDumper::STATS_ARG = "--stats"; -const char* DBDumper::HEX_OUTPUT_ARG = "--output_hex"; +const string DBDumperCommand::ARG_COUNT_ONLY = "count_only"; +const string DBDumperCommand::ARG_STATS = "stats"; -DBDumper::DBDumper(std::string& db_name, std::vector& args) : - LDBCommand(db_name, args), +DBDumperCommand::DBDumperCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, true, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + ARG_FROM, ARG_TO, ARG_MAX_KEYS, + ARG_COUNT_ONLY, ARG_STATS})), null_from_(true), null_to_(true), max_keys_(-1), count_only_(false), - print_stats_(false), - hex_(false), - hex_output_(false) { - for (unsigned int i = 0; i < args.size(); i++) { - std::string& arg = args.at(i); - if (arg.find(FROM_ARG) == 0) { - null_from_ = false; - from_ = arg.substr(strlen(FROM_ARG)); - } else if (arg.find(END_ARG) == 0) { - null_to_ = false; - to_ = arg.substr(strlen(END_ARG)); - } else if (arg == HEX_ARG) { - hex_ = true; - } else if (arg.find(MAX_KEYS_ARG) == 0) { - max_keys_ = atoi(arg.substr(strlen(MAX_KEYS_ARG)).c_str()); - } else if (arg == STATS_ARG) { - print_stats_ = true; - } else if (arg == COUNT_ONLY_ARG) { - count_only_ = true; - } else if (arg == HEX_OUTPUT_ARG) { - hex_output_ = true; - } else { - exec_state_ = LDBCommandExecuteResult::FAILED("Unknown argument:" + arg); + print_stats_(false) { + + map::const_iterator itr = options.find(ARG_FROM); + if (itr != options.end()) { + null_from_ = false; + from_ = itr->second; + } + + itr = options.find(ARG_TO); + if (itr != options.end()) { + null_to_ = false; + to_ = itr->second; + } + + itr = options.find(ARG_MAX_KEYS); + if (itr != options.end()) { + try { + max_keys_ = boost::lexical_cast(itr->second); + } catch( const boost::bad_lexical_cast & ) { + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_MAX_KEYS + + " has an invalid value"); } } - if (hex_) { + print_stats_ = IsFlagPresent(flags, ARG_STATS); + count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY); + + if (is_key_hex_) { if (!null_from_) { from_ = HexToString(from_); } @@ -285,25 +431,24 @@ DBDumper::DBDumper(std::string& db_name, std::vector& args) : } } -void DBDumper::Help(std::string& ret) { - LDBCommand::Help(ret); - ret.append("[--from=START KEY] "); - ret.append("[--to=END Key] "); - ret.append("[--hex] "); - ret.append("[--output_hex] "); - ret.append("[--max_keys=NUM] "); - ret.append("[--count_only] "); - ret.append("[--stats] "); +void DBDumperCommand::Help(string& ret) { + ret.append(" "); + ret.append(DBDumperCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append(" [--" + ARG_MAX_KEYS + "=]"); + ret.append(" [--" + ARG_COUNT_ONLY + "]"); + ret.append(" [--" + ARG_STATS + "]"); + ret.append("\n"); } -void DBDumper::DoCommand() { +void DBDumperCommand::DoCommand() { if (!db_) { return; } // Parse command line args uint64_t count = 0; if (print_stats_) { - std::string stats; + string stats; if (db_->GetProperty("leveldb.stats", &stats)) { fprintf(stdout, "%s\n", stats.c_str()); } @@ -336,9 +481,9 @@ void DBDumper::DoCommand() { } ++count; if (!count_only_) { - std::string str = PrintKeyValue(iter->key().ToString(), + string str = PrintKeyValue(iter->key().ToString(), iter->value().ToString(), - hex_output_); + is_key_hex_, is_value_hex_); fprintf(stdout, "%s\n", str.c_str()); } } @@ -347,140 +492,48 @@ void DBDumper::DoCommand() { delete iter; } +const string ReduceDBLevelsCommand::ARG_NEW_LEVELS = "new_levels"; +const string ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS = "print_old_levels"; -const char* DBQuerier::HEX_ARG = "--hex"; -const char* DBQuerier::HELP_CMD = "help"; -const char* DBQuerier::GET_CMD = "get"; -const char* DBQuerier::PUT_CMD = "put"; -const char* DBQuerier::DELETE_CMD = "delete"; - -DBQuerier::DBQuerier(std::string& db_name, std::vector& args) : - LDBCommand(db_name, args), - hex_(false) { - for (unsigned int i = 0; i < args.size(); i++) { - std::string& arg = args.at(i); - if (arg == HEX_ARG) { - hex_ = true; - } else { - exec_state_ = LDBCommandExecuteResult::FAILED("Unknown argument:" + arg); - } - } -} - -void DBQuerier::Help(std::string& ret) { - LDBCommand::Help(ret); - ret.append("[--hex] "); - ret.append("(type \"help\" on stdin for details.)"); -} - -void DBQuerier::DoCommand() { - if (!db_) { - return; - } - - leveldb::ReadOptions read_options; - leveldb::WriteOptions write_options; - - std::string line; - std::string key; - std::string value; - while (std::getline(std::cin, line, '\n')) { - - // Parse line into vector - std::vector tokens; - size_t pos = 0; - while (true) { - size_t pos2 = line.find(' ', pos); - if (pos2 == std::string::npos) { - break; - } - tokens.push_back(line.substr(pos, pos2-pos)); - pos = pos2 + 1; - } - tokens.push_back(line.substr(pos)); - - const std::string& cmd = tokens[0]; - - if (cmd == HELP_CMD) { - fprintf(stdout, - "get \n" - "put \n" - "delete \n"); - } else if (cmd == DELETE_CMD && tokens.size() == 2) { - key = (hex_ ? HexToString(tokens[1]) : tokens[1]); - db_->Delete(write_options, Slice(key)); - fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str()); - } else if (cmd == PUT_CMD && tokens.size() == 3) { - key = (hex_ ? HexToString(tokens[1]) : tokens[1]); - value = (hex_ ? HexToString(tokens[2]) : tokens[2]); - db_->Put(write_options, Slice(key), Slice(value)); - fprintf(stdout, "Successfully put %s %s\n", - tokens[1].c_str(), tokens[2].c_str()); - } else if (cmd == GET_CMD && tokens.size() == 2) { - key = (hex_ ? HexToString(tokens[1]) : tokens[1]); - if (db_->Get(read_options, Slice(key), &value).ok()) { - fprintf(stdout, "%s\n", PrintKeyValue(key, value, hex_).c_str()); - } else { - fprintf(stdout, "Not found %s\n", tokens[1].c_str()); - } - } else { - fprintf(stdout, "Unknown command %s\n", line.c_str()); - } - } -} +ReduceDBLevelsCommand::ReduceDBLevelsCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_NEW_LEVELS, ARG_PRINT_OLD_LEVELS})), + old_levels_(1 << 16), + new_levels_(-1), + print_old_levels_(false) { - -const char* ReduceDBLevels::NEW_LEVLES_ARG = "--new_levels="; -const char* ReduceDBLevels::PRINT_OLD_LEVELS_ARG = "--print_old_levels"; - -ReduceDBLevels::ReduceDBLevels(std::string& db_name, - std::vector& args) -: LDBCommand(db_name, args), - old_levels_(1 << 16), - new_levels_(-1), - print_old_levels_(false) { - - for (unsigned int i = 0; i < args.size(); i++) { - std::string& arg = args.at(i); - if (arg.find(NEW_LEVLES_ARG) == 0) { - new_levels_ = atoi(arg.substr(strlen(NEW_LEVLES_ARG)).c_str()); - } else if (arg == PRINT_OLD_LEVELS_ARG) { - print_old_levels_ = true; - } else { - exec_state_ = LDBCommandExecuteResult::FAILED( - "Unknown argument." + arg); - } - } + ParseIntOption(options_, ARG_NEW_LEVELS, new_levels_, exec_state_); + print_old_levels_ = IsFlagPresent(flags, ARG_PRINT_OLD_LEVELS); if(new_levels_ <= 0) { exec_state_ = LDBCommandExecuteResult::FAILED( - " Use --new_levels to specify a new level number\n"); + " Use --" + ARG_NEW_LEVELS + " to specify a new level number\n"); } } -std::vector ReduceDBLevels::PrepareArgs(int new_levels, - bool print_old_level) { - std::vector ret; - char arg[100]; - sprintf(arg, "%s%d", NEW_LEVLES_ARG, new_levels); - ret.push_back(arg); +vector ReduceDBLevelsCommand::PrepareArgs(const string& db_path, + int new_levels, bool print_old_level) { + vector ret; + ret.push_back("reduce_levels"); + ret.push_back("--" + ARG_DB + "=" + db_path); + ret.push_back("--" + ARG_NEW_LEVELS + "=" + std::to_string(new_levels)); if(print_old_level) { - sprintf(arg, "%s", PRINT_OLD_LEVELS_ARG); - ret.push_back(arg); + ret.push_back("--" + ARG_PRINT_OLD_LEVELS); } return ret; } -void ReduceDBLevels::Help(std::string& msg) { - LDBCommand::Help(msg); - msg.append("[--new_levels=New number of levels] "); - msg.append("[--print_old_levels] "); - msg.append("[--compression=none|snappy|zlib|bzip2] "); - msg.append("[--file_size= per-file size] "); +void ReduceDBLevelsCommand::Help(string& ret) { + ret.append(" "); + ret.append(ReduceDBLevelsCommand::Name()); + ret.append(" --" + ARG_NEW_LEVELS + "="); + ret.append(" [--" + ARG_PRINT_OLD_LEVELS + "]"); + ret.append("\n"); } -leveldb::Options ReduceDBLevels::PrepareOptionsForOpenDB() { +leveldb::Options ReduceDBLevelsCommand::PrepareOptionsForOpenDB() { leveldb::Options opt = LDBCommand::PrepareOptionsForOpenDB(); opt.num_levels = old_levels_; // Disable size compaction @@ -490,7 +543,8 @@ leveldb::Options ReduceDBLevels::PrepareOptionsForOpenDB() { return opt; } -Status ReduceDBLevels::GetOldNumOfLevels(leveldb::Options& opt, int* levels) { +Status ReduceDBLevelsCommand::GetOldNumOfLevels(leveldb::Options& opt, + int* levels) { TableCache* tc = new TableCache(db_path_, &opt, 10); const InternalKeyComparator* cmp = new InternalKeyComparator( opt.comparator); @@ -515,7 +569,7 @@ Status ReduceDBLevels::GetOldNumOfLevels(leveldb::Options& opt, int* levels) { return st; } -void ReduceDBLevels::DoCommand() { +void ReduceDBLevelsCommand::DoCommand() { if (new_levels_ <= 1) { exec_state_ = LDBCommandExecuteResult::FAILED( "Invalid number of levels.\n"); @@ -575,31 +629,39 @@ void ReduceDBLevels::DoCommand() { } } -const char* WALDumper::WAL_FILE_ARG = "--walfile="; -WALDumper::WALDumper(std::vector& args) : - LDBCommand(args), print_header_(false) { +const string WALDumperCommand::ARG_WAL_FILE = "walfile"; +const string WALDumperCommand::ARG_PRINT_HEADER = "header"; + +WALDumperCommand::WALDumperCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, true, + BuildCmdLineOptions({ARG_WAL_FILE, ARG_PRINT_HEADER})), + print_header_(false) { + wal_file_.clear(); - for (unsigned int i = 0; i < args.size(); i++) { - std::string& arg = args.at(i); - if (arg == "--header") { - print_header_ = true; - } else if (arg.find(WAL_FILE_ARG) == 0) { - wal_file_ = arg.substr(strlen(WAL_FILE_ARG)); - } else { - exec_state_ = LDBCommandExecuteResult::FAILED("Unknown argument " + arg); - } + + map::const_iterator itr = options.find(ARG_WAL_FILE); + if (itr != options.end()) { + wal_file_ = itr->second; } + + print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER); + if (wal_file_.empty()) { - exec_state_ = LDBCommandExecuteResult::FAILED("Argument --walfile reqd."); + exec_state_ = LDBCommandExecuteResult::FAILED( + "Argument " + ARG_WAL_FILE + " must be specified."); } } -void WALDumper::Help(std::string& ret) { - ret.append("--walfile write_ahead_log "); - ret.append("[--header print's a header] "); +void WALDumperCommand::Help(string& ret) { + ret.append(" "); + ret.append(WALDumperCommand::Name()); + ret.append(" --" + ARG_WAL_FILE + "="); + ret.append(" --[" + ARG_PRINT_HEADER + "] "); + ret.append("\n"); } -void WALDumper::DoCommand() { +void WALDumperCommand::DoCommand() { struct StdErrReporter : public log::Reader::Reporter { virtual void Corruption(size_t bytes, const Status& s) { std::cerr<<"Corruption detected in log file "<& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, true, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::FAILED( + " must be specified for the get command"); + } else { + key_ = params.at(0); + } + + if (is_key_hex_) { + key_ = HexToString(key_); + } +} + +void GetCommand::Help(string& ret) { + ret.append(" "); + ret.append(GetCommand::Name()); + ret.append(" "); + ret.append("\n"); +} + +void GetCommand::DoCommand() { + string value; + leveldb::Status st = db_->Get(leveldb::ReadOptions(), key_, &value); + if (st.ok()) { + fprintf(stdout, "%s\n", + (is_value_hex_ ? StringToHex(value) : value).c_str()); + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString()); + } +} + + +ApproxSizeCommand::ApproxSizeCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, true, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + ARG_FROM, ARG_TO})) { + + if (options.find(ARG_FROM) != options.end()) { + start_key_ = options.find(ARG_FROM)->second; + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_FROM + + " must be specified for approxsize command"); + return; + } + + if (options.find(ARG_TO) != options.end()) { + end_key_ = options.find(ARG_TO)->second; + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_TO + + " must be specified for approxsize command"); + return; + } + + if (is_key_hex_) { + start_key_ = HexToString(start_key_); + end_key_ = HexToString(end_key_); + } +} + +void ApproxSizeCommand::Help(string& ret) { + ret.append(" "); + ret.append(ApproxSizeCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append("\n"); +} + +void ApproxSizeCommand::DoCommand() { + + leveldb::Range ranges[1]; + ranges[0] = leveldb::Range(start_key_, end_key_); + uint64_t sizes[1]; + db_->GetApproximateSizes(ranges, 1, sizes); + fprintf(stdout, "%ld\n", sizes[0]); + /* Wierd that GetApproximateSizes() returns void, although documentation + * says that it returns a Status object. + if (!st.ok()) { + exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString()); + } + */ +} + + +BatchPutCommand::BatchPutCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + ARG_CREATE_IF_MISSING})) { + + if (params.size() < 2) { + exec_state_ = LDBCommandExecuteResult::FAILED( + "At least one pair must be specified batchput."); + } else if (params.size() % 2 != 0) { + exec_state_ = LDBCommandExecuteResult::FAILED( + "Equal number of s and s must be specified for batchput."); + } else { + for (size_t i = 0; i < params.size(); i += 2) { + string key = params.at(i); + string value = params.at(i+1); + key_values_.push_back(std::pair( + is_key_hex_ ? HexToString(key) : key, + is_value_hex_ ? HexToString(value) : value)); + } + } +} + +void BatchPutCommand::Help(string& ret) { + ret.append(" "); + ret.append(BatchPutCommand::Name()); + ret.append(" [ ] [..]"); + ret.append("\n"); +} + +void BatchPutCommand::DoCommand() { + leveldb::WriteBatch batch; + + for (vector>::const_iterator itr + = key_values_.begin(); itr != key_values_.end(); itr++) { + batch.Put(itr->first, itr->second); + } + leveldb::Status st = db_->Write(leveldb::WriteOptions(), &batch); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString()); + } +} + +leveldb::Options BatchPutCommand::PrepareOptionsForOpenDB() { + leveldb::Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); + return opt; +} + + +ScanCommand::ScanCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, true, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + ARG_FROM, ARG_TO, ARG_MAX_KEYS})), + start_key_specified_(false), + end_key_specified_(false), + max_keys_scanned_(-1) { + + map::const_iterator itr = options.find(ARG_FROM); + if (itr != options.end()) { + start_key_ = itr->second; + if (is_key_hex_) { + start_key_ = HexToString(start_key_); + } + start_key_specified_ = true; + } + itr = options.find(ARG_TO); + if (itr != options.end()) { + end_key_ = itr->second; + if (is_key_hex_) { + end_key_ = HexToString(end_key_); + } + end_key_specified_ = true; + } + + itr = options.find(ARG_MAX_KEYS); + if (itr != options.end()) { + try { + max_keys_scanned_ = boost::lexical_cast< int >(itr->second); + } catch( const boost::bad_lexical_cast & ) { + exec_state_ = LDBCommandExecuteResult::FAILED(ARG_MAX_KEYS + + " has an invalid value"); + } + } +} + +void ScanCommand::Help(string& ret) { + ret.append(" "); + ret.append(ScanCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append("--" + ARG_MAX_KEYS + "=N] "); + ret.append("\n"); +} + +void ScanCommand::DoCommand() { + + int num_keys_scanned = 0; + Iterator* it = db_->NewIterator(leveldb::ReadOptions()); + if (start_key_specified_) { + it->Seek(start_key_); + } else { + it->SeekToFirst(); + } + for ( ; + it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_); + it->Next()) { + string key = it->key().ToString(); + string value = it->value().ToString(); + fprintf(stdout, "%s : %s\n", + (is_key_hex_ ? StringToHex(key) : key).c_str(), + (is_value_hex_ ? StringToHex(value) : value).c_str() + ); + num_keys_scanned++; + if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) { + break; + } + } + if (!it->status().ok()) { // Check for any errors found during the scan + exec_state_ = LDBCommandExecuteResult::FAILED(it->status().ToString()); + } + delete it; +} + + +DeleteCommand::DeleteCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::FAILED( + "KEY must be specified for the delete command"); + } else { + key_ = params.at(0); + if (is_key_hex_) { + key_ = HexToString(key_); + } + } +} + +void DeleteCommand::Help(string& ret) { + ret.append(" "); + ret.append(DeleteCommand::Name() + " "); + ret.append("\n"); +} + +void DeleteCommand::DoCommand() { + leveldb::Status st = db_->Delete(leveldb::WriteOptions(), key_); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString()); + } +} + + +PutCommand::PutCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + ARG_CREATE_IF_MISSING})) { + + if (params.size() != 2) { + exec_state_ = LDBCommandExecuteResult::FAILED( + " and must be specified for the put command"); + } else { + key_ = params.at(0); + value_ = params.at(1); + } + + if (is_key_hex_) { + key_ = HexToString(key_); + } + + if (is_value_hex_) { + value_ = HexToString(value_); + } +} + +void PutCommand::Help(string& ret) { + ret.append(" "); + ret.append(PutCommand::Name()); + ret.append(" "); + ret.append("\n"); +} + +void PutCommand::DoCommand() { + leveldb::Status st = db_->Put(leveldb::WriteOptions(), key_, value_); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString()); + } +} + +leveldb::Options PutCommand::PrepareOptionsForOpenDB() { + leveldb::Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); + return opt; +} + + +const char* DBQuerierCommand::HELP_CMD = "help"; +const char* DBQuerierCommand::GET_CMD = "get"; +const char* DBQuerierCommand::PUT_CMD = "put"; +const char* DBQuerierCommand::DELETE_CMD = "delete"; + +DBQuerierCommand::DBQuerierCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + +} + +void DBQuerierCommand::Help(string& ret) { + ret.append(" "); + ret.append(DBQuerierCommand::Name()); + ret.append("\n"); + ret.append(" Starts a REPL shell. Type help for list of available " + "commands."); + ret.append("\n"); +} + +void DBQuerierCommand::DoCommand() { + if (!db_) { + return; + } + + leveldb::ReadOptions read_options; + leveldb::WriteOptions write_options; + + string line; + string key; + string value; + while (getline(std::cin, line, '\n')) { + + // Parse line into vector + vector tokens; + size_t pos = 0; + while (true) { + size_t pos2 = line.find(' ', pos); + if (pos2 == string::npos) { + break; + } + tokens.push_back(line.substr(pos, pos2-pos)); + pos = pos2 + 1; + } + tokens.push_back(line.substr(pos)); + + const string& cmd = tokens[0]; + + if (cmd == HELP_CMD) { + fprintf(stdout, + "get \n" + "put \n" + "delete \n"); + } else if (cmd == DELETE_CMD && tokens.size() == 2) { + key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]); + db_->Delete(write_options, Slice(key)); + fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str()); + } else if (cmd == PUT_CMD && tokens.size() == 3) { + key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]); + value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]); + db_->Put(write_options, Slice(key), Slice(value)); + fprintf(stdout, "Successfully put %s %s\n", + tokens[1].c_str(), tokens[2].c_str()); + } else if (cmd == GET_CMD && tokens.size() == 2) { + key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]); + if (db_->Get(read_options, Slice(key), &value).ok()) { + fprintf(stdout, "%s\n", PrintKeyValue(key, value, + is_key_hex_, is_value_hex_).c_str()); + } else { + fprintf(stdout, "Not found %s\n", tokens[1].c_str()); + } + } else { + fprintf(stdout, "Unknown command %s\n", line.c_str()); + } + } +} + + } diff --git a/util/ldb_cmd.h b/util/ldb_cmd.h index 06e0e5b9c..aac9905af 100644 --- a/util/ldb_cmd.h +++ b/util/ldb_cmd.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef LEVELDB_UTIL_LDB_H_ -#define LEVELDB_UTIL_LDB_H_ +#ifndef LEVELDB_UTIL_LDB_CMD_H_ +#define LEVELDB_UTIL_LDB_CMD_H_ #include #include @@ -12,6 +12,9 @@ #include #include +#include +#include + #include "leveldb/db.h" #include "leveldb/env.h" #include "leveldb/options.h" @@ -19,90 +22,37 @@ #include "leveldb/slice.h" #include "db/version_set.h" #include "util/logging.h" +#include "util/ldb_cmd_execute_result.h" + +using std::string; +using std::map; +using std::vector; +using std::ostringstream; namespace leveldb { -class LDBCommandExecuteResult { -public: - enum State { - EXEC_NOT_STARTED = 0, EXEC_SUCCEED = 1, EXEC_FAILED = 2, - }; - - LDBCommandExecuteResult() { - state_ = EXEC_NOT_STARTED; - message_ = ""; - } - - LDBCommandExecuteResult(State state, std::string& msg) { - state_ = state; - message_ = msg; - } - - std::string ToString() { - std::string ret; - switch (state_) { - case EXEC_SUCCEED: - break; - case EXEC_FAILED: - ret.append("Failed: "); - break; - case EXEC_NOT_STARTED: - ret.append("Not started: "); - } - if (!message_.empty()) { - ret.append(message_); - } - return ret; - } - - void Reset() { - state_ = EXEC_NOT_STARTED; - message_ = ""; - } - - bool IsSucceed() { - return state_ == EXEC_SUCCEED; - } - - bool IsNotStarted() { - return state_ == EXEC_NOT_STARTED; - } - - bool IsFailed() { - return state_ == EXEC_FAILED; - } - - static LDBCommandExecuteResult SUCCEED(std::string msg) { - return LDBCommandExecuteResult(EXEC_SUCCEED, msg); - } - - static LDBCommandExecuteResult FAILED(std::string msg) { - return LDBCommandExecuteResult(EXEC_FAILED, msg); - } - -private: - State state_; - std::string message_; - - bool operator==(const LDBCommandExecuteResult&); - bool operator!=(const LDBCommandExecuteResult&); -}; - class LDBCommand { public: - /* Constructor */ - LDBCommand(std::string& db_name, std::vector& args) : - db_path_(db_name), - db_(NULL) { - parse_open_args(args); - } + // Command-line arguments + static const string ARG_DB; + static const string ARG_HEX; + static const string ARG_KEY_HEX; + static const string ARG_VALUE_HEX; + static const string ARG_FROM; + static const string ARG_TO; + static const string ARG_MAX_KEYS; + static const string ARG_BLOOM_BITS; + static const string ARG_COMPRESSION_TYPE; + static const string ARG_BLOCK_SIZE; + static const string ARG_AUTO_COMPACTION; + static const string ARG_WRITE_BUFFER_SIZE; + static const string ARG_FILE_SIZE; + static const string ARG_CREATE_IF_MISSING; - LDBCommand(std::vector& args) : - db_path_(""), - db_(NULL) { - parse_open_args(args); - } + static LDBCommand* InitFromCmdLineArgs(const vector& args); + static LDBCommand* InitFromCmdLineArgs(int argc, char** argv); + bool ValidateCmdLineOptions(); virtual leveldb::Options PrepareOptionsForOpenDB(); @@ -117,23 +67,6 @@ public: } } - /* Print the help message */ - static void Help(std::string& ret) { - ret.append("--db=DB_PATH ["); - ret.append(LDBCommand::BLOOM_ARG); - ret.append("] ["); - ret.append(LDBCommand::COMPRESSION_TYPE_ARG); - ret.append("] ["); - ret.append(LDBCommand::BLOCK_SIZE); - ret.append("] ["); - ret.append(LDBCommand::AUTO_COMPACTION); - ret.append("] ["); - ret.append(LDBCommand::WRITE_BUFFER_SIZE_ARG); - ret.append("] ["); - ret.append(LDBCommand::FILE_SIZE_ARG); - ret.append("] "); - } - /* Run the command, and return the execute result. */ void Run() { if (!exec_state_.IsNotStarted()) { @@ -146,7 +79,7 @@ public: return; } } - + DoCommand(); if (exec_state_.IsNotStarted()) { exec_state_ = LDBCommandExecuteResult::SUCCEED(""); @@ -167,9 +100,15 @@ public: exec_state_.Reset(); } - static std::string HexToString(const std::string& str) { - std::string parsed; - for (unsigned int i = 0; i < str.length();) { + static string HexToString(const string& str) { + string parsed; + if (!boost::starts_with(str, "0x")) { + fprintf(stderr, "Invalid hex input %s. Must start with 0x\n", + str.c_str()); + throw "Invalid hex input"; + } + + for (unsigned int i = 2; i < str.length();) { int c; sscanf(str.c_str() + i, "%2X", &c); parsed.push_back(c); @@ -178,8 +117,8 @@ public: return parsed; } - static std::string StringToHex(const std::string& str) { - std::string result; + static string StringToHex(const string& str) { + string result = "0x"; char buf[10]; for (size_t i = 0; i < str.length(); i++) { snprintf(buf, 10, "%02X", (unsigned char)str[i]); @@ -189,43 +128,74 @@ public: } static const char* DELIM; - static bool ParseKeyValue(const std::string& line, - std::string* key, - std::string* value, - bool hex) { - size_t pos = line.find(DELIM); - if (pos != std::string::npos) { - (*key) = line.substr(0, pos); - (*value) = line.substr(pos + strlen(DELIM)); - if (hex) { - (*key) = HexToString(*key); - (*value) = HexToString(*value); - } - return true; - } else { - return false; - } - } - - static std::string PrintKeyValue(const std::string& key, - const std::string& value, - bool hex) { - std::string result; - result.append(hex ? StringToHex(key) : key); - result.append(DELIM); - result.append(hex ? StringToHex(value) : value); - return result; - } protected: + LDBCommandExecuteResult exec_state_; + std::string db_path_; + leveldb::DB* db_; + + /** + * true implies that this command can work if the db is opened in read-only + * mode. + */ + bool is_read_only_; + + /** If true, the key is input/output as hex in get/put/scan/delete etc. */ + bool is_key_hex_; + + /** If true, the value is input/output as hex in get/put/scan/delete etc. */ + bool is_value_hex_; + + /** + * Map of options passed on the command-line. + */ + const map options_; + + /** + * Flags passed on the command-line. + */ + const vector flags_; + + /** List of command-line options valid for this command */ + const vector valid_cmd_line_options_; + + bool ParseKeyValue(const string& line, string* key, string* value, + bool is_key_hex, bool is_value_hex); + + LDBCommand(const map& options, const vector& flags, + bool is_read_only, const vector& valid_cmd_line_options) : + db_(NULL), + is_read_only_(is_read_only), + is_key_hex_(false), + is_value_hex_(false), + options_(options), + flags_(flags), + valid_cmd_line_options_(valid_cmd_line_options) { + + map::const_iterator itr = options.find(ARG_DB); + if (itr != options.end()) { + db_path_ = itr->second; + } + + is_key_hex_ = IsKeyHex(options, flags); + is_value_hex_ = IsValueHex(options, flags); + } + void OpenDB() { leveldb::Options opt = PrepareOptionsForOpenDB(); if (!exec_state_.IsNotStarted()) { return; } // Open the DB. - leveldb::Status st = leveldb::DB::Open(opt, db_path_, &db_); + leveldb::Status st; + if (is_read_only_) { + //st = leveldb::DB::OpenForReadOnly(opt, db_path_, &db_); + // Could not get this to work + st = leveldb::DB::Open(opt, db_path_, &db_); + } else { + st = leveldb::DB::Open(opt, db_path_, &db_); + } if (!st.ok()) { std::string msg = st.ToString(); exec_state_ = LDBCommandExecuteResult::FAILED(msg); @@ -239,109 +209,181 @@ protected: } } - static const char* FROM_ARG; - static const char* END_ARG; - static const char* HEX_ARG; - LDBCommandExecuteResult exec_state_; - std::string db_path_; - leveldb::DB* db_; + static string PrintKeyValue(const string& key, const string& value, + bool is_key_hex, bool is_value_hex) { + string result; + result.append(is_key_hex ? StringToHex(key) : key); + result.append(DELIM); + result.append(is_value_hex ? StringToHex(value) : value); + return result; + } + + static string PrintKeyValue(const string& key, const string& value, + bool is_hex) { + return PrintKeyValue(key, value, is_hex, is_hex); + } + + /** + * Return true if the specified flag is present in the specified flags vector + */ + static bool IsFlagPresent(const vector& flags, const string& flag) { + return (std::find(flags.begin(), flags.end(), flag) != flags.end()); + } + + static string HelpRangeCmdArgs() { + ostringstream str_stream; + str_stream << " "; + str_stream << "[--" << ARG_FROM << "] "; + str_stream << "[--" << ARG_TO << "] "; + return str_stream.str(); + } + + /** + * A helper function that returns a list of command line options + * used by this command. It includes the common options and the ones + * passed in. + */ + vector BuildCmdLineOptions(vector options) { + vector ret = {ARG_DB, ARG_BLOOM_BITS, ARG_BLOCK_SIZE, + ARG_AUTO_COMPACTION, ARG_COMPRESSION_TYPE, + ARG_WRITE_BUFFER_SIZE, ARG_FILE_SIZE}; + ret.insert(ret.end(), options.begin(), options.end()); + return ret; + } + + bool ParseIntOption(const map& options, string option, + int& value, LDBCommandExecuteResult& exec_state); private: - static const char* BLOOM_ARG; - static const char* COMPRESSION_TYPE_ARG; - static const char* BLOCK_SIZE; - static const char* AUTO_COMPACTION; - static const char* WRITE_BUFFER_SIZE_ARG; - static const char* FILE_SIZE_ARG; - std::vector open_args_; - void parse_open_args(std::vector& args); + /** + * Interpret command line options and flags to determine if the key + * should be input/output in hex. + */ + bool IsKeyHex(const map& options, + const vector& flags) { + return (IsFlagPresent(flags, ARG_HEX) || + IsFlagPresent(flags, ARG_KEY_HEX) || + ParseBooleanOption(options, ARG_HEX, false) || + ParseBooleanOption(options, ARG_KEY_HEX, false)); + } + + /** + * Interpret command line options and flags to determine if the value + * should be input/output in hex. + */ + bool IsValueHex(const map& options, + const vector& flags) { + return (IsFlagPresent(flags, ARG_HEX) || + IsFlagPresent(flags, ARG_VALUE_HEX) || + ParseBooleanOption(options, ARG_HEX, false) || + ParseBooleanOption(options, ARG_VALUE_HEX, false)); + } + + /** + * Returns the value of the specified option as a boolean. + * default_val is used if the option is not found in options. + * Throws an exception if the value of the option is not + * "true" or "false" (case insensitive). + */ + bool ParseBooleanOption(const map& options, + const string& option, bool default_val) { + + map::const_iterator itr = options.find(option); + if (itr != options.end()) { + string option_val = itr->second; + return StringToBool(itr->second); + } + return default_val; + } + + /** + * Converts val to a boolean. + * val must be either true or false (case insensitive). + * Otherwise an exception is thrown. + */ + bool StringToBool(string val) { + boost::algorithm::to_lower(val); + if (val == "true") { + return true; + } else if (val == "false") { + return false; + } else { + throw "Invalid value for boolean argument"; + } + } + }; -class Compactor: public LDBCommand { +class CompactorCommand: public LDBCommand { public: - Compactor(std::string& db_name, std::vector& args); + static string Name() { return "compact"; } - virtual ~Compactor() {} + CompactorCommand(const vector& params, + const map& options, const vector& flags); - static void Help(std::string& ret); + static void Help(string& ret); virtual void DoCommand(); private: bool null_from_; - std::string from_; + string from_; bool null_to_; - std::string to_; - bool hex_; + string to_; }; -class DBDumper: public LDBCommand { +class DBDumperCommand: public LDBCommand { public: - DBDumper(std::string& db_name, std::vector& args); - virtual ~DBDumper() {} - static void Help(std::string& ret); + static string Name() { return "dump"; } + + DBDumperCommand(const vector& params, + const map& options, const vector& flags); + + static void Help(string& ret); + virtual void DoCommand(); + private: bool null_from_; - std::string from_; + string from_; bool null_to_; - std::string to_; + string to_; int max_keys_; bool count_only_; bool print_stats_; - bool hex_; - bool hex_output_; - static const char* MAX_KEYS_ARG; - static const char* COUNT_ONLY_ARG; - static const char* STATS_ARG; - static const char* HEX_OUTPUT_ARG; + static const string ARG_COUNT_ONLY; + static const string ARG_STATS; }; -class DBLoader: public LDBCommand { +class DBLoaderCommand: public LDBCommand { public: - DBLoader(std::string& db_name, std::vector& args); - virtual ~DBLoader() {} - static void Help(std::string& ret); + static string Name() { return "load"; } + + DBLoaderCommand(string& db_name, vector& args); + + DBLoaderCommand(const vector& params, + const map& options, const vector& flags); + + static void Help(string& ret); virtual void DoCommand(); virtual leveldb::Options PrepareOptionsForOpenDB(); private: - bool hex_input_; bool create_if_missing_; bool disable_wal_; - static const char* HEX_INPUT_ARG; - static const char* CREATE_IF_MISSING_ARG; - static const char* DISABLE_WAL_ARG; + static const string ARG_DISABLE_WAL; }; -class DBQuerier: public LDBCommand { +class ReduceDBLevelsCommand : public LDBCommand { public: - DBQuerier(std::string& db_name, std::vector& args); - virtual ~DBQuerier() {} - static void Help(std::string& ret); - virtual void DoCommand(); + static string Name() { return "reduce_levels"; } -private: - bool hex_; - - static const char* HEX_ARG; - - static const char* HELP_CMD; - static const char* GET_CMD; - static const char* PUT_CMD; - static const char* DELETE_CMD; -}; - -class ReduceDBLevels : public LDBCommand { -public: - - ReduceDBLevels (std::string& db_name, std::vector& args); - - ~ReduceDBLevels() {} + ReduceDBLevelsCommand(const vector& params, + const map& options, const vector& flags); virtual leveldb::Options PrepareOptionsForOpenDB(); @@ -351,8 +393,9 @@ public: return true; } - static void Help(std::string& msg); - static std::vector PrepareArgs(int new_levels, + static void Help(string& msg); + + static vector PrepareArgs(const string& db_path, int new_levels, bool print_old_level = false); private: @@ -360,30 +403,159 @@ private: int new_levels_; bool print_old_levels_; - static const char* NEW_LEVLES_ARG; - static const char* PRINT_OLD_LEVELS_ARG; + static const string ARG_NEW_LEVELS; + static const string ARG_PRINT_OLD_LEVELS; Status GetOldNumOfLevels(leveldb::Options& opt, int* levels); }; -class WALDumper : public LDBCommand { +class WALDumperCommand : public LDBCommand { public: + static string Name() { return "dump_wal"; } - WALDumper (std::vector& args); - - ~WALDumper() {} + WALDumperCommand(const vector& params, + const map& options, const vector& flags); virtual bool NoDBOpen() { return true; } - static void Help(std::string& ret); + static void Help(string& ret); virtual void DoCommand(); + private: bool print_header_; - std::string wal_file_; + string wal_file_; - static const char* WAL_FILE_ARG; + static const string ARG_WAL_FILE; + static const string ARG_PRINT_HEADER; }; + + +class GetCommand : public LDBCommand { +public: + static string Name() { return "get"; } + + GetCommand(const vector& params, const map& options, + const vector& flags); + + virtual void DoCommand(); + + static void Help(string& ret); + +private: + string key_; +}; + +class ApproxSizeCommand : public LDBCommand { +public: + static string Name() { return "approxsize"; } + + ApproxSizeCommand(const vector& params, + const map& options, const vector& flags); + + virtual void DoCommand(); + + static void Help(string& ret); + +private: + string start_key_; + string end_key_; +}; + +class BatchPutCommand : public LDBCommand { +public: + static string Name() { return "batchput"; } + + BatchPutCommand(const vector& params, + const map& options, const vector& flags); + + virtual void DoCommand(); + + static void Help(string& ret); + + virtual leveldb::Options PrepareOptionsForOpenDB(); + +private: + /** + * The key-values to be inserted. + */ + vector> key_values_; +}; + +class ScanCommand : public LDBCommand { +public: + static string Name() { return "scan"; } + + ScanCommand(const vector& params, const map& options, + const vector& flags); + + virtual void DoCommand(); + + static void Help(string& ret); + +private: + string start_key_; + string end_key_; + bool start_key_specified_; + bool end_key_specified_; + int max_keys_scanned_; +}; + +class DeleteCommand : public LDBCommand { +public: + static string Name() { return "delete"; } + + DeleteCommand(const vector& params, + const map& options, const vector& flags); + + virtual void DoCommand(); + + static void Help(string& ret); + +private: + string key_; +}; + +class PutCommand : public LDBCommand { +public: + static string Name() { return "put"; } + + PutCommand(const vector& params, const map& options, + const vector& flags); + + virtual void DoCommand(); + + static void Help(string& ret); + + virtual leveldb::Options PrepareOptionsForOpenDB(); + +private: + string key_; + string value_; +}; + +/** + * Command that starts up a REPL shell that allows + * get/put/delete. + */ +class DBQuerierCommand: public LDBCommand { +public: + static string Name() { return "query"; } + + DBQuerierCommand(const vector& params, + const map& options, const vector& flags); + + static void Help(string& ret); + + virtual void DoCommand(); + +private: + static const char* HELP_CMD; + static const char* GET_CMD; + static const char* PUT_CMD; + static const char* DELETE_CMD; +}; + } #endif diff --git a/util/ldb_cmd_execute_result.h b/util/ldb_cmd_execute_result.h new file mode 100644 index 000000000..fb94a20b9 --- /dev/null +++ b/util/ldb_cmd_execute_result.h @@ -0,0 +1,74 @@ +#ifndef LEVELDB_UTIL_LDB_CMD_EXECUTE_RESULT_H_ +#define LEVELDB_UTIL_LDB_CMD_EXECUTE_RESULT_H_ + +namespace leveldb { + +class LDBCommandExecuteResult { +public: + enum State { + EXEC_NOT_STARTED = 0, EXEC_SUCCEED = 1, EXEC_FAILED = 2, + }; + + LDBCommandExecuteResult() { + state_ = EXEC_NOT_STARTED; + message_ = ""; + } + + LDBCommandExecuteResult(State state, std::string& msg) { + state_ = state; + message_ = msg; + } + + std::string ToString() { + std::string ret; + switch (state_) { + case EXEC_SUCCEED: + break; + case EXEC_FAILED: + ret.append("Failed: "); + break; + case EXEC_NOT_STARTED: + ret.append("Not started: "); + } + if (!message_.empty()) { + ret.append(message_); + } + return ret; + } + + void Reset() { + state_ = EXEC_NOT_STARTED; + message_ = ""; + } + + bool IsSucceed() { + return state_ == EXEC_SUCCEED; + } + + bool IsNotStarted() { + return state_ == EXEC_NOT_STARTED; + } + + bool IsFailed() { + return state_ == EXEC_FAILED; + } + + static LDBCommandExecuteResult SUCCEED(std::string msg) { + return LDBCommandExecuteResult(EXEC_SUCCEED, msg); + } + + static LDBCommandExecuteResult FAILED(std::string msg) { + return LDBCommandExecuteResult(EXEC_FAILED, msg); + } + +private: + State state_; + std::string message_; + + bool operator==(const LDBCommandExecuteResult&); + bool operator!=(const LDBCommandExecuteResult&); +}; + +} + +#endif