diff --git a/tools/ldb_test.py b/tools/ldb_test.py index a76e9c9ab..a549b89c3 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -145,6 +145,24 @@ class LDBTestCase(unittest.TestCase): self.assertRunFAIL("batchput k1") self.assertRunFAIL("batchput k1 v1 k2") + def testCountDelimDump(self): + print "Running testCountDelimDump..." + self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK") + self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK") + self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") + self.assertRunOK("dump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") + self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK") + self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") + + def testCountDelimIDump(self): + print "Running testCountDelimIDump..." + self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK") + self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK") + self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") + self.assertRunOK("dump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") + self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK") + self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") + def testInvalidCmdLines(self): print "Running testInvalidCmdLines..." # db not specified diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index 7eff62fd9..bda67620a 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -580,6 +580,7 @@ void PrintBucketCounts(const vector& bucket_counts, int ttl_start, } const string InternalDumpCommand::ARG_COUNT_ONLY = "count_only"; +const string InternalDumpCommand::ARG_COUNT_DELIM = "count_delim"; const string InternalDumpCommand::ARG_STATS = "stats"; const string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex"; @@ -589,12 +590,14 @@ InternalDumpCommand::InternalDumpCommand(const vector& params, LDBCommand(options, flags, true, BuildCmdLineOptions({ ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO, ARG_MAX_KEYS, - ARG_COUNT_ONLY, ARG_STATS, + ARG_COUNT_ONLY, ARG_COUNT_DELIM, ARG_STATS, ARG_INPUT_KEY_HEX})), has_from_(false), has_to_(false), max_keys_(-1), + delim_("."), count_only_(false), + count_delim_(false), print_stats_(false), is_input_key_hex_(false) { @@ -602,6 +605,15 @@ InternalDumpCommand::InternalDumpCommand(const vector& params, has_to_ = ParseStringOption(options, ARG_TO, &to_); ParseIntOption(options, ARG_MAX_KEYS, max_keys_, exec_state_); + map::const_iterator itr = options.find(ARG_COUNT_DELIM); + if (itr != options.end()) { + delim_ = itr->second; + count_delim_ = true; + // fprintf(stdout,"delim = %c\n",delim_[0]); + } else { + count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM); + delim_="."; + } print_stats_ = IsFlagPresent(flags, ARG_STATS); count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY); @@ -624,6 +636,7 @@ void InternalDumpCommand::Help(string& ret) { ret.append(" [--" + ARG_INPUT_KEY_HEX + "]"); ret.append(" [--" + ARG_MAX_KEYS + "=]"); ret.append(" [--" + ARG_COUNT_ONLY + "]"); + ret.append(" [--" + ARG_COUNT_DELIM + "=]"); ret.append(" [--" + ARG_STATS + "]"); ret.append("\n"); } @@ -646,7 +659,10 @@ void InternalDumpCommand::DoCommand() { exec_state_ = LDBCommandExecuteResult::FAILED("DB is not DBImpl"); return; } - + string rtype1,rtype2,row,val; + rtype2 = ""; + uint64_t c=0; + uint64_t s1=0,s2=0; // Setup internal key iterator auto iter = unique_ptr(idb->TEST_NewInternalIterator()); Status st = iter->status(); @@ -678,8 +694,32 @@ void InternalDumpCommand::DoCommand() { } ++count; + int k; + if (count_delim_) { + rtype1 = ""; + s1=0; + row = iter->key().ToString(); + val = iter->value().ToString(); + for(k=0;row[k]!='\x01' && row[k]!='\0';k++) + s1++; + for(k=0;val[k]!='\x01' && val[k]!='\0';k++) + s1++; + for(int j=0;row[j]!=delim_[0] && row[j]!='\0' && row[j]!='\x01';j++) + rtype1+=row[j]; + if(rtype2.compare("") && rtype2.compare(rtype1)!=0) { + fprintf(stdout,"%s => count:%lld\tsize:%lld\n",rtype2.c_str(), + (long long)c,(long long)s2); + c=1; + s2=s1; + rtype2 = rtype1; + } else { + c++; + s2+=s1; + rtype2=rtype1; + } + } - if (!count_only_) { + if (!count_only_ && !count_delim_) { string key = ikey.DebugString(is_key_hex_); string value = iter->value().ToString(is_value_hex_); std::cout << key << " => " << value << "\n"; @@ -688,12 +728,16 @@ void InternalDumpCommand::DoCommand() { // Terminate if maximum number of keys have been dumped if (max_keys_ > 0 && count >= max_keys_) break; } - + if(count_delim_) { + fprintf(stdout,"%s => count:%lld\tsize:%lld\n", rtype2.c_str(), + (long long)c,(long long)s2); + } else fprintf(stdout, "Internal keys in range: %lld\n", (long long) count); } const string DBDumperCommand::ARG_COUNT_ONLY = "count_only"; +const string DBDumperCommand::ARG_COUNT_DELIM = "count_delim"; const string DBDumperCommand::ARG_STATS = "stats"; const string DBDumperCommand::ARG_TTL_BUCKET = "bucket"; @@ -702,13 +746,15 @@ DBDumperCommand::DBDumperCommand(const vector& params, LDBCommand(options, flags, true, BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO, - ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_STATS, - ARG_TTL_START, ARG_TTL_END, - ARG_TTL_BUCKET, ARG_TIMESTAMP})), + ARG_MAX_KEYS, ARG_COUNT_ONLY, + ARG_COUNT_DELIM, ARG_STATS, ARG_TTL_START, + ARG_TTL_END, ARG_TTL_BUCKET, + ARG_TIMESTAMP})), null_from_(true), null_to_(true), max_keys_(-1), count_only_(false), + count_delim_(false), print_stats_(false) { map::const_iterator itr = options.find(ARG_FROM); @@ -735,6 +781,14 @@ DBDumperCommand::DBDumperCommand(const vector& params, " has a value out-of-range"); } } + itr = options.find(ARG_COUNT_DELIM); + if (itr != options.end()) { + delim_ = itr->second; + count_delim_ = true; + } else { + count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM); + delim_="."; + } print_stats_ = IsFlagPresent(flags, ARG_STATS); count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY); @@ -757,6 +811,7 @@ void DBDumperCommand::Help(string& ret) { ret.append(" [--" + ARG_MAX_KEYS + "=]"); ret.append(" [--" + ARG_TIMESTAMP + "]"); ret.append(" [--" + ARG_COUNT_ONLY + "]"); + ret.append(" [--" + ARG_COUNT_DELIM + "=]"); ret.append(" [--" + ARG_STATS + "]"); ret.append(" [--" + ARG_TTL_BUCKET + "=]"); ret.append(" [--" + ARG_TTL_START + "=:- is inclusive]"); @@ -811,11 +866,17 @@ void DBDumperCommand::DoCommand() { bucket_size <= 0) { bucket_size = time_range; // Will have just 1 bucket by default } + //cretaing variables for row count of each type + string rtype1,rtype2,row,val; + rtype2 = ""; + uint64_t c=0; + uint64_t s1=0,s2=0; + // At this point, bucket_size=0 => time_range=0 uint64_t num_buckets = (bucket_size >= time_range) ? 1 : ((time_range + bucket_size - 1) / bucket_size); vector bucket_counts(num_buckets, 0); - if (is_db_ttl_ && !count_only_ && timestamp_) { + if (is_db_ttl_ && !count_only_ && timestamp_ && !count_delim_) { fprintf(stdout, "Dumping key-values from %s to %s\n", ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str()); } @@ -844,7 +905,30 @@ void DBDumperCommand::DoCommand() { rawtime, num_buckets); } ++count; - if (!count_only_) { + if (count_delim_) { + rtype1 = ""; + row = iter->key().ToString(); + val = iter->value().ToString(); + s1 = row.size()+val.size(); + for(int j=0;row[j]!=delim_[0] && row[j]!='\0';j++) + rtype1+=row[j]; + if(rtype2.compare("") && rtype2.compare(rtype1)!=0) { + fprintf(stdout,"%s => count:%lld\tsize:%lld\n",rtype2.c_str(), + (long long )c,(long long)s2); + c=1; + s2=s1; + rtype2 = rtype1; + } else { + c++; + s2+=s1; + rtype2=rtype1; + } + + } + + + + if (!count_only_ && !count_delim_) { if (is_db_ttl_ && timestamp_) { fprintf(stdout, "%s ", ReadableTime(rawtime).c_str()); } @@ -854,9 +938,13 @@ void DBDumperCommand::DoCommand() { fprintf(stdout, "%s\n", str.c_str()); } } + if (num_buckets > 1 && is_db_ttl_) { PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size, num_buckets); + } else if(count_delim_) { + fprintf(stdout,"%s => count:%lld\tsize:%lld\n",rtype2.c_str(), + (long long )c,(long long)s2); } else { fprintf(stdout, "Keys in range: %lld\n", (long long) count); } diff --git a/util/ldb_cmd.h b/util/ldb_cmd.h index 78ffa7c76..022f5fadc 100644 --- a/util/ldb_cmd.h +++ b/util/ldb_cmd.h @@ -395,10 +395,13 @@ private: bool null_to_; string to_; int max_keys_; + string delim_; bool count_only_; + bool count_delim_; bool print_stats_; static const string ARG_COUNT_ONLY; + static const string ARG_COUNT_DELIM; static const string ARG_STATS; static const string ARG_TTL_BUCKET; }; @@ -421,11 +424,15 @@ private: bool has_to_; string to_; int max_keys_; + string delim_; bool count_only_; + bool count_delim_; bool print_stats_; bool is_input_key_hex_; + static const string ARG_DELIM; static const string ARG_COUNT_ONLY; + static const string ARG_COUNT_DELIM; static const string ARG_STATS; static const string ARG_INPUT_KEY_HEX; };