diff --git a/CMakeLists.txt b/CMakeLists.txt index aa56585b5..de4e865ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -458,6 +458,7 @@ set(SOURCES utilities/column_aware_encoding_util.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/date_tiered/date_tiered_db_impl.cc + utilities/debug.cc utilities/document/document_db.cc utilities/document/json_document.cc utilities/document/json_document_builder.cc diff --git a/HISTORY.md b/HISTORY.md index e70af5459..5874d7b03 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -8,6 +8,7 @@ * Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly. * Support dynamically change `max_open_files` option via SetDBOptions() * Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families. +* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys. ## 5.4.0 (04/11/2017) ### Public API Change diff --git a/TARGETS b/TARGETS index 506cdc789..cf0419f0d 100644 --- a/TARGETS +++ b/TARGETS @@ -207,6 +207,7 @@ cpp_library( "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc", "utilities/convenience/info_log_finder.cc", "utilities/date_tiered/date_tiered_db_impl.cc", + "utilities/debug.cc", "utilities/document/document_db.cc", "utilities/document/json_document.cc", "utilities/document/json_document_builder.cc", diff --git a/include/rocksdb/utilities/debug.h b/include/rocksdb/utilities/debug.h new file mode 100644 index 000000000..1ef52c102 --- /dev/null +++ b/include/rocksdb/utilities/debug.h @@ -0,0 +1,41 @@ +// Copyright (c) 2017-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// This source code is also licensed under the GPLv2 license found in the +// COPYING file in the root directory of this source tree. + +#pragma once + +#ifndef ROCKSDB_LITE + +#include "rocksdb/db.h" +#include "rocksdb/types.h" + +namespace rocksdb { + +// Data associated with a particular version of a key. A database may internally +// store multiple versions of a same user key due to snapshots, compaction not +// happening yet, etc. +struct KeyVersion { + KeyVersion(const std::string& _user_key, const std::string& _value, + SequenceNumber _sequence, int _type) + : user_key(_user_key), value(_value), sequence(_sequence), type(_type) {} + + std::string user_key; + std::string value; + SequenceNumber sequence; + // TODO(ajkr): we should provide a helper function that converts the int to a + // string describing the type for easier debugging. + int type; +}; + +// Returns listing of all versions of keys in the provided user key range. +// The range is inclusive-inclusive, i.e., [`begin_key`, `end_key`]. +// The result is inserted into the provided vector, `key_versions`. +Status GetAllKeyVersions(DB* db, Slice begin_key, Slice end_key, + std::vector* key_versions); + +} // namespace rocksdb + +#endif // ROCKSDB_LITE diff --git a/src.mk b/src.mk index 677b18b18..18251d0ba 100644 --- a/src.mk +++ b/src.mk @@ -160,6 +160,7 @@ LIB_SOURCES = \ utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc \ utilities/convenience/info_log_finder.cc \ utilities/date_tiered/date_tiered_db_impl.cc \ + utilities/debug.cc \ utilities/document/document_db.cc \ utilities/document/json_document.cc \ utilities/document/json_document_builder.cc \ diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index e36c4fcc8..7bd841823 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -23,6 +23,7 @@ #include "rocksdb/table_properties.h" #include "rocksdb/utilities/backupable_db.h" #include "rocksdb/utilities/checkpoint.h" +#include "rocksdb/utilities/debug.h" #include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/options_util.h" #include "rocksdb/write_batch.h" @@ -1212,56 +1213,33 @@ void InternalDumpCommand::DoCommand() { } // Cast as DBImpl to get internal iterator - DBImpl* idb = dynamic_cast(db_); - if (!idb) { - exec_state_ = LDBCommandExecuteResult::Failed("DB is not DBImpl"); + std::vector key_versions; + Status st = GetAllKeyVersions(db_, from_, to_, &key_versions); + if (!st.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); return; } std::string rtype1, rtype2, row, val; rtype2 = ""; uint64_t c=0; uint64_t s1=0,s2=0; - // Setup internal key iterator - Arena arena; - auto icmp = InternalKeyComparator(options_.comparator); - RangeDelAggregator range_del_agg(icmp, {} /* snapshots */); - ScopedArenaIterator iter(idb->NewInternalIterator(&arena, &range_del_agg)); - Status st = iter->status(); - if (!st.ok()) { - exec_state_ = - LDBCommandExecuteResult::Failed("Iterator error:" + st.ToString()); - } - - if (has_from_) { - InternalKey ikey; - ikey.SetMaxPossibleForUserKey(from_); - iter->Seek(ikey.Encode()); - } else { - iter->SeekToFirst(); - } long long count = 0; - for (; iter->Valid(); iter->Next()) { - ParsedInternalKey ikey; - if (!ParseInternalKey(iter->key(), &ikey)) { - fprintf(stderr, "Internal Key [%s] parse error!\n", - iter->key().ToString(true /* in hex*/).data()); - // TODO: add error counter - continue; - } - - // If end marker was specified, we stop before it - if (has_to_ && options_.comparator->Compare(ikey.user_key, to_) >= 0) { + for (auto& key_version : key_versions) { + InternalKey ikey(key_version.user_key, key_version.sequence, + static_cast(key_version.type)); + if (has_to_ && ikey.user_key() == to_) { + // GetAllKeyVersions() includes keys with user key `to_`, but idump has + // traditionally excluded such keys. break; } - ++count; int k; if (count_delim_) { rtype1 = ""; s1=0; - row = iter->key().ToString(); - val = iter->value().ToString(); + row = ikey.Encode().ToString(); + val = key_version.value; for(k=0;row[k]!='\x01' && row[k]!='\0';k++) s1++; for(k=0;val[k]!='\x01' && val[k]!='\0';k++) @@ -1278,12 +1256,12 @@ void InternalDumpCommand::DoCommand() { c++; s2+=s1; rtype2=rtype1; + } } - } if (!count_only_ && !count_delim_) { std::string key = ikey.DebugString(is_key_hex_); - std::string value = iter->value().ToString(is_value_hex_); + std::string value = Slice(key_version.value).ToString(is_value_hex_); std::cout << key << " => " << value << "\n"; } diff --git a/tools/ldb_test.py b/tools/ldb_test.py index 2fa77bbb5..5d52d79c3 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -169,10 +169,10 @@ class LDBTestCase(unittest.TestCase): print "Running testCountDelimIDump..." self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK") self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK") - self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") - self.assertRunOK("dump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") + self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") + self.assertRunOK("idump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK") - self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") + self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") def testInvalidCmdLines(self): print "Running testInvalidCmdLines..." @@ -331,6 +331,18 @@ class LDBTestCase(unittest.TestCase): self.assertFalse(self.dumpDb( "--db=%s --create_if_missing" % origDbPath, dumpFilePath)) + def testIDumpBasics(self): + print "Running testIDumpBasics..." + self.assertRunOK("put a val --create_if_missing", "OK") + self.assertRunOK("put b val", "OK") + self.assertRunOK( + "idump", "'a' seq:1, type:1 => val\n" + "'b' seq:2, type:1 => val\nInternal keys in range: 2") + self.assertRunOK( + "idump --input_key_hex --from=%s --to=%s" % (hex(ord('a')), + hex(ord('b'))), + "'a' seq:1, type:1 => val\nInternal keys in range: 1") + def testMiscAdminTask(self): print "Running testMiscAdminTask..." # These tests need to be improved; for example with asserts about diff --git a/utilities/debug.cc b/utilities/debug.cc new file mode 100644 index 000000000..8bd79bae1 --- /dev/null +++ b/utilities/debug.cc @@ -0,0 +1,57 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// This source code is also licensed under the GPLv2 license found in the +// COPYING file in the root directory of this source tree. + +#ifndef ROCKSDB_LITE + +#include "rocksdb/utilities/debug.h" + +#include "db/db_impl.h" + +namespace rocksdb { + +Status GetAllKeyVersions(DB* db, Slice begin_key, Slice end_key, + std::vector* key_versions) { + assert(key_versions != nullptr); + key_versions->clear(); + + DBImpl* idb = static_cast(db->GetRootDB()); + auto icmp = InternalKeyComparator(idb->GetOptions().comparator); + RangeDelAggregator range_del_agg(icmp, {} /* snapshots */); + Arena arena; + ScopedArenaIterator iter(idb->NewInternalIterator(&arena, &range_del_agg)); + + if (!begin_key.empty()) { + InternalKey ikey; + ikey.SetMaxPossibleForUserKey(begin_key); + iter->Seek(ikey.Encode()); + } else { + iter->SeekToFirst(); + } + + for (; iter->Valid(); iter->Next()) { + ParsedInternalKey ikey; + if (!ParseInternalKey(iter->key(), &ikey)) { + return Status::Corruption("Internal Key [" + iter->key().ToString() + + "] parse error!"); + } + + if (!end_key.empty() && + icmp.user_comparator()->Compare(ikey.user_key, end_key) > 0) { + break; + } + + key_versions->emplace_back(ikey.user_key.ToString() /* _user_key */, + iter->value().ToString() /* _value */, + ikey.sequence /* _sequence */, + static_cast(ikey.type) /* _type */); + } + return Status::OK(); +} + +} // namespace rocksdb + +#endif // ROCKSDB_LITE