andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Print blob file checksums as hex (#8437)

Browse Source 
Summary:
Currently, blob file checksums are incorrectly dumped as raw bytes
in the `ldb manifest_dump` output (i.e. they are not printed as hex).
The patch fixes this and also updates some test cases to reflect that
the checksum value field in `BlobFileAddition` and `SharedBlobFileMetaData`
contains the raw checksum and not a hex string.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8437

Test Plan:
`make check`
Tested using `ldb manifest_dump`

Reviewed By: akankshamahajan15

Differential Revision: D29284170

Pulled By: ltamasi

fbshipit-source-id: d11cfb3435b14cd73c8a3d3eb14fa0f9fa1d2228
This commit is contained in:
Levi Tamasi 2021-06-22 09:48:50 -07:00 committed by Facebook GitHub Bot
parent 54d73d6429
commit cbb3b25915
7 changed files with 39 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
db/blob/blob_file_addition.cc
View File

@ -135,7 +135,8 @@ std::ostream& operator<<(std::ostream& os,
<< " total_blob_count: " << blob_file_addition.GetTotalBlobCount() << " total_blob_count: " << blob_file_addition.GetTotalBlobCount()
<< " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes() << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes()
<< " checksum_method: " << blob_file_addition.GetChecksumMethod() << " checksum_method: " << blob_file_addition.GetChecksumMethod()
<< " checksum_value: " << blob_file_addition.GetChecksumValue(); << " checksum_value: "
<< Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true);
return os; return os;
} }
@ -146,7 +147,8 @@ JSONWriter& operator<<(JSONWriter& jw,
<< "TotalBlobCount" << blob_file_addition.GetTotalBlobCount() << "TotalBlobCount" << blob_file_addition.GetTotalBlobCount()
<< "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes() << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes()
<< "ChecksumMethod" << blob_file_addition.GetChecksumMethod() << "ChecksumMethod" << blob_file_addition.GetChecksumMethod()
<< "ChecksumValue" << blob_file_addition.GetChecksumValue(); << "ChecksumValue"
<< Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true);
return jw; return jw;
} }

12
db/blob/blob_file_addition_test.cc
View File

@ -46,7 +46,9 @@ TEST_F(BlobFileAdditionTest, NonEmpty) {
constexpr uint64_t total_blob_count = 2; constexpr uint64_t total_blob_count = 2;
constexpr uint64_t total_blob_bytes = 123456; constexpr uint64_t total_blob_bytes = 123456;
const std::string checksum_method("SHA1"); const std::string checksum_method("SHA1");
const std::string checksum_value("bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"); const std::string checksum_value(
"\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
"\x5c\xbd");
BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
total_blob_bytes, checksum_method, total_blob_bytes, checksum_method,
@ -113,7 +115,9 @@ TEST_F(BlobFileAdditionTest, DecodeErrors) {
ASSERT_TRUE(std::strstr(s.getState(), "checksum value")); ASSERT_TRUE(std::strstr(s.getState(), "checksum value"));
} }
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; constexpr char checksum_value[] =
"\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
"\x5c\xbd";
PutLengthPrefixedSlice(&str, checksum_value); PutLengthPrefixedSlice(&str, checksum_value);
slice = str; slice = str;
@ -150,7 +154,7 @@ TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) {
constexpr uint64_t total_blob_count = 9999; constexpr uint64_t total_blob_count = 9999;
constexpr uint64_t total_blob_bytes = 100000000; constexpr uint64_t total_blob_bytes = 100000000;
const std::string checksum_method("CRC32"); const std::string checksum_method("CRC32");
const std::string checksum_value("3d87ff57"); const std::string checksum_value("\x3d\x87\xff\x57");
BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
total_blob_bytes, checksum_method, total_blob_bytes, checksum_method,
@ -178,7 +182,7 @@ TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) {
constexpr uint64_t total_blob_count = 100; constexpr uint64_t total_blob_count = 100;
constexpr uint64_t total_blob_bytes = 2000000; constexpr uint64_t total_blob_bytes = 2000000;
const std::string checksum_method("CRC32B"); const std::string checksum_method("CRC32B");
const std::string checksum_value("6dbdf23a"); const std::string checksum_value("\x6d\xbd\xf2\x3a");
BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
total_blob_bytes, checksum_method, total_blob_bytes, checksum_method,

5
db/blob/blob_file_meta.cc
View File

@ -8,6 +8,8 @@
#include <ostream> #include <ostream>
#include <sstream> #include <sstream>
#include "rocksdb/slice.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
std::string SharedBlobFileMetaData::DebugString() const { std::string SharedBlobFileMetaData::DebugString() const {
@ -23,7 +25,8 @@ std::ostream& operator<<(std::ostream& os,
<< " total_blob_count: " << shared_meta.GetTotalBlobCount() << " total_blob_count: " << shared_meta.GetTotalBlobCount()
<< " total_blob_bytes: " << shared_meta.GetTotalBlobBytes() << " total_blob_bytes: " << shared_meta.GetTotalBlobBytes()
<< " checksum_method: " << shared_meta.GetChecksumMethod() << " checksum_method: " << shared_meta.GetChecksumMethod()
<< " checksum_value: " << shared_meta.GetChecksumValue(); << " checksum_value: "
<< Slice(shared_meta.GetChecksumValue()).ToString(/* hex */ true);
return os; return os;
} }

2
db/db_test.cc
View File

@ -2364,7 +2364,7 @@ TEST_F(DBTest, GetLiveBlobFiles) {
constexpr uint64_t total_blob_count = 555; constexpr uint64_t total_blob_count = 555;
constexpr uint64_t total_blob_bytes = 66666; constexpr uint64_t total_blob_bytes = 66666;
constexpr char checksum_method[] = "CRC32"; constexpr char checksum_method[] = "CRC32";
constexpr char checksum_value[] = "3d87ff57"; constexpr char checksum_value[] = "\x3d\x87\xff\x57";
auto shared_meta = SharedBlobFileMetaData::Create( auto shared_meta = SharedBlobFileMetaData::Create(
blob_file_number, total_blob_count, total_blob_bytes, checksum_method, blob_file_number, total_blob_count, total_blob_bytes, checksum_method,

2
db/obsolete_files_test.cc
View File

@ -228,7 +228,7 @@ TEST_F(ObsoleteFilesTest, BlobFiles) {
constexpr uint64_t second_total_blob_count = 100; constexpr uint64_t second_total_blob_count = 100;
constexpr uint64_t second_total_blob_bytes = 2000000; constexpr uint64_t second_total_blob_bytes = 2000000;
constexpr char second_checksum_method[] = "CRC32B"; constexpr char second_checksum_method[] = "CRC32B";
constexpr char second_checksum_value[] = "6dbdf23a"; constexpr char second_checksum_value[] = "\x6d\xbd\xf2\x3a";
auto shared_meta = SharedBlobFileMetaData::Create( auto shared_meta = SharedBlobFileMetaData::Create(
second_blob_file_number, second_total_blob_count, second_total_blob_bytes, second_blob_file_number, second_total_blob_count, second_total_blob_bytes,

20
db/version_builder_test.cc
View File

@ -659,7 +659,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_count = 5678;
constexpr uint64_t total_blob_bytes = 999999; constexpr uint64_t total_blob_bytes = 999999;
constexpr char checksum_method[] = "SHA1"; constexpr char checksum_method[] = "SHA1";
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; constexpr char checksum_value[] =
"\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
"\x5c\xbd";
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value); checksum_method, checksum_value);
@ -703,7 +705,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) {
constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_count = 5678;
constexpr uint64_t total_blob_bytes = 999999; constexpr uint64_t total_blob_bytes = 999999;
constexpr char checksum_method[] = "SHA1"; constexpr char checksum_method[] = "SHA1";
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; constexpr char checksum_value[] =
"\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
"\x5c\xbd";
constexpr uint64_t garbage_blob_count = 123; constexpr uint64_t garbage_blob_count = 123;
constexpr uint64_t garbage_blob_bytes = 456789; constexpr uint64_t garbage_blob_bytes = 456789;
@ -744,7 +748,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyApplied) {
constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_count = 5678;
constexpr uint64_t total_blob_bytes = 999999; constexpr uint64_t total_blob_bytes = 999999;
constexpr char checksum_method[] = "SHA1"; constexpr char checksum_method[] = "SHA1";
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; constexpr char checksum_value[] =
"\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
"\x5c\xbd";
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value); checksum_method, checksum_value);
@ -764,7 +770,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_count = 5678;
constexpr uint64_t total_blob_bytes = 999999; constexpr uint64_t total_blob_bytes = 999999;
constexpr char checksum_method[] = "SHA1"; constexpr char checksum_method[] = "SHA1";
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; constexpr char checksum_value[] =
"\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
"\x5c\xbd";
constexpr uint64_t garbage_blob_count = 123; constexpr uint64_t garbage_blob_count = 123;
constexpr uint64_t garbage_blob_bytes = 456789; constexpr uint64_t garbage_blob_bytes = 456789;
@ -841,7 +849,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) {
constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_count = 5678;
constexpr uint64_t total_blob_bytes = 999999; constexpr uint64_t total_blob_bytes = 999999;
constexpr char checksum_method[] = "SHA1"; constexpr char checksum_method[] = "SHA1";
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; constexpr char checksum_value[] =
"\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
"\x5c\xbd";
addition.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, addition.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value); checksum_method, checksum_value);

11
db/version_set_test.cc
View File

@ -989,7 +989,8 @@ TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) {
constexpr uint64_t total_blob_bytes = 77777777; constexpr uint64_t total_blob_bytes = 77777777;
constexpr char checksum_method[] = "SHA1"; constexpr char checksum_method[] = "SHA1";
constexpr char checksum_value[] = constexpr char checksum_value[] =
"bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c"
"\x52\x5c\xbd";
auto shared_meta = SharedBlobFileMetaData::Create( auto shared_meta = SharedBlobFileMetaData::Create(
blob_file_number, total_blob_count, total_blob_bytes, checksum_method, blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
@ -1010,7 +1011,7 @@ TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) {
constexpr uint64_t total_blob_count = 555; constexpr uint64_t total_blob_count = 555;
constexpr uint64_t total_blob_bytes = 66666; constexpr uint64_t total_blob_bytes = 66666;
constexpr char checksum_method[] = "CRC32"; constexpr char checksum_method[] = "CRC32";
constexpr char checksum_value[] = "3d87ff57"; constexpr char checksum_value[] = "\x3d\x87\xff\x57";
auto shared_meta = SharedBlobFileMetaData::Create( auto shared_meta = SharedBlobFileMetaData::Create(
blob_file_number, total_blob_count, total_blob_bytes, checksum_method, blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
@ -1068,7 +1069,7 @@ TEST_F(VersionSetTest, AddLiveBlobFiles) {
constexpr uint64_t first_total_blob_count = 555; constexpr uint64_t first_total_blob_count = 555;
constexpr uint64_t first_total_blob_bytes = 66666; constexpr uint64_t first_total_blob_bytes = 66666;
constexpr char first_checksum_method[] = "CRC32"; constexpr char first_checksum_method[] = "CRC32";
constexpr char first_checksum_value[] = "3d87ff57"; constexpr char first_checksum_value[] = "\x3d\x87\xff\x57";
auto first_shared_meta = SharedBlobFileMetaData::Create( auto first_shared_meta = SharedBlobFileMetaData::Create(
first_blob_file_number, first_total_blob_count, first_total_blob_bytes, first_blob_file_number, first_total_blob_count, first_total_blob_bytes,
@ -1111,7 +1112,7 @@ TEST_F(VersionSetTest, AddLiveBlobFiles) {
constexpr uint64_t second_total_blob_count = 100; constexpr uint64_t second_total_blob_count = 100;
constexpr uint64_t second_total_blob_bytes = 2000000; constexpr uint64_t second_total_blob_bytes = 2000000;
constexpr char second_checksum_method[] = "CRC32B"; constexpr char second_checksum_method[] = "CRC32B";
constexpr char second_checksum_value[] = "6dbdf23a"; constexpr char second_checksum_value[] = "\x6d\xbd\xf2\x3a";
auto second_shared_meta = SharedBlobFileMetaData::Create( auto second_shared_meta = SharedBlobFileMetaData::Create(
second_blob_file_number, second_total_blob_count, second_total_blob_bytes, second_blob_file_number, second_total_blob_count, second_total_blob_bytes,
@ -1151,7 +1152,7 @@ TEST_F(VersionSetTest, ObsoleteBlobFile) {
constexpr uint64_t total_blob_count = 555; constexpr uint64_t total_blob_count = 555;
constexpr uint64_t total_blob_bytes = 66666; constexpr uint64_t total_blob_bytes = 66666;
constexpr char checksum_method[] = "CRC32"; constexpr char checksum_method[] = "CRC32";
constexpr char checksum_value[] = "3d87ff57"; constexpr char checksum_value[] = "\x3d\x87\xff\x57";
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value); checksum_method, checksum_value);