andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
6c50fe1ec9
rocksdb/db/version_edit_test.cc
Zhichao Cao e10553f2a6 Added the safe-to-ignore tag to version_edit (#6530) 
Summary:
Each time RocksDB switches to a new MANIFEST file from old one, it calls WriteCurrentStateToManifest() which writes a 'snapshot' of the current in-memory state of versions to the beginning of the new manifest as a bunch of version edits. We can distinguish these version edits from other version edits written during normal operations with a custom, safe-to-ignore tag.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6530

Test Plan: added test to version_edit_test, pass make asan_check

Reviewed By: riversand963

Differential Revision: D20524516

Pulled By: zhichao-cao

fbshipit-source-id: f1de102f5499bfa88dae3caa2f32c7f42cf904db
2020-03-19 11:30:26 -07:00

353 lines
12 KiB
C++
Raw Blame History

// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/version_edit.h"
#include "test_util/sync_point.h"
#include "test_util/testharness.h"
#include "util/coding.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
static void TestEncodeDecode(const VersionEdit& edit) {
std::string encoded, encoded2;
edit.EncodeTo(&encoded);
VersionEdit parsed;
Status s = parsed.DecodeFrom(encoded);
ASSERT_TRUE(s.ok()) << s.ToString();
parsed.EncodeTo(&encoded2);
ASSERT_EQ(encoded, encoded2);
}
class VersionEditTest : public testing::Test {};
TEST_F(VersionEditTest, EncodeDecode) {
static const uint64_t kBig = 1ull << 50;
static const uint32_t kBig32Bit = 1ull << 30;
VersionEdit edit;
for (int i = 0; i < 4; i++) {
TestEncodeDecode(edit);
edit.AddFile(3, kBig + 300 + i, kBig32Bit + 400 + i, 0,
InternalKey("foo", kBig + 500 + i, kTypeValue),
InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
kBig + 500 + i, kBig + 600 + i, false, kInvalidBlobFileNumber,
888, 678, "234", "crc32c");
edit.DeleteFile(4, kBig + 700 + i);
}
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
edit.SetNextFile(kBig + 200);
edit.SetLastSequence(kBig + 1000);
edit.SetStateUponManifestSwitch(true);
TestEncodeDecode(edit);
}
TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
static const uint64_t kBig = 1ull << 50;
VersionEdit edit;
edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue),
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue),
InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502,
kBig + 602, true, kInvalidBlobFileNumber, 666, 888,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex),
InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503,
kBig + 603, true, 1001, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
;
edit.DeleteFile(4, 700);
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
edit.SetNextFile(kBig + 200);
edit.SetLastSequence(kBig + 1000);
edit.SetStateUponManifestSwitch(true);
TestEncodeDecode(edit);
std::string encoded, encoded2;
edit.EncodeTo(&encoded);
VersionEdit parsed;
Status s = parsed.DecodeFrom(encoded);
ASSERT_TRUE(s.ok()) << s.ToString();
auto& new_files = parsed.GetNewFiles();
ASSERT_TRUE(new_files[0].second.marked_for_compaction);
ASSERT_TRUE(!new_files[1].second.marked_for_compaction);
ASSERT_TRUE(new_files[2].second.marked_for_compaction);
ASSERT_TRUE(new_files[3].second.marked_for_compaction);
ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
ASSERT_EQ(0u, new_files[2].second.fd.GetPathId());
ASSERT_EQ(0u, new_files[3].second.fd.GetPathId());
ASSERT_EQ(kInvalidBlobFileNumber,
new_files[0].second.oldest_blob_file_number);
ASSERT_EQ(kInvalidBlobFileNumber,
new_files[1].second.oldest_blob_file_number);
ASSERT_EQ(kInvalidBlobFileNumber,
new_files[2].second.oldest_blob_file_number);
ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number);
ASSERT_TRUE(parsed.GetStateUponManifestSwitch());
}
TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
static const uint64_t kBig = 1ull << 50;
VersionEdit edit;
edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue),
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, kInvalidBlobFileNumber, 686, 868, "234",
"crc32c");
edit.DeleteFile(4, 700);
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
edit.SetNextFile(kBig + 200);
edit.SetLastSequence(kBig + 1000);
std::string encoded;
// Call back function to add extra customized builds.
bool first = true;
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"VersionEdit::EncodeTo:NewFile4:CustomizeFields", [&](void* arg) {
std::string* str = reinterpret_cast<std::string*>(arg);
PutVarint32(str, 33);
const std::string str1 = "random_string";
PutLengthPrefixedSlice(str, str1);
if (first) {
first = false;
PutVarint32(str, 22);
const std::string str2 = "s";
PutLengthPrefixedSlice(str, str2);
}
});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
edit.EncodeTo(&encoded);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
VersionEdit parsed;
Status s = parsed.DecodeFrom(encoded);
ASSERT_TRUE(s.ok()) << s.ToString();
ASSERT_TRUE(!first);
auto& new_files = parsed.GetNewFiles();
ASSERT_TRUE(new_files[0].second.marked_for_compaction);
ASSERT_TRUE(!new_files[1].second.marked_for_compaction);
ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
ASSERT_EQ(1u, parsed.GetDeletedFiles().size());
}
TEST_F(VersionEditTest, NewFile4NotSupportedField) {
static const uint64_t kBig = 1ull << 50;
VersionEdit edit;
edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue),
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
edit.SetNextFile(kBig + 200);
edit.SetLastSequence(kBig + 1000);
std::string encoded;
// Call back function to add extra customized builds.
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"VersionEdit::EncodeTo:NewFile4:CustomizeFields", [&](void* arg) {
std::string* str = reinterpret_cast<std::string*>(arg);
const std::string str1 = "s";
PutLengthPrefixedSlice(str, str1);
});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
edit.EncodeTo(&encoded);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
VersionEdit parsed;
Status s = parsed.DecodeFrom(encoded);
ASSERT_NOK(s);
}
TEST_F(VersionEditTest, EncodeEmptyFile) {
VersionEdit edit;
edit.AddFile(0, 0, 0, 0, InternalKey(), InternalKey(), 0, 0, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
std::string buffer;
ASSERT_TRUE(!edit.EncodeTo(&buffer));
}
TEST_F(VersionEditTest, ColumnFamilyTest) {
VersionEdit edit;
edit.SetColumnFamily(2);
edit.AddColumnFamily("column_family");
edit.SetMaxColumnFamily(5);
TestEncodeDecode(edit);
edit.Clear();
edit.SetColumnFamily(3);
edit.DropColumnFamily();
TestEncodeDecode(edit);
}
TEST_F(VersionEditTest, MinLogNumberToKeep) {
VersionEdit edit;
edit.SetMinLogNumberToKeep(13);
TestEncodeDecode(edit);
edit.Clear();
edit.SetMinLogNumberToKeep(23);
TestEncodeDecode(edit);
}
TEST_F(VersionEditTest, AtomicGroupTest) {
VersionEdit edit;
edit.MarkAtomicGroup(1);
TestEncodeDecode(edit);
}
TEST_F(VersionEditTest, IgnorableField) {
VersionEdit ve;
std::string encoded;
// Size of ignorable field is too large
PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66);
// This is a customized ignorable tag
PutVarint32Varint64(&encoded,
0x2710 /* A field with kTagSafeIgnoreMask set */,
5 /* fieldlength 5 */);
encoded += "abc"; // Only fills 3 bytes,
ASSERT_NOK(ve.DecodeFrom(encoded));
encoded.clear();
// Error when seeing unidentified tag that is not ignorable
PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66);
// This is a customized ignorable tag
PutVarint32Varint64(&encoded, 666 /* A field with kTagSafeIgnoreMask unset */,
3 /* fieldlength 3 */);
encoded += "abc"; // Fill 3 bytes
PutVarint32Varint64(&encoded, 3 /* next file number */, 88);
ASSERT_NOK(ve.DecodeFrom(encoded));
// Safely ignore an identified but safely ignorable entry
encoded.clear();
PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66);
// This is a customized ignorable tag
PutVarint32Varint64(&encoded,
0x2710 /* A field with kTagSafeIgnoreMask set */,
3 /* fieldlength 3 */);
encoded += "abc"; // Fill 3 bytes
PutVarint32Varint64(&encoded, 3 /* kNextFileNumber */, 88);
ASSERT_OK(ve.DecodeFrom(encoded));
ASSERT_TRUE(ve.HasLogNumber());
ASSERT_TRUE(ve.HasNextFile());
ASSERT_EQ(66, ve.GetLogNumber());
ASSERT_EQ(88, ve.GetNextFile());
}
TEST_F(VersionEditTest, DbId) {
VersionEdit edit;
edit.SetDBId("ab34-cd12-435f-er00");
TestEncodeDecode(edit);
edit.Clear();
edit.SetDBId("34ba-cd12-435f-er01");
TestEncodeDecode(edit);
}
TEST_F(VersionEditTest, ManifestSwitchTag) {
VersionEdit edit1, decode1;
edit1.SetStateUponManifestSwitch(true);
TestEncodeDecode(edit1);
std::string encoded1;
edit1.EncodeTo(&encoded1);
ASSERT_OK(decode1.DecodeFrom(encoded1));
ASSERT_TRUE(decode1.GetStateUponManifestSwitch());
ASSERT_TRUE(!decode1.GetManifestSwitched());
VersionEdit edit2, decode2;
edit2.SetManifestSwitched(true);
TestEncodeDecode(edit2);
std::string encoded2;
edit2.EncodeTo(&encoded2);
ASSERT_OK(decode2.DecodeFrom(encoded2));
ASSERT_TRUE(!decode2.GetStateUponManifestSwitch());
ASSERT_TRUE(decode2.GetManifestSwitched());
VersionEdit edit3, decode3;
edit3.SetStateUponManifestSwitch(true);
edit3.SetManifestSwitched(true);
TestEncodeDecode(edit3);
std::string encoded3;
edit3.EncodeTo(&encoded3);
ASSERT_OK(decode3.DecodeFrom(encoded3));
ASSERT_TRUE(decode3.GetStateUponManifestSwitch());
ASSERT_TRUE(decode3.GetManifestSwitched());
}
TEST_F(VersionEditTest, BlobFileAdditionAndGarbage) {
VersionEdit edit;
const std::string checksum_method_prefix = "Hash";
const std::string checksum_value_prefix = "Value";
for (uint64_t blob_file_number = 1; blob_file_number <= 10;
++blob_file_number) {
const uint64_t total_blob_count = blob_file_number << 10;
const uint64_t total_blob_bytes = blob_file_number << 20;
std::string checksum_method(checksum_method_prefix);
AppendNumberTo(&checksum_method, blob_file_number);
std::string checksum_value(checksum_value_prefix);
AppendNumberTo(&checksum_value, blob_file_number);
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value);
const uint64_t garbage_blob_count = total_blob_count >> 2;
const uint64_t garbage_blob_bytes = total_blob_bytes >> 1;
edit.AddBlobFileGarbage(blob_file_number, garbage_blob_count,
garbage_blob_bytes);
}
TestEncodeDecode(edit);
}
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
Reference in New Issue View Git Blame Copy Permalink