937fbcbddc
Summary: Track per-SST user-defined timestamp information in MANIFEST https://github.com/facebook/rocksdb/issues/8957 Rockdb has supported user-defined timestamp feature. Application can specify a timestamp when writing each k-v pair. When data flush from memory to disk file called SST files, file creation activity will commit to MANIFEST. This commit is for tracking timestamp info in the MANIFEST for each file. The changes involved are as follows: 1) Track max/min timestamp in FileMetaData, and fix invoved codes. 2) Add NewFileCustomTag::kMinTimestamp and NewFileCustomTag::kMinTimestamp in NewFileCustomTag ( in the kNewFile4 part ), and support invoved codes such as VersionEdit Encode and Decode etc. 3) Add unit test code for VersionEdit EncodeDecodeNewFile4, and fix invoved test codes. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9092 Reviewed By: ajkr, akankshamahajan15 Differential Revision: D32252323 Pulled By: riversand963 fbshipit-source-id: d2642898d6e3ad1fef0eb866b98045408bd4e162
616 lines
21 KiB
C++
616 lines
21 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#include "db/version_edit.h"
|
|
|
|
#include "test_util/sync_point.h"
|
|
#include "test_util/testharness.h"
|
|
#include "test_util/testutil.h"
|
|
#include "util/coding.h"
|
|
#include "util/string_util.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
static void TestEncodeDecode(const VersionEdit& edit) {
|
|
std::string encoded, encoded2;
|
|
edit.EncodeTo(&encoded);
|
|
VersionEdit parsed;
|
|
Status s = parsed.DecodeFrom(encoded);
|
|
ASSERT_TRUE(s.ok()) << s.ToString();
|
|
parsed.EncodeTo(&encoded2);
|
|
ASSERT_EQ(encoded, encoded2);
|
|
}
|
|
|
|
class VersionEditTest : public testing::Test {};
|
|
|
|
TEST_F(VersionEditTest, EncodeDecode) {
|
|
static const uint64_t kBig = 1ull << 50;
|
|
static const uint32_t kBig32Bit = 1ull << 30;
|
|
|
|
VersionEdit edit;
|
|
for (int i = 0; i < 4; i++) {
|
|
TestEncodeDecode(edit);
|
|
edit.AddFile(3, kBig + 300 + i, kBig32Bit + 400 + i, 0,
|
|
InternalKey("foo", kBig + 500 + i, kTypeValue),
|
|
InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
|
|
kBig + 500 + i, kBig + 600 + i, false, kInvalidBlobFileNumber,
|
|
888, 678, "234", "crc32c", "123", "345");
|
|
edit.DeleteFile(4, kBig + 700 + i);
|
|
}
|
|
|
|
edit.SetComparatorName("foo");
|
|
edit.SetLogNumber(kBig + 100);
|
|
edit.SetNextFile(kBig + 200);
|
|
edit.SetLastSequence(kBig + 1000);
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
|
|
static const uint64_t kBig = 1ull << 50;
|
|
|
|
VersionEdit edit;
|
|
edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue),
|
|
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
|
|
kBig + 600, true, kInvalidBlobFileNumber,
|
|
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
|
|
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123",
|
|
"234");
|
|
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
|
|
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
|
|
kBig + 601, false, kInvalidBlobFileNumber,
|
|
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
|
|
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "345",
|
|
"543");
|
|
edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue),
|
|
InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502,
|
|
kBig + 602, true, kInvalidBlobFileNumber, 666, 888,
|
|
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "456",
|
|
"567");
|
|
edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex),
|
|
InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503,
|
|
kBig + 603, true, 1001, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName, "678", "789");
|
|
;
|
|
|
|
edit.DeleteFile(4, 700);
|
|
|
|
edit.SetComparatorName("foo");
|
|
edit.SetLogNumber(kBig + 100);
|
|
edit.SetNextFile(kBig + 200);
|
|
edit.SetLastSequence(kBig + 1000);
|
|
TestEncodeDecode(edit);
|
|
|
|
std::string encoded, encoded2;
|
|
edit.EncodeTo(&encoded);
|
|
VersionEdit parsed;
|
|
Status s = parsed.DecodeFrom(encoded);
|
|
ASSERT_TRUE(s.ok()) << s.ToString();
|
|
auto& new_files = parsed.GetNewFiles();
|
|
ASSERT_TRUE(new_files[0].second.marked_for_compaction);
|
|
ASSERT_TRUE(!new_files[1].second.marked_for_compaction);
|
|
ASSERT_TRUE(new_files[2].second.marked_for_compaction);
|
|
ASSERT_TRUE(new_files[3].second.marked_for_compaction);
|
|
ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
|
|
ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
|
|
ASSERT_EQ(0u, new_files[2].second.fd.GetPathId());
|
|
ASSERT_EQ(0u, new_files[3].second.fd.GetPathId());
|
|
ASSERT_EQ(kInvalidBlobFileNumber,
|
|
new_files[0].second.oldest_blob_file_number);
|
|
ASSERT_EQ(kInvalidBlobFileNumber,
|
|
new_files[1].second.oldest_blob_file_number);
|
|
ASSERT_EQ(kInvalidBlobFileNumber,
|
|
new_files[2].second.oldest_blob_file_number);
|
|
ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number);
|
|
ASSERT_EQ("123", new_files[0].second.min_timestamp);
|
|
ASSERT_EQ("234", new_files[0].second.max_timestamp);
|
|
ASSERT_EQ("345", new_files[1].second.min_timestamp);
|
|
ASSERT_EQ("543", new_files[1].second.max_timestamp);
|
|
ASSERT_EQ("456", new_files[2].second.min_timestamp);
|
|
ASSERT_EQ("567", new_files[2].second.max_timestamp);
|
|
ASSERT_EQ("678", new_files[3].second.min_timestamp);
|
|
ASSERT_EQ("789", new_files[3].second.max_timestamp);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
|
|
static const uint64_t kBig = 1ull << 50;
|
|
VersionEdit edit;
|
|
edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue),
|
|
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
|
|
kBig + 600, true, kInvalidBlobFileNumber,
|
|
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
|
|
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123",
|
|
"234");
|
|
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
|
|
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
|
|
kBig + 601, false, kInvalidBlobFileNumber, 686, 868, "234",
|
|
"crc32c", kDisableUserTimestamp, kDisableUserTimestamp);
|
|
edit.DeleteFile(4, 700);
|
|
|
|
edit.SetComparatorName("foo");
|
|
edit.SetLogNumber(kBig + 100);
|
|
edit.SetNextFile(kBig + 200);
|
|
edit.SetLastSequence(kBig + 1000);
|
|
|
|
std::string encoded;
|
|
|
|
// Call back function to add extra customized builds.
|
|
bool first = true;
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
"VersionEdit::EncodeTo:NewFile4:CustomizeFields", [&](void* arg) {
|
|
std::string* str = reinterpret_cast<std::string*>(arg);
|
|
PutVarint32(str, 33);
|
|
const std::string str1 = "random_string";
|
|
PutLengthPrefixedSlice(str, str1);
|
|
if (first) {
|
|
first = false;
|
|
PutVarint32(str, 22);
|
|
const std::string str2 = "s";
|
|
PutLengthPrefixedSlice(str, str2);
|
|
}
|
|
});
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
|
edit.EncodeTo(&encoded);
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
|
|
VersionEdit parsed;
|
|
Status s = parsed.DecodeFrom(encoded);
|
|
ASSERT_TRUE(s.ok()) << s.ToString();
|
|
ASSERT_TRUE(!first);
|
|
auto& new_files = parsed.GetNewFiles();
|
|
ASSERT_TRUE(new_files[0].second.marked_for_compaction);
|
|
ASSERT_TRUE(!new_files[1].second.marked_for_compaction);
|
|
ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
|
|
ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
|
|
ASSERT_EQ(1u, parsed.GetDeletedFiles().size());
|
|
ASSERT_EQ("123", new_files[0].second.min_timestamp);
|
|
ASSERT_EQ("234", new_files[0].second.max_timestamp);
|
|
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.min_timestamp);
|
|
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.max_timestamp);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, NewFile4NotSupportedField) {
|
|
static const uint64_t kBig = 1ull << 50;
|
|
VersionEdit edit;
|
|
edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue),
|
|
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
|
|
kBig + 600, true, kInvalidBlobFileNumber,
|
|
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
|
|
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
|
|
kDisableUserTimestamp, kDisableUserTimestamp);
|
|
|
|
edit.SetComparatorName("foo");
|
|
edit.SetLogNumber(kBig + 100);
|
|
edit.SetNextFile(kBig + 200);
|
|
edit.SetLastSequence(kBig + 1000);
|
|
|
|
std::string encoded;
|
|
|
|
// Call back function to add extra customized builds.
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
|
|
"VersionEdit::EncodeTo:NewFile4:CustomizeFields", [&](void* arg) {
|
|
std::string* str = reinterpret_cast<std::string*>(arg);
|
|
const std::string str1 = "s";
|
|
PutLengthPrefixedSlice(str, str1);
|
|
});
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
|
edit.EncodeTo(&encoded);
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
|
|
VersionEdit parsed;
|
|
Status s = parsed.DecodeFrom(encoded);
|
|
ASSERT_NOK(s);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, EncodeEmptyFile) {
|
|
VersionEdit edit;
|
|
edit.AddFile(0, 0, 0, 0, InternalKey(), InternalKey(), 0, 0, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
|
|
kDisableUserTimestamp);
|
|
std::string buffer;
|
|
ASSERT_TRUE(!edit.EncodeTo(&buffer));
|
|
}
|
|
|
|
TEST_F(VersionEditTest, ColumnFamilyTest) {
|
|
VersionEdit edit;
|
|
edit.SetColumnFamily(2);
|
|
edit.AddColumnFamily("column_family");
|
|
edit.SetMaxColumnFamily(5);
|
|
TestEncodeDecode(edit);
|
|
|
|
edit.Clear();
|
|
edit.SetColumnFamily(3);
|
|
edit.DropColumnFamily();
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, MinLogNumberToKeep) {
|
|
VersionEdit edit;
|
|
edit.SetMinLogNumberToKeep(13);
|
|
TestEncodeDecode(edit);
|
|
|
|
edit.Clear();
|
|
edit.SetMinLogNumberToKeep(23);
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, AtomicGroupTest) {
|
|
VersionEdit edit;
|
|
edit.MarkAtomicGroup(1);
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, IgnorableField) {
|
|
VersionEdit ve;
|
|
std::string encoded;
|
|
|
|
// Size of ignorable field is too large
|
|
PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66);
|
|
// This is a customized ignorable tag
|
|
PutVarint32Varint64(&encoded,
|
|
0x2710 /* A field with kTagSafeIgnoreMask set */,
|
|
5 /* fieldlength 5 */);
|
|
encoded += "abc"; // Only fills 3 bytes,
|
|
ASSERT_NOK(ve.DecodeFrom(encoded));
|
|
|
|
encoded.clear();
|
|
// Error when seeing unidentified tag that is not ignorable
|
|
PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66);
|
|
// This is a customized ignorable tag
|
|
PutVarint32Varint64(&encoded, 666 /* A field with kTagSafeIgnoreMask unset */,
|
|
3 /* fieldlength 3 */);
|
|
encoded += "abc"; // Fill 3 bytes
|
|
PutVarint32Varint64(&encoded, 3 /* next file number */, 88);
|
|
ASSERT_NOK(ve.DecodeFrom(encoded));
|
|
|
|
// Safely ignore an identified but safely ignorable entry
|
|
encoded.clear();
|
|
PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66);
|
|
// This is a customized ignorable tag
|
|
PutVarint32Varint64(&encoded,
|
|
0x2710 /* A field with kTagSafeIgnoreMask set */,
|
|
3 /* fieldlength 3 */);
|
|
encoded += "abc"; // Fill 3 bytes
|
|
PutVarint32Varint64(&encoded, 3 /* kNextFileNumber */, 88);
|
|
|
|
ASSERT_OK(ve.DecodeFrom(encoded));
|
|
|
|
ASSERT_TRUE(ve.HasLogNumber());
|
|
ASSERT_TRUE(ve.HasNextFile());
|
|
ASSERT_EQ(66, ve.GetLogNumber());
|
|
ASSERT_EQ(88, ve.GetNextFile());
|
|
}
|
|
|
|
TEST_F(VersionEditTest, DbId) {
|
|
VersionEdit edit;
|
|
edit.SetDBId("ab34-cd12-435f-er00");
|
|
TestEncodeDecode(edit);
|
|
|
|
edit.Clear();
|
|
edit.SetDBId("34ba-cd12-435f-er01");
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, BlobFileAdditionAndGarbage) {
|
|
VersionEdit edit;
|
|
|
|
const std::string checksum_method_prefix = "Hash";
|
|
const std::string checksum_value_prefix = "Value";
|
|
|
|
for (uint64_t blob_file_number = 1; blob_file_number <= 10;
|
|
++blob_file_number) {
|
|
const uint64_t total_blob_count = blob_file_number << 10;
|
|
const uint64_t total_blob_bytes = blob_file_number << 20;
|
|
|
|
std::string checksum_method(checksum_method_prefix);
|
|
AppendNumberTo(&checksum_method, blob_file_number);
|
|
|
|
std::string checksum_value(checksum_value_prefix);
|
|
AppendNumberTo(&checksum_value, blob_file_number);
|
|
|
|
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
|
|
checksum_method, checksum_value);
|
|
|
|
const uint64_t garbage_blob_count = total_blob_count >> 2;
|
|
const uint64_t garbage_blob_bytes = total_blob_bytes >> 1;
|
|
|
|
edit.AddBlobFileGarbage(blob_file_number, garbage_blob_count,
|
|
garbage_blob_bytes);
|
|
}
|
|
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, AddWalEncodeDecode) {
|
|
VersionEdit edit;
|
|
for (uint64_t log_number = 1; log_number <= 20; log_number++) {
|
|
WalMetadata meta;
|
|
bool has_size = rand() % 2 == 0;
|
|
if (has_size) {
|
|
meta.SetSyncedSizeInBytes(rand() % 1000);
|
|
}
|
|
edit.AddWal(log_number, meta);
|
|
}
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
static std::string PrefixEncodedWalAdditionWithLength(
|
|
const std::string& encoded) {
|
|
std::string ret;
|
|
PutVarint32(&ret, Tag::kWalAddition2);
|
|
PutLengthPrefixedSlice(&ret, encoded);
|
|
return ret;
|
|
}
|
|
|
|
TEST_F(VersionEditTest, AddWalDecodeBadLogNumber) {
|
|
std::string encoded;
|
|
|
|
{
|
|
// No log number.
|
|
std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded);
|
|
VersionEdit edit;
|
|
Status s = edit.DecodeFrom(encoded_edit);
|
|
ASSERT_TRUE(s.IsCorruption());
|
|
ASSERT_TRUE(s.ToString().find("Error decoding WAL log number") !=
|
|
std::string::npos)
|
|
<< s.ToString();
|
|
}
|
|
|
|
{
|
|
// log number should be varint64,
|
|
// but we only encode 128 which is not a valid representation of varint64.
|
|
char c = 0;
|
|
unsigned char* ptr = reinterpret_cast<unsigned char*>(&c);
|
|
*ptr = 128;
|
|
encoded.append(1, c);
|
|
|
|
std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded);
|
|
VersionEdit edit;
|
|
Status s = edit.DecodeFrom(encoded_edit);
|
|
ASSERT_TRUE(s.IsCorruption());
|
|
ASSERT_TRUE(s.ToString().find("Error decoding WAL log number") !=
|
|
std::string::npos)
|
|
<< s.ToString();
|
|
}
|
|
}
|
|
|
|
TEST_F(VersionEditTest, AddWalDecodeBadTag) {
|
|
constexpr WalNumber kLogNumber = 100;
|
|
constexpr uint64_t kSizeInBytes = 100;
|
|
|
|
std::string encoded;
|
|
PutVarint64(&encoded, kLogNumber);
|
|
|
|
{
|
|
// No tag.
|
|
std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded);
|
|
VersionEdit edit;
|
|
Status s = edit.DecodeFrom(encoded_edit);
|
|
ASSERT_TRUE(s.IsCorruption());
|
|
ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos)
|
|
<< s.ToString();
|
|
}
|
|
|
|
{
|
|
// Only has size tag, no terminate tag.
|
|
std::string encoded_with_size = encoded;
|
|
PutVarint32(&encoded_with_size,
|
|
static_cast<uint32_t>(WalAdditionTag::kSyncedSize));
|
|
PutVarint64(&encoded_with_size, kSizeInBytes);
|
|
|
|
std::string encoded_edit =
|
|
PrefixEncodedWalAdditionWithLength(encoded_with_size);
|
|
VersionEdit edit;
|
|
Status s = edit.DecodeFrom(encoded_edit);
|
|
ASSERT_TRUE(s.IsCorruption());
|
|
ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos)
|
|
<< s.ToString();
|
|
}
|
|
|
|
{
|
|
// Only has terminate tag.
|
|
std::string encoded_with_terminate = encoded;
|
|
PutVarint32(&encoded_with_terminate,
|
|
static_cast<uint32_t>(WalAdditionTag::kTerminate));
|
|
|
|
std::string encoded_edit =
|
|
PrefixEncodedWalAdditionWithLength(encoded_with_terminate);
|
|
VersionEdit edit;
|
|
ASSERT_OK(edit.DecodeFrom(encoded_edit));
|
|
auto& wal_addition = edit.GetWalAdditions()[0];
|
|
ASSERT_EQ(wal_addition.GetLogNumber(), kLogNumber);
|
|
ASSERT_FALSE(wal_addition.GetMetadata().HasSyncedSize());
|
|
}
|
|
}
|
|
|
|
TEST_F(VersionEditTest, AddWalDecodeNoSize) {
|
|
constexpr WalNumber kLogNumber = 100;
|
|
|
|
std::string encoded;
|
|
PutVarint64(&encoded, kLogNumber);
|
|
PutVarint32(&encoded, static_cast<uint32_t>(WalAdditionTag::kSyncedSize));
|
|
// No real size after the size tag.
|
|
|
|
{
|
|
// Without terminate tag.
|
|
std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded);
|
|
VersionEdit edit;
|
|
Status s = edit.DecodeFrom(encoded_edit);
|
|
ASSERT_TRUE(s.IsCorruption());
|
|
ASSERT_TRUE(s.ToString().find("Error decoding WAL file size") !=
|
|
std::string::npos)
|
|
<< s.ToString();
|
|
}
|
|
|
|
{
|
|
// With terminate tag.
|
|
PutVarint32(&encoded, static_cast<uint32_t>(WalAdditionTag::kTerminate));
|
|
|
|
std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded);
|
|
VersionEdit edit;
|
|
Status s = edit.DecodeFrom(encoded_edit);
|
|
ASSERT_TRUE(s.IsCorruption());
|
|
// The terminate tag is misunderstood as the size.
|
|
ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos)
|
|
<< s.ToString();
|
|
}
|
|
}
|
|
|
|
TEST_F(VersionEditTest, AddWalDebug) {
|
|
constexpr int n = 2;
|
|
constexpr std::array<uint64_t, n> kLogNumbers{{10, 20}};
|
|
constexpr std::array<uint64_t, n> kSizeInBytes{{100, 200}};
|
|
|
|
VersionEdit edit;
|
|
for (int i = 0; i < n; i++) {
|
|
edit.AddWal(kLogNumbers[i], WalMetadata(kSizeInBytes[i]));
|
|
}
|
|
|
|
const WalAdditions& wals = edit.GetWalAdditions();
|
|
|
|
ASSERT_TRUE(edit.IsWalAddition());
|
|
ASSERT_EQ(wals.size(), n);
|
|
for (int i = 0; i < n; i++) {
|
|
const WalAddition& wal = wals[i];
|
|
ASSERT_EQ(wal.GetLogNumber(), kLogNumbers[i]);
|
|
ASSERT_EQ(wal.GetMetadata().GetSyncedSizeInBytes(), kSizeInBytes[i]);
|
|
}
|
|
|
|
std::string expected_str = "VersionEdit {\n";
|
|
for (int i = 0; i < n; i++) {
|
|
std::stringstream ss;
|
|
ss << " WalAddition: log_number: " << kLogNumbers[i]
|
|
<< " synced_size_in_bytes: " << kSizeInBytes[i] << "\n";
|
|
expected_str += ss.str();
|
|
}
|
|
expected_str += " ColumnFamily: 0\n}\n";
|
|
ASSERT_EQ(edit.DebugString(true), expected_str);
|
|
|
|
std::string expected_json = "{\"EditNumber\": 4, \"WalAdditions\": [";
|
|
for (int i = 0; i < n; i++) {
|
|
std::stringstream ss;
|
|
ss << "{\"LogNumber\": " << kLogNumbers[i] << ", "
|
|
<< "\"SyncedSizeInBytes\": " << kSizeInBytes[i] << "}";
|
|
if (i < n - 1) ss << ", ";
|
|
expected_json += ss.str();
|
|
}
|
|
expected_json += "], \"ColumnFamily\": 0}";
|
|
ASSERT_EQ(edit.DebugJSON(4, true), expected_json);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, DeleteWalEncodeDecode) {
|
|
VersionEdit edit;
|
|
edit.DeleteWalsBefore(rand() % 100);
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, DeleteWalDebug) {
|
|
constexpr int n = 2;
|
|
constexpr std::array<uint64_t, n> kLogNumbers{{10, 20}};
|
|
|
|
VersionEdit edit;
|
|
edit.DeleteWalsBefore(kLogNumbers[n - 1]);
|
|
|
|
const WalDeletion& wal = edit.GetWalDeletion();
|
|
|
|
ASSERT_TRUE(edit.IsWalDeletion());
|
|
ASSERT_EQ(wal.GetLogNumber(), kLogNumbers[n - 1]);
|
|
|
|
std::string expected_str = "VersionEdit {\n";
|
|
{
|
|
std::stringstream ss;
|
|
ss << " WalDeletion: log_number: " << kLogNumbers[n - 1] << "\n";
|
|
expected_str += ss.str();
|
|
}
|
|
expected_str += " ColumnFamily: 0\n}\n";
|
|
ASSERT_EQ(edit.DebugString(true), expected_str);
|
|
|
|
std::string expected_json = "{\"EditNumber\": 4, \"WalDeletion\": ";
|
|
{
|
|
std::stringstream ss;
|
|
ss << "{\"LogNumber\": " << kLogNumbers[n - 1] << "}";
|
|
expected_json += ss.str();
|
|
}
|
|
expected_json += ", \"ColumnFamily\": 0}";
|
|
ASSERT_EQ(edit.DebugJSON(4, true), expected_json);
|
|
}
|
|
|
|
TEST_F(VersionEditTest, FullHistoryTsLow) {
|
|
VersionEdit edit;
|
|
ASSERT_FALSE(edit.HasFullHistoryTsLow());
|
|
std::string ts = test::EncodeInt(0);
|
|
edit.SetFullHistoryTsLow(ts);
|
|
TestEncodeDecode(edit);
|
|
}
|
|
|
|
// Tests that if RocksDB is downgraded, the new types of VersionEdits
|
|
// that have a tag larger than kTagSafeIgnoreMask can be safely ignored.
|
|
TEST_F(VersionEditTest, IgnorableTags) {
|
|
SyncPoint::GetInstance()->SetCallBack(
|
|
"VersionEdit::EncodeTo:IgnoreIgnorableTags", [&](void* arg) {
|
|
bool* ignore = static_cast<bool*>(arg);
|
|
*ignore = true;
|
|
});
|
|
SyncPoint::GetInstance()->EnableProcessing();
|
|
|
|
constexpr uint64_t kPrevLogNumber = 100;
|
|
constexpr uint64_t kLogNumber = 200;
|
|
constexpr uint64_t kNextFileNumber = 300;
|
|
constexpr uint64_t kColumnFamilyId = 400;
|
|
|
|
VersionEdit edit;
|
|
// Add some ignorable entries.
|
|
for (int i = 0; i < 2; i++) {
|
|
edit.AddWal(i + 1, WalMetadata(i + 2));
|
|
}
|
|
edit.SetDBId("db_id");
|
|
// Add unignorable entries.
|
|
edit.SetPrevLogNumber(kPrevLogNumber);
|
|
edit.SetLogNumber(kLogNumber);
|
|
// Add more ignorable entries.
|
|
edit.DeleteWalsBefore(100);
|
|
// Add unignorable entry.
|
|
edit.SetNextFile(kNextFileNumber);
|
|
// Add more ignorable entries.
|
|
edit.SetFullHistoryTsLow("ts");
|
|
// Add unignorable entry.
|
|
edit.SetColumnFamily(kColumnFamilyId);
|
|
|
|
std::string encoded;
|
|
ASSERT_TRUE(edit.EncodeTo(&encoded));
|
|
|
|
VersionEdit decoded;
|
|
ASSERT_OK(decoded.DecodeFrom(encoded));
|
|
|
|
// Check that all ignorable entries are ignored.
|
|
ASSERT_FALSE(decoded.HasDbId());
|
|
ASSERT_FALSE(decoded.HasFullHistoryTsLow());
|
|
ASSERT_FALSE(decoded.IsWalAddition());
|
|
ASSERT_FALSE(decoded.IsWalDeletion());
|
|
ASSERT_TRUE(decoded.GetWalAdditions().empty());
|
|
ASSERT_TRUE(decoded.GetWalDeletion().IsEmpty());
|
|
|
|
// Check that unignorable entries are still present.
|
|
ASSERT_EQ(edit.GetPrevLogNumber(), kPrevLogNumber);
|
|
ASSERT_EQ(edit.GetLogNumber(), kLogNumber);
|
|
ASSERT_EQ(edit.GetNextFile(), kNextFileNumber);
|
|
ASSERT_EQ(edit.GetColumnFamily(), kColumnFamilyId);
|
|
|
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
}
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
|
|
|
int main(int argc, char** argv) {
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
return RUN_ALL_TESTS();
|
|
}
|