4369f2c7bb
Summary: In the current code base, RocksDB generate the checksum for each block and verify the checksum at usage. Current PR enable SST file checksum. After a SST file is generated by Flush or Compaction, RocksDB generate the SST file checksum and store the checksum value and checksum method name in the vs_info and MANIFEST as part for the FileMetadata. Added the enable_sst_file_checksum to Options to enable or disable file checksum. Added sst_file_checksum to Options such that user can plugin their own SST file checksum calculate method via overriding the SstFileChecksum class. The checksum information inlcuding uint32_t checksum value and a checksum name (string). A new tool is added to LDB such that user can dump out a list of file checksum information from MANIFEST. If user enables the file checksum but does not provide the sst_file_checksum instance, RocksDB will use the default crc32checksum implemented in table/sst_file_checksum_crc32c.h Pull Request resolved: https://github.com/facebook/rocksdb/pull/6216 Test Plan: Added the testing case in table_test and ldb_cmd_test to verify checksum is correct in different level. Pass make asan_check. Differential Revision: D19171461 Pulled By: zhichao-cao fbshipit-source-id: b2e53479eefc5bb0437189eaa1941670e5ba8b87
350 lines
13 KiB
C++
350 lines
13 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#include <string>
|
|
#include "db/version_edit.h"
|
|
#include "db/version_set.h"
|
|
#include "logging/logging.h"
|
|
#include "test_util/testharness.h"
|
|
#include "test_util/testutil.h"
|
|
#include "util/string_util.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class VersionBuilderTest : public testing::Test {
|
|
public:
|
|
const Comparator* ucmp_;
|
|
InternalKeyComparator icmp_;
|
|
Options options_;
|
|
ImmutableCFOptions ioptions_;
|
|
MutableCFOptions mutable_cf_options_;
|
|
VersionStorageInfo vstorage_;
|
|
uint32_t file_num_;
|
|
CompactionOptionsFIFO fifo_options_;
|
|
std::vector<uint64_t> size_being_compacted_;
|
|
|
|
VersionBuilderTest()
|
|
: ucmp_(BytewiseComparator()),
|
|
icmp_(ucmp_),
|
|
ioptions_(options_),
|
|
mutable_cf_options_(options_),
|
|
vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
|
|
nullptr, false),
|
|
file_num_(1) {
|
|
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
|
|
size_being_compacted_.resize(options_.num_levels);
|
|
}
|
|
|
|
~VersionBuilderTest() override {
|
|
for (int i = 0; i < vstorage_.num_levels(); i++) {
|
|
for (auto* f : vstorage_.LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
InternalKey GetInternalKey(const char* ukey,
|
|
SequenceNumber smallest_seq = 100) {
|
|
return InternalKey(ukey, smallest_seq, kTypeValue);
|
|
}
|
|
|
|
void Add(int level, uint32_t file_number, const char* smallest,
|
|
const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
|
|
SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100,
|
|
uint64_t num_entries = 0, uint64_t num_deletions = 0,
|
|
bool sampled = false, SequenceNumber smallest_seqno = 0,
|
|
SequenceNumber largest_seqno = 0) {
|
|
assert(level < vstorage_.num_levels());
|
|
FileMetaData* f = new FileMetaData(
|
|
file_number, path_id, file_size, GetInternalKey(smallest, smallest_seq),
|
|
GetInternalKey(largest, largest_seq), smallest_seqno, largest_seqno,
|
|
/* marked_for_compact */ false, kInvalidBlobFileNumber,
|
|
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
|
|
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
|
|
f->compensated_file_size = file_size;
|
|
f->num_entries = num_entries;
|
|
f->num_deletions = num_deletions;
|
|
vstorage_.AddFile(level, f);
|
|
if (sampled) {
|
|
f->init_stats_from_file = true;
|
|
vstorage_.UpdateAccumulatedStats(f);
|
|
}
|
|
}
|
|
|
|
void UpdateVersionStorageInfo() {
|
|
vstorage_.UpdateFilesByCompactionPri(ioptions_.compaction_pri);
|
|
vstorage_.UpdateNumNonEmptyLevels();
|
|
vstorage_.GenerateFileIndexer();
|
|
vstorage_.GenerateLevelFilesBrief();
|
|
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
|
|
vstorage_.GenerateLevel0NonOverlapping();
|
|
vstorage_.SetFinalized();
|
|
}
|
|
};
|
|
|
|
void UnrefFilesInVersion(VersionStorageInfo* new_vstorage) {
|
|
for (int i = 0; i < new_vstorage->num_levels(); i++) {
|
|
for (auto* f : new_vstorage->LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
|
|
Add(0, 1U, "150", "200", 100U);
|
|
|
|
Add(1, 66U, "150", "200", 100U);
|
|
Add(1, 88U, "201", "300", 100U);
|
|
|
|
Add(2, 6U, "150", "179", 100U);
|
|
Add(2, 7U, "180", "220", 100U);
|
|
Add(2, 8U, "221", "300", 100U);
|
|
|
|
Add(3, 26U, "150", "170", 100U);
|
|
Add(3, 27U, "171", "179", 100U);
|
|
Add(3, 28U, "191", "220", 100U);
|
|
Add(3, 29U, "221", "300", 100U);
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.DeleteFile(3, 27U);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr, false);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2));
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
|
|
Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
|
|
|
|
Add(4, 6U, "150", "179", 100U);
|
|
Add(4, 7U, "180", "220", 100U);
|
|
Add(4, 8U, "221", "300", 100U);
|
|
|
|
Add(5, 26U, "150", "170", 100U);
|
|
Add(5, 27U, "171", "179", 100U);
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(3, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.DeleteFile(0, 1U);
|
|
version_edit.DeleteFile(0, 88U);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr, false);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
|
|
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(3));
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(4));
|
|
ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
|
|
Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
|
|
|
|
Add(4, 6U, "150", "179", 100U);
|
|
Add(4, 7U, "180", "220", 100U);
|
|
Add(4, 8U, "221", "300", 100U);
|
|
|
|
Add(5, 26U, "150", "170", 100U);
|
|
Add(5, 27U, "171", "179", 100U);
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(4, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.DeleteFile(0, 1U);
|
|
version_edit.DeleteFile(0, 88U);
|
|
version_edit.DeleteFile(4, 6U);
|
|
version_edit.DeleteFile(4, 7U);
|
|
version_edit.DeleteFile(4, 8U);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr, false);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
|
|
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(4));
|
|
ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
|
|
GetInternalKey("450"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
|
|
GetInternalKey("650"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
|
|
GetInternalKey("550"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
|
|
GetInternalKey("750"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr, false);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
|
|
UpdateVersionStorageInfo();
|
|
|
|
EnvOptions env_options;
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr, false);
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
|
|
GetInternalKey("450"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
|
|
GetInternalKey("650"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
|
|
GetInternalKey("550"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
|
|
GetInternalKey("750"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_builder.Apply(&version_edit);
|
|
|
|
VersionEdit version_edit2;
|
|
version_edit.AddFile(2, 808, 0, 100U, GetInternalKey("901"),
|
|
GetInternalKey("950"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_edit2.DeleteFile(2, 616);
|
|
version_edit2.DeleteFile(2, 636);
|
|
version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"),
|
|
GetInternalKey("850"), 200, 200, false,
|
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
kUnknownFileChecksumFuncName);
|
|
version_builder.Apply(&version_edit2);
|
|
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, EstimatedActiveKeys) {
|
|
const uint32_t kTotalSamples = 20;
|
|
const uint32_t kNumLevels = 5;
|
|
const uint32_t kFilesPerLevel = 8;
|
|
const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
|
|
const uint32_t kEntriesPerFile = 1000;
|
|
const uint32_t kDeletionsPerFile = 100;
|
|
for (uint32_t i = 0; i < kNumFiles; ++i) {
|
|
Add(static_cast<int>(i / kFilesPerLevel), i + 1,
|
|
ToString((i + 100) * 1000).c_str(),
|
|
ToString((i + 100) * 1000 + 999).c_str(),
|
|
100U, 0, 100, 100,
|
|
kEntriesPerFile, kDeletionsPerFile,
|
|
(i < kTotalSamples));
|
|
}
|
|
// minus 2X for the number of deletion entries because:
|
|
// 1x for deletion entry does not count as a data entry.
|
|
// 1x for each deletion entry will actually remove one data entry.
|
|
ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
|
|
(kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
|
|
}
|
|
|
|
} // namespace rocksdb
|
|
|
|
int main(int argc, char** argv) {
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
return RUN_ALL_TESTS();
|
|
}
|