4369f2c7bb
Summary: In the current code base, RocksDB generate the checksum for each block and verify the checksum at usage. Current PR enable SST file checksum. After a SST file is generated by Flush or Compaction, RocksDB generate the SST file checksum and store the checksum value and checksum method name in the vs_info and MANIFEST as part for the FileMetadata. Added the enable_sst_file_checksum to Options to enable or disable file checksum. Added sst_file_checksum to Options such that user can plugin their own SST file checksum calculate method via overriding the SstFileChecksum class. The checksum information inlcuding uint32_t checksum value and a checksum name (string). A new tool is added to LDB such that user can dump out a list of file checksum information from MANIFEST. If user enables the file checksum but does not provide the sst_file_checksum instance, RocksDB will use the default crc32checksum implemented in table/sst_file_checksum_crc32c.h Pull Request resolved: https://github.com/facebook/rocksdb/pull/6216 Test Plan: Added the testing case in table_test and ldb_cmd_test to verify checksum is correct in different level. Pass make asan_check. Differential Revision: D19171461 Pulled By: zhichao-cao fbshipit-source-id: b2e53479eefc5bb0437189eaa1941670e5ba8b87
86 lines
2.7 KiB
C++
86 lines
2.7 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#include "util/file_checksum_helper.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
void FileChecksumListImpl::reset() { checksum_map_.clear(); }
|
|
|
|
size_t FileChecksumListImpl::size() const { return checksum_map_.size(); }
|
|
|
|
Status FileChecksumListImpl::GetAllFileChecksums(
|
|
std::vector<uint64_t>* file_numbers, std::vector<std::string>* checksums,
|
|
std::vector<std::string>* checksum_func_names) {
|
|
if (file_numbers == nullptr || checksums == nullptr ||
|
|
checksum_func_names == nullptr) {
|
|
return Status::InvalidArgument("Pointer has not been initiated");
|
|
}
|
|
|
|
for (auto i : checksum_map_) {
|
|
file_numbers->push_back(i.first);
|
|
checksums->push_back(i.second.first);
|
|
checksum_func_names->push_back(i.second.second);
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
Status FileChecksumListImpl::SearchOneFileChecksum(
|
|
uint64_t file_number, std::string* checksum,
|
|
std::string* checksum_func_name) {
|
|
if (checksum == nullptr || checksum_func_name == nullptr) {
|
|
return Status::InvalidArgument("Pointer has not been initiated");
|
|
}
|
|
|
|
auto it = checksum_map_.find(file_number);
|
|
if (it == checksum_map_.end()) {
|
|
return Status::NotFound();
|
|
} else {
|
|
*checksum = it->second.first;
|
|
*checksum_func_name = it->second.second;
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
Status FileChecksumListImpl::InsertOneFileChecksum(
|
|
uint64_t file_number, const std::string& checksum,
|
|
const std::string& checksum_func_name) {
|
|
auto it = checksum_map_.find(file_number);
|
|
if (it == checksum_map_.end()) {
|
|
checksum_map_.insert(std::make_pair(
|
|
file_number, std::make_pair(checksum, checksum_func_name)));
|
|
} else {
|
|
it->second.first = checksum;
|
|
it->second.second = checksum_func_name;
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
Status FileChecksumListImpl::RemoveOneFileChecksum(uint64_t file_number) {
|
|
auto it = checksum_map_.find(file_number);
|
|
if (it == checksum_map_.end()) {
|
|
return Status::NotFound();
|
|
} else {
|
|
checksum_map_.erase(it);
|
|
}
|
|
return Status::OK();
|
|
}
|
|
|
|
FileChecksumList* NewFileChecksumList() {
|
|
FileChecksumListImpl* checksum_list = new FileChecksumListImpl();
|
|
return checksum_list;
|
|
}
|
|
|
|
FileChecksumFunc* CreateFileChecksumFuncCrc32c() {
|
|
FileChecksumFunc* file_checksum_crc32c = new FileChecksumFuncCrc32c();
|
|
return file_checksum_crc32c;
|
|
}
|
|
|
|
} // namespace rocksdb
|