Summary: In the current code base, RocksDB generate the checksum for each block and verify the checksum at usage. Current PR enable SST file checksum. After a SST file is generated by Flush or Compaction, RocksDB generate the SST file checksum and store the checksum value and checksum method name in the vs_info and MANIFEST as part for the FileMetadata. Added the enable_sst_file_checksum to Options to enable or disable file checksum. Added sst_file_checksum to Options such that user can plugin their own SST file checksum calculate method via overriding the SstFileChecksum class. The checksum information inlcuding uint32_t checksum value and a checksum name (string). A new tool is added to LDB such that user can dump out a list of file checksum information from MANIFEST. If user enables the file checksum but does not provide the sst_file_checksum instance, RocksDB will use the default crc32checksum implemented in table/sst_file_checksum_crc32c.h Pull Request resolved: https://github.com/facebook/rocksdb/pull/6216 Test Plan: Added the testing case in table_test and ldb_cmd_test to verify checksum is correct in different level. Pass make asan_check. Differential Revision: D19171461 Pulled By: zhichao-cao fbshipit-source-id: b2e53479eefc5bb0437189eaa1941670e5ba8b87
152 lines
5.2 KiB
C++
152 lines
5.2 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
#include <stdint.h>
|
|
#include <string>
|
|
#include <vector>
|
|
#include "db/version_edit.h"
|
|
#include "rocksdb/options.h"
|
|
#include "rocksdb/status.h"
|
|
#include "rocksdb/table.h"
|
|
#include "rocksdb/table_properties.h"
|
|
#include "table/plain/plain_table_bloom.h"
|
|
#include "table/plain/plain_table_index.h"
|
|
#include "table/plain/plain_table_key_coding.h"
|
|
#include "table/table_builder.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class BlockBuilder;
|
|
class BlockHandle;
|
|
class WritableFile;
|
|
class TableBuilder;
|
|
|
|
// The builder class of PlainTable. For description of PlainTable format
|
|
// See comments of class PlainTableFactory, where instances of
|
|
// PlainTableReader are created.
|
|
class PlainTableBuilder: public TableBuilder {
|
|
public:
|
|
// Create a builder that will store the contents of the table it is
|
|
// building in *file. Does not close the file. It is up to the
|
|
// caller to close the file after calling Finish(). The output file
|
|
// will be part of level specified by 'level'. A value of -1 means
|
|
// that the caller does not know which level the output file will reside.
|
|
PlainTableBuilder(
|
|
const ImmutableCFOptions& ioptions, const MutableCFOptions& moptions,
|
|
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
|
int_tbl_prop_collector_factories,
|
|
uint32_t column_family_id, WritableFileWriter* file,
|
|
uint32_t user_key_size, EncodingType encoding_type,
|
|
size_t index_sparseness, uint32_t bloom_bits_per_key,
|
|
const std::string& column_family_name, uint32_t num_probes = 6,
|
|
size_t huge_page_tlb_size = 0, double hash_table_ratio = 0,
|
|
bool store_index_in_file = false);
|
|
// No copying allowed
|
|
PlainTableBuilder(const PlainTableBuilder&) = delete;
|
|
void operator=(const PlainTableBuilder&) = delete;
|
|
|
|
// REQUIRES: Either Finish() or Abandon() has been called.
|
|
~PlainTableBuilder();
|
|
|
|
// Add key,value to the table being constructed.
|
|
// REQUIRES: key is after any previously added key according to comparator.
|
|
// REQUIRES: Finish(), Abandon() have not been called
|
|
void Add(const Slice& key, const Slice& value) override;
|
|
|
|
// Return non-ok iff some error has been detected.
|
|
Status status() const override;
|
|
|
|
// Finish building the table. Stops using the file passed to the
|
|
// constructor after this function returns.
|
|
// REQUIRES: Finish(), Abandon() have not been called
|
|
Status Finish() override;
|
|
|
|
// Indicate that the contents of this builder should be abandoned. Stops
|
|
// using the file passed to the constructor after this function returns.
|
|
// If the caller is not going to call Finish(), it must call Abandon()
|
|
// before destroying this builder.
|
|
// REQUIRES: Finish(), Abandon() have not been called
|
|
void Abandon() override;
|
|
|
|
// Number of calls to Add() so far.
|
|
uint64_t NumEntries() const override;
|
|
|
|
// Size of the file generated so far. If invoked after a successful
|
|
// Finish() call, returns the size of the final generated file.
|
|
uint64_t FileSize() const override;
|
|
|
|
TableProperties GetTableProperties() const override { return properties_; }
|
|
|
|
bool SaveIndexInFile() const { return store_index_in_file_; }
|
|
|
|
// Get file checksum
|
|
const std::string& GetFileChecksum() const override { return file_checksum_; }
|
|
|
|
// Get file checksum function name
|
|
const char* GetFileChecksumFuncName() const override;
|
|
|
|
private:
|
|
Arena arena_;
|
|
const ImmutableCFOptions& ioptions_;
|
|
const MutableCFOptions& moptions_;
|
|
std::vector<std::unique_ptr<IntTblPropCollector>>
|
|
table_properties_collectors_;
|
|
|
|
BloomBlockBuilder bloom_block_;
|
|
std::unique_ptr<PlainTableIndexBuilder> index_builder_;
|
|
|
|
WritableFileWriter* file_;
|
|
uint64_t offset_ = 0;
|
|
uint32_t bloom_bits_per_key_;
|
|
size_t huge_page_tlb_size_;
|
|
Status status_;
|
|
TableProperties properties_;
|
|
PlainTableKeyEncoder encoder_;
|
|
|
|
bool store_index_in_file_;
|
|
|
|
std::vector<uint32_t> keys_or_prefixes_hashes_;
|
|
bool closed_ = false; // Either Finish() or Abandon() has been called.
|
|
|
|
const SliceTransform* prefix_extractor_;
|
|
|
|
// Store file checksum. If checksum is disabled, its value is "0".
|
|
std::string file_checksum_ = kUnknownFileChecksum;
|
|
|
|
Slice GetPrefix(const Slice& target) const {
|
|
assert(target.size() >= 8); // target is internal key
|
|
return GetPrefixFromUserKey(GetUserKey(target));
|
|
}
|
|
|
|
Slice GetPrefix(const ParsedInternalKey& target) const {
|
|
return GetPrefixFromUserKey(target.user_key);
|
|
}
|
|
|
|
Slice GetUserKey(const Slice& key) const {
|
|
return Slice(key.data(), key.size() - 8);
|
|
}
|
|
|
|
Slice GetPrefixFromUserKey(const Slice& user_key) const {
|
|
if (!IsTotalOrderMode()) {
|
|
return prefix_extractor_->Transform(user_key);
|
|
} else {
|
|
// Use empty slice as prefix if prefix_extractor is not set.
|
|
// In that case,
|
|
// it falls back to pure binary search and
|
|
// total iterator seek is supported.
|
|
return Slice();
|
|
}
|
|
}
|
|
|
|
bool IsTotalOrderMode() const { return (prefix_extractor_ == nullptr); }
|
|
};
|
|
|
|
} // namespace rocksdb
|
|
|
|
#endif // ROCKSDB_LITE
|