04db764831
Summary: I very recently realized that with https://github.com/facebook/rocksdb/issues/8669 we cannot later add file numbers to external SST files (so that more can share db session ids for better uniqueness properties), because of forward compatibility. We would have a version of RocksDB that assumes session IDs are unique on external SST files and therefore can't really break that invariant in future files. This change adds a table property for "orig_file_number" which is populated by normal SST files and also external SST files generated by SstFileWriter. SstFileWriter now keeps a db_session_id for life of the object and increments its own file numbers for embedding in table properties. (They are arguably "fake" file numbers because these numbers and not embedded in the file name.) While updating block_based_table_builder, I removed several unnecessary fields from Rep, because following the pattern would have created another unnecessary field. This change also updates block_based_table_reader to use this new property when available, which means that for newer SST files, we can determine the stable/original <db_session_id,file_number> unique identifier using just the file contents, not the file name. (It's a bit complicated; detailed comments in block_based_table_reader.) Also added DB host id to properties listing by sst_dump, which could be useful in debugging. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8686 Test Plan: majorly overhauled StableCacheKeys test for this change Reviewed By: zhichao-cao Differential Revision: D30457742 Pulled By: pdillinger fbshipit-source-id: 2e5ae7dddeb94fb9d8eac8a928486aed8b8cd445
105 lines
4.0 KiB
C++
105 lines
4.0 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
#include "table/cuckoo/cuckoo_table_factory.h"
|
|
|
|
#include "db/dbformat.h"
|
|
#include "options/configurable_helper.h"
|
|
#include "rocksdb/utilities/options_type.h"
|
|
#include "table/cuckoo/cuckoo_table_builder.h"
|
|
#include "table/cuckoo/cuckoo_table_reader.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
Status CuckooTableFactory::NewTableReader(
|
|
const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options,
|
|
std::unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
|
|
std::unique_ptr<TableReader>* table,
|
|
bool /*prefetch_index_and_filter_in_cache*/) const {
|
|
std::unique_ptr<CuckooTableReader> new_reader(new CuckooTableReader(
|
|
table_reader_options.ioptions, std::move(file), file_size,
|
|
table_reader_options.internal_comparator.user_comparator(), nullptr));
|
|
Status s = new_reader->status();
|
|
if (s.ok()) {
|
|
*table = std::move(new_reader);
|
|
}
|
|
return s;
|
|
}
|
|
|
|
TableBuilder* CuckooTableFactory::NewTableBuilder(
|
|
const TableBuilderOptions& table_builder_options,
|
|
WritableFileWriter* file) const {
|
|
// TODO: change builder to take the option struct
|
|
return new CuckooTableBuilder(
|
|
file, table_options_.hash_table_ratio, 64,
|
|
table_options_.max_search_depth,
|
|
table_builder_options.internal_comparator.user_comparator(),
|
|
table_options_.cuckoo_block_size, table_options_.use_module_hash,
|
|
table_options_.identity_as_first_hash, nullptr /* get_slice_hash */,
|
|
table_builder_options.column_family_id,
|
|
table_builder_options.column_family_name, table_builder_options.db_id,
|
|
table_builder_options.db_session_id, table_builder_options.cur_file_num);
|
|
}
|
|
|
|
std::string CuckooTableFactory::GetPrintableOptions() const {
|
|
std::string ret;
|
|
ret.reserve(2000);
|
|
const int kBufferSize = 200;
|
|
char buffer[kBufferSize];
|
|
|
|
snprintf(buffer, kBufferSize, " hash_table_ratio: %lf\n",
|
|
table_options_.hash_table_ratio);
|
|
ret.append(buffer);
|
|
snprintf(buffer, kBufferSize, " max_search_depth: %u\n",
|
|
table_options_.max_search_depth);
|
|
ret.append(buffer);
|
|
snprintf(buffer, kBufferSize, " cuckoo_block_size: %u\n",
|
|
table_options_.cuckoo_block_size);
|
|
ret.append(buffer);
|
|
snprintf(buffer, kBufferSize, " identity_as_first_hash: %d\n",
|
|
table_options_.identity_as_first_hash);
|
|
ret.append(buffer);
|
|
return ret;
|
|
}
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo> cuckoo_table_type_info =
|
|
{
|
|
#ifndef ROCKSDB_LITE
|
|
{"hash_table_ratio",
|
|
{offsetof(struct CuckooTableOptions, hash_table_ratio),
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
OptionTypeFlags::kNone}},
|
|
{"max_search_depth",
|
|
{offsetof(struct CuckooTableOptions, max_search_depth),
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
OptionTypeFlags::kNone}},
|
|
{"cuckoo_block_size",
|
|
{offsetof(struct CuckooTableOptions, cuckoo_block_size),
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
OptionTypeFlags::kNone}},
|
|
{"identity_as_first_hash",
|
|
{offsetof(struct CuckooTableOptions, identity_as_first_hash),
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
OptionTypeFlags::kNone}},
|
|
{"use_module_hash",
|
|
{offsetof(struct CuckooTableOptions, use_module_hash),
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
OptionTypeFlags::kNone}},
|
|
#endif // ROCKSDB_LITE
|
|
};
|
|
|
|
CuckooTableFactory::CuckooTableFactory(const CuckooTableOptions& table_options)
|
|
: table_options_(table_options) {
|
|
RegisterOptions(&table_options_, &cuckoo_table_type_info);
|
|
}
|
|
|
|
TableFactory* NewCuckooTableFactory(const CuckooTableOptions& table_options) {
|
|
return new CuckooTableFactory(table_options);
|
|
}
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
|
#endif // ROCKSDB_LITE
|