rocksdb/table/cuckoo/cuckoo_table_factory.cc
Peter Dillinger 04db764831 Embed original file number in SST table properties (#8686)
Summary:
I very recently realized that with https://github.com/facebook/rocksdb/issues/8669 we cannot later add
file numbers to external SST files (so that more can share db session
ids for better uniqueness properties), because of forward compatibility.
We would have a version of RocksDB that assumes session IDs are unique
on external SST files and therefore can't really break that invariant in
future files.

This change adds a table property for "orig_file_number" which is
populated by normal SST files and also external SST files generated by
SstFileWriter. SstFileWriter now keeps a db_session_id for life of the
object and increments its own file numbers for embedding in table
properties. (They are arguably "fake" file numbers because these numbers
and not embedded in the file name.)

While updating block_based_table_builder, I removed several unnecessary
fields from Rep, because following the pattern would have created
another unnecessary field.

This change also updates block_based_table_reader to use this new
property when available, which means that for newer SST files, we can
determine the stable/original <db_session_id,file_number> unique
identifier using just the file contents, not the file name. (It's a bit
complicated; detailed comments in block_based_table_reader.)

Also added DB host id to properties listing by sst_dump, which could be
useful in debugging.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8686

Test Plan: majorly overhauled StableCacheKeys test for this change

Reviewed By: zhichao-cao

Differential Revision: D30457742

Pulled By: pdillinger

fbshipit-source-id: 2e5ae7dddeb94fb9d8eac8a928486aed8b8cd445
2021-08-20 20:40:48 -07:00

105 lines
4.0 KiB
C++

// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include "table/cuckoo/cuckoo_table_factory.h"
#include "db/dbformat.h"
#include "options/configurable_helper.h"
#include "rocksdb/utilities/options_type.h"
#include "table/cuckoo/cuckoo_table_builder.h"
#include "table/cuckoo/cuckoo_table_reader.h"
namespace ROCKSDB_NAMESPACE {
Status CuckooTableFactory::NewTableReader(
const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options,
std::unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
std::unique_ptr<TableReader>* table,
bool /*prefetch_index_and_filter_in_cache*/) const {
std::unique_ptr<CuckooTableReader> new_reader(new CuckooTableReader(
table_reader_options.ioptions, std::move(file), file_size,
table_reader_options.internal_comparator.user_comparator(), nullptr));
Status s = new_reader->status();
if (s.ok()) {
*table = std::move(new_reader);
}
return s;
}
TableBuilder* CuckooTableFactory::NewTableBuilder(
const TableBuilderOptions& table_builder_options,
WritableFileWriter* file) const {
// TODO: change builder to take the option struct
return new CuckooTableBuilder(
file, table_options_.hash_table_ratio, 64,
table_options_.max_search_depth,
table_builder_options.internal_comparator.user_comparator(),
table_options_.cuckoo_block_size, table_options_.use_module_hash,
table_options_.identity_as_first_hash, nullptr /* get_slice_hash */,
table_builder_options.column_family_id,
table_builder_options.column_family_name, table_builder_options.db_id,
table_builder_options.db_session_id, table_builder_options.cur_file_num);
}
std::string CuckooTableFactory::GetPrintableOptions() const {
std::string ret;
ret.reserve(2000);
const int kBufferSize = 200;
char buffer[kBufferSize];
snprintf(buffer, kBufferSize, " hash_table_ratio: %lf\n",
table_options_.hash_table_ratio);
ret.append(buffer);
snprintf(buffer, kBufferSize, " max_search_depth: %u\n",
table_options_.max_search_depth);
ret.append(buffer);
snprintf(buffer, kBufferSize, " cuckoo_block_size: %u\n",
table_options_.cuckoo_block_size);
ret.append(buffer);
snprintf(buffer, kBufferSize, " identity_as_first_hash: %d\n",
table_options_.identity_as_first_hash);
ret.append(buffer);
return ret;
}
static std::unordered_map<std::string, OptionTypeInfo> cuckoo_table_type_info =
{
#ifndef ROCKSDB_LITE
{"hash_table_ratio",
{offsetof(struct CuckooTableOptions, hash_table_ratio),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"max_search_depth",
{offsetof(struct CuckooTableOptions, max_search_depth),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"cuckoo_block_size",
{offsetof(struct CuckooTableOptions, cuckoo_block_size),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"identity_as_first_hash",
{offsetof(struct CuckooTableOptions, identity_as_first_hash),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"use_module_hash",
{offsetof(struct CuckooTableOptions, use_module_hash),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
CuckooTableFactory::CuckooTableFactory(const CuckooTableOptions& table_options)
: table_options_(table_options) {
RegisterOptions(&table_options_, &cuckoo_table_type_info);
}
TableFactory* NewCuckooTableFactory(const CuckooTableOptions& table_options) {
return new CuckooTableFactory(table_options);
}
} // namespace ROCKSDB_NAMESPACE
#endif // ROCKSDB_LITE