andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Apply sample_for_compression to all block-based tables (#8105)

Browse Source 
Summary:
Previously it only applied to block-based tables generated by flush. This restriction
was undocumented and blocked a new use case. Now compression sampling
applies to all block-based tables we generate when it is enabled.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8105

Test Plan: new unit test

Reviewed By: riversand963

Differential Revision: D27317275

Pulled By: ajkr

fbshipit-source-id: cd9fcc5178d6515e8cb59c6facb5ac01893cb5b0
This commit is contained in:
Andrew Kryczka 2021-03-25 14:58:23 -07:00 committed by Facebook GitHub Bot
parent 45c65d6dcf
commit c20a7cd6c7
28 changed files with 211 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
HISTORY.md
View File

@ -1,5 +1,8 @@
# Rocksdb Change Log
## Unreleased
### Behavior Changes
* `ColumnFamilyOptions::sample_for_compression` now takes effect for creation of all block-based tables. Previously it only took effect for block-based tables created by flush.
### Bug Fixes
* Use thread-safe `strerror_r()` to get error messages.

26
db/builder.cc
View File

@ -52,8 +52,8 @@ TableBuilder* NewTableBuilder(
int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type,
uint64_t sample_for_compression, const CompressionOptions& compression_opts,
int level, const bool skip_filters, const uint64_t creation_time,
const CompressionOptions& compression_opts, int level,
const bool skip_filters, const uint64_t creation_time,
const uint64_t oldest_key_time, const uint64_t target_file_size,
const uint64_t file_creation_time, const std::string& db_id,
const std::string& db_session_id) {
@ -63,10 +63,10 @@ TableBuilder* NewTableBuilder(
return ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, internal_comparator,
int_tbl_prop_collector_factories, compression_type,
sample_for_compression, compression_opts,
skip_filters, column_family_name, level,
creation_time, oldest_key_time, target_file_size,
file_creation_time, db_id, db_session_id),
compression_opts, skip_filters, column_family_name,
level, creation_time, oldest_key_time,
target_file_size, file_creation_time, db_id,
db_session_id),
column_family_id, file);
}
@ -85,11 +85,10 @@ Status BuildTable(
std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot,
SnapshotChecker* snapshot_checker, const CompressionType compression,
uint64_t sample_for_compression, const CompressionOptions& compression_opts,
bool paranoid_file_checks, InternalStats* internal_stats,
TableFileCreationReason reason, IOStatus* io_status,
const std::shared_ptr<IOTracer>& io_tracer, EventLogger* event_logger,
int job_id, const Env::IOPriority io_priority,
const CompressionOptions& compression_opts, bool paranoid_file_checks,
InternalStats* internal_stats, TableFileCreationReason reason,
IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
TableProperties* table_properties, int level, const uint64_t creation_time,
const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint,
const uint64_t file_creation_time, const std::string& db_id,
@ -163,9 +162,8 @@ Status BuildTable(
builder = NewTableBuilder(
ioptions, mutable_cf_options, internal_comparator,
int_tbl_prop_collector_factories, column_family_id,
column_family_name, file_writer.get(), compression,
sample_for_compression, compression_opts, level,
false /* skip_filters */, creation_time, oldest_key_time,
column_family_name, file_writer.get(), compression, compression_opts,
level, false /* skip_filters */, creation_time, oldest_key_time,
0 /*target_file_size*/, file_creation_time, db_id, db_session_id);
}

2
db/builder.h
View File

@ -50,7 +50,6 @@ TableBuilder* NewTableBuilder(
int_tbl_prop_collector_factories,
uint32_t column_family_id, const std::string& column_family_name,
WritableFileWriter* file, const CompressionType compression_type,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, int level,
const bool skip_filters = false, const uint64_t creation_time = 0,
const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0,
@ -80,7 +79,6 @@ extern Status BuildTable(
std::vector<SequenceNumber> snapshots,
SequenceNumber earliest_write_conflict_snapshot,
SnapshotChecker* snapshot_checker, const CompressionType compression,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, bool paranoid_file_checks,
InternalStats* internal_stats, TableFileCreationReason reason,
IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,

1
db/compaction/compaction_job.cc
View File

@ -1770,7 +1770,6 @@ Status CompactionJob::OpenCompactionOutputFile(
cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
cfd->GetID(), cfd->GetName(), sub_compact->outfile.get(),
sub_compact->compaction->output_compression(),
0 /*sample_for_compression */,
sub_compact->compaction->output_compression_opts(),
sub_compact->compaction->output_level(), skip_filters,
oldest_ancester_time, 0 /* oldest_key_time */,

1
db/db_impl/db_impl_open.cc
View File

@ -1358,7 +1358,6 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
cfd->GetID(), cfd->GetName(), snapshot_seqs,
earliest_write_conflict_snapshot, snapshot_checker,
GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
mutable_cf_options.sample_for_compression,
mutable_cf_options.compression_opts, paranoid_file_checks,
cfd->internal_stats(), TableFileCreationReason::kRecovery, &io_s,
io_tracer_, &event_logger_, job_id, Env::IO_HIGH,

107
db/db_properties_test.cc
View File

@ -1175,6 +1175,61 @@ class CountingDeleteTabPropCollectorFactory
}
};
class BlockCountingTablePropertiesCollector : public TablePropertiesCollector {
public:
static const std::string kNumSampledBlocksPropertyName;
const char* Name() const override {
return "BlockCountingTablePropertiesCollector";
}
Status Finish(UserCollectedProperties* properties) override {
(*properties)[kNumSampledBlocksPropertyName] =
ToString(num_sampled_blocks_);
return Status::OK();
}
Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
EntryType /*type*/, SequenceNumber /*seq*/,
uint64_t /*file_size*/) override {
return Status::OK();
}
void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t block_compressed_bytes_fast,
uint64_t block_compressed_bytes_slow) override {
if (block_compressed_bytes_fast > 0 || block_compressed_bytes_slow > 0) {
num_sampled_blocks_++;
}
}
UserCollectedProperties GetReadableProperties() const override {
return UserCollectedProperties{
{kNumSampledBlocksPropertyName, ToString(num_sampled_blocks_)},
};
}
private:
uint32_t num_sampled_blocks_ = 0;
};
const std::string
BlockCountingTablePropertiesCollector::kNumSampledBlocksPropertyName =
"NumSampledBlocks";
class BlockCountingTablePropertiesCollectorFactory
: public TablePropertiesCollectorFactory {
public:
const char* Name() const override {
return "BlockCountingTablePropertiesCollectorFactory";
}
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context /* context */) override {
return new BlockCountingTablePropertiesCollector();
}
};
#ifndef ROCKSDB_LITE
TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
Options options = CurrentOptions();
@ -1413,6 +1468,58 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
}
}
// Excluded from RocksDB lite tests due to `GetPropertiesOfAllTables()` usage.
TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) {
// Sampled compression requires at least one of the following four types.
if (!Snappy_Supported() && !Zlib_Supported() && !LZ4_Supported() &&
!ZSTD_Supported()) {
return;
}
Options options = CurrentOptions();
options.disable_auto_compactions = true;
options.table_properties_collector_factories.emplace_back(
std::make_shared<BlockCountingTablePropertiesCollectorFactory>());
for (bool sample_for_compression : {false, true}) {
// For simplicity/determinism, sample 100% when enabled, or 0% when disabled
options.sample_for_compression = sample_for_compression ? 1 : 0;
DestroyAndReopen(options);
// Setup the following LSM:
//
// L0_0 ["a", "b"]
// L1_0 ["a", "b"]
//
// L0_0 was created by flush. L1_0 was created by compaction. Each file
// contains one data block.
for (int i = 0; i < 3; ++i) {
ASSERT_OK(Put("a", "val"));
ASSERT_OK(Put("b", "val"));
ASSERT_OK(Flush());
if (i == 1) {
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
}
}
// A `BlockAdd()` should have been seen for files generated by flush or
// compaction when `sample_for_compression` is enabled.
TablePropertiesCollection file_to_props;
ASSERT_OK(db_->GetPropertiesOfAllTables(&file_to_props));
ASSERT_EQ(2, file_to_props.size());
for (const auto& file_and_props : file_to_props) {
auto& user_props = file_and_props.second->user_collected_properties;
ASSERT_TRUE(user_props.find(BlockCountingTablePropertiesCollector::
kNumSampledBlocksPropertyName) !=
user_props.end());
ASSERT_EQ(user_props.at(BlockCountingTablePropertiesCollector::
kNumSampledBlocksPropertyName),
ToString(sample_for_compression ? 1 : 0));
}
}
}
TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
Options options = CurrentOptions();
Reopen(options);

3
db/flush_job.cc
View File

@ -411,8 +411,7 @@ Status FlushJob::WriteLevel0Table() {
cfd_->internal_comparator(), cfd_->int_tbl_prop_collector_factories(),
cfd_->GetID(), cfd_->GetName(), existing_snapshots_,
earliest_write_conflict_snapshot_, snapshot_checker_,
output_compression_, mutable_cf_options_.sample_for_compression,
mutable_cf_options_.compression_opts,
output_compression_, mutable_cf_options_.compression_opts,
mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
TableFileCreationReason::kFlush, &io_s, io_tracer_, event_logger_,
job_context_->job_id, Env::IO_HIGH, &table_properties_, 0 /* level */,

13
db/repair.cc
View File

@ -447,13 +447,12 @@ class Repairer {
nullptr /* blob_file_additions */, cfd->internal_comparator(),
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
{}, kMaxSequenceNumber, snapshot_checker, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(), false,
nullptr /* internal_stats */, TableFileCreationReason::kRecovery,
&io_s, nullptr /*IOTracer*/, nullptr /* event_logger */,
0 /* job_id */, Env::IO_HIGH, nullptr /* table_properties */,
-1 /* level */, current_time, 0 /* oldest_key_time */, write_hint,
0 /* file_creation_time */, "DB Repairer" /* db_id */,
db_session_id_);
CompressionOptions(), false, nullptr /* internal_stats */,
TableFileCreationReason::kRecovery, &io_s, nullptr /*IOTracer*/,
nullptr /* event_logger */, 0 /* job_id */, Env::IO_HIGH,
nullptr /* table_properties */, -1 /* level */, current_time,
0 /* oldest_key_time */, write_hint, 0 /* file_creation_time */,
"DB Repairer" /* db_id */, db_session_id_);
ROCKS_LOG_INFO(db_options_.info_log,
"Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s",
log, counter, meta.fd.GetNumber(),

8
db/table_properties_collector.cc
View File

@ -43,10 +43,10 @@ Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
}
void UserKeyTablePropertiesCollector::BlockAdd(
uint64_t bLockRawBytes, uint64_t blockCompressedBytesFast,
uint64_t blockCompressedBytesSlow) {
return collector_->BlockAdd(bLockRawBytes, blockCompressedBytesFast,
blockCompressedBytesSlow);
uint64_t block_raw_bytes, uint64_t block_compressed_bytes_fast,
uint64_t block_compressed_bytes_slow) {
return collector_->BlockAdd(block_raw_bytes, block_compressed_bytes_fast,
block_compressed_bytes_slow);
}
Status UserKeyTablePropertiesCollector::Finish(

12
db/table_properties_collector.h
View File

@ -27,9 +27,9 @@ class IntTblPropCollector {
virtual Status InternalAdd(const Slice& key, const Slice& value,
uint64_t file_size) = 0;
virtual void BlockAdd(uint64_t blockRawBytes,
uint64_t blockCompressedBytesFast,
uint64_t blockCompressedBytesSlow) = 0;
virtual void BlockAdd(uint64_t block_raw_bytes,
uint64_t block_compressed_bytes_fast,
uint64_t block_compressed_bytes_slow) = 0;
virtual UserCollectedProperties GetReadableProperties() const = 0;
@ -64,9 +64,9 @@ class UserKeyTablePropertiesCollector : public IntTblPropCollector {
virtual Status InternalAdd(const Slice& key, const Slice& value,
uint64_t file_size) override;
virtual void BlockAdd(uint64_t blockRawBytes,
uint64_t blockCompressedBytesFast,
uint64_t blockCompressedBytesSlow) override;
virtual void BlockAdd(uint64_t block_raw_bytes,
uint64_t block_compressed_bytes_fast,
uint64_t block_compressed_bytes_slow) override;
virtual Status Finish(UserCollectedProperties* properties) override;

9
db/table_properties_collector_test.cc
View File

@ -55,8 +55,7 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
builder->reset(NewTableBuilder(
ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories,
kTestColumnFamilyId, kTestColumnFamilyName, writable->get(),
options.compression, options.sample_for_compression,
options.compression_opts, unknown_level));
options.compression, options.compression_opts, unknown_level));
}
} // namespace
@ -176,9 +175,9 @@ class RegularKeysStartWithAInternal : public IntTblPropCollector {
return Status::OK();
}
void BlockAdd(uint64_t /* blockRawBytes */,
uint64_t /* blockCompressedBytesFast */,
uint64_t /* blockCompressedBytesSlow */) override {
void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* block_compressed_bytes_fast */,
uint64_t /* block_compressed_bytes_slow */) override {
// Nothing to do.
return;
}

2
db/version_set_test.cc
View File

@ -2781,7 +2781,7 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
TableBuilderOptions(
immutable_cf_options_, mutable_cf_options_, *internal_comparator_,
&int_tbl_prop_collector_factories, kNoCompression,
/*_sample_for_compression=*/0, CompressionOptions(),
CompressionOptions(),
/*_skip_filters=*/false, info.column_family, info.level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
fwriter.get()));

6
include/rocksdb/table_properties.h
View File

@ -99,9 +99,9 @@ class TablePropertiesCollector {
}
// Called after each new block is cut
virtual void BlockAdd(uint64_t /* blockRawBytes */,
uint64_t /* blockCompressedBytesFast */,
uint64_t /* blockCompressedBytesSlow */) {
virtual void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* block_compressed_bytes_fast */,
uint64_t /* block_compressed_bytes_slow */) {
// Nothing to do here. Callback registers can override.
return;
}

22
table/block_based/block_based_table_builder.cc
View File

@ -210,9 +210,9 @@ class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector
return Status::OK();
}
virtual void BlockAdd(uint64_t /* blockRawBytes */,
uint64_t /* blockCompressedBytesFast */,
uint64_t /* blockCompressedBytesSlow */) override {
virtual void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* block_compressed_bytes_fast */,
uint64_t /* block_compressed_bytes_slow */) override {
// Intentionally left blank. No interest in collecting stats for
// blocks.
return;
@ -405,7 +405,6 @@ struct BlockBasedTableBuilder::Rep {
int_tbl_prop_collector_factories,
uint32_t _column_family_id, WritableFileWriter* f,
const CompressionType _compression_type,
const uint64_t _sample_for_compression,
const CompressionOptions& _compression_opts, const bool skip_filters,
const int _level_at_creation, const std::string& _column_family_name,
const uint64_t _creation_time, const uint64_t _oldest_key_time,
@ -431,7 +430,7 @@ struct BlockBasedTableBuilder::Rep {
range_del_block(1 /* block_restart_interval */),
internal_prefix_transform(_moptions.prefix_extractor.get()),
compression_type(_compression_type),
sample_for_compression(_sample_for_compression),
sample_for_compression(_moptions.sample_for_compression),
compression_opts(_compression_opts),
compression_dict(),
compression_ctxs(_compression_opts.parallel_threads),
@ -841,7 +840,6 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, const bool skip_filters,
const std::string& column_family_name, const int level_at_creation,
const uint64_t creation_time, const uint64_t oldest_key_time,
@ -859,12 +857,12 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
sanitized_table_options.format_version = 1;
}
rep_ = new Rep(
ioptions, moptions, sanitized_table_options, internal_comparator,
int_tbl_prop_collector_factories, column_family_id, file,
compression_type, sample_for_compression, compression_opts, skip_filters,
level_at_creation, column_family_name, creation_time, oldest_key_time,
target_file_size, file_creation_time, db_id, db_session_id);
rep_ = new Rep(ioptions, moptions, sanitized_table_options,
internal_comparator, int_tbl_prop_collector_factories,
column_family_id, file, compression_type, compression_opts,
skip_filters, level_at_creation, column_family_name,
creation_time, oldest_key_time, target_file_size,
file_creation_time, db_id, db_session_id);
if (rep_->filter_builder != nullptr) {
rep_->filter_builder->StartBlock(0);

1
table/block_based/block_based_table_builder.h
View File

@ -46,7 +46,6 @@ class BlockBasedTableBuilder : public TableBuilder {
int_tbl_prop_collector_factories,
uint32_t column_family_id, WritableFileWriter* file,
const CompressionType compression_type,
const uint64_t sample_for_compression,
const CompressionOptions& compression_opts, const bool skip_filters,
const std::string& column_family_name, const int level_at_creation,
const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0,

1
table/block_based/block_based_table_factory.cc
View File

@ -500,7 +500,6 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder(
table_options_, table_builder_options.internal_comparator,
table_builder_options.int_tbl_prop_collector_factories, column_family_id,
file, table_builder_options.compression_type,
table_builder_options.sample_for_compression,
table_builder_options.compression_opts,
table_builder_options.skip_filters,
table_builder_options.column_family_name, table_builder_options.level,

6
table/block_based/block_based_table_reader_test.cc
View File

@ -66,9 +66,9 @@ class BlockBasedTableReaderTest
std::vector<std::unique_ptr<IntTblPropCollectorFactory>> factories;
std::unique_ptr<TableBuilder> table_builder(table_factory_->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, comparator, &factories,
compression_type, 0 /* sample_for_compression */,
CompressionOptions(), false /* skip_filters */,
kDefaultColumnFamilyName, -1 /* level */),
compression_type, CompressionOptions(),
false /* skip_filters */, kDefaultColumnFamilyName,
-1 /* level */),
0 /* column_family_id */, writer.get()));
// Build table.

5
table/block_based/data_block_hash_index_test.cc
View File

@ -557,9 +557,8 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2,
builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, internal_comparator,
&int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression,
CompressionOptions(), false /* skip_filters */,
column_family_name, level_),
options.compression, CompressionOptions(),
false /* skip_filters */, column_family_name, level_),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));

6
table/block_fetcher_test.cc
View File

@ -100,9 +100,9 @@ class BlockFetcherTest : public testing::Test {
std::vector<std::unique_ptr<IntTblPropCollectorFactory>> factories;
std::unique_ptr<TableBuilder> table_builder(table_factory_.NewTableBuilder(
TableBuilderOptions(ioptions, moptions, comparator, &factories,
compression_type, 0 /* sample_for_compression */,
CompressionOptions(), false /* skip_filters */,
kDefaultColumnFamilyName, -1 /* level */),
compression_type, CompressionOptions(),
false /* skip_filters */, kDefaultColumnFamilyName,
-1 /* level */),
0 /* column_family_id */, writer.get()));
// Build table.

8
table/meta_blocks.cc
View File

@ -172,11 +172,11 @@ bool NotifyCollectTableCollectorsOnAdd(
void NotifyCollectTableCollectorsOnBlockAdd(
const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors,
const uint64_t blockRawBytes, const uint64_t blockCompressedBytesFast,
const uint64_t blockCompressedBytesSlow) {
const uint64_t block_raw_bytes, const uint64_t block_compressed_bytes_fast,
const uint64_t block_compressed_bytes_slow) {
for (auto& collector : collectors) {
collector->BlockAdd(blockRawBytes, blockCompressedBytesFast,
blockCompressedBytesSlow);
collector->BlockAdd(block_raw_bytes, block_compressed_bytes_fast,
block_compressed_bytes_slow);
}
}

4
table/meta_blocks.h
View File

@ -86,8 +86,8 @@ bool NotifyCollectTableCollectorsOnAdd(
void NotifyCollectTableCollectorsOnBlockAdd(
const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors,
uint64_t blockRawBytes, uint64_t blockCompressedBytesFast,
uint64_t blockCompressedBytesSlow);
uint64_t block_raw_bytes, uint64_t block_compressed_bytes_fast,
uint64_t block_compressed_bytes_slow);
// NotifyCollectTableCollectorsOnFinish() triggers the `Finish` event for all
// property collectors. The collected properties will be added to `builder`.

6
table/sst_file_dumper.cc
View File

@ -274,9 +274,9 @@ Status SstFileDumper::ShowCompressionSize(
std::string column_family_name;
int unknown_level = -1;
TableBuilderOptions tb_opts(
imoptions, moptions, ikc, &block_based_table_factories, compress_type,
0 /* sample_for_compression */, compress_opt, false /* skip_filters */,
TableBuilderOptions tb_opts(imoptions, moptions, ikc,
&block_based_table_factories, compress_type,
compress_opt, false /* skip_filters */,
column_family_name, unknown_level);
uint64_t num_data_blocks = 0;
std::chrono::steady_clock::time_point start =

9
table/sst_file_writer.cc
View File

@ -210,8 +210,6 @@ Status SstFileWriter::Open(const std::string& file_path) {
compression_type = r->mutable_cf_options.compression;
compression_opts = r->mutable_cf_options.compression_opts;
}
uint64_t sample_for_compression =
r->mutable_cf_options.sample_for_compression;
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories;
@ -252,10 +250,9 @@ Status SstFileWriter::Open(const std::string& file_path) {
}
TableBuilderOptions table_builder_options(
r->ioptions, r->mutable_cf_options, r->internal_comparator,
&int_tbl_prop_collector_factories, compression_type,
sample_for_compression, compression_opts, r->skip_filters,
r->column_family_name, unknown_level, 0 /* creation_time */,
0 /* oldest_key_time */, 0 /* target_file_size */,
&int_tbl_prop_collector_factories, compression_type, compression_opts,
r->skip_filters, r->column_family_name, unknown_level,
0 /* creation_time */, 0 /* oldest_key_time */, 0 /* target_file_size */,
0 /* file_creation_time */, "SST Writer" /* db_id */, db_session_id);
FileTypeSet tmp_set = r->ioptions.checksum_handoff_file_types;
r->file_writer.reset(new WritableFileWriter(

6
table/sst_file_writer_collectors.h
View File

@ -35,9 +35,9 @@ class SstFileWriterPropertiesCollector : public IntTblPropCollector {
return Status::OK();
}
virtual void BlockAdd(uint64_t /* blockRawBytes */,
uint64_t /* blockCompressedBytesFast */,
uint64_t /* blockCompressedBytesSlow */) override {
virtual void BlockAdd(uint64_t /* block_raw_bytes */,
uint64_t /* block_compressed_bytes_fast */,
uint64_t /* block_compressed_bytes_slow */) override {
// Intentionally left blank. No interest in collecting stats for
// blocks.
return;

4
table/table_builder.h
View File

@ -92,7 +92,7 @@ struct TableBuilderOptions {
const InternalKeyComparator& _internal_comparator,
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
_int_tbl_prop_collector_factories,
CompressionType _compression_type, uint64_t _sample_for_compression,
CompressionType _compression_type,
const CompressionOptions& _compression_opts, bool _skip_filters,
const std::string& _column_family_name, int _level,
const uint64_t _creation_time = 0, const int64_t _oldest_key_time = 0,
@ -104,7 +104,6 @@ struct TableBuilderOptions {
internal_comparator(_internal_comparator),
int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories),
compression_type(_compression_type),
sample_for_compression(_sample_for_compression),
compression_opts(_compression_opts),
skip_filters(_skip_filters),
column_family_name(_column_family_name),
@ -122,7 +121,6 @@ struct TableBuilderOptions {
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
int_tbl_prop_collector_factories;
CompressionType compression_type;
uint64_t sample_for_compression;
const CompressionOptions& compression_opts;
bool skip_filters; // only used by BlockBasedTableBuilder
const std::string& column_family_name;

5
table/table_reader_bench.cc
View File

@ -102,9 +102,8 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
tb = opts.table_factory->NewTableBuilder(
TableBuilderOptions(
ioptions, moptions, ikc, &int_tbl_prop_collector_factories,
CompressionType::kNoCompression, 0 /* sample_for_compression */,
CompressionOptions(), false /* skip_filters */,
kDefaultColumnFamilyName, unknown_level),
CompressionType::kNoCompression, CompressionOptions(),
false /* skip_filters */, kDefaultColumnFamilyName, unknown_level),
0 /* column_family_id */, file_writer.get());
} else {
s = DB::Open(opts, dbname, &db);

52
table/table_test.cc
View File

@ -363,9 +363,9 @@ class TableConstructor : public Constructor {
builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, internal_comparator,
&int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression,
options.compression_opts, false /* skip_filters */,
column_family_name, level_),
options.compression, options.compression_opts,
false /* skip_filters */, column_family_name,
level_),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer_.get()));
@ -3325,9 +3325,8 @@ TEST_P(BlockBasedTableTest, NoFileChecksum) {
builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, *comparator,
&int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression,
options.compression_opts, false /* skip_filters */,
column_family_name, level),
options.compression, options.compression_opts,
false /* skip_filters */, column_family_name, level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -3364,9 +3363,8 @@ TEST_P(BlockBasedTableTest, Crc32cFileChecksum) {
builder.reset(ioptions.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, *comparator,
&int_tbl_prop_collector_factories,
options.compression, options.sample_for_compression,
options.compression_opts, false /* skip_filters */,
column_family_name, level),
options.compression, options.compression_opts,
false /* skip_filters */, column_family_name, level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -3413,10 +3411,10 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
std::string column_family_name;
int unknown_level = -1;
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
TableBuilderOptions(
ioptions, moptions, ikc, &int_tbl_prop_collector_factories,
kNoCompression, 0 /* sample_for_compression */, CompressionOptions(),
false /* skip_filters */, column_family_name, unknown_level),
TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression,
CompressionOptions(), false /* skip_filters */,
column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));
@ -3470,10 +3468,10 @@ TEST_F(PlainTableTest, NoFileChecksum) {
f.CreateWriteableFile();
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
TableBuilderOptions(
ioptions, moptions, ikc, &int_tbl_prop_collector_factories,
kNoCompression, 0 /* sample_for_compression */, CompressionOptions(),
false /* skip_filters */, column_family_name, unknown_level),
TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression,
CompressionOptions(), false /* skip_filters */,
column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -3512,10 +3510,10 @@ TEST_F(PlainTableTest, Crc32cFileChecksum) {
f.SetFileChecksumGenerator(checksum_crc32c_gen1.release());
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
TableBuilderOptions(
ioptions, moptions, ikc, &int_tbl_prop_collector_factories,
kNoCompression, 0 /* sample_for_compression */, CompressionOptions(),
false /* skip_filters */, column_family_name, unknown_level),
TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression,
CompressionOptions(), false /* skip_filters */,
column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
f.GetFileWriter()));
ASSERT_OK(f.ResetTableBuilder(std::move(builder)));
@ -4081,8 +4079,8 @@ TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) {
std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(),
false /* skip_filters */, column_family_name, -1),
CompressionOptions(), false /* skip_filters */,
column_family_name, -1),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));
@ -4267,8 +4265,8 @@ TEST_P(BlockBasedTableTest, BlockAlignTest) {
std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(),
false /* skip_filters */, column_family_name, -1),
CompressionOptions(), false /* skip_filters */,
column_family_name, -1),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));
@ -4362,8 +4360,8 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
std::unique_ptr<TableBuilder> builder(options.table_factory->NewTableBuilder(
TableBuilderOptions(ioptions, moptions, ikc,
&int_tbl_prop_collector_factories, kNoCompression,
0 /* sample_for_compression */, CompressionOptions(),
false /* skip_filters */, column_family_name, -1),
CompressionOptions(), false /* skip_filters */,
column_family_name, -1),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));

5
tools/sst_dump_test.cc
View File

@ -113,9 +113,8 @@ class SSTDumpToolTest : public testing::Test {
tb.reset(opts.table_factory->NewTableBuilder(
TableBuilderOptions(
imoptions, moptions, ikc, &int_tbl_prop_collector_factories,
CompressionType::kNoCompression, 0 /* sample_for_compression */,
CompressionOptions(), false /* skip_filters */, column_family_name,
unknown_level),
CompressionType::kNoCompression, CompressionOptions(),
false /* skip_filters */, column_family_name, unknown_level),
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
file_writer.get()));