Add TableBuilderOptions::level and relevant changes (#1335)
This commit is contained in:
parent
3edb9461b7
commit
4c3f4496b5
|
@ -45,6 +45,7 @@ TableBuilder* NewTableBuilder(
|
||||||
uint32_t column_family_id, const std::string& column_family_name,
|
uint32_t column_family_id, const std::string& column_family_name,
|
||||||
WritableFileWriter* file, const CompressionType compression_type,
|
WritableFileWriter* file, const CompressionType compression_type,
|
||||||
const CompressionOptions& compression_opts,
|
const CompressionOptions& compression_opts,
|
||||||
|
int level,
|
||||||
const std::string* compression_dict, const bool skip_filters) {
|
const std::string* compression_dict, const bool skip_filters) {
|
||||||
assert((column_family_id ==
|
assert((column_family_id ==
|
||||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
|
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
|
||||||
|
@ -53,7 +54,7 @@ TableBuilder* NewTableBuilder(
|
||||||
TableBuilderOptions(ioptions, internal_comparator,
|
TableBuilderOptions(ioptions, internal_comparator,
|
||||||
int_tbl_prop_collector_factories, compression_type,
|
int_tbl_prop_collector_factories, compression_type,
|
||||||
compression_opts, compression_dict, skip_filters,
|
compression_opts, compression_dict, skip_filters,
|
||||||
column_family_name),
|
column_family_name, level),
|
||||||
column_family_id, file);
|
column_family_id, file);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -108,7 +109,7 @@ Status BuildTable(
|
||||||
builder = NewTableBuilder(
|
builder = NewTableBuilder(
|
||||||
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
||||||
column_family_id, column_family_name, file_writer.get(), compression,
|
column_family_id, column_family_name, file_writer.get(), compression,
|
||||||
compression_opts);
|
compression_opts, level);
|
||||||
}
|
}
|
||||||
|
|
||||||
MergeHelper merge(env, internal_comparator.user_comparator(),
|
MergeHelper merge(env, internal_comparator.user_comparator(),
|
||||||
|
|
|
@ -48,6 +48,7 @@ TableBuilder* NewTableBuilder(
|
||||||
uint32_t column_family_id, const std::string& column_family_name,
|
uint32_t column_family_id, const std::string& column_family_name,
|
||||||
WritableFileWriter* file, const CompressionType compression_type,
|
WritableFileWriter* file, const CompressionType compression_type,
|
||||||
const CompressionOptions& compression_opts,
|
const CompressionOptions& compression_opts,
|
||||||
|
int level,
|
||||||
const std::string* compression_dict = nullptr,
|
const std::string* compression_dict = nullptr,
|
||||||
const bool skip_filters = false);
|
const bool skip_filters = false);
|
||||||
|
|
||||||
|
|
|
@ -1136,7 +1136,9 @@ Status CompactionJob::OpenCompactionOutputFile(
|
||||||
*cfd->ioptions(), cfd->internal_comparator(),
|
*cfd->ioptions(), cfd->internal_comparator(),
|
||||||
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
|
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
|
||||||
sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
|
sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
|
||||||
cfd->ioptions()->compression_opts, &sub_compact->compression_dict,
|
cfd->ioptions()->compression_opts,
|
||||||
|
sub_compact->compaction->output_level(),
|
||||||
|
&sub_compact->compression_dict,
|
||||||
skip_filters));
|
skip_filters));
|
||||||
LogFlush(db_options_.info_log);
|
LogFlush(db_options_.info_log);
|
||||||
return s;
|
return s;
|
||||||
|
|
|
@ -46,11 +46,12 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
|
||||||
std::unique_ptr<TableBuilder>* builder) {
|
std::unique_ptr<TableBuilder>* builder) {
|
||||||
unique_ptr<WritableFile> wf(new test::StringSink);
|
unique_ptr<WritableFile> wf(new test::StringSink);
|
||||||
writable->reset(new WritableFileWriter(std::move(wf), EnvOptions()));
|
writable->reset(new WritableFileWriter(std::move(wf), EnvOptions()));
|
||||||
|
int unknown_level = -1;
|
||||||
builder->reset(NewTableBuilder(
|
builder->reset(NewTableBuilder(
|
||||||
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
||||||
kTestColumnFamilyId, kTestColumnFamilyName,
|
kTestColumnFamilyId, kTestColumnFamilyName,
|
||||||
writable->get(), options.compression, options.compression_opts));
|
writable->get(), options.compression, options.compression_opts,
|
||||||
|
unknown_level));
|
||||||
}
|
}
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
|
|
|
@ -130,12 +130,12 @@ Status SstFileWriter::Open(const std::string& file_path) {
|
||||||
new UserKeyTablePropertiesCollectorFactory(
|
new UserKeyTablePropertiesCollectorFactory(
|
||||||
user_collector_factories[i]));
|
user_collector_factories[i]));
|
||||||
}
|
}
|
||||||
|
int unknown_level = -1;
|
||||||
TableBuilderOptions table_builder_options(
|
TableBuilderOptions table_builder_options(
|
||||||
r->ioptions, r->internal_comparator, &int_tbl_prop_collector_factories,
|
r->ioptions, r->internal_comparator, &int_tbl_prop_collector_factories,
|
||||||
compression_type, r->ioptions.compression_opts,
|
compression_type, r->ioptions.compression_opts,
|
||||||
nullptr /* compression_dict */, false /* skip_filters */,
|
nullptr /* compression_dict */, false /* skip_filters */,
|
||||||
r->column_family_name);
|
r->column_family_name, unknown_level);
|
||||||
r->file_writer.reset(
|
r->file_writer.reset(
|
||||||
new WritableFileWriter(std::move(sst_file), r->env_options));
|
new WritableFileWriter(std::move(sst_file), r->env_options));
|
||||||
r->builder.reset(r->ioptions.table_factory->NewTableBuilder(
|
r->builder.reset(r->ioptions.table_factory->NewTableBuilder(
|
||||||
|
|
|
@ -54,7 +54,7 @@ struct TableBuilderOptions {
|
||||||
CompressionType _compression_type,
|
CompressionType _compression_type,
|
||||||
const CompressionOptions& _compression_opts,
|
const CompressionOptions& _compression_opts,
|
||||||
const std::string* _compression_dict, bool _skip_filters,
|
const std::string* _compression_dict, bool _skip_filters,
|
||||||
const std::string& _column_family_name)
|
const std::string& _column_family_name, int _level)
|
||||||
: ioptions(_ioptions),
|
: ioptions(_ioptions),
|
||||||
internal_comparator(_internal_comparator),
|
internal_comparator(_internal_comparator),
|
||||||
int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories),
|
int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories),
|
||||||
|
@ -62,7 +62,8 @@ struct TableBuilderOptions {
|
||||||
compression_opts(_compression_opts),
|
compression_opts(_compression_opts),
|
||||||
compression_dict(_compression_dict),
|
compression_dict(_compression_dict),
|
||||||
skip_filters(_skip_filters),
|
skip_filters(_skip_filters),
|
||||||
column_family_name(_column_family_name) {}
|
column_family_name(_column_family_name),
|
||||||
|
level(_level) {}
|
||||||
const ImmutableCFOptions& ioptions;
|
const ImmutableCFOptions& ioptions;
|
||||||
const InternalKeyComparator& internal_comparator;
|
const InternalKeyComparator& internal_comparator;
|
||||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||||
|
@ -73,6 +74,7 @@ struct TableBuilderOptions {
|
||||||
const std::string* compression_dict;
|
const std::string* compression_dict;
|
||||||
bool skip_filters; // only used by BlockBasedTableBuilder
|
bool skip_filters; // only used by BlockBasedTableBuilder
|
||||||
const std::string& column_family_name;
|
const std::string& column_family_name;
|
||||||
|
int level; // what level this table/file is on, -1 for "not set, don't know"
|
||||||
};
|
};
|
||||||
|
|
||||||
// TableBuilder provides the interface used to build a Table
|
// TableBuilder provides the interface used to build a Table
|
||||||
|
|
|
@ -94,13 +94,14 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
|
||||||
int_tbl_prop_collector_factories;
|
int_tbl_prop_collector_factories;
|
||||||
|
|
||||||
file_writer.reset(new WritableFileWriter(std::move(file), env_options));
|
file_writer.reset(new WritableFileWriter(std::move(file), env_options));
|
||||||
|
int unknown_level = -1;
|
||||||
tb = opts.table_factory->NewTableBuilder(
|
tb = opts.table_factory->NewTableBuilder(
|
||||||
TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
|
TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
|
||||||
CompressionType::kNoCompression,
|
CompressionType::kNoCompression,
|
||||||
CompressionOptions(),
|
CompressionOptions(),
|
||||||
nullptr /* compression_dict */,
|
nullptr /* compression_dict */,
|
||||||
false /* skip_filters */, kDefaultColumnFamilyName),
|
false /* skip_filters */, kDefaultColumnFamilyName,
|
||||||
|
unknown_level),
|
||||||
0 /* column_family_id */, file_writer.get());
|
0 /* column_family_id */, file_writer.get());
|
||||||
} else {
|
} else {
|
||||||
s = DB::Open(opts, dbname, &db);
|
s = DB::Open(opts, dbname, &db);
|
||||||
|
|
|
@ -307,12 +307,14 @@ class TableConstructor: public Constructor {
|
||||||
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
|
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
|
||||||
int_tbl_prop_collector_factories;
|
int_tbl_prop_collector_factories;
|
||||||
std::string column_family_name;
|
std::string column_family_name;
|
||||||
|
int unknown_level = -1;
|
||||||
builder.reset(ioptions.table_factory->NewTableBuilder(
|
builder.reset(ioptions.table_factory->NewTableBuilder(
|
||||||
TableBuilderOptions(ioptions, internal_comparator,
|
TableBuilderOptions(ioptions, internal_comparator,
|
||||||
&int_tbl_prop_collector_factories,
|
&int_tbl_prop_collector_factories,
|
||||||
options.compression, CompressionOptions(),
|
options.compression, CompressionOptions(),
|
||||||
nullptr /* compression_dict */,
|
nullptr /* compression_dict */,
|
||||||
false /* skip_filters */, column_family_name),
|
false /* skip_filters */, column_family_name,
|
||||||
|
unknown_level),
|
||||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
|
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
|
||||||
file_writer_.get()));
|
file_writer_.get()));
|
||||||
|
|
||||||
|
@ -2210,11 +2212,13 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
|
||||||
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
|
std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
|
||||||
int_tbl_prop_collector_factories;
|
int_tbl_prop_collector_factories;
|
||||||
std::string column_family_name;
|
std::string column_family_name;
|
||||||
|
int unknown_level = -1;
|
||||||
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
|
std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
|
||||||
TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
|
TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
|
||||||
kNoCompression, CompressionOptions(),
|
kNoCompression, CompressionOptions(),
|
||||||
nullptr /* compression_dict */,
|
nullptr /* compression_dict */,
|
||||||
false /* skip_filters */, column_family_name),
|
false /* skip_filters */, column_family_name,
|
||||||
|
unknown_level),
|
||||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
|
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
|
||||||
file_writer.get()));
|
file_writer.get()));
|
||||||
|
|
||||||
|
|
|
@ -59,11 +59,13 @@ void createSST(const std::string& file_name,
|
||||||
unique_ptr<WritableFileWriter> file_writer(
|
unique_ptr<WritableFileWriter> file_writer(
|
||||||
new WritableFileWriter(std::move(file), EnvOptions()));
|
new WritableFileWriter(std::move(file), EnvOptions()));
|
||||||
std::string column_family_name;
|
std::string column_family_name;
|
||||||
|
int unknown_level = -1;
|
||||||
tb.reset(opts.table_factory->NewTableBuilder(
|
tb.reset(opts.table_factory->NewTableBuilder(
|
||||||
TableBuilderOptions(imoptions, ikc, &int_tbl_prop_collector_factories,
|
TableBuilderOptions(imoptions, ikc, &int_tbl_prop_collector_factories,
|
||||||
CompressionType::kNoCompression, CompressionOptions(),
|
CompressionType::kNoCompression, CompressionOptions(),
|
||||||
nullptr /* compression_dict */,
|
nullptr /* compression_dict */,
|
||||||
false /* skip_filters */, column_family_name),
|
false /* skip_filters */, column_family_name,
|
||||||
|
unknown_level),
|
||||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
|
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
|
||||||
file_writer.get()));
|
file_writer.get()));
|
||||||
|
|
||||||
|
|
|
@ -199,10 +199,12 @@ int SstFileReader::ShowAllCompressionSizes(size_t block_size) {
|
||||||
if (CompressionTypeSupported(i.first)) {
|
if (CompressionTypeSupported(i.first)) {
|
||||||
CompressionOptions compress_opt;
|
CompressionOptions compress_opt;
|
||||||
std::string column_family_name;
|
std::string column_family_name;
|
||||||
|
int unknown_level = -1;
|
||||||
TableBuilderOptions tb_opts(imoptions, ikc, &block_based_table_factories,
|
TableBuilderOptions tb_opts(imoptions, ikc, &block_based_table_factories,
|
||||||
i.first, compress_opt,
|
i.first, compress_opt,
|
||||||
nullptr /* compression_dict */,
|
nullptr /* compression_dict */,
|
||||||
false /* skip_filters */, column_family_name);
|
false /* skip_filters */, column_family_name,
|
||||||
|
unknown_level);
|
||||||
uint64_t file_size = CalculateCompressedTableSize(tb_opts, block_size);
|
uint64_t file_size = CalculateCompressedTableSize(tb_opts, block_size);
|
||||||
fprintf(stdout, "Compression: %s", i.second);
|
fprintf(stdout, "Compression: %s", i.second);
|
||||||
fprintf(stdout, " Size: %" PRIu64 "\n", file_size);
|
fprintf(stdout, " Size: %" PRIu64 "\n", file_size);
|
||||||
|
|
Loading…
Reference in New Issue
Block a user