From 4c3f4496b5bb1fca208e7efebbf2b0d3ae66ae41 Mon Sep 17 00:00:00 2001 From: rockeet Date: Sun, 18 Sep 2016 13:30:43 +0800 Subject: [PATCH] Add TableBuilderOptions::level and relevant changes (#1335) --- db/builder.cc | 5 +++-- db/builder.h | 1 + db/compaction_job.cc | 4 +++- db/table_properties_collector_test.cc | 5 +++-- table/sst_file_writer.cc | 4 ++-- table/table_builder.h | 6 ++++-- table/table_reader_bench.cc | 5 +++-- table/table_test.cc | 8 ++++++-- tools/sst_dump_test.cc | 4 +++- tools/sst_dump_tool.cc | 4 +++- 10 files changed, 31 insertions(+), 15 deletions(-) diff --git a/db/builder.cc b/db/builder.cc index ab255e493..29eda01e5 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -45,6 +45,7 @@ TableBuilder* NewTableBuilder( uint32_t column_family_id, const std::string& column_family_name, WritableFileWriter* file, const CompressionType compression_type, const CompressionOptions& compression_opts, + int level, const std::string* compression_dict, const bool skip_filters) { assert((column_family_id == TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) == @@ -53,7 +54,7 @@ TableBuilder* NewTableBuilder( TableBuilderOptions(ioptions, internal_comparator, int_tbl_prop_collector_factories, compression_type, compression_opts, compression_dict, skip_filters, - column_family_name), + column_family_name, level), column_family_id, file); } @@ -108,7 +109,7 @@ Status BuildTable( builder = NewTableBuilder( ioptions, internal_comparator, int_tbl_prop_collector_factories, column_family_id, column_family_name, file_writer.get(), compression, - compression_opts); + compression_opts, level); } MergeHelper merge(env, internal_comparator.user_comparator(), diff --git a/db/builder.h b/db/builder.h index e7e023623..6de015eca 100644 --- a/db/builder.h +++ b/db/builder.h @@ -48,6 +48,7 @@ TableBuilder* NewTableBuilder( uint32_t column_family_id, const std::string& column_family_name, WritableFileWriter* file, const CompressionType compression_type, const CompressionOptions& compression_opts, + int level, const std::string* compression_dict = nullptr, const bool skip_filters = false); diff --git a/db/compaction_job.cc b/db/compaction_job.cc index 82dd6d83f..fee5cdc9a 100644 --- a/db/compaction_job.cc +++ b/db/compaction_job.cc @@ -1136,7 +1136,9 @@ Status CompactionJob::OpenCompactionOutputFile( *cfd->ioptions(), cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(), sub_compact->outfile.get(), sub_compact->compaction->output_compression(), - cfd->ioptions()->compression_opts, &sub_compact->compression_dict, + cfd->ioptions()->compression_opts, + sub_compact->compaction->output_level(), + &sub_compact->compression_dict, skip_filters)); LogFlush(db_options_.info_log); return s; diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index 6b0cf45a3..c5c43760c 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -46,11 +46,12 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions, std::unique_ptr* builder) { unique_ptr wf(new test::StringSink); writable->reset(new WritableFileWriter(std::move(wf), EnvOptions())); - + int unknown_level = -1; builder->reset(NewTableBuilder( ioptions, internal_comparator, int_tbl_prop_collector_factories, kTestColumnFamilyId, kTestColumnFamilyName, - writable->get(), options.compression, options.compression_opts)); + writable->get(), options.compression, options.compression_opts, + unknown_level)); } } // namespace diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index 9e073806f..b6450431c 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -130,12 +130,12 @@ Status SstFileWriter::Open(const std::string& file_path) { new UserKeyTablePropertiesCollectorFactory( user_collector_factories[i])); } - + int unknown_level = -1; TableBuilderOptions table_builder_options( r->ioptions, r->internal_comparator, &int_tbl_prop_collector_factories, compression_type, r->ioptions.compression_opts, nullptr /* compression_dict */, false /* skip_filters */, - r->column_family_name); + r->column_family_name, unknown_level); r->file_writer.reset( new WritableFileWriter(std::move(sst_file), r->env_options)); r->builder.reset(r->ioptions.table_factory->NewTableBuilder( diff --git a/table/table_builder.h b/table/table_builder.h index aafd3ad8f..2b421657f 100644 --- a/table/table_builder.h +++ b/table/table_builder.h @@ -54,7 +54,7 @@ struct TableBuilderOptions { CompressionType _compression_type, const CompressionOptions& _compression_opts, const std::string* _compression_dict, bool _skip_filters, - const std::string& _column_family_name) + const std::string& _column_family_name, int _level) : ioptions(_ioptions), internal_comparator(_internal_comparator), int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories), @@ -62,7 +62,8 @@ struct TableBuilderOptions { compression_opts(_compression_opts), compression_dict(_compression_dict), skip_filters(_skip_filters), - column_family_name(_column_family_name) {} + column_family_name(_column_family_name), + level(_level) {} const ImmutableCFOptions& ioptions; const InternalKeyComparator& internal_comparator; const std::vector>* @@ -73,6 +74,7 @@ struct TableBuilderOptions { const std::string* compression_dict; bool skip_filters; // only used by BlockBasedTableBuilder const std::string& column_family_name; + int level; // what level this table/file is on, -1 for "not set, don't know" }; // TableBuilder provides the interface used to build a Table diff --git a/table/table_reader_bench.cc b/table/table_reader_bench.cc index 0a227d1e3..b04c5657b 100644 --- a/table/table_reader_bench.cc +++ b/table/table_reader_bench.cc @@ -94,13 +94,14 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options, int_tbl_prop_collector_factories; file_writer.reset(new WritableFileWriter(std::move(file), env_options)); - + int unknown_level = -1; tb = opts.table_factory->NewTableBuilder( TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories, CompressionType::kNoCompression, CompressionOptions(), nullptr /* compression_dict */, - false /* skip_filters */, kDefaultColumnFamilyName), + false /* skip_filters */, kDefaultColumnFamilyName, + unknown_level), 0 /* column_family_id */, file_writer.get()); } else { s = DB::Open(opts, dbname, &db); diff --git a/table/table_test.cc b/table/table_test.cc index 572394bb5..fb1d24b06 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -307,12 +307,14 @@ class TableConstructor: public Constructor { std::vector> int_tbl_prop_collector_factories; std::string column_family_name; + int unknown_level = -1; builder.reset(ioptions.table_factory->NewTableBuilder( TableBuilderOptions(ioptions, internal_comparator, &int_tbl_prop_collector_factories, options.compression, CompressionOptions(), nullptr /* compression_dict */, - false /* skip_filters */, column_family_name), + false /* skip_filters */, column_family_name, + unknown_level), TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer_.get())); @@ -2210,11 +2212,13 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) { std::vector> int_tbl_prop_collector_factories; std::string column_family_name; + int unknown_level = -1; std::unique_ptr builder(factory.NewTableBuilder( TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories, kNoCompression, CompressionOptions(), nullptr /* compression_dict */, - false /* skip_filters */, column_family_name), + false /* skip_filters */, column_family_name, + unknown_level), TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer.get())); diff --git a/tools/sst_dump_test.cc b/tools/sst_dump_test.cc index bef88bc23..3ba8a931a 100644 --- a/tools/sst_dump_test.cc +++ b/tools/sst_dump_test.cc @@ -59,11 +59,13 @@ void createSST(const std::string& file_name, unique_ptr file_writer( new WritableFileWriter(std::move(file), EnvOptions())); std::string column_family_name; + int unknown_level = -1; tb.reset(opts.table_factory->NewTableBuilder( TableBuilderOptions(imoptions, ikc, &int_tbl_prop_collector_factories, CompressionType::kNoCompression, CompressionOptions(), nullptr /* compression_dict */, - false /* skip_filters */, column_family_name), + false /* skip_filters */, column_family_name, + unknown_level), TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, file_writer.get())); diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index 01e3066c4..d2468e4d1 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -199,10 +199,12 @@ int SstFileReader::ShowAllCompressionSizes(size_t block_size) { if (CompressionTypeSupported(i.first)) { CompressionOptions compress_opt; std::string column_family_name; + int unknown_level = -1; TableBuilderOptions tb_opts(imoptions, ikc, &block_based_table_factories, i.first, compress_opt, nullptr /* compression_dict */, - false /* skip_filters */, column_family_name); + false /* skip_filters */, column_family_name, + unknown_level); uint64_t file_size = CalculateCompressedTableSize(tb_opts, block_size); fprintf(stdout, "Compression: %s", i.second); fprintf(stdout, " Size: %" PRIu64 "\n", file_size);