diff --git a/db/blob/blob_file_builder.cc b/db/blob/blob_file_builder.cc index 1fbc636b8..1bf027c17 100644 --- a/db/blob/blob_file_builder.cc +++ b/db/blob/blob_file_builder.cc @@ -190,7 +190,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() { std::move(file), blob_file_paths_->back(), *file_options_, immutable_cf_options_->clock, io_tracer_, statistics, immutable_cf_options_->listeners, - immutable_cf_options_->file_checksum_gen_factory, + immutable_cf_options_->file_checksum_gen_factory.get(), tmp_set.Contains(FileType::kBlobFile))); constexpr bool do_flush = false; diff --git a/db/blob/blob_file_reader.cc b/db/blob/blob_file_reader.cc index 6e1f78d5a..74ff58f5a 100644 --- a/db/blob/blob_file_reader.cc +++ b/db/blob/blob_file_reader.cc @@ -120,7 +120,7 @@ Status BlobFileReader::OpenFile( file_reader->reset(new RandomAccessFileReader( std::move(file), blob_file_path, immutable_cf_options.clock, io_tracer, immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS, - blob_file_read_hist, immutable_cf_options.rate_limiter, + blob_file_read_hist, immutable_cf_options.rate_limiter.get(), immutable_cf_options.listeners)); return Status::OK(); diff --git a/db/builder.cc b/db/builder.cc index 0e41884ae..45d43f42d 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -156,7 +156,7 @@ Status BuildTable( file_writer.reset(new WritableFileWriter( std::move(file), fname, file_options, ioptions.clock, io_tracer, ioptions.statistics, ioptions.listeners, - ioptions.file_checksum_gen_factory, + ioptions.file_checksum_gen_factory.get(), tmp_set.Contains(FileType::kTableFile))); builder = NewTableBuilder( @@ -168,7 +168,7 @@ Status BuildTable( } MergeHelper merge(env, internal_comparator.user_comparator(), - ioptions.merge_operator, nullptr, ioptions.info_log, + ioptions.merge_operator.get(), nullptr, ioptions.info_log, true /* internal key corruption is not ok */, snapshots.empty() ? 0 : snapshots.back(), snapshot_checker); diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 3f750289e..1342537ed 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -965,7 +965,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { } MergeHelper merge( - env_, cfd->user_comparator(), cfd->ioptions()->merge_operator, + env_, cfd->user_comparator(), cfd->ioptions()->merge_operator.get(), compaction_filter, db_options_.info_log.get(), false /* internal key corruption is expected */, existing_snapshots_.empty() ? 0 : existing_snapshots_.back(), diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index a3e631b68..58f54b52e 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -697,8 +697,8 @@ void DBImpl::MaybeIgnoreError(Status* s) const { } const Status DBImpl::CreateArchivalDirectory() { - if (immutable_db_options_.wal_ttl_seconds > 0 || - immutable_db_options_.wal_size_limit_mb > 0) { + if (immutable_db_options_.WAL_ttl_seconds > 0 || + immutable_db_options_.WAL_size_limit_MB > 0) { std::string archivalPath = ArchivalDirectory(immutable_db_options_.wal_dir); return env_->CreateDirIfMissing(archivalPath); } diff --git a/db/db_impl/db_impl_files.cc b/db/db_impl/db_impl_files.cc index 926734f38..c0405d6bf 100644 --- a/db/db_impl/db_impl_files.cc +++ b/db/db_impl/db_impl_files.cc @@ -558,8 +558,8 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) { } #ifndef ROCKSDB_LITE - if (type == kWalFile && (immutable_db_options_.wal_ttl_seconds > 0 || - immutable_db_options_.wal_size_limit_mb > 0)) { + if (type == kWalFile && (immutable_db_options_.WAL_ttl_seconds > 0 || + immutable_db_options_.WAL_size_limit_MB > 0)) { wal_manager_.ArchiveWALFile(fname, number); continue; } diff --git a/db/db_iter.cc b/db/db_iter.cc index 9daca967a..dd06e71ea 100644 --- a/db/db_iter.cc +++ b/db/db_iter.cc @@ -45,10 +45,10 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options, ColumnFamilyData* cfd, bool expose_blob_index) : prefix_extractor_(mutable_cf_options.prefix_extractor.get()), env_(_env), - clock_(_env->GetSystemClock().get()), + clock_(cf_options.clock), logger_(cf_options.info_log), user_comparator_(cmp), - merge_operator_(cf_options.merge_operator), + merge_operator_(cf_options.merge_operator.get()), iter_(iter), version_(version), read_callback_(read_callback), diff --git a/db/internal_stats.cc b/db/internal_stats.cc index a5e2b09df..cfe6adf16 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -961,7 +961,7 @@ bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/, bool InternalStats::HandleBlockCacheStat(Cache** block_cache) { assert(block_cache != nullptr); - auto* table_factory = cfd_->ioptions()->table_factory; + auto* table_factory = cfd_->ioptions()->table_factory.get(); assert(table_factory != nullptr); *block_cache = table_factory->GetOptions(TableFactory::kBlockCacheOpts()); diff --git a/db/memtable.cc b/db/memtable.cc index 5789956fa..ca5174294 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -59,7 +59,7 @@ ImmutableMemTableOptions::ImmutableMemTableOptions( inplace_callback(ioptions.inplace_callback), max_successive_merges(mutable_cf_options.max_successive_merges), statistics(ioptions.statistics), - merge_operator(ioptions.merge_operator), + merge_operator(ioptions.merge_operator.get()), info_log(ioptions.info_log), allow_data_in_errors(ioptions.allow_data_in_errors) {} @@ -106,7 +106,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp, flush_state_(FLUSH_NOT_REQUESTED), clock_(ioptions.clock), insert_with_hint_prefix_extractor_( - ioptions.memtable_insert_with_hint_prefix_extractor), + ioptions.memtable_insert_with_hint_prefix_extractor.get()), oldest_key_time_(std::numeric_limits::max()), atomic_flush_seqno_(kMaxSequenceNumber), approximate_memory_usage_(0) { diff --git a/db/table_cache.cc b/db/table_cache.cc index 1cd3d4fcf..401ef8077 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -130,7 +130,7 @@ Status TableCache::GetTableReader( new RandomAccessFileReader( std::move(file), fname, ioptions_.clock, io_tracer_, record_read_stats ? ioptions_.statistics : nullptr, SST_READ_MICROS, - file_read_hist, ioptions_.rate_limiter, ioptions_.listeners)); + file_read_hist, ioptions_.rate_limiter.get(), ioptions_.listeners)); s = ioptions_.table_factory->NewTableReader( ro, TableReaderOptions(ioptions_, prefix_extractor, file_options, diff --git a/db/version_set.cc b/db/version_set.cc index 09ece799d..737f6b6ad 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1768,8 +1768,8 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset, : cfd_->ioptions()->statistics), table_cache_((cfd_ == nullptr) ? nullptr : cfd_->table_cache()), blob_file_cache_(cfd_ ? cfd_->blob_file_cache() : nullptr), - merge_operator_((cfd_ == nullptr) ? nullptr - : cfd_->ioptions()->merge_operator), + merge_operator_( + (cfd_ == nullptr) ? nullptr : cfd_->ioptions()->merge_operator.get()), storage_info_( (cfd_ == nullptr) ? nullptr : &cfd_->internal_comparator(), (cfd_ == nullptr) ? nullptr : cfd_->user_comparator(), diff --git a/db/wal_manager.cc b/db/wal_manager.cc index 7c7c27c12..359cc1546 100644 --- a/db/wal_manager.cc +++ b/db/wal_manager.cc @@ -134,8 +134,8 @@ Status WalManager::GetUpdatesSince( // b. get sorted non-empty archived logs // c. delete what should be deleted void WalManager::PurgeObsoleteWALFiles() { - bool const ttl_enabled = db_options_.wal_ttl_seconds > 0; - bool const size_limit_enabled = db_options_.wal_size_limit_mb > 0; + bool const ttl_enabled = db_options_.WAL_ttl_seconds > 0; + bool const size_limit_enabled = db_options_.WAL_size_limit_MB > 0; if (!ttl_enabled && !size_limit_enabled) { return; } @@ -150,7 +150,7 @@ void WalManager::PurgeObsoleteWALFiles() { } uint64_t const now_seconds = static_cast(current_time); uint64_t const time_to_check = (ttl_enabled && !size_limit_enabled) - ? db_options_.wal_ttl_seconds / 2 + ? db_options_.WAL_ttl_seconds / 2 : kDefaultIntervalToDeleteObsoleteWAL; if (purge_wal_files_last_run_ + time_to_check > now_seconds) { @@ -185,7 +185,7 @@ void WalManager::PurgeObsoleteWALFiles() { s.ToString().c_str()); continue; } - if (now_seconds - file_m_time > db_options_.wal_ttl_seconds) { + if (now_seconds - file_m_time > db_options_.WAL_ttl_seconds) { s = DeleteDBFile(&db_options_, file_path, archival_dir, false, /*force_fg=*/!wal_in_db_path_); if (!s.ok()) { @@ -234,8 +234,8 @@ void WalManager::PurgeObsoleteWALFiles() { return; } - size_t const files_keep_num = - static_cast(db_options_.wal_size_limit_mb * 1024 * 1024 / log_file_size); + size_t const files_keep_num = static_cast( + db_options_.WAL_size_limit_MB * 1024 * 1024 / log_file_size); if (log_files_num <= files_keep_num) { return; } diff --git a/db/wal_manager_test.cc b/db/wal_manager_test.cc index 5ea5d0c58..f49f47fb2 100644 --- a/db/wal_manager_test.cc +++ b/db/wal_manager_test.cc @@ -217,8 +217,8 @@ int CountRecords(TransactionLogIterator* iter) { } // namespace TEST_F(WalManagerTest, WALArchivalSizeLimit) { - db_options_.wal_ttl_seconds = 0; - db_options_.wal_size_limit_mb = 1000; + db_options_.WAL_ttl_seconds = 0; + db_options_.WAL_size_limit_MB = 1000; Init(); // TEST : Create WalManager with huge size limit and no ttl. @@ -226,7 +226,7 @@ TEST_F(WalManagerTest, WALArchivalSizeLimit) { // Count the archived log files that survived. // Assert that all of them did. // Change size limit. Re-open WalManager. - // Assert that archive is not greater than wal_size_limit_mb after + // Assert that archive is not greater than WAL_size_limit_MB after // PurgeObsoleteWALFiles() // Set ttl and time_to_check_ to small values. Re-open db. // Assert that there are no archived logs left. @@ -238,14 +238,14 @@ TEST_F(WalManagerTest, WALArchivalSizeLimit) { ListSpecificFiles(env_.get(), archive_dir, kWalFile); ASSERT_EQ(log_files.size(), 20U); - db_options_.wal_size_limit_mb = 8; + db_options_.WAL_size_limit_MB = 8; Reopen(); wal_manager_->PurgeObsoleteWALFiles(); uint64_t archive_size = GetLogDirSize(archive_dir, env_.get()); - ASSERT_TRUE(archive_size <= db_options_.wal_size_limit_mb * 1024 * 1024); + ASSERT_TRUE(archive_size <= db_options_.WAL_size_limit_MB * 1024 * 1024); - db_options_.wal_ttl_seconds = 1; + db_options_.WAL_ttl_seconds = 1; env_->FakeSleepForMicroseconds(2 * 1000 * 1000); Reopen(); wal_manager_->PurgeObsoleteWALFiles(); @@ -255,7 +255,7 @@ TEST_F(WalManagerTest, WALArchivalSizeLimit) { } TEST_F(WalManagerTest, WALArchivalTtl) { - db_options_.wal_ttl_seconds = 1000; + db_options_.WAL_ttl_seconds = 1000; Init(); // TEST : Create WalManager with a ttl and no size limit. @@ -271,7 +271,7 @@ TEST_F(WalManagerTest, WALArchivalTtl) { ListSpecificFiles(env_.get(), archive_dir, kWalFile); ASSERT_GT(log_files.size(), 0U); - db_options_.wal_ttl_seconds = 1; + db_options_.WAL_ttl_seconds = 1; env_->FakeSleepForMicroseconds(3 * 1000 * 1000); Reopen(); wal_manager_->PurgeObsoleteWALFiles(); diff --git a/options/cf_options.cc b/options/cf_options.cc index 96ac51541..64a5ae7fb 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -38,13 +38,9 @@ namespace ROCKSDB_NAMESPACE { // http://en.cppreference.com/w/cpp/concept/StandardLayoutType // https://gist.github.com/graphitemaster/494f21190bb2c63c5516 #ifndef ROCKSDB_LITE -static ColumnFamilyOptions dummy_cf_options; +static ImmutableCFOptions dummy_cf_options; template -int offset_of(T1 ColumnFamilyOptions::*member) { - return int(size_t(&(dummy_cf_options.*member)) - size_t(&dummy_cf_options)); -} -template -int offset_of(T1 AdvancedColumnFamilyOptions::*member) { +int offset_of(T1 ImmutableCFOptions::*member) { return int(size_t(&(dummy_cf_options.*member)) - size_t(&dummy_cf_options)); } @@ -492,57 +488,58 @@ static std::unordered_map {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}, {"inplace_update_support", - {offset_of(&ColumnFamilyOptions::inplace_update_support), + {offset_of(&ImmutableCFOptions::inplace_update_support), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"level_compaction_dynamic_level_bytes", - {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), + {offset_of(&ImmutableCFOptions::level_compaction_dynamic_level_bytes), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"optimize_filters_for_hits", - {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), + {offset_of(&ImmutableCFOptions::optimize_filters_for_hits), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"force_consistency_checks", - {offset_of(&ColumnFamilyOptions::force_consistency_checks), + {offset_of(&ImmutableCFOptions::force_consistency_checks), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"purge_redundant_kvs_while_flush", - {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), + {offset_of(&ImmutableCFOptions::purge_redundant_kvs_while_flush), OptionType::kBoolean, OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}, {"max_mem_compaction_level", {0, OptionType::kInt, OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}, {"max_write_buffer_number_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), + {offset_of(&ImmutableCFOptions::max_write_buffer_number_to_maintain), OptionType::kInt, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, {"max_write_buffer_size_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain), + {offset_of(&ImmutableCFOptions::max_write_buffer_size_to_maintain), OptionType::kInt64T, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"min_write_buffer_number_to_merge", - {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), + {offset_of(&ImmutableCFOptions::min_write_buffer_number_to_merge), OptionType::kInt, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, {"num_levels", - {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, + {offset_of(&ImmutableCFOptions::num_levels), OptionType::kInt, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"bloom_locality", - {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, + {offset_of(&ImmutableCFOptions::bloom_locality), OptionType::kUInt32T, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"rate_limit_delay_max_milliseconds", {0, OptionType::kUInt, OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}, {"compression_per_level", OptionTypeInfo::Vector( - offset_of(&ColumnFamilyOptions::compression_per_level), + offset_of(&ImmutableCFOptions::compression_per_level), OptionVerificationType::kNormal, OptionTypeFlags::kNone, {0, OptionType::kCompressionType})}, {"comparator", - {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, - OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose, + {offset_of(&ImmutableCFOptions::user_comparator), + OptionType::kComparator, OptionVerificationType::kByName, + OptionTypeFlags::kCompareLoose, // Parses the string and sets the corresponding comparator [](const ConfigOptions& /*opts*/, const std::string& /*name*/, const std::string& value, char* addr) { @@ -558,15 +555,15 @@ static std::unordered_map }}}, {"memtable_insert_with_hint_prefix_extractor", {offset_of( - &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), + &ImmutableCFOptions::memtable_insert_with_hint_prefix_extractor), OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, OptionTypeFlags::kNone}}, {"memtable_factory", - {offset_of(&ColumnFamilyOptions::memtable_factory), + {offset_of(&ImmutableCFOptions::memtable_factory), OptionType::kMemTableRepFactory, OptionVerificationType::kByName, OptionTypeFlags::kNone}}, {"memtable", - {offset_of(&ColumnFamilyOptions::memtable_factory), + {offset_of(&ImmutableCFOptions::memtable_factory), OptionType::kMemTableRepFactory, OptionVerificationType::kAlias, OptionTypeFlags::kNone, // Parses the value string and updates the memtable_factory @@ -582,13 +579,13 @@ static std::unordered_map return s; }}}, {"table_factory", OptionTypeInfo::AsCustomSharedPtr( - offset_of(&ColumnFamilyOptions::table_factory), + offset_of(&ImmutableCFOptions::table_factory), OptionVerificationType::kByName, (OptionTypeFlags::kCompareLoose | OptionTypeFlags::kStringNameOnly | OptionTypeFlags::kDontPrepare))}, {"block_based_table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), + {offset_of(&ImmutableCFOptions::table_factory), OptionType::kCustomizable, OptionVerificationType::kAlias, OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, // Parses the input value and creates a BlockBasedTableFactory @@ -620,7 +617,7 @@ static std::unordered_map } }}}, {"plain_table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), + {offset_of(&ImmutableCFOptions::table_factory), OptionType::kCustomizable, OptionVerificationType::kAlias, OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, // Parses the input value and creates a PlainTableFactory @@ -651,15 +648,15 @@ static std::unordered_map } }}}, {"compaction_filter", - {offset_of(&ColumnFamilyOptions::compaction_filter), + {offset_of(&ImmutableCFOptions::compaction_filter), OptionType::kCompactionFilter, OptionVerificationType::kByName, OptionTypeFlags::kNone}}, {"compaction_filter_factory", - {offset_of(&ColumnFamilyOptions::compaction_filter_factory), + {offset_of(&ImmutableCFOptions::compaction_filter_factory), OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, OptionTypeFlags::kNone}}, {"merge_operator", - {offset_of(&ColumnFamilyOptions::merge_operator), + {offset_of(&ImmutableCFOptions::merge_operator), OptionType::kMergeOperator, OptionVerificationType::kByNameAllowFromNull, OptionTypeFlags::kCompareLoose, @@ -677,11 +674,11 @@ static std::unordered_map return Status::OK(); }}}, {"compaction_style", - {offset_of(&ColumnFamilyOptions::compaction_style), + {offset_of(&ImmutableCFOptions::compaction_style), OptionType::kCompactionStyle, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"compaction_pri", - {offset_of(&ColumnFamilyOptions::compaction_pri), + {offset_of(&ImmutableCFOptions::compaction_pri), OptionType::kCompactionPri, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, }; @@ -705,11 +702,10 @@ class ConfigurableCFOptions : public ConfigurableMutableCFOptions { ConfigurableCFOptions(const ColumnFamilyOptions& opts, const std::unordered_map* map) : ConfigurableMutableCFOptions(MutableCFOptions(opts)), - immutable_(opts), + immutable_(ImmutableDBOptions(), opts), cf_options_(opts), opt_map_(map) { - ConfigurableHelper::RegisterOptions(*this, OptionsHelper::kCFOptionsName, - &immutable_, + ConfigurableHelper::RegisterOptions(*this, &immutable_, &cf_immutable_options_type_info); } @@ -774,7 +770,7 @@ class ConfigurableCFOptions : public ConfigurableMutableCFOptions { } private: - ColumnFamilyOptions immutable_; + ImmutableCFOptions immutable_; ColumnFamilyOptions cf_options_; const std::unordered_map* opt_map_; }; @@ -792,18 +788,22 @@ std::unique_ptr CFOptionsAsConfigurable( } #endif // ROCKSDB_LITE +ImmutableCFOptions::ImmutableCFOptions() : ImmutableCFOptions(Options()) {} + ImmutableCFOptions::ImmutableCFOptions(const Options& options) : ImmutableCFOptions(ImmutableDBOptions(options), options) {} ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options, const ColumnFamilyOptions& cf_options) - : compaction_style(cf_options.compaction_style), + : logger(db_options.info_log), + stats(db_options.statistics), + compaction_style(cf_options.compaction_style), compaction_pri(cf_options.compaction_pri), user_comparator(cf_options.comparator), internal_comparator(InternalKeyComparator(cf_options.comparator)), - merge_operator(cf_options.merge_operator.get()), + merge_operator(cf_options.merge_operator), compaction_filter(cf_options.compaction_filter), - compaction_filter_factory(cf_options.compaction_filter_factory.get()), + compaction_filter_factory(cf_options.compaction_filter_factory), min_write_buffer_number_to_merge( cf_options.min_write_buffer_number_to_merge), max_write_buffer_number_to_maintain( @@ -812,9 +812,9 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options, cf_options.max_write_buffer_size_to_maintain), inplace_update_support(cf_options.inplace_update_support), inplace_callback(cf_options.inplace_callback), - info_log(db_options.info_log.get()), - statistics(db_options.statistics.get()), - rate_limiter(db_options.rate_limiter.get()), + info_log(logger.get()), + statistics(stats.get()), + rate_limiter(db_options.rate_limiter), info_log_level(db_options.info_log_level), env(db_options.env), fs(db_options.fs.get()), @@ -822,8 +822,8 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options, allow_mmap_reads(db_options.allow_mmap_reads), allow_mmap_writes(db_options.allow_mmap_writes), db_paths(db_options.db_paths), - memtable_factory(cf_options.memtable_factory.get()), - table_factory(cf_options.table_factory.get()), + memtable_factory(cf_options.memtable_factory), + table_factory(cf_options.table_factory), table_properties_collector_factories( cf_options.table_properties_collector_factories), advise_random_on_open(db_options.advise_random_on_open), @@ -846,10 +846,10 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options, listeners(db_options.listeners), row_cache(db_options.row_cache), memtable_insert_with_hint_prefix_extractor( - cf_options.memtable_insert_with_hint_prefix_extractor.get()), + cf_options.memtable_insert_with_hint_prefix_extractor), cf_paths(cf_options.cf_paths), compaction_thread_limiter(cf_options.compaction_thread_limiter), - file_checksum_gen_factory(db_options.file_checksum_gen_factory.get()), + file_checksum_gen_factory(db_options.file_checksum_gen_factory), sst_partitioner_factory(cf_options.sst_partitioner_factory), allow_data_in_errors(db_options.allow_data_in_errors), db_host_id(db_options.db_host_id), diff --git a/options/cf_options.h b/options/cf_options.h index 3aae6c8c4..c0ce9f410 100644 --- a/options/cf_options.h +++ b/options/cf_options.h @@ -20,7 +20,12 @@ namespace ROCKSDB_NAMESPACE { // of DB. Raw pointers defined in this struct do not have ownership to the data // they point to. Options contains std::shared_ptr to these data. struct ImmutableCFOptions { + private: + std::shared_ptr logger; // ImmutableDBOptions + std::shared_ptr stats; // ImmutableDBOptions + public: static const char* kName() { return "ImmutableCFOptions"; } + explicit ImmutableCFOptions(); explicit ImmutableCFOptions(const Options& options); ImmutableCFOptions(const ImmutableDBOptions& db_options, @@ -31,13 +36,13 @@ struct ImmutableCFOptions { CompactionPri compaction_pri; const Comparator* user_comparator; - InternalKeyComparator internal_comparator; + InternalKeyComparator internal_comparator; // Only in Immutable - MergeOperator* merge_operator; + std::shared_ptr merge_operator; const CompactionFilter* compaction_filter; - CompactionFilterFactory* compaction_filter_factory; + std::shared_ptr compaction_filter_factory; int min_write_buffer_number_to_merge; @@ -52,36 +57,36 @@ struct ImmutableCFOptions { Slice delta_value, std::string* merged_value); - Logger* info_log; + Logger* info_log; // ImmutableDBOptions - Statistics* statistics; + Statistics* statistics; // ImmutableDBOptions - RateLimiter* rate_limiter; + std::shared_ptr rate_limiter; // ImmutableDBOptions - InfoLogLevel info_log_level; + InfoLogLevel info_log_level; // ImmutableDBOptions - Env* env; + Env* env; // ImmutableDBOptions - FileSystem* fs; + FileSystem* fs; // ImmutableDBOptions - SystemClock* clock; + SystemClock* clock; // ImmutableDBOptions // Allow the OS to mmap file for reading sst tables. Default: false - bool allow_mmap_reads; + bool allow_mmap_reads; // ImmutableDBOptions // Allow the OS to mmap file for writing. Default: false - bool allow_mmap_writes; + bool allow_mmap_writes; // ImmutableDBOptions - std::vector db_paths; + std::vector db_paths; // ImmutableDBOptions - MemTableRepFactory* memtable_factory; + std::shared_ptr memtable_factory; - TableFactory* table_factory; + std::shared_ptr table_factory; Options::TablePropertiesCollectorFactories table_properties_collector_factories; - bool advise_random_on_open; + bool advise_random_on_open; // ImmutableDBOptions // This options is required by PlainTableReader. May need to move it // to PlainTableOptions just like bloom_bits_per_key @@ -89,15 +94,15 @@ struct ImmutableCFOptions { bool purge_redundant_kvs_while_flush; - bool use_fsync; + bool use_fsync; // ImmutableDBOptions std::vector compression_per_level; bool level_compaction_dynamic_level_bytes; - Options::AccessHint access_hint_on_compaction_start; + Options::AccessHint access_hint_on_compaction_start; // ImmutableDBOptions - bool new_table_reader_for_compaction_inputs; + bool new_table_reader_for_compaction_inputs; // ImmutableDBOptions int num_levels; @@ -105,31 +110,33 @@ struct ImmutableCFOptions { bool force_consistency_checks; - bool allow_ingest_behind; + bool allow_ingest_behind; // ImmutableDBOptions - bool preserve_deletes; + bool preserve_deletes; // ImmutableDBOptions // A vector of EventListeners which callback functions will be called // when specific RocksDB event happens. - std::vector> listeners; + std::vector> listeners; // ImmutableDBOptions - std::shared_ptr row_cache; + std::shared_ptr row_cache; // ImmutableDBOptions - const SliceTransform* memtable_insert_with_hint_prefix_extractor; + std::shared_ptr + memtable_insert_with_hint_prefix_extractor; std::vector cf_paths; std::shared_ptr compaction_thread_limiter; - FileChecksumGenFactory* file_checksum_gen_factory; + std::shared_ptr + file_checksum_gen_factory; // ImmutableDBOptions std::shared_ptr sst_partitioner_factory; - bool allow_data_in_errors; + bool allow_data_in_errors; // ImmutableDBOptions - std::string db_host_id; + std::string db_host_id; // ImmutableDBOptions - FileTypeSet checksum_handoff_file_types; + FileTypeSet checksum_handoff_file_types; // ImmutableDBOptions }; struct MutableCFOptions { diff --git a/options/db_options.cc b/options/db_options.cc index 69596bf1f..af88e41d4 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -272,11 +272,11 @@ static std::unordered_map {offsetof(struct ImmutableDBOptions, wal_dir), OptionType::kString, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"WAL_size_limit_MB", - {offsetof(struct ImmutableDBOptions, wal_size_limit_mb), + {offsetof(struct ImmutableDBOptions, WAL_size_limit_MB), OptionType::kUInt64T, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"WAL_ttl_seconds", - {offsetof(struct ImmutableDBOptions, wal_ttl_seconds), + {offsetof(struct ImmutableDBOptions, WAL_ttl_seconds), OptionType::kUInt64T, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"max_manifest_file_size", @@ -509,7 +509,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) track_and_verify_wals_in_manifest( options.track_and_verify_wals_in_manifest), env(options.env), - fs(options.env->GetFileSystem()), rate_limiter(options.rate_limiter), sst_file_manager(options.sst_file_manager), info_log(options.info_log), @@ -526,8 +525,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) recycle_log_file_num(options.recycle_log_file_num), max_manifest_file_size(options.max_manifest_file_size), table_cache_numshardbits(options.table_cache_numshardbits), - wal_ttl_seconds(options.WAL_ttl_seconds), - wal_size_limit_mb(options.WAL_size_limit_MB), + WAL_ttl_seconds(options.WAL_ttl_seconds), + WAL_size_limit_MB(options.WAL_size_limit_MB), max_write_batch_group_size_bytes( options.max_write_batch_group_size_bytes), manifest_preallocation_size(options.manifest_preallocation_size), @@ -583,6 +582,7 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) allow_data_in_errors(options.allow_data_in_errors), db_host_id(options.db_host_id), checksum_handoff_file_types(options.checksum_handoff_file_types) { + fs = env->GetFileSystem(); if (env != nullptr) { clock = env->GetSystemClock().get(); } else { @@ -650,10 +650,10 @@ void ImmutableDBOptions::Dump(Logger* log) const { table_cache_numshardbits); ROCKS_LOG_HEADER(log, " Options.WAL_ttl_seconds: %" PRIu64, - wal_ttl_seconds); + WAL_ttl_seconds); ROCKS_LOG_HEADER(log, " Options.WAL_size_limit_MB: %" PRIu64, - wal_size_limit_mb); + WAL_size_limit_MB); ROCKS_LOG_HEADER(log, " " "Options.max_write_batch_group_size_bytes: %" PRIu64, diff --git a/options/db_options.h b/options/db_options.h index b7979dc86..49aa170dd 100644 --- a/options/db_options.h +++ b/options/db_options.h @@ -26,8 +26,6 @@ struct ImmutableDBOptions { bool paranoid_checks; bool track_and_verify_wals_in_manifest; Env* env; - std::shared_ptr fs; - SystemClock* clock; std::shared_ptr rate_limiter; std::shared_ptr sst_file_manager; std::shared_ptr info_log; @@ -44,8 +42,8 @@ struct ImmutableDBOptions { size_t recycle_log_file_num; uint64_t max_manifest_file_size; int table_cache_numshardbits; - uint64_t wal_ttl_seconds; - uint64_t wal_size_limit_mb; + uint64_t WAL_ttl_seconds; + uint64_t WAL_size_limit_MB; uint64_t max_write_batch_group_size_bytes; size_t manifest_preallocation_size; bool allow_mmap_reads; @@ -96,6 +94,9 @@ struct ImmutableDBOptions { bool allow_data_in_errors; std::string db_host_id; FileTypeSet checksum_handoff_file_types; + // Convenience/Helper objects that are not part of the base DBOptions + std::shared_ptr fs; + SystemClock* clock; }; struct MutableDBOptions { diff --git a/options/options_helper.cc b/options/options_helper.cc index e4e8d4ee8..d39c3f3df 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -86,8 +86,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.max_manifest_file_size = immutable_db_options.max_manifest_file_size; options.table_cache_numshardbits = immutable_db_options.table_cache_numshardbits; - options.WAL_ttl_seconds = immutable_db_options.wal_ttl_seconds; - options.WAL_size_limit_MB = immutable_db_options.wal_size_limit_mb; + options.WAL_ttl_seconds = immutable_db_options.WAL_ttl_seconds; + options.WAL_size_limit_MB = immutable_db_options.WAL_size_limit_MB; options.manifest_preallocation_size = immutable_db_options.manifest_preallocation_size; options.allow_mmap_reads = immutable_db_options.allow_mmap_reads; @@ -257,6 +257,111 @@ ColumnFamilyOptions BuildColumnFamilyOptions( return cf_opts; } +ColumnFamilyOptions BuildColumnFamilyOptions(const ImmutableCFOptions& ioptions, + const MutableCFOptions& moptions) { + ColumnFamilyOptions cf_opts; + + // Memtable related options + cf_opts.write_buffer_size = moptions.write_buffer_size; + cf_opts.max_write_buffer_number = moptions.max_write_buffer_number; + cf_opts.arena_block_size = moptions.arena_block_size; + cf_opts.memtable_prefix_bloom_size_ratio = + moptions.memtable_prefix_bloom_size_ratio; + cf_opts.memtable_whole_key_filtering = moptions.memtable_whole_key_filtering; + cf_opts.memtable_huge_page_size = moptions.memtable_huge_page_size; + cf_opts.max_successive_merges = moptions.max_successive_merges; + cf_opts.inplace_update_num_locks = moptions.inplace_update_num_locks; + cf_opts.prefix_extractor = moptions.prefix_extractor; + + // Compaction related options + cf_opts.disable_auto_compactions = moptions.disable_auto_compactions; + cf_opts.soft_pending_compaction_bytes_limit = + moptions.soft_pending_compaction_bytes_limit; + cf_opts.hard_pending_compaction_bytes_limit = + moptions.hard_pending_compaction_bytes_limit; + cf_opts.level0_file_num_compaction_trigger = + moptions.level0_file_num_compaction_trigger; + cf_opts.level0_slowdown_writes_trigger = + moptions.level0_slowdown_writes_trigger; + cf_opts.level0_stop_writes_trigger = moptions.level0_stop_writes_trigger; + cf_opts.max_compaction_bytes = moptions.max_compaction_bytes; + cf_opts.target_file_size_base = moptions.target_file_size_base; + cf_opts.target_file_size_multiplier = moptions.target_file_size_multiplier; + cf_opts.max_bytes_for_level_base = moptions.max_bytes_for_level_base; + cf_opts.max_bytes_for_level_multiplier = + moptions.max_bytes_for_level_multiplier; + cf_opts.ttl = moptions.ttl; + cf_opts.periodic_compaction_seconds = moptions.periodic_compaction_seconds; + + cf_opts.max_bytes_for_level_multiplier_additional.clear(); + for (auto value : moptions.max_bytes_for_level_multiplier_additional) { + cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value); + } + + cf_opts.compaction_options_fifo = moptions.compaction_options_fifo; + cf_opts.compaction_options_universal = moptions.compaction_options_universal; + + // Blob file related options + cf_opts.enable_blob_files = moptions.enable_blob_files; + cf_opts.min_blob_size = moptions.min_blob_size; + cf_opts.blob_file_size = moptions.blob_file_size; + cf_opts.blob_compression_type = moptions.blob_compression_type; + cf_opts.enable_blob_garbage_collection = + moptions.enable_blob_garbage_collection; + cf_opts.blob_garbage_collection_age_cutoff = + moptions.blob_garbage_collection_age_cutoff; + + // Misc options + cf_opts.max_sequential_skip_in_iterations = + moptions.max_sequential_skip_in_iterations; + cf_opts.check_flush_compaction_key_order = + moptions.check_flush_compaction_key_order; + cf_opts.paranoid_file_checks = moptions.paranoid_file_checks; + cf_opts.report_bg_io_stats = moptions.report_bg_io_stats; + cf_opts.compression = moptions.compression; + cf_opts.compression_opts = moptions.compression_opts; + cf_opts.bottommost_compression = moptions.bottommost_compression; + cf_opts.bottommost_compression_opts = moptions.bottommost_compression_opts; + cf_opts.sample_for_compression = moptions.sample_for_compression; + + cf_opts.compaction_style = ioptions.compaction_style; + cf_opts.compaction_pri = ioptions.compaction_pri; + cf_opts.comparator = ioptions.user_comparator; + cf_opts.merge_operator = ioptions.merge_operator; + cf_opts.compaction_filter = ioptions.compaction_filter; + cf_opts.compaction_filter_factory = ioptions.compaction_filter_factory; + cf_opts.min_write_buffer_number_to_merge = + ioptions.min_write_buffer_number_to_merge; + cf_opts.max_write_buffer_number_to_maintain = + ioptions.max_write_buffer_number_to_maintain; + cf_opts.max_write_buffer_size_to_maintain = + ioptions.max_write_buffer_size_to_maintain; + cf_opts.inplace_update_support = ioptions.inplace_update_support; + cf_opts.inplace_callback = ioptions.inplace_callback; + cf_opts.memtable_factory = ioptions.memtable_factory; + cf_opts.table_factory = ioptions.table_factory; + cf_opts.table_properties_collector_factories = + ioptions.table_properties_collector_factories; + cf_opts.bloom_locality = ioptions.bloom_locality; + cf_opts.purge_redundant_kvs_while_flush = + ioptions.purge_redundant_kvs_while_flush; + cf_opts.compression_per_level = ioptions.compression_per_level; + cf_opts.level_compaction_dynamic_level_bytes = + ioptions.level_compaction_dynamic_level_bytes; + cf_opts.num_levels = ioptions.num_levels; + cf_opts.optimize_filters_for_hits = ioptions.optimize_filters_for_hits; + cf_opts.force_consistency_checks = ioptions.force_consistency_checks; + cf_opts.memtable_insert_with_hint_prefix_extractor = + ioptions.memtable_insert_with_hint_prefix_extractor; + cf_opts.cf_paths = ioptions.cf_paths; + cf_opts.compaction_thread_limiter = ioptions.compaction_thread_limiter; + cf_opts.sst_partitioner_factory = ioptions.sst_partitioner_factory; + + // TODO(yhchiang): find some way to handle the following derived options + // * max_file_size + + return cf_opts; +} std::map OptionsHelper::compaction_style_to_string = { diff --git a/options/options_helper.h b/options/options_helper.h index 43ddbdfc3..f27d8f182 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -18,6 +18,7 @@ namespace ROCKSDB_NAMESPACE { struct ColumnFamilyOptions; struct ConfigOptions; struct DBOptions; +struct ImmutableCFOptions; struct ImmutableDBOptions; struct MutableDBOptions; struct MutableCFOptions; @@ -37,6 +38,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, ColumnFamilyOptions BuildColumnFamilyOptions( const ColumnFamilyOptions& ioptions, const MutableCFOptions& mutable_cf_options); +ColumnFamilyOptions BuildColumnFamilyOptions(const ImmutableCFOptions& ioptions, + const MutableCFOptions& moptions); #ifndef ROCKSDB_LITE std::unique_ptr DBOptionsAsConfigurable( diff --git a/options/options_test.cc b/options/options_test.cc index bb2f34146..21c2df28d 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -1298,6 +1298,36 @@ TEST_F(OptionsTest, OptionsComposeDecompose) { delete new_cf_opts.compaction_filter; } +TEST_F(OptionsTest, DBOptionsComposeImmutable) { + // Build a DBOptions from an Immutable/Mutable one and verify that + // we get same constituent options. + ConfigOptions config_options; + Random rnd(301); + DBOptions base_opts, new_opts; + test::RandomInitDBOptions(&base_opts, &rnd); + MutableDBOptions m_opts(base_opts); + ImmutableDBOptions i_opts(base_opts); + new_opts = BuildDBOptions(i_opts, m_opts); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_opts, + new_opts)); +} + +TEST_F(OptionsTest, CFOptionsComposeImmutable) { + // Build a DBOptions from an Immutable/Mutable one and verify that + // we get same constituent options. + ConfigOptions config_options; + Random rnd(301); + ColumnFamilyOptions base_opts, new_opts; + DBOptions dummy; // Needed to create ImmutableCFOptions + test::RandomInitCFOptions(&base_opts, dummy, &rnd); + MutableCFOptions m_opts(base_opts); + ImmutableCFOptions i_opts(ImmutableDBOptions(dummy), base_opts); + new_opts = BuildColumnFamilyOptions(i_opts, m_opts); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_opts, + new_opts)); + delete new_opts.compaction_filter; +} + TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { Options options; ColumnFamilyOptions base_opt, new_opt; diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc index 7fcf4676d..a4fad90a1 100644 --- a/table/meta_blocks.cc +++ b/table/meta_blocks.cc @@ -152,8 +152,8 @@ Slice PropertyBlockBuilder::Finish() { return properties_block_->Finish(); } -void LogPropertiesCollectionError( - Logger* info_log, const std::string& method, const std::string& name) { +void LogPropertiesCollectionError(Logger* info_log, const std::string& method, + const std::string& name) { assert(method == "Add" || method == "Finish"); std::string msg = diff --git a/table/meta_blocks.h b/table/meta_blocks.h index 4477246c6..aaaa2d5ed 100644 --- a/table/meta_blocks.h +++ b/table/meta_blocks.h @@ -70,8 +70,8 @@ class PropertyBlockBuilder { // Were we encounter any error occurs during user-defined statistics collection, // we'll write the warning message to info log. -void LogPropertiesCollectionError( - Logger* info_log, const std::string& method, const std::string& name); +void LogPropertiesCollectionError(Logger* info_log, const std::string& method, + const std::string& name); // Utility functions help table builder to trigger batch events for user // defined property collectors. diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index 47cdbb69e..2f215947f 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -260,7 +260,7 @@ Status SstFileWriter::Open(const std::string& file_path) { r->file_writer.reset(new WritableFileWriter( std::move(sst_file), file_path, r->env_options, r->ioptions.clock, nullptr /* io_tracer */, nullptr /* stats */, r->ioptions.listeners, - r->ioptions.file_checksum_gen_factory, + r->ioptions.file_checksum_gen_factory.get(), tmp_set.Contains(FileType::kTableFile))); // TODO(tec) : If table_factory is using compressed block cache, we will diff --git a/table/table_reader_bench.cc b/table/table_reader_bench.cc index 76559e5f3..09e28cb37 100644 --- a/table/table_reader_bench.cc +++ b/table/table_reader_bench.cc @@ -175,11 +175,11 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options, PinnableSlice value; MergeContext merge_context; SequenceNumber max_covering_tombstone_seq = 0; - GetContext get_context(ioptions.user_comparator, - ioptions.merge_operator, ioptions.info_log, - ioptions.statistics, GetContext::kNotFound, - Slice(key), &value, nullptr, &merge_context, - true, &max_covering_tombstone_seq, clock); + GetContext get_context( + ioptions.user_comparator, ioptions.merge_operator.get(), + ioptions.info_log, ioptions.statistics, GetContext::kNotFound, + Slice(key), &value, nullptr, &merge_context, true, + &max_covering_tombstone_seq, clock); s = table_reader->Get(read_options, key, &get_context, nullptr); } else { s = db->Get(read_options, key, &result); diff --git a/util/filter_bench.cc b/util/filter_bench.cc index da9ae49b9..76057e125 100644 --- a/util/filter_bench.cc +++ b/util/filter_bench.cc @@ -270,8 +270,8 @@ struct FilterBench : public MockBlockBasedTableTester { Random32 random_; std::ostringstream fp_rate_report_; Arena arena_; - StderrLogger stderr_logger_; double m_queries_; + StderrLogger stderr_logger_; FilterBench() : MockBlockBasedTableTester(new BloomFilterPolicy( diff --git a/utilities/memory/memory_test.cc b/utilities/memory/memory_test.cc index 07dab4fb8..bef22fc2f 100644 --- a/utilities/memory/memory_test.cc +++ b/utilities/memory/memory_test.cc @@ -68,7 +68,8 @@ class MemoryTest : public testing::Test { ASSERT_OK(db_impl->TEST_GetAllImmutableCFOptions(&iopts_map)); } for (auto pair : iopts_map) { - GetCachePointersFromTableFactory(pair.second->table_factory, cache_set); + GetCachePointersFromTableFactory(pair.second->table_factory.get(), + cache_set); } } } diff --git a/utilities/write_batch_with_index/write_batch_with_index_internal.cc b/utilities/write_batch_with_index/write_batch_with_index_internal.cc index c41ccd2fa..fb0bea091 100644 --- a/utilities/write_batch_with_index/write_batch_with_index_internal.cc +++ b/utilities/write_batch_with_index/write_batch_with_index_internal.cc @@ -199,7 +199,7 @@ Status WriteBatchWithIndexInternal::MergeKey(const Slice& key, Slice* result_operand) { if (column_family_ != nullptr) { auto cfh = static_cast_with_check(column_family_); - const auto merge_operator = cfh->cfd()->ioptions()->merge_operator; + const auto merge_operator = cfh->cfd()->ioptions()->merge_operator.get(); if (merge_operator == nullptr) { return Status::InvalidArgument( "Merge_operator must be set for column_family");