diff --git a/db/compaction.cc b/db/compaction.cc index cc9538533..6126b4642 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -229,7 +229,7 @@ bool Compaction::IsTrivialMove() const { // Used in universal compaction, where trivial move can be done if the // input files are non overlapping - if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) && + if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) && (output_level_ != 0)) { return is_trivial_move_; } diff --git a/db/compaction_picker_test.cc b/db/compaction_picker_test.cc index 37c24841b..069308bcc 100644 --- a/db/compaction_picker_test.cc +++ b/db/compaction_picker_test.cc @@ -437,7 +437,7 @@ TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) { TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) { const uint64_t kFileSize = 100000; - ioptions_.compaction_options_universal.allow_trivial_move = true; + mutable_cf_options_.compaction_options_universal.allow_trivial_move = true; NewVersionStorage(1, kCompactionStyleUniversal); UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); // must return false when there's no files. @@ -468,7 +468,7 @@ TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) { TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) { const uint64_t kFileSize = 100000; - ioptions_.compaction_options_universal.allow_trivial_move = true; + mutable_cf_options_.compaction_options_universal.allow_trivial_move = true; UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); NewVersionStorage(3, kCompactionStyleUniversal); diff --git a/db/compaction_picker_universal.cc b/db/compaction_picker_universal.cc index 29632b745..9975ef80c 100644 --- a/db/compaction_picker_universal.cc +++ b/db/compaction_picker_universal.cc @@ -204,7 +204,8 @@ void UniversalCompactionPicker::SortedRun::DumpSizeInfo( std::vector UniversalCompactionPicker::CalculateSortedRuns( - const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) { + const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options) { std::vector ret; for (FileMetaData* f : vstorage.LevelFiles(0)) { ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size, @@ -218,7 +219,8 @@ UniversalCompactionPicker::CalculateSortedRuns( for (FileMetaData* f : vstorage.LevelFiles(level)) { total_compensated_size += f->compensated_file_size; total_size += f->fd.GetFileSize(); - if (ioptions.compaction_options_universal.allow_trivial_move == true) { + if (mutable_cf_options.compaction_options_universal.allow_trivial_move == + true) { if (f->being_compacted) { being_compacted = f->being_compacted; } @@ -227,7 +229,8 @@ UniversalCompactionPicker::CalculateSortedRuns( // non-zero level, all the files should share the same being_compacted // value. // This assumption is only valid when - // ioptions.compaction_options_universal.allow_trivial_move is false + // mutable_cf_options.compaction_options_universal.allow_trivial_move is + // false assert(is_first || f->being_compacted == being_compacted); } if (is_first) { @@ -245,14 +248,13 @@ UniversalCompactionPicker::CalculateSortedRuns( // Universal style of compaction. Pick files that are contiguous in // time-range to compact. -// Compaction* UniversalCompactionPicker::PickCompaction( const std::string& cf_name, const MutableCFOptions& mutable_cf_options, VersionStorageInfo* vstorage, LogBuffer* log_buffer) { const int kLevel0 = 0; double score = vstorage->CompactionScore(kLevel0); std::vector sorted_runs = - CalculateSortedRuns(*vstorage, ioptions_); + CalculateSortedRuns(*vstorage, ioptions_, mutable_cf_options); if (sorted_runs.size() == 0 || sorted_runs.size() < @@ -279,7 +281,8 @@ Compaction* UniversalCompactionPicker::PickCompaction( } else { // Size amplification is within limits. Try reducing read // amplification while maintaining file size ratios. - unsigned int ratio = ioptions_.compaction_options_universal.size_ratio; + unsigned int ratio = + mutable_cf_options.compaction_options_universal.size_ratio; if ((c = PickCompactionToReduceSortedRuns( cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX, @@ -327,7 +330,8 @@ Compaction* UniversalCompactionPicker::PickCompaction( return nullptr; } - if (ioptions_.compaction_options_universal.allow_trivial_move == true) { + if (mutable_cf_options.compaction_options_universal.allow_trivial_move == + true) { c->set_is_trivial_move(IsInputFilesNonOverlapping(c)); } @@ -381,7 +385,8 @@ Compaction* UniversalCompactionPicker::PickCompaction( } uint32_t UniversalCompactionPicker::GetPathId( - const ImmutableCFOptions& ioptions, uint64_t file_size) { + const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options, uint64_t file_size) { // Two conditions need to be satisfied: // (1) the target path needs to be able to hold the file's size // (2) Total size left in this and previous paths need to be not @@ -398,8 +403,8 @@ uint32_t UniversalCompactionPicker::GetPathId( // that case. We need to improve it. uint64_t accumulated_size = 0; uint64_t future_size = - file_size * (100 - ioptions.compaction_options_universal.size_ratio) / - 100; + file_size * + (100 - mutable_cf_options.compaction_options_universal.size_ratio) / 100; uint32_t p = 0; assert(!ioptions.db_paths.empty()); for (; p < ioptions.db_paths.size() - 1; p++) { @@ -423,9 +428,9 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns( unsigned int max_number_of_files_to_compact, const std::vector& sorted_runs, LogBuffer* log_buffer) { unsigned int min_merge_width = - ioptions_.compaction_options_universal.min_merge_width; + mutable_cf_options.compaction_options_universal.min_merge_width; unsigned int max_merge_width = - ioptions_.compaction_options_universal.max_merge_width; + mutable_cf_options.compaction_options_universal.max_merge_width; const SortedRun* sr = nullptr; bool done = false; @@ -492,7 +497,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns( if (sz < static_cast(succeeding_sr->size)) { break; } - if (ioptions_.compaction_options_universal.stop_style == + if (mutable_cf_options.compaction_options_universal.stop_style == kCompactionStopStyleSimilarSize) { // Similar-size stopping rule: also check the last picked file isn't // far larger than the next candidate file. @@ -535,7 +540,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns( // size ratio of compression. bool enable_compression = true; int ratio_to_compress = - ioptions_.compaction_options_universal.compression_size_percent; + mutable_cf_options.compaction_options_universal.compression_size_percent; if (ratio_to_compress >= 0) { uint64_t total_size = 0; for (auto& sorted_run : sorted_runs) { @@ -556,7 +561,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns( for (unsigned int i = 0; i < first_index_after; i++) { estimated_total_size += sorted_runs[i].size; } - uint32_t path_id = GetPathId(ioptions_, estimated_total_size); + uint32_t path_id = + GetPathId(ioptions_, mutable_cf_options, estimated_total_size); int start_level = sorted_runs[start_index].level; int output_level; if (first_index_after == sorted_runs.size()) { @@ -621,8 +627,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp( VersionStorageInfo* vstorage, double score, const std::vector& sorted_runs, LogBuffer* log_buffer) { // percentage flexibility while reducing size amplification - uint64_t ratio = - ioptions_.compaction_options_universal.max_size_amplification_percent; + uint64_t ratio = mutable_cf_options.compaction_options_universal + .max_size_amplification_percent; unsigned int candidate_count = 0; uint64_t candidate_size = 0; @@ -700,7 +706,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp( for (size_t loop = start_index; loop < sorted_runs.size(); loop++) { estimated_total_size += sorted_runs[loop].size; } - uint32_t path_id = GetPathId(ioptions_, estimated_total_size); + uint32_t path_id = + GetPathId(ioptions_, mutable_cf_options, estimated_total_size); int start_level = sorted_runs[start_index].level; std::vector inputs(vstorage->num_levels()); diff --git a/db/compaction_picker_universal.h b/db/compaction_picker_universal.h index 3f2bed3e6..ac4fb10bb 100644 --- a/db/compaction_picker_universal.h +++ b/db/compaction_picker_universal.h @@ -80,11 +80,13 @@ class UniversalCompactionPicker : public CompactionPicker { bool IsInputFilesNonOverlapping(Compaction* c); static std::vector CalculateSortedRuns( - const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions); + const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options); // Pick a path ID to place a newly generated file, with its estimated file // size. static uint32_t GetPathId(const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options, uint64_t file_size); }; } // namespace rocksdb diff --git a/db/db_impl.cc b/db/db_impl.cc index ba1be480b..4440ed74d 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -106,7 +106,8 @@ CompressionType GetCompressionFlush( // optimization is used for leveled compaction. Otherwise the CPU and // latency overhead is not offset by saving much space. if (ioptions.compaction_style == kCompactionStyleUniversal) { - if (ioptions.compaction_options_universal.compression_size_percent < 0) { + if (mutable_cf_options.compaction_options_universal + .compression_size_percent < 0) { return mutable_cf_options.compression; } else { return kNoCompression; diff --git a/db/db_test.cc b/db/db_test.cc index 0d573631b..e7265f74f 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -4377,6 +4377,74 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) { } #endif // ROCKSDB_LITE +TEST_F(DBTest, DynamicUniversalCompactionOptions) { + Options options; + options.create_if_missing = true; + DestroyAndReopen(options); + + // Initial defaults + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 1); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width, + 2); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width, + UINT_MAX); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.max_size_amplification_percent, + 200); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.compression_size_percent, + -1); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style, + kCompactionStopStyleTotalSize); + ASSERT_EQ( + dbfull()->GetOptions().compaction_options_universal.allow_trivial_move, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_universal", "{size_ratio=7;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width, + 2); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width, + UINT_MAX); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.max_size_amplification_percent, + 200); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.compression_size_percent, + -1); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style, + kCompactionStopStyleTotalSize); + ASSERT_EQ( + dbfull()->GetOptions().compaction_options_universal.allow_trivial_move, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_universal", "{min_merge_width=11;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width, + 11); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width, + UINT_MAX); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.max_size_amplification_percent, + 200); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.compression_size_percent, + -1); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style, + kCompactionStopStyleTotalSize); + ASSERT_EQ( + dbfull()->GetOptions().compaction_options_universal.allow_trivial_move, + false); +} + TEST_F(DBTest, FileCreationRandomFailure) { Options options; options.env = env_; diff --git a/db/db_universal_compaction_test.cc b/db/db_universal_compaction_test.cc index 58fda80d5..dbb212b2f 100644 --- a/db/db_universal_compaction_test.cc +++ b/db/db_universal_compaction_test.cc @@ -374,6 +374,79 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) { ASSERT_EQ(NumSortedRuns(1), 1); } +TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = 3; + // Initial setup of compaction_options_universal will prevent universal + // compaction from happening + options.compaction_options_universal.size_ratio = 100; + options.compaction_options_universal.min_merge_width = 100; + DestroyAndReopen(options); + + int total_picked_compactions = 0; + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionPicker::PickCompaction:Return", [&](void* arg) { + if (arg) { + total_picked_compactions++; + } + }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); + + MutableCFOptions mutable_cf_options; + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + int key_idx = 0; + + // Generate two files in Level 0. Both files are approx the same size. + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000))); + key_idx++; + } + dbfull()->TEST_WaitForFlushMemTable(handles_[1]); + ASSERT_EQ(NumSortedRuns(1), num + 1); + } + ASSERT_EQ(NumSortedRuns(1), 2); + + // Flush whatever is remaining in memtable. This is typically + // small, which should not trigger size ratio based compaction + // but could instead trigger size amplification if it's set + // to 110. + ASSERT_OK(Flush(1)); + dbfull()->TEST_WaitForCompact(); + // Verify compaction did not happen + ASSERT_EQ(NumSortedRuns(1), 3); + + // Trigger compaction if size amplification exceeds 110% without reopening DB + ASSERT_EQ(dbfull() + ->GetOptions(handles_[1]) + .compaction_options_universal.max_size_amplification_percent, + 200); + ASSERT_OK(dbfull()->SetOptions(handles_[1], + {{"compaction_options_universal", + "{max_size_amplification_percent=110;}"}})); + ASSERT_EQ(dbfull() + ->GetOptions(handles_[1]) + .compaction_options_universal.max_size_amplification_percent, + 110); + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1], + &mutable_cf_options)); + ASSERT_EQ(110, mutable_cf_options.compaction_options_universal + .max_size_amplification_percent); + + dbfull()->TEST_WaitForCompact(); + // Verify that size amplification did happen + ASSERT_EQ(NumSortedRuns(1), 1); + ASSERT_EQ(total_picked_compactions, 1); +} + TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) { const int kTestKeySize = 16; const int kTestValueSize = 984; diff --git a/options/cf_options.cc b/options/cf_options.cc index f8e4e8dfe..bd9a1f8cf 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -27,7 +27,6 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options, const ColumnFamilyOptions& cf_options) : compaction_style(cf_options.compaction_style), compaction_pri(cf_options.compaction_pri), - compaction_options_universal(cf_options.compaction_options_universal), prefix_extractor(cf_options.prefix_extractor.get()), user_comparator(cf_options.comparator), internal_comparator(InternalKeyComparator(cf_options.comparator)), diff --git a/options/cf_options.h b/options/cf_options.h index e24a756fe..40db7c0ea 100644 --- a/options/cf_options.h +++ b/options/cf_options.h @@ -30,8 +30,6 @@ struct ImmutableCFOptions { CompactionPri compaction_pri; - CompactionOptionsUniversal compaction_options_universal; - const SliceTransform* prefix_extractor; const Comparator* user_comparator; @@ -149,6 +147,7 @@ struct MutableCFOptions { max_bytes_for_level_multiplier_additional( options.max_bytes_for_level_multiplier_additional), compaction_options_fifo(options.compaction_options_fifo), + compaction_options_universal(options.compaction_options_universal), max_sequential_skip_in_iterations( options.max_sequential_skip_in_iterations), paranoid_file_checks(options.paranoid_file_checks), @@ -224,6 +223,7 @@ struct MutableCFOptions { double max_bytes_for_level_multiplier; std::vector max_bytes_for_level_multiplier_additional; CompactionOptionsFIFO compaction_options_fifo; + CompactionOptionsUniversal compaction_options_universal; // Misc options uint64_t max_sequential_skip_in_iterations; diff --git a/options/options_helper.cc b/options/options_helper.cc index 39141d9a1..4d34769ff 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -168,6 +168,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions( } cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo; + cf_opts.compaction_options_universal = + mutable_cf_options.compaction_options_universal; // Misc options cf_opts.max_sequential_skip_in_iterations = @@ -496,6 +498,14 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, reinterpret_cast(opt_address), lru_cache_options_type_info); } + case OptionType::kCompactionOptionsUniversal: + return ParseStructOptions( + value, reinterpret_cast(opt_address), + universal_compaction_options_type_info); + case OptionType::kCompactionStopStyle: + return ParseEnum( + compaction_stop_style_string_map, value, + reinterpret_cast(opt_address)); default: return false; } @@ -660,11 +670,18 @@ bool SerializeSingleOptionHelper(const char* opt_address, return SerializeEnum( info_log_level_string_map, *reinterpret_cast(opt_address), value); - case OptionType::kCompactionOptionsFIFO: { + case OptionType::kCompactionOptionsFIFO: return SerializeStruct( *reinterpret_cast(opt_address), value, fifo_compaction_options_type_info); - } + case OptionType::kCompactionOptionsUniversal: + return SerializeStruct( + *reinterpret_cast(opt_address), + value, universal_compaction_options_type_info); + case OptionType::kCompactionStopStyle: + return SerializeEnum( + compaction_stop_style_string_map, + *reinterpret_cast(opt_address), value); default: return false; } @@ -1525,6 +1542,7 @@ std::unordered_map ColumnFamilyOptions OptionsHelper::dummy_cf_options; CompactionOptionsFIFO OptionsHelper::dummy_comp_options; LRUCacheOptions OptionsHelper::dummy_lru_cache_options; +CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal; // offset_of is used to get the offset of a class data member // ex: offset_of(&ColumnFamilyOptions::num_levels) @@ -1555,6 +1573,11 @@ int offset_of(T1 LRUCacheOptions::*member) { return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - size_t(&OptionsHelper::dummy_lru_cache_options)); } +template +int offset_of(T1 CompactionOptionsUniversal::*member) { + return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) - + size_t(&OptionsHelper::dummy_comp_options_universal)); +} std::unordered_map OptionsHelper::cf_options_type_info = { @@ -1782,7 +1805,12 @@ std::unordered_map {"compaction_options_fifo", {offset_of(&ColumnFamilyOptions::compaction_options_fifo), OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, - true, offsetof(struct MutableCFOptions, compaction_options_fifo)}}}; + true, offsetof(struct MutableCFOptions, compaction_options_fifo)}}, + {"compaction_options_universal", + {offset_of(&ColumnFamilyOptions::compaction_options_universal), + OptionType::kCompactionOptionsUniversal, + OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, compaction_options_universal)}}}; std::unordered_map OptionsHelper::fifo_compaction_options_type_info = { @@ -1800,20 +1828,61 @@ std::unordered_map offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; std::unordered_map - OptionsHelper::lru_cache_options_type_info = { - {"capacity", {offset_of(&LRUCacheOptions::capacity), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct LRUCacheOptions, capacity)}}, - {"num_shard_bits", {offset_of(&LRUCacheOptions::num_shard_bits), + OptionsHelper::universal_compaction_options_type_info = { + {"size_ratio", + {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt, + OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, size_ratio)}}, + {"min_merge_width", + {offset_of(&CompactionOptionsUniversal::min_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, min_merge_width)}}, + {"max_merge_width", + {offset_of(&CompactionOptionsUniversal::max_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, max_merge_width)}}, + {"max_size_amplification_percent", + {offset_of( + &CompactionOptionsUniversal::max_size_amplification_percent), + OptionType::kUInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, + max_size_amplification_percent)}}, + {"compression_size_percent", + {offset_of(&CompactionOptionsUniversal::compression_size_percent), OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, + compression_size_percent)}}, + {"stop_style", + {offset_of(&CompactionOptionsUniversal::stop_style), + OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, + true, offsetof(class CompactionOptionsUniversal, stop_style)}}, + {"allow_trivial_move", + {offset_of(&CompactionOptionsUniversal::allow_trivial_move), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}}; + +std::unordered_map + OptionsHelper::compaction_stop_style_string_map = { + {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize}, + {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}}; + +std::unordered_map + OptionsHelper::lru_cache_options_type_info = { + {"capacity", + {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT, + OptionVerificationType::kNormal, true, + offsetof(struct LRUCacheOptions, capacity)}}, + {"num_shard_bits", + {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt, + OptionVerificationType::kNormal, true, offsetof(struct LRUCacheOptions, num_shard_bits)}}, {"strict_capacity_limit", {offset_of(&LRUCacheOptions::strict_capacity_limit), OptionType::kBoolean, OptionVerificationType::kNormal, true, offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, {"high_pri_pool_ratio", - {offset_of(&LRUCacheOptions::high_pri_pool_ratio), - OptionType::kDouble, OptionVerificationType::kNormal, true, + {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble, + OptionVerificationType::kNormal, true, offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; #endif // !ROCKSDB_LITE diff --git a/options/options_helper.h b/options/options_helper.h index e7aff37c5..002e1303b 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -15,6 +15,7 @@ #include "rocksdb/options.h" #include "rocksdb/status.h" #include "rocksdb/table.h" +#include "rocksdb/universal_compaction.h" namespace rocksdb { @@ -63,6 +64,8 @@ enum class OptionType { kCompactionFilter, kCompactionFilterFactory, kCompactionOptionsFIFO, + kCompactionOptionsUniversal, + kCompactionStopStyle, kMergeOperator, kMemTableRepFactory, kBlockBasedTableIndexType, @@ -145,6 +148,10 @@ struct OptionsHelper { static std::unordered_map cf_options_type_info; static std::unordered_map fifo_compaction_options_type_info; + static std::unordered_map + universal_compaction_options_type_info; + static std::unordered_map + compaction_stop_style_string_map; static std::unordered_map db_options_type_info; static std::unordered_map lru_cache_options_type_info; @@ -166,6 +173,7 @@ struct OptionsHelper { static ColumnFamilyOptions dummy_cf_options; static CompactionOptionsFIFO dummy_comp_options; static LRUCacheOptions dummy_lru_cache_options; + static CompactionOptionsUniversal dummy_comp_options_universal; #endif // !ROCKSDB_LITE }; @@ -180,6 +188,10 @@ static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map; static auto& cf_options_type_info = OptionsHelper::cf_options_type_info; static auto& fifo_compaction_options_type_info = OptionsHelper::fifo_compaction_options_type_info; +static auto& universal_compaction_options_type_info = + OptionsHelper::universal_compaction_options_type_info; +static auto& compaction_stop_style_string_map = + OptionsHelper::compaction_stop_style_string_map; static auto& db_options_type_info = OptionsHelper::db_options_type_info; static auto& lru_cache_options_type_info = OptionsHelper::lru_cache_options_type_info; diff --git a/options/options_parser.cc b/options/options_parser.cc index f0eaa8045..e68295981 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -598,6 +598,23 @@ bool AreEqualOptions( } return false; } + case OptionType::kCompactionOptionsUniversal: { + CompactionOptionsUniversal lhs = + *reinterpret_cast(offset1); + CompactionOptionsUniversal rhs = + *reinterpret_cast(offset2); + if (lhs.size_ratio == rhs.size_ratio && + lhs.min_merge_width == rhs.min_merge_width && + lhs.max_merge_width == rhs.max_merge_width && + lhs.max_size_amplification_percent == + rhs.max_size_amplification_percent && + lhs.compression_size_percent == rhs.compression_size_percent && + lhs.stop_style == rhs.stop_style && + lhs.allow_trivial_move == rhs.allow_trivial_move) { + return true; + } + return false; + } default: if (type_info.verification == OptionVerificationType::kByName || type_info.verification ==