Make Universal compaction options dynamic

Summary:
Let me know if more test coverage is needed
Closes https://github.com/facebook/rocksdb/pull/3213

Differential Revision: D6457165

Pulled By: miasantreble

fbshipit-source-id: 3f944abff28aa7775237f1c4f61c64ccbad4eea9
This commit is contained in:
Zhongyi Xie 2017-12-11 13:12:12 -08:00 committed by Facebook Github Bot
parent 250a51a3f9
commit fcc8a6574d
12 changed files with 284 additions and 36 deletions

View File

@ -229,7 +229,7 @@ bool Compaction::IsTrivialMove() const {
// Used in universal compaction, where trivial move can be done if the // Used in universal compaction, where trivial move can be done if the
// input files are non overlapping // input files are non overlapping
if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) && if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
(output_level_ != 0)) { (output_level_ != 0)) {
return is_trivial_move_; return is_trivial_move_;
} }

View File

@ -437,7 +437,7 @@ TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) { TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
const uint64_t kFileSize = 100000; const uint64_t kFileSize = 100000;
ioptions_.compaction_options_universal.allow_trivial_move = true; mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
NewVersionStorage(1, kCompactionStyleUniversal); NewVersionStorage(1, kCompactionStyleUniversal);
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
// must return false when there's no files. // must return false when there's no files.
@ -468,7 +468,7 @@ TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) { TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) {
const uint64_t kFileSize = 100000; const uint64_t kFileSize = 100000;
ioptions_.compaction_options_universal.allow_trivial_move = true; mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
NewVersionStorage(3, kCompactionStyleUniversal); NewVersionStorage(3, kCompactionStyleUniversal);

View File

@ -204,7 +204,8 @@ void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
std::vector<UniversalCompactionPicker::SortedRun> std::vector<UniversalCompactionPicker::SortedRun>
UniversalCompactionPicker::CalculateSortedRuns( UniversalCompactionPicker::CalculateSortedRuns(
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) { const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options) {
std::vector<UniversalCompactionPicker::SortedRun> ret; std::vector<UniversalCompactionPicker::SortedRun> ret;
for (FileMetaData* f : vstorage.LevelFiles(0)) { for (FileMetaData* f : vstorage.LevelFiles(0)) {
ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size, ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
@ -218,7 +219,8 @@ UniversalCompactionPicker::CalculateSortedRuns(
for (FileMetaData* f : vstorage.LevelFiles(level)) { for (FileMetaData* f : vstorage.LevelFiles(level)) {
total_compensated_size += f->compensated_file_size; total_compensated_size += f->compensated_file_size;
total_size += f->fd.GetFileSize(); total_size += f->fd.GetFileSize();
if (ioptions.compaction_options_universal.allow_trivial_move == true) { if (mutable_cf_options.compaction_options_universal.allow_trivial_move ==
true) {
if (f->being_compacted) { if (f->being_compacted) {
being_compacted = f->being_compacted; being_compacted = f->being_compacted;
} }
@ -227,7 +229,8 @@ UniversalCompactionPicker::CalculateSortedRuns(
// non-zero level, all the files should share the same being_compacted // non-zero level, all the files should share the same being_compacted
// value. // value.
// This assumption is only valid when // This assumption is only valid when
// ioptions.compaction_options_universal.allow_trivial_move is false // mutable_cf_options.compaction_options_universal.allow_trivial_move is
// false
assert(is_first || f->being_compacted == being_compacted); assert(is_first || f->being_compacted == being_compacted);
} }
if (is_first) { if (is_first) {
@ -245,14 +248,13 @@ UniversalCompactionPicker::CalculateSortedRuns(
// Universal style of compaction. Pick files that are contiguous in // Universal style of compaction. Pick files that are contiguous in
// time-range to compact. // time-range to compact.
//
Compaction* UniversalCompactionPicker::PickCompaction( Compaction* UniversalCompactionPicker::PickCompaction(
const std::string& cf_name, const MutableCFOptions& mutable_cf_options, const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, LogBuffer* log_buffer) { VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
const int kLevel0 = 0; const int kLevel0 = 0;
double score = vstorage->CompactionScore(kLevel0); double score = vstorage->CompactionScore(kLevel0);
std::vector<SortedRun> sorted_runs = std::vector<SortedRun> sorted_runs =
CalculateSortedRuns(*vstorage, ioptions_); CalculateSortedRuns(*vstorage, ioptions_, mutable_cf_options);
if (sorted_runs.size() == 0 || if (sorted_runs.size() == 0 ||
sorted_runs.size() < sorted_runs.size() <
@ -279,7 +281,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
} else { } else {
// Size amplification is within limits. Try reducing read // Size amplification is within limits. Try reducing read
// amplification while maintaining file size ratios. // amplification while maintaining file size ratios.
unsigned int ratio = ioptions_.compaction_options_universal.size_ratio; unsigned int ratio =
mutable_cf_options.compaction_options_universal.size_ratio;
if ((c = PickCompactionToReduceSortedRuns( if ((c = PickCompactionToReduceSortedRuns(
cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX, cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
@ -327,7 +330,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
return nullptr; return nullptr;
} }
if (ioptions_.compaction_options_universal.allow_trivial_move == true) { if (mutable_cf_options.compaction_options_universal.allow_trivial_move ==
true) {
c->set_is_trivial_move(IsInputFilesNonOverlapping(c)); c->set_is_trivial_move(IsInputFilesNonOverlapping(c));
} }
@ -381,7 +385,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
} }
uint32_t UniversalCompactionPicker::GetPathId( uint32_t UniversalCompactionPicker::GetPathId(
const ImmutableCFOptions& ioptions, uint64_t file_size) { const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options, uint64_t file_size) {
// Two conditions need to be satisfied: // Two conditions need to be satisfied:
// (1) the target path needs to be able to hold the file's size // (1) the target path needs to be able to hold the file's size
// (2) Total size left in this and previous paths need to be not // (2) Total size left in this and previous paths need to be not
@ -398,8 +403,8 @@ uint32_t UniversalCompactionPicker::GetPathId(
// that case. We need to improve it. // that case. We need to improve it.
uint64_t accumulated_size = 0; uint64_t accumulated_size = 0;
uint64_t future_size = uint64_t future_size =
file_size * (100 - ioptions.compaction_options_universal.size_ratio) / file_size *
100; (100 - mutable_cf_options.compaction_options_universal.size_ratio) / 100;
uint32_t p = 0; uint32_t p = 0;
assert(!ioptions.db_paths.empty()); assert(!ioptions.db_paths.empty());
for (; p < ioptions.db_paths.size() - 1; p++) { for (; p < ioptions.db_paths.size() - 1; p++) {
@ -423,9 +428,9 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
unsigned int max_number_of_files_to_compact, unsigned int max_number_of_files_to_compact,
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) { const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
unsigned int min_merge_width = unsigned int min_merge_width =
ioptions_.compaction_options_universal.min_merge_width; mutable_cf_options.compaction_options_universal.min_merge_width;
unsigned int max_merge_width = unsigned int max_merge_width =
ioptions_.compaction_options_universal.max_merge_width; mutable_cf_options.compaction_options_universal.max_merge_width;
const SortedRun* sr = nullptr; const SortedRun* sr = nullptr;
bool done = false; bool done = false;
@ -492,7 +497,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
if (sz < static_cast<double>(succeeding_sr->size)) { if (sz < static_cast<double>(succeeding_sr->size)) {
break; break;
} }
if (ioptions_.compaction_options_universal.stop_style == if (mutable_cf_options.compaction_options_universal.stop_style ==
kCompactionStopStyleSimilarSize) { kCompactionStopStyleSimilarSize) {
// Similar-size stopping rule: also check the last picked file isn't // Similar-size stopping rule: also check the last picked file isn't
// far larger than the next candidate file. // far larger than the next candidate file.
@ -535,7 +540,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
// size ratio of compression. // size ratio of compression.
bool enable_compression = true; bool enable_compression = true;
int ratio_to_compress = int ratio_to_compress =
ioptions_.compaction_options_universal.compression_size_percent; mutable_cf_options.compaction_options_universal.compression_size_percent;
if (ratio_to_compress >= 0) { if (ratio_to_compress >= 0) {
uint64_t total_size = 0; uint64_t total_size = 0;
for (auto& sorted_run : sorted_runs) { for (auto& sorted_run : sorted_runs) {
@ -556,7 +561,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
for (unsigned int i = 0; i < first_index_after; i++) { for (unsigned int i = 0; i < first_index_after; i++) {
estimated_total_size += sorted_runs[i].size; estimated_total_size += sorted_runs[i].size;
} }
uint32_t path_id = GetPathId(ioptions_, estimated_total_size); uint32_t path_id =
GetPathId(ioptions_, mutable_cf_options, estimated_total_size);
int start_level = sorted_runs[start_index].level; int start_level = sorted_runs[start_index].level;
int output_level; int output_level;
if (first_index_after == sorted_runs.size()) { if (first_index_after == sorted_runs.size()) {
@ -621,8 +627,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
VersionStorageInfo* vstorage, double score, VersionStorageInfo* vstorage, double score,
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) { const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
// percentage flexibility while reducing size amplification // percentage flexibility while reducing size amplification
uint64_t ratio = uint64_t ratio = mutable_cf_options.compaction_options_universal
ioptions_.compaction_options_universal.max_size_amplification_percent; .max_size_amplification_percent;
unsigned int candidate_count = 0; unsigned int candidate_count = 0;
uint64_t candidate_size = 0; uint64_t candidate_size = 0;
@ -700,7 +706,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
for (size_t loop = start_index; loop < sorted_runs.size(); loop++) { for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
estimated_total_size += sorted_runs[loop].size; estimated_total_size += sorted_runs[loop].size;
} }
uint32_t path_id = GetPathId(ioptions_, estimated_total_size); uint32_t path_id =
GetPathId(ioptions_, mutable_cf_options, estimated_total_size);
int start_level = sorted_runs[start_index].level; int start_level = sorted_runs[start_index].level;
std::vector<CompactionInputFiles> inputs(vstorage->num_levels()); std::vector<CompactionInputFiles> inputs(vstorage->num_levels());

View File

@ -80,11 +80,13 @@ class UniversalCompactionPicker : public CompactionPicker {
bool IsInputFilesNonOverlapping(Compaction* c); bool IsInputFilesNonOverlapping(Compaction* c);
static std::vector<SortedRun> CalculateSortedRuns( static std::vector<SortedRun> CalculateSortedRuns(
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions); const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options);
// Pick a path ID to place a newly generated file, with its estimated file // Pick a path ID to place a newly generated file, with its estimated file
// size. // size.
static uint32_t GetPathId(const ImmutableCFOptions& ioptions, static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options,
uint64_t file_size); uint64_t file_size);
}; };
} // namespace rocksdb } // namespace rocksdb

View File

@ -106,7 +106,8 @@ CompressionType GetCompressionFlush(
// optimization is used for leveled compaction. Otherwise the CPU and // optimization is used for leveled compaction. Otherwise the CPU and
// latency overhead is not offset by saving much space. // latency overhead is not offset by saving much space.
if (ioptions.compaction_style == kCompactionStyleUniversal) { if (ioptions.compaction_style == kCompactionStyleUniversal) {
if (ioptions.compaction_options_universal.compression_size_percent < 0) { if (mutable_cf_options.compaction_options_universal
.compression_size_percent < 0) {
return mutable_cf_options.compression; return mutable_cf_options.compression;
} else { } else {
return kNoCompression; return kNoCompression;

View File

@ -4377,6 +4377,74 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
TEST_F(DBTest, DynamicUniversalCompactionOptions) {
Options options;
options.create_if_missing = true;
DestroyAndReopen(options);
// Initial defaults
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 1);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width,
2);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width,
UINT_MAX);
ASSERT_EQ(dbfull()
->GetOptions()
.compaction_options_universal.max_size_amplification_percent,
200);
ASSERT_EQ(dbfull()
->GetOptions()
.compaction_options_universal.compression_size_percent,
-1);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style,
kCompactionStopStyleTotalSize);
ASSERT_EQ(
dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
false);
ASSERT_OK(dbfull()->SetOptions(
{{"compaction_options_universal", "{size_ratio=7;}"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width,
2);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width,
UINT_MAX);
ASSERT_EQ(dbfull()
->GetOptions()
.compaction_options_universal.max_size_amplification_percent,
200);
ASSERT_EQ(dbfull()
->GetOptions()
.compaction_options_universal.compression_size_percent,
-1);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style,
kCompactionStopStyleTotalSize);
ASSERT_EQ(
dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
false);
ASSERT_OK(dbfull()->SetOptions(
{{"compaction_options_universal", "{min_merge_width=11;}"}}));
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width,
11);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width,
UINT_MAX);
ASSERT_EQ(dbfull()
->GetOptions()
.compaction_options_universal.max_size_amplification_percent,
200);
ASSERT_EQ(dbfull()
->GetOptions()
.compaction_options_universal.compression_size_percent,
-1);
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style,
kCompactionStopStyleTotalSize);
ASSERT_EQ(
dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
false);
}
TEST_F(DBTest, FileCreationRandomFailure) { TEST_F(DBTest, FileCreationRandomFailure) {
Options options; Options options;
options.env = env_; options.env = env_;

View File

@ -374,6 +374,79 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
ASSERT_EQ(NumSortedRuns(1), 1); ASSERT_EQ(NumSortedRuns(1), 1);
} }
TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
options.num_levels = 1;
options.write_buffer_size = 100 << 10; // 100KB
options.target_file_size_base = 32 << 10; // 32KB
options.level0_file_num_compaction_trigger = 3;
// Initial setup of compaction_options_universal will prevent universal
// compaction from happening
options.compaction_options_universal.size_ratio = 100;
options.compaction_options_universal.min_merge_width = 100;
DestroyAndReopen(options);
int total_picked_compactions = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"UniversalCompactionPicker::PickCompaction:Return", [&](void* arg) {
if (arg) {
total_picked_compactions++;
}
});
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
MutableCFOptions mutable_cf_options;
CreateAndReopenWithCF({"pikachu"}, options);
Random rnd(301);
int key_idx = 0;
// Generate two files in Level 0. Both files are approx the same size.
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
num++) {
// Write 110KB (11 values, each 10K)
for (int i = 0; i < 11; i++) {
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
ASSERT_EQ(NumSortedRuns(1), num + 1);
}
ASSERT_EQ(NumSortedRuns(1), 2);
// Flush whatever is remaining in memtable. This is typically
// small, which should not trigger size ratio based compaction
// but could instead trigger size amplification if it's set
// to 110.
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
// Verify compaction did not happen
ASSERT_EQ(NumSortedRuns(1), 3);
// Trigger compaction if size amplification exceeds 110% without reopening DB
ASSERT_EQ(dbfull()
->GetOptions(handles_[1])
.compaction_options_universal.max_size_amplification_percent,
200);
ASSERT_OK(dbfull()->SetOptions(handles_[1],
{{"compaction_options_universal",
"{max_size_amplification_percent=110;}"}}));
ASSERT_EQ(dbfull()
->GetOptions(handles_[1])
.compaction_options_universal.max_size_amplification_percent,
110);
ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
&mutable_cf_options));
ASSERT_EQ(110, mutable_cf_options.compaction_options_universal
.max_size_amplification_percent);
dbfull()->TEST_WaitForCompact();
// Verify that size amplification did happen
ASSERT_EQ(NumSortedRuns(1), 1);
ASSERT_EQ(total_picked_compactions, 1);
}
TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) { TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
const int kTestKeySize = 16; const int kTestKeySize = 16;
const int kTestValueSize = 984; const int kTestValueSize = 984;

View File

@ -27,7 +27,6 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options,
const ColumnFamilyOptions& cf_options) const ColumnFamilyOptions& cf_options)
: compaction_style(cf_options.compaction_style), : compaction_style(cf_options.compaction_style),
compaction_pri(cf_options.compaction_pri), compaction_pri(cf_options.compaction_pri),
compaction_options_universal(cf_options.compaction_options_universal),
prefix_extractor(cf_options.prefix_extractor.get()), prefix_extractor(cf_options.prefix_extractor.get()),
user_comparator(cf_options.comparator), user_comparator(cf_options.comparator),
internal_comparator(InternalKeyComparator(cf_options.comparator)), internal_comparator(InternalKeyComparator(cf_options.comparator)),

View File

@ -30,8 +30,6 @@ struct ImmutableCFOptions {
CompactionPri compaction_pri; CompactionPri compaction_pri;
CompactionOptionsUniversal compaction_options_universal;
const SliceTransform* prefix_extractor; const SliceTransform* prefix_extractor;
const Comparator* user_comparator; const Comparator* user_comparator;
@ -149,6 +147,7 @@ struct MutableCFOptions {
max_bytes_for_level_multiplier_additional( max_bytes_for_level_multiplier_additional(
options.max_bytes_for_level_multiplier_additional), options.max_bytes_for_level_multiplier_additional),
compaction_options_fifo(options.compaction_options_fifo), compaction_options_fifo(options.compaction_options_fifo),
compaction_options_universal(options.compaction_options_universal),
max_sequential_skip_in_iterations( max_sequential_skip_in_iterations(
options.max_sequential_skip_in_iterations), options.max_sequential_skip_in_iterations),
paranoid_file_checks(options.paranoid_file_checks), paranoid_file_checks(options.paranoid_file_checks),
@ -224,6 +223,7 @@ struct MutableCFOptions {
double max_bytes_for_level_multiplier; double max_bytes_for_level_multiplier;
std::vector<int> max_bytes_for_level_multiplier_additional; std::vector<int> max_bytes_for_level_multiplier_additional;
CompactionOptionsFIFO compaction_options_fifo; CompactionOptionsFIFO compaction_options_fifo;
CompactionOptionsUniversal compaction_options_universal;
// Misc options // Misc options
uint64_t max_sequential_skip_in_iterations; uint64_t max_sequential_skip_in_iterations;

View File

@ -168,6 +168,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
} }
cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo; cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo;
cf_opts.compaction_options_universal =
mutable_cf_options.compaction_options_universal;
// Misc options // Misc options
cf_opts.max_sequential_skip_in_iterations = cf_opts.max_sequential_skip_in_iterations =
@ -496,6 +498,14 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
reinterpret_cast<LRUCacheOptions*>(opt_address), reinterpret_cast<LRUCacheOptions*>(opt_address),
lru_cache_options_type_info); lru_cache_options_type_info);
} }
case OptionType::kCompactionOptionsUniversal:
return ParseStructOptions<CompactionOptionsUniversal>(
value, reinterpret_cast<CompactionOptionsUniversal*>(opt_address),
universal_compaction_options_type_info);
case OptionType::kCompactionStopStyle:
return ParseEnum<CompactionStopStyle>(
compaction_stop_style_string_map, value,
reinterpret_cast<CompactionStopStyle*>(opt_address));
default: default:
return false; return false;
} }
@ -660,11 +670,18 @@ bool SerializeSingleOptionHelper(const char* opt_address,
return SerializeEnum<InfoLogLevel>( return SerializeEnum<InfoLogLevel>(
info_log_level_string_map, info_log_level_string_map,
*reinterpret_cast<const InfoLogLevel*>(opt_address), value); *reinterpret_cast<const InfoLogLevel*>(opt_address), value);
case OptionType::kCompactionOptionsFIFO: { case OptionType::kCompactionOptionsFIFO:
return SerializeStruct<CompactionOptionsFIFO>( return SerializeStruct<CompactionOptionsFIFO>(
*reinterpret_cast<const CompactionOptionsFIFO*>(opt_address), value, *reinterpret_cast<const CompactionOptionsFIFO*>(opt_address), value,
fifo_compaction_options_type_info); fifo_compaction_options_type_info);
} case OptionType::kCompactionOptionsUniversal:
return SerializeStruct<CompactionOptionsUniversal>(
*reinterpret_cast<const CompactionOptionsUniversal*>(opt_address),
value, universal_compaction_options_type_info);
case OptionType::kCompactionStopStyle:
return SerializeEnum<CompactionStopStyle>(
compaction_stop_style_string_map,
*reinterpret_cast<const CompactionStopStyle*>(opt_address), value);
default: default:
return false; return false;
} }
@ -1525,6 +1542,7 @@ std::unordered_map<std::string, InfoLogLevel>
ColumnFamilyOptions OptionsHelper::dummy_cf_options; ColumnFamilyOptions OptionsHelper::dummy_cf_options;
CompactionOptionsFIFO OptionsHelper::dummy_comp_options; CompactionOptionsFIFO OptionsHelper::dummy_comp_options;
LRUCacheOptions OptionsHelper::dummy_lru_cache_options; LRUCacheOptions OptionsHelper::dummy_lru_cache_options;
CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal;
// offset_of is used to get the offset of a class data member // offset_of is used to get the offset of a class data member
// ex: offset_of(&ColumnFamilyOptions::num_levels) // ex: offset_of(&ColumnFamilyOptions::num_levels)
@ -1555,6 +1573,11 @@ int offset_of(T1 LRUCacheOptions::*member) {
return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) -
size_t(&OptionsHelper::dummy_lru_cache_options)); size_t(&OptionsHelper::dummy_lru_cache_options));
} }
template <typename T1>
int offset_of(T1 CompactionOptionsUniversal::*member) {
return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) -
size_t(&OptionsHelper::dummy_comp_options_universal));
}
std::unordered_map<std::string, OptionTypeInfo> std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::cf_options_type_info = { OptionsHelper::cf_options_type_info = {
@ -1782,7 +1805,12 @@ std::unordered_map<std::string, OptionTypeInfo>
{"compaction_options_fifo", {"compaction_options_fifo",
{offset_of(&ColumnFamilyOptions::compaction_options_fifo), {offset_of(&ColumnFamilyOptions::compaction_options_fifo),
OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal,
true, offsetof(struct MutableCFOptions, compaction_options_fifo)}}}; true, offsetof(struct MutableCFOptions, compaction_options_fifo)}},
{"compaction_options_universal",
{offset_of(&ColumnFamilyOptions::compaction_options_universal),
OptionType::kCompactionOptionsUniversal,
OptionVerificationType::kNormal, true,
offsetof(struct MutableCFOptions, compaction_options_universal)}}};
std::unordered_map<std::string, OptionTypeInfo> std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::fifo_compaction_options_type_info = { OptionsHelper::fifo_compaction_options_type_info = {
@ -1800,20 +1828,61 @@ std::unordered_map<std::string, OptionTypeInfo>
offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; offsetof(struct CompactionOptionsFIFO, allow_compaction)}}};
std::unordered_map<std::string, OptionTypeInfo> std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::lru_cache_options_type_info = { OptionsHelper::universal_compaction_options_type_info = {
{"capacity", {offset_of(&LRUCacheOptions::capacity), {"size_ratio",
OptionType::kSizeT, OptionVerificationType::kNormal, true, {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt,
offsetof(struct LRUCacheOptions, capacity)}}, OptionVerificationType::kNormal, true,
{"num_shard_bits", {offset_of(&LRUCacheOptions::num_shard_bits), offsetof(class CompactionOptionsUniversal, size_ratio)}},
{"min_merge_width",
{offset_of(&CompactionOptionsUniversal::min_merge_width),
OptionType::kUInt, OptionVerificationType::kNormal, true,
offsetof(class CompactionOptionsUniversal, min_merge_width)}},
{"max_merge_width",
{offset_of(&CompactionOptionsUniversal::max_merge_width),
OptionType::kUInt, OptionVerificationType::kNormal, true,
offsetof(class CompactionOptionsUniversal, max_merge_width)}},
{"max_size_amplification_percent",
{offset_of(
&CompactionOptionsUniversal::max_size_amplification_percent),
OptionType::kUInt, OptionVerificationType::kNormal, true,
offsetof(class CompactionOptionsUniversal,
max_size_amplification_percent)}},
{"compression_size_percent",
{offset_of(&CompactionOptionsUniversal::compression_size_percent),
OptionType::kInt, OptionVerificationType::kNormal, true, OptionType::kInt, OptionVerificationType::kNormal, true,
offsetof(class CompactionOptionsUniversal,
compression_size_percent)}},
{"stop_style",
{offset_of(&CompactionOptionsUniversal::stop_style),
OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
true, offsetof(class CompactionOptionsUniversal, stop_style)}},
{"allow_trivial_move",
{offset_of(&CompactionOptionsUniversal::allow_trivial_move),
OptionType::kBoolean, OptionVerificationType::kNormal, true,
offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}};
std::unordered_map<std::string, CompactionStopStyle>
OptionsHelper::compaction_stop_style_string_map = {
{"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
{"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::lru_cache_options_type_info = {
{"capacity",
{offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT,
OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, capacity)}},
{"num_shard_bits",
{offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt,
OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, num_shard_bits)}}, offsetof(struct LRUCacheOptions, num_shard_bits)}},
{"strict_capacity_limit", {"strict_capacity_limit",
{offset_of(&LRUCacheOptions::strict_capacity_limit), {offset_of(&LRUCacheOptions::strict_capacity_limit),
OptionType::kBoolean, OptionVerificationType::kNormal, true, OptionType::kBoolean, OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
{"high_pri_pool_ratio", {"high_pri_pool_ratio",
{offset_of(&LRUCacheOptions::high_pri_pool_ratio), {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble,
OptionType::kDouble, OptionVerificationType::kNormal, true, OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}};
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE

View File

@ -15,6 +15,7 @@
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/universal_compaction.h"
namespace rocksdb { namespace rocksdb {
@ -63,6 +64,8 @@ enum class OptionType {
kCompactionFilter, kCompactionFilter,
kCompactionFilterFactory, kCompactionFilterFactory,
kCompactionOptionsFIFO, kCompactionOptionsFIFO,
kCompactionOptionsUniversal,
kCompactionStopStyle,
kMergeOperator, kMergeOperator,
kMemTableRepFactory, kMemTableRepFactory,
kBlockBasedTableIndexType, kBlockBasedTableIndexType,
@ -145,6 +148,10 @@ struct OptionsHelper {
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info; static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info;
static std::unordered_map<std::string, OptionTypeInfo> static std::unordered_map<std::string, OptionTypeInfo>
fifo_compaction_options_type_info; fifo_compaction_options_type_info;
static std::unordered_map<std::string, OptionTypeInfo>
universal_compaction_options_type_info;
static std::unordered_map<std::string, CompactionStopStyle>
compaction_stop_style_string_map;
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info; static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info;
static std::unordered_map<std::string, OptionTypeInfo> static std::unordered_map<std::string, OptionTypeInfo>
lru_cache_options_type_info; lru_cache_options_type_info;
@ -166,6 +173,7 @@ struct OptionsHelper {
static ColumnFamilyOptions dummy_cf_options; static ColumnFamilyOptions dummy_cf_options;
static CompactionOptionsFIFO dummy_comp_options; static CompactionOptionsFIFO dummy_comp_options;
static LRUCacheOptions dummy_lru_cache_options; static LRUCacheOptions dummy_lru_cache_options;
static CompactionOptionsUniversal dummy_comp_options_universal;
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
}; };
@ -180,6 +188,10 @@ static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map;
static auto& cf_options_type_info = OptionsHelper::cf_options_type_info; static auto& cf_options_type_info = OptionsHelper::cf_options_type_info;
static auto& fifo_compaction_options_type_info = static auto& fifo_compaction_options_type_info =
OptionsHelper::fifo_compaction_options_type_info; OptionsHelper::fifo_compaction_options_type_info;
static auto& universal_compaction_options_type_info =
OptionsHelper::universal_compaction_options_type_info;
static auto& compaction_stop_style_string_map =
OptionsHelper::compaction_stop_style_string_map;
static auto& db_options_type_info = OptionsHelper::db_options_type_info; static auto& db_options_type_info = OptionsHelper::db_options_type_info;
static auto& lru_cache_options_type_info = static auto& lru_cache_options_type_info =
OptionsHelper::lru_cache_options_type_info; OptionsHelper::lru_cache_options_type_info;

View File

@ -598,6 +598,23 @@ bool AreEqualOptions(
} }
return false; return false;
} }
case OptionType::kCompactionOptionsUniversal: {
CompactionOptionsUniversal lhs =
*reinterpret_cast<const CompactionOptionsUniversal*>(offset1);
CompactionOptionsUniversal rhs =
*reinterpret_cast<const CompactionOptionsUniversal*>(offset2);
if (lhs.size_ratio == rhs.size_ratio &&
lhs.min_merge_width == rhs.min_merge_width &&
lhs.max_merge_width == rhs.max_merge_width &&
lhs.max_size_amplification_percent ==
rhs.max_size_amplification_percent &&
lhs.compression_size_percent == rhs.compression_size_percent &&
lhs.stop_style == rhs.stop_style &&
lhs.allow_trivial_move == rhs.allow_trivial_move) {
return true;
}
return false;
}
default: default:
if (type_info.verification == OptionVerificationType::kByName || if (type_info.verification == OptionVerificationType::kByName ||
type_info.verification == type_info.verification ==