Make Universal compaction options dynamic
Summary: Let me know if more test coverage is needed Closes https://github.com/facebook/rocksdb/pull/3213 Differential Revision: D6457165 Pulled By: miasantreble fbshipit-source-id: 3f944abff28aa7775237f1c4f61c64ccbad4eea9
This commit is contained in:
parent
250a51a3f9
commit
fcc8a6574d
@ -229,7 +229,7 @@ bool Compaction::IsTrivialMove() const {
|
||||
|
||||
// Used in universal compaction, where trivial move can be done if the
|
||||
// input files are non overlapping
|
||||
if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
|
||||
if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
|
||||
(output_level_ != 0)) {
|
||||
return is_trivial_move_;
|
||||
}
|
||||
|
@ -437,7 +437,7 @@ TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
|
||||
TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
|
||||
const uint64_t kFileSize = 100000;
|
||||
|
||||
ioptions_.compaction_options_universal.allow_trivial_move = true;
|
||||
mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
|
||||
NewVersionStorage(1, kCompactionStyleUniversal);
|
||||
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
|
||||
// must return false when there's no files.
|
||||
@ -468,7 +468,7 @@ TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
|
||||
TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) {
|
||||
const uint64_t kFileSize = 100000;
|
||||
|
||||
ioptions_.compaction_options_universal.allow_trivial_move = true;
|
||||
mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
|
||||
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
|
||||
|
||||
NewVersionStorage(3, kCompactionStyleUniversal);
|
||||
|
@ -204,7 +204,8 @@ void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
|
||||
|
||||
std::vector<UniversalCompactionPicker::SortedRun>
|
||||
UniversalCompactionPicker::CalculateSortedRuns(
|
||||
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) {
|
||||
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions,
|
||||
const MutableCFOptions& mutable_cf_options) {
|
||||
std::vector<UniversalCompactionPicker::SortedRun> ret;
|
||||
for (FileMetaData* f : vstorage.LevelFiles(0)) {
|
||||
ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
|
||||
@ -218,7 +219,8 @@ UniversalCompactionPicker::CalculateSortedRuns(
|
||||
for (FileMetaData* f : vstorage.LevelFiles(level)) {
|
||||
total_compensated_size += f->compensated_file_size;
|
||||
total_size += f->fd.GetFileSize();
|
||||
if (ioptions.compaction_options_universal.allow_trivial_move == true) {
|
||||
if (mutable_cf_options.compaction_options_universal.allow_trivial_move ==
|
||||
true) {
|
||||
if (f->being_compacted) {
|
||||
being_compacted = f->being_compacted;
|
||||
}
|
||||
@ -227,7 +229,8 @@ UniversalCompactionPicker::CalculateSortedRuns(
|
||||
// non-zero level, all the files should share the same being_compacted
|
||||
// value.
|
||||
// This assumption is only valid when
|
||||
// ioptions.compaction_options_universal.allow_trivial_move is false
|
||||
// mutable_cf_options.compaction_options_universal.allow_trivial_move is
|
||||
// false
|
||||
assert(is_first || f->being_compacted == being_compacted);
|
||||
}
|
||||
if (is_first) {
|
||||
@ -245,14 +248,13 @@ UniversalCompactionPicker::CalculateSortedRuns(
|
||||
|
||||
// Universal style of compaction. Pick files that are contiguous in
|
||||
// time-range to compact.
|
||||
//
|
||||
Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
|
||||
const int kLevel0 = 0;
|
||||
double score = vstorage->CompactionScore(kLevel0);
|
||||
std::vector<SortedRun> sorted_runs =
|
||||
CalculateSortedRuns(*vstorage, ioptions_);
|
||||
CalculateSortedRuns(*vstorage, ioptions_, mutable_cf_options);
|
||||
|
||||
if (sorted_runs.size() == 0 ||
|
||||
sorted_runs.size() <
|
||||
@ -279,7 +281,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
} else {
|
||||
// Size amplification is within limits. Try reducing read
|
||||
// amplification while maintaining file size ratios.
|
||||
unsigned int ratio = ioptions_.compaction_options_universal.size_ratio;
|
||||
unsigned int ratio =
|
||||
mutable_cf_options.compaction_options_universal.size_ratio;
|
||||
|
||||
if ((c = PickCompactionToReduceSortedRuns(
|
||||
cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
|
||||
@ -327,7 +330,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (ioptions_.compaction_options_universal.allow_trivial_move == true) {
|
||||
if (mutable_cf_options.compaction_options_universal.allow_trivial_move ==
|
||||
true) {
|
||||
c->set_is_trivial_move(IsInputFilesNonOverlapping(c));
|
||||
}
|
||||
|
||||
@ -381,7 +385,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
}
|
||||
|
||||
uint32_t UniversalCompactionPicker::GetPathId(
|
||||
const ImmutableCFOptions& ioptions, uint64_t file_size) {
|
||||
const ImmutableCFOptions& ioptions,
|
||||
const MutableCFOptions& mutable_cf_options, uint64_t file_size) {
|
||||
// Two conditions need to be satisfied:
|
||||
// (1) the target path needs to be able to hold the file's size
|
||||
// (2) Total size left in this and previous paths need to be not
|
||||
@ -398,8 +403,8 @@ uint32_t UniversalCompactionPicker::GetPathId(
|
||||
// that case. We need to improve it.
|
||||
uint64_t accumulated_size = 0;
|
||||
uint64_t future_size =
|
||||
file_size * (100 - ioptions.compaction_options_universal.size_ratio) /
|
||||
100;
|
||||
file_size *
|
||||
(100 - mutable_cf_options.compaction_options_universal.size_ratio) / 100;
|
||||
uint32_t p = 0;
|
||||
assert(!ioptions.db_paths.empty());
|
||||
for (; p < ioptions.db_paths.size() - 1; p++) {
|
||||
@ -423,9 +428,9 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
|
||||
unsigned int max_number_of_files_to_compact,
|
||||
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
|
||||
unsigned int min_merge_width =
|
||||
ioptions_.compaction_options_universal.min_merge_width;
|
||||
mutable_cf_options.compaction_options_universal.min_merge_width;
|
||||
unsigned int max_merge_width =
|
||||
ioptions_.compaction_options_universal.max_merge_width;
|
||||
mutable_cf_options.compaction_options_universal.max_merge_width;
|
||||
|
||||
const SortedRun* sr = nullptr;
|
||||
bool done = false;
|
||||
@ -492,7 +497,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
|
||||
if (sz < static_cast<double>(succeeding_sr->size)) {
|
||||
break;
|
||||
}
|
||||
if (ioptions_.compaction_options_universal.stop_style ==
|
||||
if (mutable_cf_options.compaction_options_universal.stop_style ==
|
||||
kCompactionStopStyleSimilarSize) {
|
||||
// Similar-size stopping rule: also check the last picked file isn't
|
||||
// far larger than the next candidate file.
|
||||
@ -535,7 +540,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
|
||||
// size ratio of compression.
|
||||
bool enable_compression = true;
|
||||
int ratio_to_compress =
|
||||
ioptions_.compaction_options_universal.compression_size_percent;
|
||||
mutable_cf_options.compaction_options_universal.compression_size_percent;
|
||||
if (ratio_to_compress >= 0) {
|
||||
uint64_t total_size = 0;
|
||||
for (auto& sorted_run : sorted_runs) {
|
||||
@ -556,7 +561,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
|
||||
for (unsigned int i = 0; i < first_index_after; i++) {
|
||||
estimated_total_size += sorted_runs[i].size;
|
||||
}
|
||||
uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
|
||||
uint32_t path_id =
|
||||
GetPathId(ioptions_, mutable_cf_options, estimated_total_size);
|
||||
int start_level = sorted_runs[start_index].level;
|
||||
int output_level;
|
||||
if (first_index_after == sorted_runs.size()) {
|
||||
@ -621,8 +627,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
|
||||
VersionStorageInfo* vstorage, double score,
|
||||
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
|
||||
// percentage flexibility while reducing size amplification
|
||||
uint64_t ratio =
|
||||
ioptions_.compaction_options_universal.max_size_amplification_percent;
|
||||
uint64_t ratio = mutable_cf_options.compaction_options_universal
|
||||
.max_size_amplification_percent;
|
||||
|
||||
unsigned int candidate_count = 0;
|
||||
uint64_t candidate_size = 0;
|
||||
@ -700,7 +706,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
|
||||
for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
|
||||
estimated_total_size += sorted_runs[loop].size;
|
||||
}
|
||||
uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
|
||||
uint32_t path_id =
|
||||
GetPathId(ioptions_, mutable_cf_options, estimated_total_size);
|
||||
int start_level = sorted_runs[start_index].level;
|
||||
|
||||
std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
|
||||
|
@ -80,11 +80,13 @@ class UniversalCompactionPicker : public CompactionPicker {
|
||||
bool IsInputFilesNonOverlapping(Compaction* c);
|
||||
|
||||
static std::vector<SortedRun> CalculateSortedRuns(
|
||||
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
|
||||
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions,
|
||||
const MutableCFOptions& mutable_cf_options);
|
||||
|
||||
// Pick a path ID to place a newly generated file, with its estimated file
|
||||
// size.
|
||||
static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
uint64_t file_size);
|
||||
};
|
||||
} // namespace rocksdb
|
||||
|
@ -106,7 +106,8 @@ CompressionType GetCompressionFlush(
|
||||
// optimization is used for leveled compaction. Otherwise the CPU and
|
||||
// latency overhead is not offset by saving much space.
|
||||
if (ioptions.compaction_style == kCompactionStyleUniversal) {
|
||||
if (ioptions.compaction_options_universal.compression_size_percent < 0) {
|
||||
if (mutable_cf_options.compaction_options_universal
|
||||
.compression_size_percent < 0) {
|
||||
return mutable_cf_options.compression;
|
||||
} else {
|
||||
return kNoCompression;
|
||||
|
@ -4377,6 +4377,74 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
TEST_F(DBTest, DynamicUniversalCompactionOptions) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// Initial defaults
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 1);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width,
|
||||
2);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width,
|
||||
UINT_MAX);
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions()
|
||||
.compaction_options_universal.max_size_amplification_percent,
|
||||
200);
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions()
|
||||
.compaction_options_universal.compression_size_percent,
|
||||
-1);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style,
|
||||
kCompactionStopStyleTotalSize);
|
||||
ASSERT_EQ(
|
||||
dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
|
||||
false);
|
||||
|
||||
ASSERT_OK(dbfull()->SetOptions(
|
||||
{{"compaction_options_universal", "{size_ratio=7;}"}}));
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width,
|
||||
2);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width,
|
||||
UINT_MAX);
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions()
|
||||
.compaction_options_universal.max_size_amplification_percent,
|
||||
200);
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions()
|
||||
.compaction_options_universal.compression_size_percent,
|
||||
-1);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style,
|
||||
kCompactionStopStyleTotalSize);
|
||||
ASSERT_EQ(
|
||||
dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
|
||||
false);
|
||||
|
||||
ASSERT_OK(dbfull()->SetOptions(
|
||||
{{"compaction_options_universal", "{min_merge_width=11;}"}}));
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width,
|
||||
11);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width,
|
||||
UINT_MAX);
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions()
|
||||
.compaction_options_universal.max_size_amplification_percent,
|
||||
200);
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions()
|
||||
.compaction_options_universal.compression_size_percent,
|
||||
-1);
|
||||
ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style,
|
||||
kCompactionStopStyleTotalSize);
|
||||
ASSERT_EQ(
|
||||
dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
|
||||
false);
|
||||
}
|
||||
|
||||
TEST_F(DBTest, FileCreationRandomFailure) {
|
||||
Options options;
|
||||
options.env = env_;
|
||||
|
@ -374,6 +374,79 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
|
||||
ASSERT_EQ(NumSortedRuns(1), 1);
|
||||
}
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) {
|
||||
Options options = CurrentOptions();
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.num_levels = 1;
|
||||
options.write_buffer_size = 100 << 10; // 100KB
|
||||
options.target_file_size_base = 32 << 10; // 32KB
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
// Initial setup of compaction_options_universal will prevent universal
|
||||
// compaction from happening
|
||||
options.compaction_options_universal.size_ratio = 100;
|
||||
options.compaction_options_universal.min_merge_width = 100;
|
||||
DestroyAndReopen(options);
|
||||
|
||||
int total_picked_compactions = 0;
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"UniversalCompactionPicker::PickCompaction:Return", [&](void* arg) {
|
||||
if (arg) {
|
||||
total_picked_compactions++;
|
||||
}
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
MutableCFOptions mutable_cf_options;
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
Random rnd(301);
|
||||
int key_idx = 0;
|
||||
|
||||
// Generate two files in Level 0. Both files are approx the same size.
|
||||
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
||||
num++) {
|
||||
// Write 110KB (11 values, each 10K)
|
||||
for (int i = 0; i < 11; i++) {
|
||||
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
||||
key_idx++;
|
||||
}
|
||||
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
|
||||
ASSERT_EQ(NumSortedRuns(1), num + 1);
|
||||
}
|
||||
ASSERT_EQ(NumSortedRuns(1), 2);
|
||||
|
||||
// Flush whatever is remaining in memtable. This is typically
|
||||
// small, which should not trigger size ratio based compaction
|
||||
// but could instead trigger size amplification if it's set
|
||||
// to 110.
|
||||
ASSERT_OK(Flush(1));
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
// Verify compaction did not happen
|
||||
ASSERT_EQ(NumSortedRuns(1), 3);
|
||||
|
||||
// Trigger compaction if size amplification exceeds 110% without reopening DB
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions(handles_[1])
|
||||
.compaction_options_universal.max_size_amplification_percent,
|
||||
200);
|
||||
ASSERT_OK(dbfull()->SetOptions(handles_[1],
|
||||
{{"compaction_options_universal",
|
||||
"{max_size_amplification_percent=110;}"}}));
|
||||
ASSERT_EQ(dbfull()
|
||||
->GetOptions(handles_[1])
|
||||
.compaction_options_universal.max_size_amplification_percent,
|
||||
110);
|
||||
ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
|
||||
&mutable_cf_options));
|
||||
ASSERT_EQ(110, mutable_cf_options.compaction_options_universal
|
||||
.max_size_amplification_percent);
|
||||
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
// Verify that size amplification did happen
|
||||
ASSERT_EQ(NumSortedRuns(1), 1);
|
||||
ASSERT_EQ(total_picked_compactions, 1);
|
||||
}
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||
const int kTestKeySize = 16;
|
||||
const int kTestValueSize = 984;
|
||||
|
@ -27,7 +27,6 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options,
|
||||
const ColumnFamilyOptions& cf_options)
|
||||
: compaction_style(cf_options.compaction_style),
|
||||
compaction_pri(cf_options.compaction_pri),
|
||||
compaction_options_universal(cf_options.compaction_options_universal),
|
||||
prefix_extractor(cf_options.prefix_extractor.get()),
|
||||
user_comparator(cf_options.comparator),
|
||||
internal_comparator(InternalKeyComparator(cf_options.comparator)),
|
||||
|
@ -30,8 +30,6 @@ struct ImmutableCFOptions {
|
||||
|
||||
CompactionPri compaction_pri;
|
||||
|
||||
CompactionOptionsUniversal compaction_options_universal;
|
||||
|
||||
const SliceTransform* prefix_extractor;
|
||||
|
||||
const Comparator* user_comparator;
|
||||
@ -149,6 +147,7 @@ struct MutableCFOptions {
|
||||
max_bytes_for_level_multiplier_additional(
|
||||
options.max_bytes_for_level_multiplier_additional),
|
||||
compaction_options_fifo(options.compaction_options_fifo),
|
||||
compaction_options_universal(options.compaction_options_universal),
|
||||
max_sequential_skip_in_iterations(
|
||||
options.max_sequential_skip_in_iterations),
|
||||
paranoid_file_checks(options.paranoid_file_checks),
|
||||
@ -224,6 +223,7 @@ struct MutableCFOptions {
|
||||
double max_bytes_for_level_multiplier;
|
||||
std::vector<int> max_bytes_for_level_multiplier_additional;
|
||||
CompactionOptionsFIFO compaction_options_fifo;
|
||||
CompactionOptionsUniversal compaction_options_universal;
|
||||
|
||||
// Misc options
|
||||
uint64_t max_sequential_skip_in_iterations;
|
||||
|
@ -168,6 +168,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
|
||||
}
|
||||
|
||||
cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo;
|
||||
cf_opts.compaction_options_universal =
|
||||
mutable_cf_options.compaction_options_universal;
|
||||
|
||||
// Misc options
|
||||
cf_opts.max_sequential_skip_in_iterations =
|
||||
@ -496,6 +498,14 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
|
||||
reinterpret_cast<LRUCacheOptions*>(opt_address),
|
||||
lru_cache_options_type_info);
|
||||
}
|
||||
case OptionType::kCompactionOptionsUniversal:
|
||||
return ParseStructOptions<CompactionOptionsUniversal>(
|
||||
value, reinterpret_cast<CompactionOptionsUniversal*>(opt_address),
|
||||
universal_compaction_options_type_info);
|
||||
case OptionType::kCompactionStopStyle:
|
||||
return ParseEnum<CompactionStopStyle>(
|
||||
compaction_stop_style_string_map, value,
|
||||
reinterpret_cast<CompactionStopStyle*>(opt_address));
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@ -660,11 +670,18 @@ bool SerializeSingleOptionHelper(const char* opt_address,
|
||||
return SerializeEnum<InfoLogLevel>(
|
||||
info_log_level_string_map,
|
||||
*reinterpret_cast<const InfoLogLevel*>(opt_address), value);
|
||||
case OptionType::kCompactionOptionsFIFO: {
|
||||
case OptionType::kCompactionOptionsFIFO:
|
||||
return SerializeStruct<CompactionOptionsFIFO>(
|
||||
*reinterpret_cast<const CompactionOptionsFIFO*>(opt_address), value,
|
||||
fifo_compaction_options_type_info);
|
||||
}
|
||||
case OptionType::kCompactionOptionsUniversal:
|
||||
return SerializeStruct<CompactionOptionsUniversal>(
|
||||
*reinterpret_cast<const CompactionOptionsUniversal*>(opt_address),
|
||||
value, universal_compaction_options_type_info);
|
||||
case OptionType::kCompactionStopStyle:
|
||||
return SerializeEnum<CompactionStopStyle>(
|
||||
compaction_stop_style_string_map,
|
||||
*reinterpret_cast<const CompactionStopStyle*>(opt_address), value);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@ -1525,6 +1542,7 @@ std::unordered_map<std::string, InfoLogLevel>
|
||||
ColumnFamilyOptions OptionsHelper::dummy_cf_options;
|
||||
CompactionOptionsFIFO OptionsHelper::dummy_comp_options;
|
||||
LRUCacheOptions OptionsHelper::dummy_lru_cache_options;
|
||||
CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal;
|
||||
|
||||
// offset_of is used to get the offset of a class data member
|
||||
// ex: offset_of(&ColumnFamilyOptions::num_levels)
|
||||
@ -1555,6 +1573,11 @@ int offset_of(T1 LRUCacheOptions::*member) {
|
||||
return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) -
|
||||
size_t(&OptionsHelper::dummy_lru_cache_options));
|
||||
}
|
||||
template <typename T1>
|
||||
int offset_of(T1 CompactionOptionsUniversal::*member) {
|
||||
return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) -
|
||||
size_t(&OptionsHelper::dummy_comp_options_universal));
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, OptionTypeInfo>
|
||||
OptionsHelper::cf_options_type_info = {
|
||||
@ -1782,7 +1805,12 @@ std::unordered_map<std::string, OptionTypeInfo>
|
||||
{"compaction_options_fifo",
|
||||
{offset_of(&ColumnFamilyOptions::compaction_options_fifo),
|
||||
OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal,
|
||||
true, offsetof(struct MutableCFOptions, compaction_options_fifo)}}};
|
||||
true, offsetof(struct MutableCFOptions, compaction_options_fifo)}},
|
||||
{"compaction_options_universal",
|
||||
{offset_of(&ColumnFamilyOptions::compaction_options_universal),
|
||||
OptionType::kCompactionOptionsUniversal,
|
||||
OptionVerificationType::kNormal, true,
|
||||
offsetof(struct MutableCFOptions, compaction_options_universal)}}};
|
||||
|
||||
std::unordered_map<std::string, OptionTypeInfo>
|
||||
OptionsHelper::fifo_compaction_options_type_info = {
|
||||
@ -1800,20 +1828,61 @@ std::unordered_map<std::string, OptionTypeInfo>
|
||||
offsetof(struct CompactionOptionsFIFO, allow_compaction)}}};
|
||||
|
||||
std::unordered_map<std::string, OptionTypeInfo>
|
||||
OptionsHelper::lru_cache_options_type_info = {
|
||||
{"capacity", {offset_of(&LRUCacheOptions::capacity),
|
||||
OptionType::kSizeT, OptionVerificationType::kNormal, true,
|
||||
offsetof(struct LRUCacheOptions, capacity)}},
|
||||
{"num_shard_bits", {offset_of(&LRUCacheOptions::num_shard_bits),
|
||||
OptionsHelper::universal_compaction_options_type_info = {
|
||||
{"size_ratio",
|
||||
{offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt,
|
||||
OptionVerificationType::kNormal, true,
|
||||
offsetof(class CompactionOptionsUniversal, size_ratio)}},
|
||||
{"min_merge_width",
|
||||
{offset_of(&CompactionOptionsUniversal::min_merge_width),
|
||||
OptionType::kUInt, OptionVerificationType::kNormal, true,
|
||||
offsetof(class CompactionOptionsUniversal, min_merge_width)}},
|
||||
{"max_merge_width",
|
||||
{offset_of(&CompactionOptionsUniversal::max_merge_width),
|
||||
OptionType::kUInt, OptionVerificationType::kNormal, true,
|
||||
offsetof(class CompactionOptionsUniversal, max_merge_width)}},
|
||||
{"max_size_amplification_percent",
|
||||
{offset_of(
|
||||
&CompactionOptionsUniversal::max_size_amplification_percent),
|
||||
OptionType::kUInt, OptionVerificationType::kNormal, true,
|
||||
offsetof(class CompactionOptionsUniversal,
|
||||
max_size_amplification_percent)}},
|
||||
{"compression_size_percent",
|
||||
{offset_of(&CompactionOptionsUniversal::compression_size_percent),
|
||||
OptionType::kInt, OptionVerificationType::kNormal, true,
|
||||
offsetof(class CompactionOptionsUniversal,
|
||||
compression_size_percent)}},
|
||||
{"stop_style",
|
||||
{offset_of(&CompactionOptionsUniversal::stop_style),
|
||||
OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
|
||||
true, offsetof(class CompactionOptionsUniversal, stop_style)}},
|
||||
{"allow_trivial_move",
|
||||
{offset_of(&CompactionOptionsUniversal::allow_trivial_move),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal, true,
|
||||
offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}};
|
||||
|
||||
std::unordered_map<std::string, CompactionStopStyle>
|
||||
OptionsHelper::compaction_stop_style_string_map = {
|
||||
{"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
|
||||
{"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
|
||||
|
||||
std::unordered_map<std::string, OptionTypeInfo>
|
||||
OptionsHelper::lru_cache_options_type_info = {
|
||||
{"capacity",
|
||||
{offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT,
|
||||
OptionVerificationType::kNormal, true,
|
||||
offsetof(struct LRUCacheOptions, capacity)}},
|
||||
{"num_shard_bits",
|
||||
{offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt,
|
||||
OptionVerificationType::kNormal, true,
|
||||
offsetof(struct LRUCacheOptions, num_shard_bits)}},
|
||||
{"strict_capacity_limit",
|
||||
{offset_of(&LRUCacheOptions::strict_capacity_limit),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal, true,
|
||||
offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
|
||||
{"high_pri_pool_ratio",
|
||||
{offset_of(&LRUCacheOptions::high_pri_pool_ratio),
|
||||
OptionType::kDouble, OptionVerificationType::kNormal, true,
|
||||
{offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble,
|
||||
OptionVerificationType::kNormal, true,
|
||||
offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}};
|
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
@ -15,6 +15,7 @@
|
||||
#include "rocksdb/options.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/universal_compaction.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
@ -63,6 +64,8 @@ enum class OptionType {
|
||||
kCompactionFilter,
|
||||
kCompactionFilterFactory,
|
||||
kCompactionOptionsFIFO,
|
||||
kCompactionOptionsUniversal,
|
||||
kCompactionStopStyle,
|
||||
kMergeOperator,
|
||||
kMemTableRepFactory,
|
||||
kBlockBasedTableIndexType,
|
||||
@ -145,6 +148,10 @@ struct OptionsHelper {
|
||||
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info;
|
||||
static std::unordered_map<std::string, OptionTypeInfo>
|
||||
fifo_compaction_options_type_info;
|
||||
static std::unordered_map<std::string, OptionTypeInfo>
|
||||
universal_compaction_options_type_info;
|
||||
static std::unordered_map<std::string, CompactionStopStyle>
|
||||
compaction_stop_style_string_map;
|
||||
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info;
|
||||
static std::unordered_map<std::string, OptionTypeInfo>
|
||||
lru_cache_options_type_info;
|
||||
@ -166,6 +173,7 @@ struct OptionsHelper {
|
||||
static ColumnFamilyOptions dummy_cf_options;
|
||||
static CompactionOptionsFIFO dummy_comp_options;
|
||||
static LRUCacheOptions dummy_lru_cache_options;
|
||||
static CompactionOptionsUniversal dummy_comp_options_universal;
|
||||
#endif // !ROCKSDB_LITE
|
||||
};
|
||||
|
||||
@ -180,6 +188,10 @@ static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map;
|
||||
static auto& cf_options_type_info = OptionsHelper::cf_options_type_info;
|
||||
static auto& fifo_compaction_options_type_info =
|
||||
OptionsHelper::fifo_compaction_options_type_info;
|
||||
static auto& universal_compaction_options_type_info =
|
||||
OptionsHelper::universal_compaction_options_type_info;
|
||||
static auto& compaction_stop_style_string_map =
|
||||
OptionsHelper::compaction_stop_style_string_map;
|
||||
static auto& db_options_type_info = OptionsHelper::db_options_type_info;
|
||||
static auto& lru_cache_options_type_info =
|
||||
OptionsHelper::lru_cache_options_type_info;
|
||||
|
@ -598,6 +598,23 @@ bool AreEqualOptions(
|
||||
}
|
||||
return false;
|
||||
}
|
||||
case OptionType::kCompactionOptionsUniversal: {
|
||||
CompactionOptionsUniversal lhs =
|
||||
*reinterpret_cast<const CompactionOptionsUniversal*>(offset1);
|
||||
CompactionOptionsUniversal rhs =
|
||||
*reinterpret_cast<const CompactionOptionsUniversal*>(offset2);
|
||||
if (lhs.size_ratio == rhs.size_ratio &&
|
||||
lhs.min_merge_width == rhs.min_merge_width &&
|
||||
lhs.max_merge_width == rhs.max_merge_width &&
|
||||
lhs.max_size_amplification_percent ==
|
||||
rhs.max_size_amplification_percent &&
|
||||
lhs.compression_size_percent == rhs.compression_size_percent &&
|
||||
lhs.stop_style == rhs.stop_style &&
|
||||
lhs.allow_trivial_move == rhs.allow_trivial_move) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
default:
|
||||
if (type_info.verification == OptionVerificationType::kByName ||
|
||||
type_info.verification ==
|
||||
|
Loading…
Reference in New Issue
Block a user