Make bottommost_temperature dynamically changeable (#9402)

Summary:
Make `AdvancedColumnFamilyOptions.bottommost_temperature`
dynamically changeable with `SetOptions` API.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9402

Test Plan: added unittest

Reviewed By: siying

Differential Revision: D33674487

Pulled By: jay-zhuang

fbshipit-source-id: 8943768156aa6197c63850a64238a8092527d517
This commit is contained in:
Jay Zhuang 2022-01-25 14:58:48 -08:00 committed by Facebook GitHub Bot
parent 5d10a53b42
commit 022b400cba
9 changed files with 65 additions and 1 deletions

View File

@ -14,6 +14,7 @@ Note: The next release will be major release 7.0. See https://github.com/faceboo
* Add ObjectLibrary::AddFactory and ObjectLibrary::PatternEntry classes. This method and associated class are the preferred mechanism for registering factories with the ObjectLibrary going forward. The ObjectLibrary::Register method, which uses regular expressions and may be problematic, is deprecated and will be in a future release.
* Changed `BlockBasedTableOptions::block_size` from `size_t` to `uint64_t`.
* Added API warning against using `Iterator::Refresh()` together with `DB::DeleteRange()`, which are incompatible and have always risked causing the refreshed iterator to return incorrect results.
* Made `AdvancedColumnFamilyOptions.bottommost_temperature` dynamically changeable with `SetOptions()`.
### Behavior Changes
* `DB::DestroyColumnFamilyHandle()` will return Status::InvalidArgument() if called with `DB::DefaultColumnFamily()`.

View File

@ -6728,6 +6728,34 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
DB::Properties::kLiveSstFilesSizeAtTemperature + std::to_string(22),
&prop));
ASSERT_EQ(std::atoi(prop.c_str()), 0);
// Update bottommost temperature dynamically with SetOptions
auto s = db_->SetOptions({{"bottommost_temperature", "kCold"}});
ASSERT_OK(s);
ASSERT_EQ(db_->GetOptions().bottommost_temperature, Temperature::kCold);
db_->GetColumnFamilyMetaData(&metadata);
// Should not impact the existing files
ASSERT_EQ(Temperature::kWarm,
metadata.levels[kBottommostLevel].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kCold);
ASSERT_EQ(size, 0);
// new generated files should have the new settings
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
db_->GetColumnFamilyMetaData(&metadata);
ASSERT_EQ(1, metadata.file_count);
ASSERT_EQ(Temperature::kCold,
metadata.levels[kBottommostLevel].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kCold);
ASSERT_GT(size, 0);
}
#endif // ROCKSDB_LITE

View File

@ -814,6 +814,8 @@ struct AdvancedColumnFamilyOptions {
// If this option is set, when creating bottommost files, pass this
// temperature to FileSystem used. Should be no-op for default FileSystem
// and users need to plug in their own FileSystem to take advantage of it.
//
// Dynamically changeable through the SetOptions() API
Temperature bottommost_temperature = Temperature::kUnknown;
// When set, large values (blobs) are written to separate blob files, and

View File

@ -45,6 +45,7 @@ enum class OptionType {
kConfigurable,
kCustomizable,
kEncodedString,
kTemperature,
kUnknown,
};

View File

@ -405,6 +405,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct MutableCFOptions, periodic_compaction_seconds),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"bottommost_temperature",
{offsetof(struct MutableCFOptions, bottommost_temperature),
OptionType::kTemperature, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"enable_blob_files",
{offsetof(struct MutableCFOptions, enable_blob_files),
OptionType::kBoolean, OptionVerificationType::kNormal,
@ -1061,6 +1065,9 @@ void MutableCFOptions::Dump(Logger* log) const {
blob_garbage_collection_force_threshold);
ROCKS_LOG_INFO(log, " blob_compaction_readahead_size: %" PRIu64,
blob_compaction_readahead_size);
ROCKS_LOG_INFO(log, " bottommost_temperature: %d",
static_cast<int>(bottommost_temperature));
}
MutableCFOptions::MutableCFOptions(const Options& options)

View File

@ -269,6 +269,7 @@ void UpdateColumnFamilyOptions(const MutableCFOptions& moptions,
cf_opts->bottommost_compression = moptions.bottommost_compression;
cf_opts->bottommost_compression_opts = moptions.bottommost_compression_opts;
cf_opts->sample_for_compression = moptions.sample_for_compression;
cf_opts->bottommost_temperature = moptions.bottommost_temperature;
}
void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions,
@ -445,6 +446,10 @@ static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type,
(Slice(value)).DecodeHex(output_addr);
break;
}
case OptionType::kTemperature: {
return ParseEnum<Temperature>(temperature_string_map, value,
static_cast<Temperature*>(opt_address));
}
default:
return false;
}
@ -538,6 +543,11 @@ bool SerializeSingleOptionHelper(const void* opt_address,
*value = (Slice(*ptr)).ToString(true);
break;
}
case OptionType::kTemperature: {
return SerializeEnum<Temperature>(
temperature_string_map, *static_cast<const Temperature*>(opt_address),
value);
}
default:
return false;
}
@ -829,6 +839,13 @@ std::unordered_map<std::string, CompactionStopStyle>
{"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
{"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
std::unordered_map<std::string, Temperature>
OptionsHelper::temperature_string_map = {
{"kUnknown", Temperature::kUnknown},
{"kHot", Temperature::kHot},
{"kWarm", Temperature::kWarm},
{"kCold", Temperature::kCold}};
Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter,
size_t pos, size_t* end, std::string* token) {
while (pos < opts.size() && isspace(opts[pos])) {
@ -1198,6 +1215,8 @@ static bool AreOptionsEqual(OptionType type, const void* this_offset,
return IsOptionEqual<EncodingType>(this_offset, that_offset);
case OptionType::kEncodedString:
return IsOptionEqual<std::string>(this_offset, that_offset);
case OptionType::kTemperature:
return IsOptionEqual<Temperature>(this_offset, that_offset);
default:
return false;
} // End switch

View File

@ -88,6 +88,7 @@ struct OptionsHelper {
compaction_style_string_map;
static std::unordered_map<std::string, CompactionPri>
compaction_pri_string_map;
static std::unordered_map<std::string, Temperature> temperature_string_map;
#endif // !ROCKSDB_LITE
};
@ -108,6 +109,7 @@ static auto& compaction_style_string_map =
OptionsHelper::compaction_style_string_map;
static auto& compaction_pri_string_map =
OptionsHelper::compaction_pri_string_map;
static auto& temperature_string_map = OptionsHelper::temperature_string_map;
#endif // !ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

View File

@ -447,7 +447,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
options->max_mem_compaction_level = 0;
options->compaction_filter = nullptr;
options->sst_partitioner_factory = nullptr;
options->bottommost_temperature = Temperature::kUnknown;
char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
ColumnFamilyOptions* new_options =
@ -519,6 +518,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"blob_garbage_collection_age_cutoff=0.5;"
"blob_garbage_collection_force_threshold=0.75;"
"blob_compaction_readahead_size=262144;"
"bottommost_temperature=kWarm;"
"compaction_options_fifo={max_table_files_size=3;allow_"
"compaction=false;age_for_warm=1;};",
new_options));

View File

@ -110,6 +110,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"blob_garbage_collection_age_cutoff", "0.5"},
{"blob_garbage_collection_force_threshold", "0.75"},
{"blob_compaction_readahead_size", "256K"},
{"bottommost_temperature", "kWarm"},
};
std::unordered_map<std::string, std::string> db_options_map = {
@ -243,6 +244,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75);
ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
cf_options_map["write_buffer_size"] = "hello";
ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
@ -2270,6 +2272,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
{"blob_garbage_collection_age_cutoff", "0.5"},
{"blob_garbage_collection_force_threshold", "0.75"},
{"blob_compaction_readahead_size", "256K"},
{"bottommost_temperature", "kWarm"},
};
std::unordered_map<std::string, std::string> db_options_map = {
@ -2395,6 +2398,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75);
ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
cf_options_map["write_buffer_size"] = "hello";
ASSERT_NOK(GetColumnFamilyOptionsFromMap(