Remove deprecated API AdvancedColumnFamilyOptions::soft_rate_limit/hard_rate_limit (#9452)

Summary:
**Context/Summary:**
AdvancedColumnFamilyOptions::soft_rate_limit/hard_rate_limit have been marked as deprecated and it's time to actually remove the code.
- Keep `soft_rate_limit`/`hard_rate_limit` in `cf_mutable_options_type_info` to prevent throwing `InvalidArgument` in `GetColumnFamilyOptionsFromMap` when reading an option file still with these options (e.g, old option file generated from RocksDB before the deprecation)
- Keep `soft_rate_limit`/`hard_rate_limit` in under `OptionsOldApiTest.GetOptionsFromMapTest` to test the case mentioned above.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9452

Test Plan: Rely on my eyeball and CI

Reviewed By: ajkr

Differential Revision: D33804938

Pulled By: hx235

fbshipit-source-id: 133d49f7ec5238d7efceeb0a3122a5792a2b9945
This commit is contained in:
Hui Xiao 2022-01-27 12:59:49 -08:00 committed by Facebook GitHub Bot
parent 7fb723f581
commit 1e0e883ca5
20 changed files with 13 additions and 112 deletions

View File

@ -6,6 +6,8 @@
* Remove deprecated API DB::AddFile from main repo.
* Remove deprecated API ObjectLibrary::Register() and the (now obsolete) Regex public API. Use ObjectLibrary::AddFactory() with PatternEntry instead.
* Remove deprecated option DBOption::table_cache_remove_scan_count_limit.
* Remove deprecated API AdvancedColumnFamilyOptions::soft_rate_limit.
* Remove deprecated API AdvancedColumnFamilyOptions::hard_rate_limit.
## 6.29.0 (01/21/2022)
Note: The next release will be major release 7.0. See https://github.com/facebook/rocksdb/issues/9390 for more info.

View File

@ -258,7 +258,6 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
--value_size=100 \
--compression_type=none \
--compression_ratio=1 \
--hard_rate_limit=2 \
--write_buffer_size=134217728 \
--max_write_buffer_number=4 \
--level0_file_num_compaction_trigger=8 \

16
db/c.cc
View File

@ -3276,22 +3276,6 @@ size_t rocksdb_options_get_recycle_log_file_num(rocksdb_options_t* opt) {
return opt->rep.recycle_log_file_num;
}
void rocksdb_options_set_soft_rate_limit(rocksdb_options_t* opt, double v) {
opt->rep.soft_rate_limit = v;
}
double rocksdb_options_get_soft_rate_limit(rocksdb_options_t* opt) {
return opt->rep.soft_rate_limit;
}
void rocksdb_options_set_hard_rate_limit(rocksdb_options_t* opt, double v) {
opt->rep.hard_rate_limit = v;
}
double rocksdb_options_get_hard_rate_limit(rocksdb_options_t* opt) {
return opt->rep.hard_rate_limit;
}
void rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, size_t v) {
opt->rep.soft_pending_compaction_bytes_limit = v;
}

View File

@ -1631,12 +1631,6 @@ int main(int argc, char** argv) {
rocksdb_options_set_recycle_log_file_num(o, 9);
CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o));
rocksdb_options_set_soft_rate_limit(o, 2.0);
CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(o));
rocksdb_options_set_hard_rate_limit(o, 4.0);
CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(o));
rocksdb_options_set_soft_pending_compaction_bytes_limit(o, 10);
CheckCondition(10 ==
rocksdb_options_get_soft_pending_compaction_bytes_limit(o));
@ -1857,8 +1851,6 @@ int main(int argc, char** argv) {
CheckCondition(7 == rocksdb_options_get_log_file_time_to_roll(copy));
CheckCondition(8 == rocksdb_options_get_keep_log_file_num(copy));
CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(copy));
CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(copy));
CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(copy));
CheckCondition(
10 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy));
CheckCondition(
@ -2083,14 +2075,6 @@ int main(int argc, char** argv) {
CheckCondition(19 == rocksdb_options_get_recycle_log_file_num(copy));
CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o));
rocksdb_options_set_soft_rate_limit(copy, 4.0);
CheckCondition(4.0 == rocksdb_options_get_soft_rate_limit(copy));
CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(o));
rocksdb_options_set_hard_rate_limit(copy, 2.0);
CheckCondition(2.0 == rocksdb_options_get_hard_rate_limit(copy));
CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(o));
rocksdb_options_set_soft_pending_compaction_bytes_limit(copy, 110);
CheckCondition(
110 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy));

View File

@ -69,7 +69,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
options.level_compaction_dynamic_level_bytes = true;
options.max_bytes_for_level_base = 10240;
options.max_bytes_for_level_multiplier = 4;
options.soft_rate_limit = 1.1;
options.max_background_compactions = max_background_compactions;
options.num_levels = 5;
@ -370,7 +369,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
options.level_compaction_dynamic_level_bytes = true;
options.max_bytes_for_level_base = 10240;
options.max_bytes_for_level_multiplier = 4;
options.soft_rate_limit = 1.1;
options.max_background_compactions = 2;
options.num_levels = 5;
options.max_compaction_bytes = 100000000;
@ -422,7 +420,6 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
options.level_compaction_dynamic_level_bytes = false;
options.max_bytes_for_level_base = 10240;
options.max_bytes_for_level_multiplier = 4;
options.soft_rate_limit = 1.1;
options.num_levels = 8;
DestroyAndReopen(options);

View File

@ -6530,6 +6530,10 @@ TEST_F(DBTest, SoftLimit) {
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Now there is one L1 file but doesn't trigger soft_rate_limit
//
// TODO: soft_rate_limit is depreciated. If this test
// relies on soft_rate_limit, then we need to change the test.
//
// The L1 file size is around 30KB.
ASSERT_EQ(NumTableFilesAtLevel(1), 1);
ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

View File

@ -456,7 +456,6 @@ Options DBTestBase::GetOptions(
options.max_manifest_file_size = 50; // 50 bytes
break;
case kPerfOptions:
options.soft_rate_limit = 2.0;
options.delayed_write_rate = 8 * 1024 * 1024;
options.report_bg_io_stats = true;
// TODO(3.13) -- test more options

View File

@ -202,8 +202,6 @@ bool StressTest::BuildOptionsTable() {
{"inplace_update_num_locks", {"100", "200", "300"}},
// TODO(ljin): enable test for this option
// {"disable_auto_compactions", {"100", "200", "300"}},
{"soft_rate_limit", {"0", "0.5", "0.9"}},
{"hard_rate_limit", {"0", "1.1", "2.0"}},
{"level0_file_num_compaction_trigger",
{
ToString(options_.level0_file_num_compaction_trigger),
@ -572,12 +570,9 @@ Status StressTest::SetOptions(ThreadState* thread) {
std::string name =
options_index_[thread->rand.Next() % options_index_.size()];
int value_idx = thread->rand.Next() % options_table_[name].size();
if (name == "soft_rate_limit" || name == "hard_rate_limit") {
opts["soft_rate_limit"] = options_table_["soft_rate_limit"][value_idx];
opts["hard_rate_limit"] = options_table_["hard_rate_limit"][value_idx];
} else if (name == "level0_file_num_compaction_trigger" ||
name == "level0_slowdown_writes_trigger" ||
name == "level0_stop_writes_trigger") {
if (name == "level0_file_num_compaction_trigger" ||
name == "level0_slowdown_writes_trigger" ||
name == "level0_stop_writes_trigger") {
opts["level0_file_num_compaction_trigger"] =
options_table_["level0_file_num_compaction_trigger"][value_idx];
opts["level0_slowdown_writes_trigger"] =

View File

@ -86,7 +86,6 @@
table_factory=BlockBasedTable
comparator=leveldb.BytewiseComparator
max_sequential_skip_in_iterations=8
soft_rate_limit=0.000000
max_bytes_for_level_base=1073741824
memtable_prefix_bloom_probes=6
memtable_prefix_bloom_bits=0

View File

@ -916,18 +916,6 @@ struct AdvancedColumnFamilyOptions {
// This does not do anything anymore.
int max_mem_compaction_level;
// NOT SUPPORTED ANYMORE -- this options is no longer used
// Puts are delayed to options.delayed_write_rate when any level has a
// compaction score that exceeds soft_rate_limit. This is ignored when == 0.0.
//
// Default: 0 (disabled)
//
// Dynamically changeable through SetOptions() API
double soft_rate_limit = 0.0;
// NOT SUPPORTED ANYMORE -- this options is no longer used
double hard_rate_limit = 0.0;
// NOT SUPPORTED ANYMORE -- this options is no longer used
unsigned int rate_limit_delay_max_milliseconds = 100;
};

View File

@ -1194,14 +1194,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num(
rocksdb_options_t*, size_t);
extern ROCKSDB_LIBRARY_API size_t
rocksdb_options_get_recycle_log_file_num(rocksdb_options_t*);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_soft_rate_limit(
rocksdb_options_t*, double);
extern ROCKSDB_LIBRARY_API double rocksdb_options_get_soft_rate_limit(
rocksdb_options_t*);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hard_rate_limit(
rocksdb_options_t*, double);
extern ROCKSDB_LIBRARY_API double rocksdb_options_get_hard_rate_limit(
rocksdb_options_t*);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_soft_pending_compaction_bytes_limit(
rocksdb_options_t* opt, size_t v);
extern ROCKSDB_LIBRARY_API size_t

View File

@ -141,8 +141,10 @@ struct ConfigOptions {
// Doubles / Floating Points are converted directly from string. Note that
// currently we do not support units.
// [Example]:
// - {"hard_rate_limit", "2.1"} in GetColumnFamilyOptionsFromMap, or
// - "hard_rate_limit=2.1" in GetColumnFamilyOptionsFromString.
// - {"memtable_prefix_bloom_size_ratio", "0.1"} in
// GetColumnFamilyOptionsFromMap, or
// - "memtable_prefix_bloom_size_ratio=0.1" in
// GetColumnFamilyOptionsFromString.
// * Array / Vectors:
// An array is specified by a list of values, where ':' is used as
// the delimiter to separate each value.

View File

@ -1350,18 +1350,6 @@ public class DbBenchmark {
return Integer.parseInt(value);
}
},
soft_rate_limit(0.0d,"") {
@Override public Object parseValue(String value) {
return Double.parseDouble(value);
}
},
hard_rate_limit(0.0d,"When not equal to 0 this make threads\n" +
"\tsleep at each stats reporting interval until the compaction\n" +
"\tscore for all levels is less than or equal to this value.") {
@Override public Object parseValue(String value) {
return Double.parseDouble(value);
}
},
rate_limit_delay_max_milliseconds(1000,
"When hard_rate_limit is set then this is the max time a put will\n" +
"\tbe stalled.") {

View File

@ -87,9 +87,7 @@ public class MutableColumnFamilyOptions
public enum CompactionOption implements MutableColumnFamilyOptionKey {
disable_auto_compactions(ValueType.BOOLEAN),
@Deprecated soft_rate_limit(ValueType.DOUBLE),
soft_pending_compaction_bytes_limit(ValueType.LONG),
@Deprecated hard_rate_limit(ValueType.DOUBLE),
hard_pending_compaction_bytes_limit(ValueType.LONG),
level0_file_num_compaction_trigger(ValueType.INT),
level0_slowdown_writes_trigger(ValueType.INT),

View File

@ -441,8 +441,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
// GetColumnFamilyOptionsFromString():
options->rate_limit_delay_max_milliseconds = 33;
options->compaction_options_universal = CompactionOptionsUniversal();
options->hard_rate_limit = 0;
options->soft_rate_limit = 0;
options->num_levels = 42; // Initialize options for MutableCF
options->max_mem_compaction_level = 0;
options->compaction_filter = nullptr;
@ -487,7 +485,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"level0_slowdown_writes_trigger=22;"
"level0_file_num_compaction_trigger=14;"
"compaction_filter=urxcqstuwnCompactionFilter;"
"soft_rate_limit=530.615385;"
"soft_pending_compaction_bytes_limit=0;"
"max_write_buffer_number_to_maintain=84;"
"max_write_buffer_size_to_maintain=2147483648;"

View File

@ -80,8 +80,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"max_bytes_for_level_multiplier", "15.0"},
{"max_bytes_for_level_multiplier_additional", "16:17:18"},
{"max_compaction_bytes", "21"},
{"soft_rate_limit", "1.1"},
{"hard_rate_limit", "2.1"},
{"hard_pending_compaction_bytes_limit", "211"},
{"arena_block_size", "22"},
{"disable_auto_compactions", "true"},

View File

@ -398,8 +398,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions& db_options,
cf_opt->enable_blob_garbage_collection = rnd->Uniform(2);
// double options
cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->memtable_prefix_bloom_size_ratio =
static_cast<double>(rnd->Uniform(10000)) / 20000.0;
cf_opt->blob_garbage_collection_age_cutoff = rnd->Uniform(10000) / 10000.0;

View File

@ -140,7 +140,6 @@ const_params="
--pin_l0_filter_and_index_blocks_in_cache=1 \
--benchmark_write_rate_limit=$(( 1024 * 1024 * $mb_written_per_sec )) \
\
--hard_rate_limit=3 \
--rate_limit_delay_max_milliseconds=1000000 \
--write_buffer_size=$((128 * M)) \
--target_file_size_base=$((128 * M)) \

View File

@ -1203,19 +1203,6 @@ DEFINE_int32(thread_status_per_interval, 0,
DEFINE_int32(perf_level, ROCKSDB_NAMESPACE::PerfLevel::kDisable,
"Level of perf collection");
static bool ValidateRateLimit(const char* flagname, double value) {
const double EPSILON = 1e-10;
if ( value < -EPSILON ) {
fprintf(stderr, "Invalid value for --%s: %12.6f, must be >= 0.0\n",
flagname, value);
return false;
}
return true;
}
DEFINE_double(soft_rate_limit, 0.0, "DEPRECATED");
DEFINE_double(hard_rate_limit, 0.0, "DEPRECATED");
DEFINE_uint64(soft_pending_compaction_bytes_limit, 64ull * 1024 * 1024 * 1024,
"Slowdown writes if pending compaction bytes exceed this number");
@ -1534,12 +1521,6 @@ DEFINE_string(secondary_cache_uri, "",
static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;
#endif // ROCKSDB_LITE
static const bool FLAGS_soft_rate_limit_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_soft_rate_limit, &ValidateRateLimit);
static const bool FLAGS_hard_rate_limit_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_hard_rate_limit, &ValidateRateLimit);
static const bool FLAGS_prefix_size_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize);
@ -4290,8 +4271,6 @@ class Benchmark {
options.compression_per_level[i] = FLAGS_compression_type_e;
}
}
options.soft_rate_limit = FLAGS_soft_rate_limit;
options.hard_rate_limit = FLAGS_hard_rate_limit;
options.soft_pending_compaction_bytes_limit =
FLAGS_soft_pending_compaction_bytes_limit;
options.hard_pending_compaction_bytes_limit =

View File

@ -252,7 +252,6 @@ const std::string options_file_content = R"OPTIONS_FILE(
level0_slowdown_writes_trigger=50
level0_file_num_compaction_trigger=10
expanded_compaction_factor=25
soft_rate_limit=0.000000
max_write_buffer_number_to_maintain=0
max_write_buffer_size_to_maintain=0
verify_checksums_in_compaction=true