andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove deprecated API AdvancedColumnFamilyOptions::soft_rate_limit/hard_rate_limit (#9452)

Browse Source 
Summary:
**Context/Summary:**
AdvancedColumnFamilyOptions::soft_rate_limit/hard_rate_limit have been marked as deprecated and it's time to actually remove the code.
- Keep `soft_rate_limit`/`hard_rate_limit` in `cf_mutable_options_type_info` to prevent throwing `InvalidArgument` in `GetColumnFamilyOptionsFromMap` when reading an option file still with these options (e.g, old option file generated from RocksDB before the deprecation)
- Keep `soft_rate_limit`/`hard_rate_limit` in under `OptionsOldApiTest.GetOptionsFromMapTest` to test the case mentioned above.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9452

Test Plan: Rely on my eyeball and CI

Reviewed By: ajkr

Differential Revision: D33804938

Pulled By: hx235

fbshipit-source-id: 133d49f7ec5238d7efceeb0a3122a5792a2b9945
This commit is contained in:
Hui Xiao 2022-01-27 12:59:49 -08:00 committed by Facebook GitHub Bot
parent 7fb723f581
commit 1e0e883ca5
20 changed files with 13 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
HISTORY.md
View File

@ -6,6 +6,8 @@
* Remove deprecated API DB::AddFile from main repo.
* Remove deprecated API ObjectLibrary::Register() and the (now obsolete) Regex public API. Use ObjectLibrary::AddFactory() with PatternEntry instead.
* Remove deprecated option DBOption::table_cache_remove_scan_count_limit.
* Remove deprecated API AdvancedColumnFamilyOptions::soft_rate_limit.
* Remove deprecated API AdvancedColumnFamilyOptions::hard_rate_limit.
## 6.29.0 (01/21/2022)
Note: The next release will be major release 7.0. See https://github.com/facebook/rocksdb/issues/9390 for more info.

1
build_tools/regression_build_test.sh
View File

@ -258,7 +258,6 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
--value_size=100 \
--compression_type=none \
--compression_ratio=1 \
--hard_rate_limit=2 \
--write_buffer_size=134217728 \
--max_write_buffer_number=4 \
--level0_file_num_compaction_trigger=8 \

16
db/c.cc
View File

@ -3276,22 +3276,6 @@ size_t rocksdb_options_get_recycle_log_file_num(rocksdb_options_t* opt) {
return opt->rep.recycle_log_file_num;
}
void rocksdb_options_set_soft_rate_limit(rocksdb_options_t* opt, double v) {
opt->rep.soft_rate_limit = v;
}
double rocksdb_options_get_soft_rate_limit(rocksdb_options_t* opt) {
return opt->rep.soft_rate_limit;
}
void rocksdb_options_set_hard_rate_limit(rocksdb_options_t* opt, double v) {
opt->rep.hard_rate_limit = v;
}
double rocksdb_options_get_hard_rate_limit(rocksdb_options_t* opt) {
return opt->rep.hard_rate_limit;
}
void rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, size_t v) {
opt->rep.soft_pending_compaction_bytes_limit = v;
}

16
db/c_test.c
View File

@ -1631,12 +1631,6 @@ int main(int argc, char** argv) {
rocksdb_options_set_recycle_log_file_num(o, 9);
CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o));
rocksdb_options_set_soft_rate_limit(o, 2.0);
CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(o));
rocksdb_options_set_hard_rate_limit(o, 4.0);
CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(o));
rocksdb_options_set_soft_pending_compaction_bytes_limit(o, 10);
CheckCondition(10 ==
rocksdb_options_get_soft_pending_compaction_bytes_limit(o));
@ -1857,8 +1851,6 @@ int main(int argc, char** argv) {
CheckCondition(7 == rocksdb_options_get_log_file_time_to_roll(copy));
CheckCondition(8 == rocksdb_options_get_keep_log_file_num(copy));
CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(copy));
CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(copy));
CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(copy));
CheckCondition(
10 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy));
CheckCondition(
@ -2083,14 +2075,6 @@ int main(int argc, char** argv) {
CheckCondition(19 == rocksdb_options_get_recycle_log_file_num(copy));
CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o));
rocksdb_options_set_soft_rate_limit(copy, 4.0);
CheckCondition(4.0 == rocksdb_options_get_soft_rate_limit(copy));
CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(o));
rocksdb_options_set_hard_rate_limit(copy, 2.0);
CheckCondition(2.0 == rocksdb_options_get_hard_rate_limit(copy));
CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(o));
rocksdb_options_set_soft_pending_compaction_bytes_limit(copy, 110);
CheckCondition(
110 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy));

3
db/db_dynamic_level_test.cc
View File

@ -69,7 +69,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
options.level_compaction_dynamic_level_bytes = true;
options.max_bytes_for_level_base = 10240;
options.max_bytes_for_level_multiplier = 4;
options.soft_rate_limit = 1.1;
options.max_background_compactions = max_background_compactions;
options.num_levels = 5;
@ -370,7 +369,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
options.level_compaction_dynamic_level_bytes = true;
options.max_bytes_for_level_base = 10240;
options.max_bytes_for_level_multiplier = 4;
options.soft_rate_limit = 1.1;
options.max_background_compactions = 2;
options.num_levels = 5;
options.max_compaction_bytes = 100000000;
@ -422,7 +420,6 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
options.level_compaction_dynamic_level_bytes = false;
options.max_bytes_for_level_base = 10240;
options.max_bytes_for_level_multiplier = 4;
options.soft_rate_limit = 1.1;
options.num_levels = 8;
DestroyAndReopen(options);

4
db/db_test.cc
View File

@ -6530,6 +6530,10 @@ TEST_F(DBTest, SoftLimit) {
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Now there is one L1 file but doesn't trigger soft_rate_limit
//
// TODO: soft_rate_limit is depreciated. If this test
// relies on soft_rate_limit, then we need to change the test.
//
// The L1 file size is around 30KB.
ASSERT_EQ(NumTableFilesAtLevel(1), 1);
ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

1
db/db_test_util.cc
View File

@ -456,7 +456,6 @@ Options DBTestBase::GetOptions(
options.max_manifest_file_size = 50; // 50 bytes
break;
case kPerfOptions:
options.soft_rate_limit = 2.0;
options.delayed_write_rate = 8 * 1024 * 1024;
options.report_bg_io_stats = true;
// TODO(3.13) -- test more options

11
db_stress_tool/db_stress_test_base.cc
View File

@ -202,8 +202,6 @@ bool StressTest::BuildOptionsTable() {
{"inplace_update_num_locks", {"100", "200", "300"}},
// TODO(ljin): enable test for this option
// {"disable_auto_compactions", {"100", "200", "300"}},
{"soft_rate_limit", {"0", "0.5", "0.9"}},
{"hard_rate_limit", {"0", "1.1", "2.0"}},
{"level0_file_num_compaction_trigger",
{
ToString(options_.level0_file_num_compaction_trigger),
@ -572,12 +570,9 @@ Status StressTest::SetOptions(ThreadState* thread) {
std::string name =
options_index_[thread->rand.Next() % options_index_.size()];
int value_idx = thread->rand.Next() % options_table_[name].size();
if (name == "soft_rate_limit" || name == "hard_rate_limit") {
opts["soft_rate_limit"] = options_table_["soft_rate_limit"][value_idx];
opts["hard_rate_limit"] = options_table_["hard_rate_limit"][value_idx];
} else if (name == "level0_file_num_compaction_trigger" ||
name == "level0_slowdown_writes_trigger" ||
name == "level0_stop_writes_trigger") {
if (name == "level0_file_num_compaction_trigger" ||
name == "level0_slowdown_writes_trigger" ||
name == "level0_stop_writes_trigger") {
opts["level0_file_num_compaction_trigger"] =
options_table_["level0_file_num_compaction_trigger"][value_idx];
opts["level0_slowdown_writes_trigger"] =

1
examples/rocksdb_option_file_example.ini
View File

@ -86,7 +86,6 @@
table_factory=BlockBasedTable
comparator=leveldb.BytewiseComparator
max_sequential_skip_in_iterations=8
soft_rate_limit=0.000000
max_bytes_for_level_base=1073741824
memtable_prefix_bloom_probes=6
memtable_prefix_bloom_bits=0

12
include/rocksdb/advanced_options.h
View File

@ -916,18 +916,6 @@ struct AdvancedColumnFamilyOptions {
// This does not do anything anymore.
int max_mem_compaction_level;
// NOT SUPPORTED ANYMORE -- this options is no longer used
// Puts are delayed to options.delayed_write_rate when any level has a
// compaction score that exceeds soft_rate_limit. This is ignored when == 0.0.
//
// Default: 0 (disabled)
//
// Dynamically changeable through SetOptions() API
double soft_rate_limit = 0.0;
// NOT SUPPORTED ANYMORE -- this options is no longer used
double hard_rate_limit = 0.0;
// NOT SUPPORTED ANYMORE -- this options is no longer used
unsigned int rate_limit_delay_max_milliseconds = 100;
};

8
include/rocksdb/c.h
View File

@ -1194,14 +1194,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num(
rocksdb_options_t*, size_t);
extern ROCKSDB_LIBRARY_API size_t
rocksdb_options_get_recycle_log_file_num(rocksdb_options_t*);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_soft_rate_limit(
rocksdb_options_t*, double);
extern ROCKSDB_LIBRARY_API double rocksdb_options_get_soft_rate_limit(
rocksdb_options_t*);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hard_rate_limit(
rocksdb_options_t*, double);
extern ROCKSDB_LIBRARY_API double rocksdb_options_get_hard_rate_limit(
rocksdb_options_t*);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_soft_pending_compaction_bytes_limit(
rocksdb_options_t* opt, size_t v);
extern ROCKSDB_LIBRARY_API size_t

6
include/rocksdb/convenience.h
View File

@ -141,8 +141,10 @@ struct ConfigOptions {
// Doubles / Floating Points are converted directly from string. Note that
// currently we do not support units.
// [Example]:
// - {"hard_rate_limit", "2.1"} in GetColumnFamilyOptionsFromMap, or
// - "hard_rate_limit=2.1" in GetColumnFamilyOptionsFromString.
// - {"memtable_prefix_bloom_size_ratio", "0.1"} in
// GetColumnFamilyOptionsFromMap, or
// - "memtable_prefix_bloom_size_ratio=0.1" in
// GetColumnFamilyOptionsFromString.
// * Array / Vectors:
// An array is specified by a list of values, where ':' is used as
// the delimiter to separate each value.

12
java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java
View File

@ -1350,18 +1350,6 @@ public class DbBenchmark {
return Integer.parseInt(value);
}
},
soft_rate_limit(0.0d,"") {
@Override public Object parseValue(String value) {
return Double.parseDouble(value);
}
},
hard_rate_limit(0.0d,"When not equal to 0 this make threads\n" +
"\tsleep at each stats reporting interval until the compaction\n" +
"\tscore for all levels is less than or equal to this value.") {
@Override public Object parseValue(String value) {
return Double.parseDouble(value);
}
},
rate_limit_delay_max_milliseconds(1000,
"When hard_rate_limit is set then this is the max time a put will\n" +
"\tbe stalled.") {

2
java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
View File

@ -87,9 +87,7 @@ public class MutableColumnFamilyOptions
public enum CompactionOption implements MutableColumnFamilyOptionKey {
disable_auto_compactions(ValueType.BOOLEAN),
@Deprecated soft_rate_limit(ValueType.DOUBLE),
soft_pending_compaction_bytes_limit(ValueType.LONG),
@Deprecated hard_rate_limit(ValueType.DOUBLE),
hard_pending_compaction_bytes_limit(ValueType.LONG),
level0_file_num_compaction_trigger(ValueType.INT),
level0_slowdown_writes_trigger(ValueType.INT),

3
options/options_settable_test.cc
View File

@ -441,8 +441,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
// GetColumnFamilyOptionsFromString():
options->rate_limit_delay_max_milliseconds = 33;
options->compaction_options_universal = CompactionOptionsUniversal();
options->hard_rate_limit = 0;
options->soft_rate_limit = 0;
options->num_levels = 42; // Initialize options for MutableCF
options->max_mem_compaction_level = 0;
options->compaction_filter = nullptr;
@ -487,7 +485,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"level0_slowdown_writes_trigger=22;"
"level0_file_num_compaction_trigger=14;"
"compaction_filter=urxcqstuwnCompactionFilter;"
"soft_rate_limit=530.615385;"
"soft_pending_compaction_bytes_limit=0;"
"max_write_buffer_number_to_maintain=84;"
"max_write_buffer_size_to_maintain=2147483648;"

2
options/options_test.cc
View File

@ -80,8 +80,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"max_bytes_for_level_multiplier", "15.0"},
{"max_bytes_for_level_multiplier_additional", "16:17:18"},
{"max_compaction_bytes", "21"},
{"soft_rate_limit", "1.1"},
{"hard_rate_limit", "2.1"},
{"hard_pending_compaction_bytes_limit", "211"},
{"arena_block_size", "22"},
{"disable_auto_compactions", "true"},

2
test_util/testutil.cc
View File

@ -398,8 +398,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions& db_options,
cf_opt->enable_blob_garbage_collection = rnd->Uniform(2);
// double options
cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->memtable_prefix_bloom_size_ratio =
static_cast<double>(rnd->Uniform(10000)) / 20000.0;
cf_opt->blob_garbage_collection_age_cutoff = rnd->Uniform(10000) / 10000.0;

1
tools/benchmark.sh
View File

@ -140,7 +140,6 @@ const_params="
--pin_l0_filter_and_index_blocks_in_cache=1 \
--benchmark_write_rate_limit=$(( 1024 * 1024 * $mb_written_per_sec )) \
\
--hard_rate_limit=3 \
--rate_limit_delay_max_milliseconds=1000000 \
--write_buffer_size=$((128 * M)) \
--target_file_size_base=$((128 * M)) \

21
tools/db_bench_tool.cc
View File

@ -1203,19 +1203,6 @@ DEFINE_int32(thread_status_per_interval, 0,
DEFINE_int32(perf_level, ROCKSDB_NAMESPACE::PerfLevel::kDisable,
"Level of perf collection");
static bool ValidateRateLimit(const char* flagname, double value) {
const double EPSILON = 1e-10;
if ( value < -EPSILON ) {
fprintf(stderr, "Invalid value for --%s: %12.6f, must be >= 0.0\n",
flagname, value);
return false;
}
return true;
}
DEFINE_double(soft_rate_limit, 0.0, "DEPRECATED");
DEFINE_double(hard_rate_limit, 0.0, "DEPRECATED");
DEFINE_uint64(soft_pending_compaction_bytes_limit, 64ull * 1024 * 1024 * 1024,
"Slowdown writes if pending compaction bytes exceed this number");
@ -1534,12 +1521,6 @@ DEFINE_string(secondary_cache_uri, "",
static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;
#endif // ROCKSDB_LITE
static const bool FLAGS_soft_rate_limit_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_soft_rate_limit, &ValidateRateLimit);
static const bool FLAGS_hard_rate_limit_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_hard_rate_limit, &ValidateRateLimit);
static const bool FLAGS_prefix_size_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize);
@ -4290,8 +4271,6 @@ class Benchmark {
options.compression_per_level[i] = FLAGS_compression_type_e;
}
}
options.soft_rate_limit = FLAGS_soft_rate_limit;
options.hard_rate_limit = FLAGS_hard_rate_limit;
options.soft_pending_compaction_bytes_limit =
FLAGS_soft_pending_compaction_bytes_limit;
options.hard_pending_compaction_bytes_limit =

1
tools/db_bench_tool_test.cc
View File

@ -252,7 +252,6 @@ const std::string options_file_content = R"OPTIONS_FILE(
level0_slowdown_writes_trigger=50
level0_file_num_compaction_trigger=10
expanded_compaction_factor=25
soft_rate_limit=0.000000
max_write_buffer_number_to_maintain=0
max_write_buffer_size_to_maintain=0
verify_checksums_in_compaction=true