andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add options.hard_pending_compaction_bytes_limit to stop writes if compaction lagging behind

Browse Source 
Summary: Add an option to stop writes if compaction lefts behind. If estimated pending compaction bytes is more than threshold specified by options.hard_pending_compaction_bytes_liimt, writes will stop until compactions are cleared to under the threshold.

Test Plan: Add unit test DBTest.HardLimit

Reviewers: rven, kradhakrishnan, anthony, IslamAbdelRahman, yhchiang, igor

Reviewed By: igor

Subscribers: MarkCallaghan, leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D45999
This commit is contained in:
sdong 2015-09-11 14:31:23 -07:00
parent 7143242d12
commit 5de807ac16
12 changed files with 121 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
db/column_family.cc
View File

@ -454,6 +454,16 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
"[%s] Stopping writes because we have %d level-0 files",
name_.c_str(), vstorage->l0_delay_trigger_count());
} else if (mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
vstorage->estimated_compaction_needed_bytes() >=
mutable_cf_options.hard_pending_compaction_bytes_limit) {
write_controller_token_ = write_controller->GetStopToken();
internal_stats_->AddCFStats(
InternalStats::HARD_PENDING_COMPACTION_BYTES_LIMIT, 1);
Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
"[%s] Stopping writes because estimated pending compaction "
"bytes exceed %" PRIu64,
name_.c_str(), vstorage->estimated_compaction_needed_bytes());
} else if (mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
vstorage->l0_delay_trigger_count() >=
mutable_cf_options.level0_slowdown_writes_trigger) {

9
db/db_bench.cc
View File

@ -611,9 +611,10 @@ static bool ValidateRateLimit(const char* flagname, double value) {
}
DEFINE_double(soft_rate_limit, 0.0, "");
DEFINE_double(hard_rate_limit, 0.0, "When not equal to 0 this make threads "
"sleep at each stats reporting interval until the compaction"
" score for all levels is less than or equal to this value.");
DEFINE_double(hard_rate_limit, 0.0, "DEPRECATED");
DEFINE_uint64(hard_pending_compaction_bytes_limit, 128u * 1024 * 1024 * 1024,
"Stop writes if pending compaction bytes exceed this number");
DEFINE_uint64(delayed_write_rate, 2097152u,
"Limited bytes allowed to DB when soft_rate_limit or "
@ -2431,6 +2432,8 @@ class Benchmark {
}
options.soft_rate_limit = FLAGS_soft_rate_limit;
options.hard_rate_limit = FLAGS_hard_rate_limit;
options.hard_pending_compaction_bytes_limit =
FLAGS_hard_pending_compaction_bytes_limit;
options.delayed_write_rate = FLAGS_delayed_write_rate;
options.rate_limit_delay_max_milliseconds =
FLAGS_rate_limit_delay_max_milliseconds;

47
db/db_test.cc
View File

@ -8513,6 +8513,53 @@ TEST_F(DBTest, DelayedWriteRate) {
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}
TEST_F(DBTest, HardLimit) {
Options options;
options.env = env_;
env_->SetBackgroundThreads(1, Env::LOW);
options = CurrentOptions(options);
options.max_write_buffer_number = 256;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 * 1024;
options.level0_file_num_compaction_trigger = 4;
options.level0_slowdown_writes_trigger = 999999;
options.level0_stop_writes_trigger = 999999;
options.hard_pending_compaction_bytes_limit = 800 << 10;
options.max_bytes_for_level_base = 10000000000u;
options.max_background_compactions = 1;
env_->SetBackgroundThreads(1, Env::LOW);
SleepingBackgroundTask sleeping_task_low;
env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
Env::Priority::LOW);
CreateAndReopenWithCF({"pikachu"}, options);
std::atomic<int> callback_count(0);
rocksdb::SyncPoint::GetInstance()->SetCallBack("DBImpl::DelayWrite:Wait",
[&](void* arg) {
callback_count.fetch_add(1);
sleeping_task_low.WakeUp();
});
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Random rnd(301);
int key_idx = 0;
for (int num = 0; num < 5; num++) {
GenerateNewFile(&rnd, &key_idx, true);
}
ASSERT_EQ(0, callback_count.load());
for (int num = 0; num < 5; num++) {
GenerateNewFile(&rnd, &key_idx, true);
dbfull()->TEST_WaitForFlushMemTable();
}
ASSERT_GE(callback_count.load(), 1);
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}
TEST_F(DBTest, SoftLimit) {
Options options;
options.env = env_;

40
db/internal_stats.cc
View File

@ -669,11 +669,13 @@ void InternalStats::DumpCFStats(std::string* value) {
total_files += files;
total_files_being_compacted += files_being_compacted[level];
if (comp_stats_[level].micros > 0 || files > 0) {
uint64_t stalls = level == 0 ? (cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
cf_stats_count_[MEMTABLE_COMPACTION])
: (stall_leveln_slowdown_count_soft_[level] +
stall_leveln_slowdown_count_hard_[level]);
uint64_t stalls =
level == 0 ? (cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
cf_stats_count_[MEMTABLE_COMPACTION])
: (stall_leveln_slowdown_count_soft_[level] +
stall_leveln_slowdown_count_hard_[level]);
stats_sum.Add(comp_stats_[level]);
total_file_size += vstorage->NumLevelBytes(level);
@ -715,20 +717,28 @@ void InternalStats::DumpCFStats(std::string* value) {
curr_ingest / kGB, interval_ingest / kGB);
value->append(buf);
snprintf(buf, sizeof(buf),
"Stalls(count): %" PRIu64 " level0_slowdown, "
"%" PRIu64 " level0_slowdown_with_compaction, "
"%" PRIu64 " level0_numfiles, "
"%" PRIu64 " level0_numfiles_with_compaction, "
"%" PRIu64 " memtable_compaction, "
"%" PRIu64 " leveln_slowdown_soft, "
"%" PRIu64 " leveln_slowdown_hard\n",
snprintf(buf, sizeof(buf), "Stalls(count): %" PRIu64
" level0_slowdown, "
"%" PRIu64
" level0_slowdown_with_compaction, "
"%" PRIu64
" level0_numfiles, "
"%" PRIu64
" level0_numfiles_with_compaction, "
"%" PRIu64
" pending_compaction_bytes, "
"%" PRIu64
" memtable_compaction, "
"%" PRIu64
" leveln_slowdown_soft, "
"%" PRIu64 " leveln_slowdown_hard\n",
cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL],
cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION],
cf_stats_count_[LEVEL0_NUM_FILES_TOTAL],
cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION],
cf_stats_count_[MEMTABLE_COMPACTION],
total_slowdown_count_soft, total_slowdown_count_hard);
cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT],
cf_stats_count_[MEMTABLE_COMPACTION], total_slowdown_count_soft,
total_slowdown_count_hard);
value->append(buf);
cf_stats_snapshot_.ingest_bytes = curr_ingest;

2
db/internal_stats.h
View File

@ -85,6 +85,7 @@ class InternalStats {
MEMTABLE_COMPACTION,
LEVEL0_NUM_FILES_TOTAL,
LEVEL0_NUM_FILES_WITH_COMPACTION,
HARD_PENDING_COMPACTION_BYTES_LIMIT,
WRITE_STALLS_ENUM_MAX,
BYTES_FLUSHED,
INTERNAL_CF_STATS_ENUM_MAX,
@ -357,6 +358,7 @@ class InternalStats {
MEMTABLE_COMPACTION,
LEVEL0_NUM_FILES_TOTAL,
LEVEL0_NUM_FILES_WITH_COMPACTION,
HARD_PENDING_COMPACTION_BYTES_LIMIT,
WRITE_STALLS_ENUM_MAX,
BYTES_FLUSHED,
INTERNAL_CF_STATS_ENUM_MAX,

8
include/rocksdb/options.h
View File

@ -499,8 +499,6 @@ struct ColumnFamilyOptions {
// Puts are delayed to options.delayed_write_rate when any level has a
// compaction score that exceeds soft_rate_limit. This is ignored when == 0.0.
// CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
// hold, RocksDB will set soft_rate_limit = hard_rate_limit
//
// Default: 0 (disabled)
//
@ -510,6 +508,12 @@ struct ColumnFamilyOptions {
// DEPRECATED -- this options is no longer usde
double hard_rate_limit;
// All writes are stopped if estimated bytes needed to be compaction exceed
// this threshold.
//
// Default: 0 (disabled)
uint64_t hard_pending_compaction_bytes_limit;
// DEPRECATED -- this options is no longer used
unsigned int rate_limit_delay_max_milliseconds;

7
util/mutable_cf_options.cc
View File

@ -3,6 +3,8 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include "util/mutable_cf_options.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
@ -15,7 +17,6 @@
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/immutable_options.h"
#include "util/mutable_cf_options.h"
namespace rocksdb {
@ -83,8 +84,8 @@ void MutableCFOptions::Dump(Logger* log) const {
disable_auto_compactions);
Log(log, " soft_rate_limit: %lf",
soft_rate_limit);
Log(log, " hard_rate_limit: %lf",
hard_rate_limit);
Log(log, " hard_pending_compaction_bytes_limit: %" PRIu64,
hard_pending_compaction_bytes_limit);
Log(log, " level0_file_num_compaction_trigger: %d",
level0_file_num_compaction_trigger);
Log(log, " level0_slowdown_writes_trigger: %d",

7
util/mutable_cf_options.h
View File

@ -25,7 +25,8 @@ struct MutableCFOptions {
inplace_update_num_locks(options.inplace_update_num_locks),
disable_auto_compactions(options.disable_auto_compactions),
soft_rate_limit(options.soft_rate_limit),
hard_rate_limit(options.hard_rate_limit),
hard_pending_compaction_bytes_limit(
options.hard_pending_compaction_bytes_limit),
level0_file_num_compaction_trigger(
options.level0_file_num_compaction_trigger),
level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
@ -61,7 +62,7 @@ struct MutableCFOptions {
inplace_update_num_locks(0),
disable_auto_compactions(false),
soft_rate_limit(0),
hard_rate_limit(0),
hard_pending_compaction_bytes_limit(0),
level0_file_num_compaction_trigger(0),
level0_slowdown_writes_trigger(0),
level0_stop_writes_trigger(0),
@ -112,7 +113,7 @@ struct MutableCFOptions {
// Compaction related options
bool disable_auto_compactions;
double soft_rate_limit;
double hard_rate_limit;
uint64_t hard_pending_compaction_bytes_limit;
int level0_file_num_compaction_trigger;
int level0_slowdown_writes_trigger;
int level0_stop_writes_trigger;

9
util/options.cc
View File

@ -102,7 +102,7 @@ ColumnFamilyOptions::ColumnFamilyOptions()
source_compaction_factor(1),
max_grandparent_overlap_factor(10),
soft_rate_limit(0.0),
hard_rate_limit(0.0),
hard_pending_compaction_bytes_limit(0),
rate_limit_delay_max_milliseconds(1000),
arena_block_size(0),
disable_auto_compactions(false),
@ -161,7 +161,8 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
source_compaction_factor(options.source_compaction_factor),
max_grandparent_overlap_factor(options.max_grandparent_overlap_factor),
soft_rate_limit(options.soft_rate_limit),
hard_rate_limit(options.hard_rate_limit),
hard_pending_compaction_bytes_limit(
options.hard_pending_compaction_bytes_limit),
rate_limit_delay_max_milliseconds(
options.rate_limit_delay_max_milliseconds),
arena_block_size(options.arena_block_size),
@ -473,8 +474,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
arena_block_size);
Warn(log, " Options.soft_rate_limit: %.2f",
soft_rate_limit);
Warn(log, " Options.hard_rate_limit: %.2f",
hard_rate_limit);
Warn(log, " Options.hard_pending_compaction_bytes_limit: %" PRIu64,
hard_pending_compaction_bytes_limit);
Warn(log, " Options.rate_limit_delay_max_milliseconds: %u",
rate_limit_delay_max_milliseconds);
Warn(log, " Options.disable_auto_compactions: %d",

5
util/options_helper.cc
View File

@ -281,8 +281,11 @@ bool ParseCompactionOptions(const std::string& name, const std::string& value,
new_options->disable_auto_compactions = ParseBoolean(name, value);
} else if (name == "soft_rate_limit") {
new_options->soft_rate_limit = ParseDouble(value);
} else if (name == "hard_pending_compaction_bytes_limit") {
new_options->hard_pending_compaction_bytes_limit = ParseUint64(value);
} else if (name == "hard_rate_limit") {
new_options->hard_rate_limit = ParseDouble(value);
// Deprecated options but still leave it here to avoid older options
// strings can be consumed.
} else if (name == "level0_file_num_compaction_trigger") {
new_options->level0_file_num_compaction_trigger = ParseInt(value);
} else if (name == "level0_slowdown_writes_trigger") {

3
util/options_helper.h
View File

@ -194,6 +194,9 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{"verify_checksums_in_compaction",
{offsetof(struct ColumnFamilyOptions, verify_checksums_in_compaction),
OptionType::kBoolean}},
{"hard_pending_compaction_bytes_limit",
{offsetof(struct ColumnFamilyOptions, hard_pending_compaction_bytes_limit),
OptionType::kUInt64T}},
{"hard_rate_limit",
{offsetof(struct ColumnFamilyOptions, hard_rate_limit),
OptionType::kDouble}},

8
util/options_test.cc
View File

@ -122,6 +122,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"max_grandparent_overlap_factor", "21"},
{"soft_rate_limit", "1.1"},
{"hard_rate_limit", "2.1"},
{"hard_pending_compaction_bytes_limit", "211"},
{"arena_block_size", "22"},
{"disable_auto_compactions", "true"},
{"compaction_style", "kCompactionStyleLevel"},
@ -214,7 +215,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.source_compaction_factor, 20);
ASSERT_EQ(new_cf_opt.max_grandparent_overlap_factor, 21);
ASSERT_EQ(new_cf_opt.soft_rate_limit, 1.1);
ASSERT_EQ(new_cf_opt.hard_rate_limit, 2.1);
ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
@ -667,7 +668,8 @@ void VerifyColumnFamilyOptions(const ColumnFamilyOptions& base_opt,
new_opt.verify_checksums_in_compaction);
// double options
VerifyDouble(base_opt.hard_rate_limit, new_opt.hard_rate_limit);
ASSERT_EQ(base_opt.hard_pending_compaction_bytes_limit,
new_opt.hard_pending_compaction_bytes_limit);
VerifyDouble(base_opt.soft_rate_limit, new_opt.soft_rate_limit);
// int options
@ -746,7 +748,6 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
base_opt.verify_checksums_in_compaction = rnd.Uniform(2);
// double options
base_opt.hard_rate_limit = static_cast<double>(rnd.Uniform(10000)) / 13;
base_opt.soft_rate_limit = static_cast<double>(rnd.Uniform(10000)) / 13;
// int options
@ -782,6 +783,7 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
base_opt.max_sequential_skip_in_iterations = uint_max + rnd.Uniform(10000);
base_opt.target_file_size_base = uint_max + rnd.Uniform(10000);
base_opt.hard_pending_compaction_bytes_limit = uint_max + rnd.Uniform(10000);
// unsigned int options
base_opt.rate_limit_delay_max_milliseconds = rnd.Uniform(10000);