andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

MaxFileSizeForLevel: adjust max_file_size for dynamic level compaction

Browse Source 
Summary:
`MutableCFOptions::RefreshDerivedOptions` always assume base level is L1, which is not true when `level_compaction_dynamic_level_bytes=true` and Level based compaction is used.
This PR fixes this by recomputing `max_file_size` at query time (in `MaxFileSizeForLevel`)
Fixes https://github.com/facebook/rocksdb/issues/3229

In master:

```
Level Files Size(MB)
--------------------
  0       14      846
  1        0        0
  2        0        0
  3        0        0
  4        0        0
  5       15      366
  6       11      481
Cumulative compaction: 3.83 GB write, 2.27 GB read
```
In branch:
```
Level Files Size(MB)
--------------------
  0        9      544
  1        0        0
  2        0        0
  3        0        0
  4        0        0
  5        0        0
  6      445      935
Cumulative compaction: 2.91 GB write, 1.46 GB read
```

db_bench command used:
```
./db_bench --benchmarks="fillrandom,deleterandom,fillrandom,levelstats,stats" --statistics -deletes=5000 -db=tmp -compression_type=none --num=20000 -value_size=100000 -level_compaction_dynamic_level_bytes=true -target_file_size_base=2097152 -target_file_size_multiplier=2
```
Closes https://github.com/facebook/rocksdb/pull/3755

Differential Revision: D7721381

Pulled By: miasantreble

fbshipit-source-id: 39afb8503190bac3b466adf9bbf2a9b3655789f8
This commit is contained in:
Zhongyi Xie 2018-05-03 16:35:46 -07:00 committed by Facebook Github Bot
parent 934f96de27
commit a703432808
9 changed files with 60 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
db/compaction.cc
View File

@ -473,4 +473,8 @@ uint64_t Compaction::MaxInputFileCreationTime() const {
return max_creation_time;
}
int Compaction::GetInputBaseLevel() const {
return input_vstorage_->base_level();
}
} // namespace rocksdb

2
db/compaction.h
View File

@ -234,6 +234,8 @@ class Compaction {
Slice GetLargestUserKey() const { return largest_user_key_; }
int GetInputBaseLevel() const;
CompactionReason compaction_reason() { return compaction_reason_; }
const std::vector<FileMetaData*>& grandparents() const {

16
db/compaction_job.cc
View File

@ -526,9 +526,12 @@ void CompactionJob::GenSubcompactionBoundaries() {
// Group the ranges into subcompactions
const double min_file_fill_percent = 4.0 / 5;
uint64_t max_output_files = static_cast<uint64_t>(
std::ceil(sum / min_file_fill_percent /
c->mutable_cf_options()->MaxFileSizeForLevel(out_lvl)));
int base_level = v->storage_info()->base_level();
uint64_t max_output_files = static_cast<uint64_t>(std::ceil(
sum / min_file_fill_percent /
MaxFileSizeForLevel(*(c->mutable_cf_options()), out_lvl,
c->immutable_cf_options()->compaction_style, base_level,
c->immutable_cf_options()->level_compaction_dynamic_level_bytes)));
uint64_t subcompactions =
std::min({static_cast<uint64_t>(ranges.size()),
static_cast<uint64_t>(c->max_subcompactions()),
@ -766,8 +769,11 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
if (bottommost_level_ && kSampleBytes > 0) {
const size_t kMaxSamples = kSampleBytes >> kSampleLenShift;
const size_t kOutFileLen =
static_cast<size_t>(mutable_cf_options->MaxFileSizeForLevel(
compact_->compaction->output_level()));
static_cast<size_t>(MaxFileSizeForLevel(*mutable_cf_options,
compact_->compaction->output_level(),
cfd->ioptions()->compaction_style,
compact_->compaction->GetInputBaseLevel(),
cfd->ioptions()->level_compaction_dynamic_level_bytes));
if (kOutFileLen != port::kMaxSizet) {
const size_t kOutFileNumSamples = kOutFileLen >> kSampleLenShift;
Random64 generator{versions_->NewFileNumber()};

1
db/compaction_job.h
View File

@ -30,6 +30,7 @@
#include "db/write_controller.h"
#include "db/write_thread.h"
#include "options/db_options.h"
#include "options/cf_options.h"
#include "port/port.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/compaction_job_stats.h"

13
db/compaction_picker.cc
View File

@ -569,7 +569,8 @@ Compaction* CompactionPicker::CompactRange(
Compaction* c = new Compaction(
vstorage, ioptions_, mutable_cf_options, std::move(inputs),
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
output_level, MaxFileSizeForLevel(mutable_cf_options, output_level,
ioptions_.compaction_style),
/* max_compaction_bytes */ LLONG_MAX, output_path_id,
GetCompressionType(ioptions_, vstorage, mutable_cf_options,
output_level, 1),
@ -676,7 +677,10 @@ Compaction* CompactionPicker::CompactRange(
GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
Compaction* compaction = new Compaction(
vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
output_level,
MaxFileSizeForLevel(mutable_cf_options, output_level,
ioptions_.compaction_style, vstorage->base_level(),
ioptions_.level_compaction_dynamic_level_bytes),
mutable_cf_options.max_compaction_bytes, output_path_id,
GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
vstorage->base_level()),
@ -1311,7 +1315,10 @@ Compaction* LevelCompactionBuilder::PickCompaction() {
Compaction* LevelCompactionBuilder::GetCompaction() {
auto c = new Compaction(
vstorage_, ioptions_, mutable_cf_options_, std::move(compaction_inputs_),
output_level_, mutable_cf_options_.MaxFileSizeForLevel(output_level_),
output_level_,
MaxFileSizeForLevel(mutable_cf_options_, output_level_,
ioptions_.compaction_style, vstorage_->base_level(),
ioptions_.level_compaction_dynamic_level_bytes),
mutable_cf_options_.max_compaction_bytes,
GetPathId(ioptions_, mutable_cf_options_, output_level_),
GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,

7
db/compaction_picker_universal.cc
View File

@ -609,7 +609,9 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
}
return new Compaction(
vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
MaxFileSizeForLevel(mutable_cf_options, output_level,
kCompactionStyleUniversal),
LLONG_MAX, path_id,
GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
1, enable_compression),
/* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
@ -742,7 +744,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
return new Compaction(
vstorage, ioptions_, mutable_cf_options, std::move(inputs),
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
output_level, MaxFileSizeForLevel(mutable_cf_options, output_level,
kCompactionStyleUniversal),
/* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
GetCompressionType(ioptions_, vstorage, mutable_cf_options,
output_level, 1),

24
options/cf_options.cc
View File

@ -89,6 +89,24 @@ uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
return static_cast<uint64_t>(op1 * op2);
}
// when level_compaction_dynamic_level_bytes is true and leveled compaction
// is used, the base level is not always L1, so precomupted max_file_size can
// no longer be used. Recompute file_size_for_level from base level.
uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
int level, CompactionStyle compaction_style, int base_level,
bool level_compaction_dynamic_level_bytes) {
if (!level_compaction_dynamic_level_bytes || level < base_level ||
compaction_style != kCompactionStyleLevel) {
assert(level >= 0);
assert(level < (int)cf_options.max_file_size.size());
return cf_options.max_file_size[level];
} else {
assert(level >= 0 && base_level >= 0);
assert(level - base_level < (int)cf_options.max_file_size.size());
return cf_options.max_file_size[level - base_level];
}
}
void MutableCFOptions::RefreshDerivedOptions(int num_levels,
CompactionStyle compaction_style) {
max_file_size.resize(num_levels);
@ -104,12 +122,6 @@ void MutableCFOptions::RefreshDerivedOptions(int num_levels,
}
}
uint64_t MutableCFOptions::MaxFileSizeForLevel(int level) const {
assert(level >= 0);
assert(level < (int)max_file_size.size());
return max_file_size[level];
}
void MutableCFOptions::Dump(Logger* log) const {
// Memtable related options
ROCKS_LOG_INFO(log,

6
options/cf_options.h
View File

@ -192,8 +192,6 @@ struct MutableCFOptions {
RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style);
}
// Get the max file size in a given level.
uint64_t MaxFileSizeForLevel(int level) const;
int MaxBytesMultiplerAdditional(int level) const {
if (level >=
static_cast<int>(max_bytes_for_level_multiplier_additional.size())) {
@ -242,4 +240,8 @@ struct MutableCFOptions {
uint64_t MultiplyCheckOverflow(uint64_t op1, double op2);
// Get the max file size in a given level.
uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
int level, CompactionStyle compaction_style, int base_level = 1,
bool level_compaction_dynamic_level_bytes = false);
} // namespace rocksdb

9
tools/db_bench_tool.cc
View File

@ -45,6 +45,7 @@
#include "rocksdb/filter_policy.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/options.h"
#include "options/cf_options.h"
#include "rocksdb/perf_context.h"
#include "rocksdb/persistent_cache.h"
#include "rocksdb/rate_limiter.h"
@ -3795,8 +3796,8 @@ void VerifyDBFromDB(std::string& truth_db_name) {
MutableCFOptions mutable_cf_options(options);
for (size_t j = 0; j < sorted_runs[i].size(); j++) {
compactionOptions.output_file_size_limit =
mutable_cf_options.MaxFileSizeForLevel(
static_cast<int>(output_level));
MaxFileSizeForLevel(mutable_cf_options,
static_cast<int>(output_level), compaction_style);
std::cout << sorted_runs[i][j].size() << std::endl;
db->CompactFiles(compactionOptions, {sorted_runs[i][j].back().name,
sorted_runs[i][j].front().name},
@ -3847,8 +3848,8 @@ void VerifyDBFromDB(std::string& truth_db_name) {
MutableCFOptions mutable_cf_options(options);
for (size_t j = 0; j < sorted_runs[i].size(); j++) {
compactionOptions.output_file_size_limit =
mutable_cf_options.MaxFileSizeForLevel(
static_cast<int>(output_level));
MaxFileSizeForLevel(mutable_cf_options,
static_cast<int>(output_level), compaction_style);
db->CompactFiles(
compactionOptions,
{sorted_runs[i][j].back().name, sorted_runs[i][j].front().name},