andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Block Universal and FIFO compactions in ROCKSDB_LITE

Browse Source 
Summary: Block Universal and FIFO compactions in ROCKSDB_LITE

Test Plan:
make shared_lib -j32
make OPT=-DROCKSDB_LITE shared_lib

Reviewers: ljin, igor

Reviewed By: igor

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D29589
This commit is contained in:
Yueh-Hsuan Chiang 2014-11-26 15:45:11 -08:00
parent b8136a7d27
commit bcf9086899
5 changed files with 68 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
db/column_family.cc
View File

@ -253,12 +253,13 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
internal_stats_.reset(
new InternalStats(ioptions_.num_levels, db_options->env, this));
table_cache_.reset(new TableCache(ioptions_, env_options, _table_cache));
if (ioptions_.compaction_style == kCompactionStyleUniversal) {
compaction_picker_.reset(
new UniversalCompactionPicker(ioptions_, &internal_comparator_));
} else if (ioptions_.compaction_style == kCompactionStyleLevel) {
if (ioptions_.compaction_style == kCompactionStyleLevel) {
compaction_picker_.reset(
new LevelCompactionPicker(ioptions_, &internal_comparator_));
#ifndef ROCKSDB_LITE
} else if (ioptions_.compaction_style == kCompactionStyleUniversal) {
compaction_picker_.reset(
new UniversalCompactionPicker(ioptions_, &internal_comparator_));
} else if (ioptions_.compaction_style == kCompactionStyleFIFO) {
compaction_picker_.reset(
new FIFOCompactionPicker(ioptions_, &internal_comparator_));
@ -269,6 +270,7 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
"Column family %s does not use any background compaction. "
"Compactions can only be done via CompactFiles\n",
GetName().c_str());
#endif // !ROCKSDB_LITE
} else {
Log(InfoLogLevel::ERROR_LEVEL, ioptions_.info_log,
"Unable to recognize the specified compaction style %d. "

6
db/compaction_picker.cc
View File

@ -387,6 +387,7 @@ Compaction* CompactionPicker::CompactRange(
begin = nullptr;
end = nullptr;
}
vstorage->GetOverlappingInputs(input_level, begin, end, &inputs);
if (inputs.empty()) {
return nullptr;
@ -676,7 +677,7 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
return Status::OK();
}
#endif // ROCKSDB_LITE
#endif // !ROCKSDB_LITE
bool LevelCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage)
const {
@ -842,6 +843,7 @@ Compaction* LevelCompactionPicker::PickCompactionBySize(
return c;
}
#ifndef ROCKSDB_LITE
bool UniversalCompactionPicker::NeedsCompaction(
const VersionStorageInfo* vstorage) const {
const int kLevel0 = 0;
@ -1325,4 +1327,6 @@ Compaction* FIFOCompactionPicker::CompactRange(
return c;
}
#endif // !ROCKSDB_LITE
} // namespace rocksdb

60
db/compaction_picker.h
View File

@ -169,6 +169,36 @@ class CompactionPicker {
const InternalKeyComparator* const icmp_;
};
class LevelCompactionPicker : public CompactionPicker {
public:
LevelCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(ioptions, icmp) {}
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) override;
// Returns current_num_levels - 2, meaning the last level cannot be
// compaction input level.
virtual int MaxInputLevel(int current_num_levels) const override {
return current_num_levels - 2;
}
virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
override;
private:
// For the specfied level, pick a compaction.
// Returns nullptr if there is no compaction to be done.
// If level is 0 and there is already a compaction on that level, this
// function will return nullptr.
Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, int level,
double score);
};
#ifndef ROCKSDB_LITE
class UniversalCompactionPicker : public CompactionPicker {
public:
UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
@ -210,35 +240,6 @@ class UniversalCompactionPicker : public CompactionPicker {
uint64_t file_size);
};
class LevelCompactionPicker : public CompactionPicker {
public:
LevelCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(ioptions, icmp) {}
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) override;
// Returns current_num_levels - 2, meaning the last level cannot be
// compaction input level.
virtual int MaxInputLevel(int current_num_levels) const override {
return current_num_levels - 2;
}
virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
override;
private:
// For the specfied level, pick a compaction.
// Returns nullptr if there is no compaction to be done.
// If level is 0 and there is already a compaction on that level, this
// function will return nullptr.
Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, int level,
double score);
};
class FIFOCompactionPicker : public CompactionPicker {
public:
FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
@ -306,6 +307,7 @@ class NullCompactionPicker : public CompactionPicker {
return false;
}
};
#endif // !ROCKSDB_LITE
// Utility function
extern uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files);

23
include/rocksdb/options.h
View File

@ -56,11 +56,18 @@ enum CompressionType : char {
};
enum CompactionStyle : char {
kCompactionStyleLevel = 0x0, // level based compaction style
kCompactionStyleUniversal = 0x1, // Universal compaction style
kCompactionStyleFIFO = 0x2, // FIFO compaction style
kCompactionStyleNone = 0x3, // Disable background compaction. Compaction
// jobs are submitted via CompactFiles()
// level based compaction style
kCompactionStyleLevel = 0x0,
// Universal compaction style
// Not supported in ROCKSDB_LITE.
kCompactionStyleUniversal = 0x1,
// FIFO compaction style
// Not supported in ROCKSDB_LITE
kCompactionStyleFIFO = 0x2,
// Disable background compaction. Compaction jobs are submitted
// via CompactFiles().
// Not supported in ROCKSDB_LITE
kCompactionStyleNone = 0x3,
};
struct CompactionOptionsFIFO {
@ -101,9 +108,10 @@ struct Options;
struct ColumnFamilyOptions {
// Some functions that make it easier to optimize RocksDB
#ifndef ROCKSDB_LITE
// Use this if you don't need to keep the data sorted, i.e. you'll never use
// an iterator, only Put() and Get() API calls
//
// Not supported in ROCKSDB_LITE
ColumnFamilyOptions* OptimizeForPointLookup(
uint64_t block_cache_size_mb);
@ -121,11 +129,12 @@ struct ColumnFamilyOptions {
// biggest performance gains.
// Note: we might use more memory than memtable_memory_budget during high
// write rate period
//
// OptimizeUniversalStyleCompaction is not supported in ROCKSDB_LITE
ColumnFamilyOptions* OptimizeLevelStyleCompaction(
uint64_t memtable_memory_budget = 512 * 1024 * 1024);
ColumnFamilyOptions* OptimizeUniversalStyleCompaction(
uint64_t memtable_memory_budget = 512 * 1024 * 1024);
#endif // ROCKSDB_LITE
// -------------------
// Parameters that affect behavior

10
util/options_builder.cc
View File

@ -28,6 +28,7 @@ CompactionStyle PickCompactionStyle(size_t write_buffer_size,
int read_amp_threshold,
int write_amp_threshold,
uint64_t target_db_size) {
#ifndef ROCKSDB_LITE
// Estimate read amplification and write amplification of two compaction
// styles. If there is hard limit to force a choice, make the choice.
// Otherwise, calculate a score based on threshold and expected value of
@ -78,6 +79,9 @@ CompactionStyle PickCompactionStyle(size_t write_buffer_size,
} else {
return kCompactionStyleUniversal;
}
#else
return kCompactionStyleLevel;
#endif // !ROCKSDB_LITE
}
// Pick mem table size
@ -107,12 +111,14 @@ void PickWriteBufferSize(size_t total_write_buffer_limit, Options* options) {
options->min_write_buffer_number_to_merge = 1;
}
#ifndef ROCKSDB_LITE
void OptimizeForUniversal(Options* options) {
options->level0_file_num_compaction_trigger = 2;
options->level0_slowdown_writes_trigger = 30;
options->level0_stop_writes_trigger = 40;
options->max_open_files = -1;
}
#endif
// Optimize parameters for level-based compaction
void OptimizeForLevel(int read_amplification_threshold,
@ -192,9 +198,13 @@ Options GetOptions(size_t total_write_buffer_limit,
options.compaction_style =
PickCompactionStyle(write_buffer_size, read_amplification_threshold,
write_amplification_threshold, target_db_size);
#ifndef ROCKSDB_LITE
if (options.compaction_style == kCompactionStyleUniversal) {
OptimizeForUniversal(&options);
} else {
#else
{
#endif // !ROCKSDB_LITE
OptimizeForLevel(read_amplification_threshold,
write_amplification_threshold, target_db_size, &options);
}