andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Renamed 'hybrid_compaction' tp be "Universal Compaction'.

Browse Source 
Summary:
All the universal compaction parameters are encapsulated in
a new file universal_compaction.h

Test Plan:
make check
This commit is contained in:
Dhruba Borthakur 2013-07-03 15:32:49 -07:00
parent a84f547084
commit 116ec527f2
8 changed files with 128 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
db/db_bench.cc
View File

@ -9,6 +9,7 @@
#include "db/db_impl.h"
#include "db/version_set.h"
#include "db/db_statistics.h"
#include "leveldb/options.h"
#include "leveldb/cache.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
@ -125,7 +126,7 @@ static int FLAGS_max_write_buffer_number = 0;
// The minimum number of write buffers that will be merged together
// before writing to storage. This is cheap because it is an
// in-memory merge. If this feature is not enabled, then all these
// write buffers are fushed to L0 as seperate files and this increases
// write buffers are fushed to L0 as separate files and this increases
// read amplification because a get request has to check in all of these
// files. Also, an in-memory merge may result in writing lesser
// data to storage if there are duplicate records in each of these
@ -137,14 +138,14 @@ static int FLAGS_min_write_buffer_number_to_merge = 0;
// This is initialized to default value of 1 in "main" function.
static int FLAGS_max_background_compactions = 0;
// Run database in hybrid mode where all data resides in L0.
static bool FLAGS_hybrid_mode = false;
// style of compaction: level-based vs universal
static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel;
// Percentage flexibilty while comparing file size.
static int FLAGS_hybrid_size_ratio = 1;
static int FLAGS_universal_size_ratio = 1;
// The minimum number of files in a single compaction run.
static int FLAGS_hybrid_min_numfiles_in_single_compaction = 2;
static int FLAGS_compaction_universal_min_merge_width = 2;
// Number of bytes to use as a cache of uncompressed data.
// Negative means use default settings.
@ -1113,10 +1114,10 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
options.min_write_buffer_number_to_merge =
FLAGS_min_write_buffer_number_to_merge;
options.max_background_compactions = FLAGS_max_background_compactions;
options.hybrid_mode = FLAGS_hybrid_mode;
options.hybrid_size_ratio = FLAGS_hybrid_size_ratio;
options.hybrid_min_numfiles_in_single_compaction =
FLAGS_hybrid_min_numfiles_in_single_compaction;
options.compaction_style = FLAGS_compaction_style;
options.compaction_options_universal.size_ratio = FLAGS_universal_size_ratio;
options.compaction_options_universal.min_merge_width =
FLAGS_compaction_universal_min_merge_width;
options.block_size = FLAGS_block_size;
options.filter_policy = filter_policy_;
options.max_open_files = FLAGS_open_files;
@ -1999,12 +2000,11 @@ int main(int argc, char** argv) {
FLAGS_open_files = leveldb::Options().max_open_files;
FLAGS_max_background_compactions =
leveldb::Options().max_background_compactions;
FLAGS_hybrid_mode =
leveldb::Options().hybrid_mode;
FLAGS_hybrid_size_ratio =
leveldb::Options().hybrid_size_ratio;
FLAGS_hybrid_min_numfiles_in_single_compaction =
leveldb::Options().hybrid_min_numfiles_in_single_compaction;
FLAGS_compaction_style = leveldb::Options().compaction_style;
FLAGS_universal_size_ratio =
leveldb::Options().compaction_options_universal.size_ratio;
FLAGS_compaction_universal_min_merge_width =
leveldb::Options().compaction_options_universal.min_merge_width;
// Compression test code above refers to FLAGS_block_size
FLAGS_block_size = leveldb::Options().block_size;
FLAGS_use_os_buffer = leveldb::EnvOptions().use_os_buffer;
@ -2063,13 +2063,13 @@ int main(int argc, char** argv) {
FLAGS_min_write_buffer_number_to_merge = n;
} else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk) == 1) {
FLAGS_max_background_compactions = n;
} else if (sscanf(argv[i], "--hybrid_mode=%d%c", &n, &junk) == 1) {
FLAGS_hybrid_mode = n;
} else if (sscanf(argv[i], "--hybrid_size_ratio=%d%c", &n, &junk) == 1) {
FLAGS_hybrid_size_ratio = n;
} else if (sscanf(argv[i], "--hybrid_min_numfiles_in_single_compaction=%d%c",
} else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) {
FLAGS_compaction_style = (leveldb::CompactionStyle)n;
} else if (sscanf(argv[i], "--universal_size_ratio=%d%c", &n, &junk) == 1) {
FLAGS_universal_size_ratio = n;
} else if (sscanf(argv[i], "--universal_min_merge_width=%d%c",
&n, &junk) == 1) {
FLAGS_hybrid_min_numfiles_in_single_compaction = n;
FLAGS_compaction_universal_min_merge_width = n;
} else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) {
FLAGS_cache_size = l;
} else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {

9
db/db_impl.cc
View File

@ -836,7 +836,7 @@ Status DBImpl::WriteLevel0Table(std::vector<MemTable*> &mems, VersionEdit* edit,
// threads could be concurrently producing compacted files for
// that key range.
if (base != nullptr && options_.max_background_compactions <= 1 &&
!options_.hybrid_mode) {
options_.compaction_style == kCompactionStyleLevel) {
level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
}
edit->AddFile(level, meta.number, meta.file_size,
@ -1578,7 +1578,8 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) {
for (size_t i = 0; i < compact->outputs.size(); i++) {
const CompactionState::Output& out = compact->outputs[i];
compact->compaction->edit()->AddFile(
options_.hybrid_mode? level : level + 1,
(options_.compaction_style == kCompactionStyleUniversal) ?
level : level + 1,
out.number, out.file_size, out.smallest, out.largest,
out.smallest_seqno, out.largest_seqno);
}
@ -1828,9 +1829,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
// If this is the bottommost level (no files in lower levels)
// and the earliest snapshot is larger than this seqno
// then we can squash the seqno to zero.
// Hybrid mode depends on the sequence number to determine
// Universal mode depends on the sequence number to determine
// time-order of files that is needed for compactions.
if (!options_.hybrid_mode &&
if (options_.compaction_style == kCompactionStyleLevel &&
bottommost_level && ikey.sequence < earliest_snapshot &&
ikey.type != kTypeMerge) {
assert(ikey.type != kTypeDeletion);

32
db/version_set.cc
View File

@ -384,7 +384,7 @@ void Version::Get(const ReadOptions& options,
}
if (tmp.empty()) continue;
if (vset_->options_->hybrid_mode) {
if (vset_->options_->compaction_style == kCompactionStyleUniversal) {
std::sort(tmp.begin(), tmp.end(), NewestFirstBySeqNo);
} else {
std::sort(tmp.begin(), tmp.end(), NewestFirst);
@ -1581,7 +1581,7 @@ static bool compareSizeDescending(const VersionSet::Fsize& first,
return (first.file->file_size > second.file->file_size);
}
// A static compator used to sort files based on their seqno
// In hybrid mode: descending seqno
// In universal style : descending seqno
static bool compareSeqnoDescending(const VersionSet::Fsize& first,
const VersionSet::Fsize& second) {
if (first.file->smallest_seqno > second.file->smallest_seqno) {
@ -1596,8 +1596,8 @@ static bool compareSeqnoDescending(const VersionSet::Fsize& first,
void VersionSet::UpdateFilesBySize(Version* v) {
// No need to sort the highest level because it is never compacted.
int max_level = options_->hybrid_mode? NumberLevels() :
NumberLevels() - 1;
int max_level = (options_->compaction_style == kCompactionStyleUniversal) ?
NumberLevels() : NumberLevels() - 1;
for (int level = 0; level < max_level; level++) {
@ -1613,7 +1613,7 @@ void VersionSet::UpdateFilesBySize(Version* v) {
}
// sort the top number_of_files_to_sort_ based on file size
if (options_->hybrid_mode) {
if (options_->compaction_style == kCompactionStyleUniversal) {
int num = temp.size();
std::partial_sort(temp.begin(), temp.begin() + num,
temp.end(), compareSeqnoDescending);
@ -2020,7 +2020,11 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
assert (level == 0);
// percentage flexibilty while comparing file sizes
uint64_t ratio = (uint64_t)options_->hybrid_size_ratio;
uint64_t ratio = options_->compaction_options_universal.size_ratio;
unsigned int min_merge_width =
options_->compaction_options_universal.min_merge_width;
unsigned int max_merge_width =
options_->compaction_options_universal.max_merge_width;
if ((current_->files_[level].size() <=
(unsigned int)options_->level0_file_num_compaction_trigger)) {
@ -2043,7 +2047,7 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
bool done = false;
assert(file_by_time.size() == current_->files_[level].size());
unsigned int max_files_to_compact = UINT_MAX;
unsigned int max_files_to_compact = std::min(max_merge_width, UINT_MAX);
// Make two pass. The first pass considers a candidate file
// only if it is smaller than the total size accumulated so far.
@ -2098,8 +2102,7 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
}
// Found a series of consecutive files that need compaction.
if (candidate_count >= (unsigned int)
options_->hybrid_min_numfiles_in_single_compaction) {
if (candidate_count >= (unsigned int)min_merge_width) {
for (unsigned int i = loop; i < loop + candidate_count; i++) {
int index = file_by_time[i];
FileMetaData* f = current_->files_[level][index];
@ -2141,8 +2144,8 @@ Compaction* VersionSet::PickCompactionHybrid(int level, double score) {
options_->level0_file_num_compaction_trigger + 1) {
done = true; // nothing more to do
} else {
max_files_to_compact = expected_num_files -
options_->level0_file_num_compaction_trigger;
max_files_to_compact = std::min((int)max_merge_width,
expected_num_files - options_->level0_file_num_compaction_trigger);
Log(options_->info_log, "Hybrid: second loop with maxfiles %d",
max_files_to_compact);
}
@ -2264,8 +2267,8 @@ Compaction* VersionSet::PickCompaction() {
current_->vset_->SizeBeingCompacted(size_being_compacted);
Finalize(current_, size_being_compacted);
// In hybrid mode compact L0 files back into L0.
if (options_->hybrid_mode) {
// In universal style of compaction, compact L0 files back into L0.
if (options_->compaction_style == kCompactionStyleUniversal) {
int level = 0;
c = PickCompactionHybrid(level, current_->compaction_score_[level]);
return c;
@ -2473,7 +2476,8 @@ Compaction* VersionSet::CompactRange(
}
}
}
int out_level = options_->hybrid_mode ? level : level+1;
int out_level = (options_->compaction_style == kCompactionStyleUniversal) ?
level : level+1;
Compaction* c = new Compaction(level, out_level, MaxFileSizeForLevel(level),
MaxGrandParentOverlapBytes(level), NumberLevels());

21
include/leveldb/options.h
View File

@ -12,6 +12,7 @@
#include <stdint.h>
#include "leveldb/slice.h"
#include "leveldb/statistics.h"
#include "leveldb/universal_compaction.h"
namespace leveldb {
@ -39,6 +40,11 @@ enum CompressionType {
kBZip2Compression = 0x3
};
enum CompactionStyle {
kCompactionStyleLevel = 0x0, // level based compaction style
kCompactionStyleUniversal = 0x1 // Universal compaction style
};
// Compression options for different compression algorithms like Zlib
struct CompressionOptions {
int window_bits;
@ -132,7 +138,7 @@ struct Options {
int max_write_buffer_number;
// The minimum number of write buffers that will be merged together
// before writing to storage. If set to 1, then
// before writing to storage. If set to 1, then
// all write buffers are fushed to L0 as individual files and this increases
// read amplification because a get request has to check in all of these
// files. Also, an in-memory merge may result in writing lesser
@ -476,17 +482,12 @@ struct Options {
// Default: 0
uint64_t bytes_per_sync;
// Hybrid Mode. There is only a single level and files in L0 are
// The compaction style
// compacted back into L0. Default: false
bool hybrid_mode;
CompactionStyle compaction_style;
// Percentage flexibilty while comparing file size. If the candidate file(s)
// size is 1% smaller than the next file's size, then include next file into
// this candidate set. // Default: 1
int hybrid_size_ratio;
// The minimum number of files in a single compaction run. Default: 2
int hybrid_min_numfiles_in_single_compaction;
// The options needed to support Universal Style compactions
CompactionOptionsUniversal compaction_options_universal;
};
// Options that control read operations

57
include/leveldb/universal_compaction.h Normal file
View File

@ -0,0 +1,57 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
#define STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
#include <stddef.h>
#include <string>
#include <memory>
#include <vector>
#include <stdint.h>
#include <climits>
#include "leveldb/slice.h"
#include "leveldb/statistics.h"
namespace leveldb {
//
// Algorithm used to make a compaction request stop picking new files
// into a single compaction run
//
enum CompactionStopStyle {
kCompactionStopStyleSimilarSize, // pick files of similar size
kCompactionStopStyleTotalSize // total size of picked files > next file
};
class CompactionOptionsUniversal {
public:
// Percentage flexibilty while comparing file size. If the candidate file(s)
// size is 1% smaller than the next file's size, then include next file into
// this candidate set. // Default: 1
unsigned int size_ratio;
// The minimum number of files in a single compaction run. Default: 2
unsigned int min_merge_width;
// The maximum number of files in a single compaction run. Default: INT_MAX
unsigned int max_merge_width;
// The algorithm used to stop picking files into a single compaction run
// Default: kCompactionStopStyleTotalSize
CompactionStopStyle stop_style;
// Default set of parameters
CompactionOptionsUniversal() :
size_ratio(1),
min_merge_width(2),
max_merge_width(UINT_MAX),
stop_style(kCompactionStopStyleTotalSize) {
}
};
} // namespace leveldb
#endif // STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H

12
tools/db_stress.cc
View File

@ -89,7 +89,7 @@ static int FLAGS_max_write_buffer_number = 0;
static int FLAGS_max_background_compactions = 0;
// This is initialized to default value of false
static bool FLAGS_hybrid_mode = false;
static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel;
// Number of bytes to use as a cache of uncompressed data.
static long FLAGS_cache_size = 2 * KB * KB * KB;
@ -933,7 +933,7 @@ class StressTest {
options.write_buffer_size = FLAGS_write_buffer_size;
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
options.max_background_compactions = FLAGS_max_background_compactions;
options.hybrid_mode = FLAGS_hybrid_mode;
options.compaction_style = FLAGS_compaction_style;
options.block_size = FLAGS_block_size;
options.filter_policy = filter_policy_;
options.max_open_files = FLAGS_open_files;
@ -1020,8 +1020,8 @@ int main(int argc, char** argv) {
FLAGS_open_files = leveldb::Options().max_open_files;
FLAGS_max_background_compactions =
leveldb::Options().max_background_compactions;
FLAGS_hybrid_mode =
leveldb::Options().hybrid_mode;
FLAGS_compaction_style =
leveldb::Options().compaction_style;
FLAGS_level0_file_num_compaction_trigger =
leveldb::Options().level0_file_num_compaction_trigger;
FLAGS_level0_slowdown_writes_trigger =
@ -1074,8 +1074,8 @@ int main(int argc, char** argv) {
FLAGS_max_write_buffer_number = n;
} else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk) == 1) {
FLAGS_max_background_compactions = n;
} else if (sscanf(argv[i], "--hybrid_mode=%d%c", &n, &junk) == 1) {
FLAGS_hybrid_mode = n;
} else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) {
FLAGS_compaction_style = (leveldb::CompactionStyle)n;
} else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) {
FLAGS_cache_size = l;
} else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {

2
util/ldb_cmd.cc
View File

@ -1070,7 +1070,7 @@ void ApproxSizeCommand::DoCommand() {
uint64_t sizes[1];
db_->GetApproximateSizes(ranges, 1, sizes);
fprintf(stdout, "%ld\n", sizes[0]);
/* Wierd that GetApproximateSizes() returns void, although documentation
/* Weird that GetApproximateSizes() returns void, although documentation
* says that it returns a Status object.
if (!st.ok()) {
exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString());

18
util/options.cc
View File

@ -77,9 +77,7 @@ Options::Options()
access_hint_on_compaction_start(NORMAL),
use_adaptive_mutex(false),
bytes_per_sync(0),
hybrid_mode(false),
hybrid_size_ratio(1),
hybrid_min_numfiles_in_single_compaction(2) {
compaction_style(kCompactionStyleLevel) {
}
static const char* const access_hints[] = {
@ -220,12 +218,14 @@ Options::Dump(Logger* log) const
use_adaptive_mutex);
Log(log," Options.bytes_per_sync: %ld",
bytes_per_sync);
Log(log," Options.hybrid_mode: %d",
hybrid_mode);
Log(log," Options.hybrid_size_ratio: %d",
hybrid_size_ratio);
Log(log,"Options.hybrid_min_numfiles_in_single_compaction: %d",
hybrid_min_numfiles_in_single_compaction);
Log(log," Options.compaction_style: %d",
compaction_style);
Log(log," Options.compaction_options_universal.size_ratio: %d",
compaction_options_universal.size_ratio);
Log(log," Options.compaction_options_universal.min_merge_width: %d",
compaction_options_universal.min_merge_width);
Log(log," Options.compaction_options_universal.max_merge_width: %d",
compaction_options_universal.max_merge_width);
} // Options::Dump
//