andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Correctly preallocate files in universal compaction

Browse Source 
Summary: In universal compaction, MaxFileSizeForLevel is ULLONG_MAX. We've been preallocation files to UULONG_MAX size all these time :)

Test Plan: make check

Reviewers: dhruba, haobo, ljin, sdong, yhchiang

Reviewed By: yhchiang

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D18915
This commit is contained in:
Igor Canadi 2014-06-05 13:19:35 -07:00
parent 166cc5b456
commit 5d870717ae
3 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
db/compaction.cc
View File

@ -250,4 +250,21 @@ void Compaction::Summary(char* output, int len) {
snprintf(output + write, len - write, "]");
}
uint64_t Compaction::OutputFilePreallocationSize() {
uint64_t preallocation_size = 0;
if (cfd_->options()->compaction_style == kCompactionStyleLevel) {
preallocation_size =
cfd_->compaction_picker()->MaxFileSizeForLevel(output_level());
} else {
for (const auto& f : inputs_[0]) {
preallocation_size += f->file_size;
}
}
// Over-estimate slightly so we don't end up just barely crossing
// the threshold
return preallocation_size * 1.1;
}
} // namespace rocksdb

5
db/compaction.h
View File

@ -91,6 +91,11 @@ class Compaction {
// Was this compaction triggered manually by the client?
bool IsManualCompaction() { return is_manual_compaction_; }
// Returns a number of byte that the output file should be preallocated to
// In level compaction, that is max_file_size_. In universal compaction, that
// is the sum of all input file sizes
uint64_t OutputFilePreallocationSize();
private:
friend class CompactionPicker;
friend class UniversalCompactionPicker;

7
db/db_impl.cc
View File

@ -2334,13 +2334,10 @@ Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
Status s = env_->NewWritableFile(fname, &compact->outfile, storage_options_);
if (s.ok()) {
// Over-estimate slightly so we don't end up just barely crossing
// the threshold.
ColumnFamilyData* cfd = compact->compaction->column_family_data();
compact->outfile->SetPreallocationBlockSize(
1.1 * cfd->compaction_picker()->MaxFileSizeForLevel(
compact->compaction->output_level()));
compact->compaction->OutputFilePreallocationSize());
ColumnFamilyData* cfd = compact->compaction->column_family_data();
CompressionType compression_type =
GetCompressionType(*cfd->options(), compact->compaction->output_level(),
compact->compaction->enable_compression());