Refactor MutableCFOptions
Summary: * Change constructor of MutableCFOptions to depends only on ColumnFamilyOptions. * Move `max_subcompactions`, `compaction_options_fifo` and `compaction_pri` to ImmutableCFOptions to make it clear that they are immutable. Test Plan: existing unit tests. Reviewers: yhchiang, IslamAbdelRahman, sdong Reviewed By: sdong Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D63945
This commit is contained in:
Yi Wu
parent
ba65c816bb
commit
81747f1be6
@ -350,7 +350,7 @@ ColumnFamilyData::ColumnFamilyData(
|
||||
options_(*db_options,
|
||||
SanitizeOptions(*db_options, &internal_comparator_, cf_options)),
|
||||
ioptions_(options_),
|
||||
mutable_cf_options_(options_, ioptions_),
|
||||
mutable_cf_options_(options_),
|
||||
write_buffer_manager_(write_buffer_manager),
|
||||
mem_(nullptr),
|
||||
imm_(options_.min_write_buffer_number_to_merge,
|
||||
|
||||
@ -2440,9 +2440,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
|
||||
|
||||
VersionStorageInfo* vstorage = cfd->current()->storage_info();
|
||||
|
||||
MutableCFOptions mutable_cf_options(
|
||||
Options(db_options_, column_family_options_),
|
||||
ImmutableCFOptions(Options(db_options_, column_family_options_)));
|
||||
MutableCFOptions mutable_cf_options(column_family_options_);
|
||||
|
||||
mutable_cf_options.level0_slowdown_writes_trigger = 20;
|
||||
mutable_cf_options.level0_stop_writes_trigger = 10000;
|
||||
@ -2631,9 +2629,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
|
||||
|
||||
VersionStorageInfo* vstorage = cfd->current()->storage_info();
|
||||
|
||||
MutableCFOptions mutable_cf_options(
|
||||
Options(db_options_, column_family_options_),
|
||||
ImmutableCFOptions(Options(db_options_, column_family_options_)));
|
||||
MutableCFOptions mutable_cf_options(column_family_options_);
|
||||
|
||||
// Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
|
||||
mutable_cf_options.level0_file_num_compaction_trigger = 4;
|
||||
@ -2702,9 +2698,7 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
|
||||
static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
|
||||
VersionStorageInfo* vstorage1 = cfd1->current()->storage_info();
|
||||
|
||||
MutableCFOptions mutable_cf_options(
|
||||
Options(db_options_, column_family_options_),
|
||||
ImmutableCFOptions(Options(db_options_, column_family_options_)));
|
||||
MutableCFOptions mutable_cf_options(column_family_options_);
|
||||
mutable_cf_options.level0_slowdown_writes_trigger = 20;
|
||||
mutable_cf_options.level0_stop_writes_trigger = 10000;
|
||||
mutable_cf_options.soft_pending_compaction_bytes_limit = 200;
|
||||
@ -2786,9 +2780,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
|
||||
static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
|
||||
VersionStorageInfo* vstorage1 = cfd1->current()->storage_info();
|
||||
|
||||
MutableCFOptions mutable_cf_options(
|
||||
Options(db_options_, column_family_options_),
|
||||
ImmutableCFOptions(Options(db_options_, column_family_options_)));
|
||||
MutableCFOptions mutable_cf_options(column_family_options_);
|
||||
// Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
|
||||
mutable_cf_options.level0_file_num_compaction_trigger = 4;
|
||||
mutable_cf_options.level0_slowdown_writes_trigger = 36;
|
||||
|
||||
@ -140,6 +140,7 @@ bool Compaction::IsFullCompaction(
|
||||
}
|
||||
|
||||
Compaction::Compaction(VersionStorageInfo* vstorage,
|
||||
const ImmutableCFOptions& _immutable_cf_options,
|
||||
const MutableCFOptions& _mutable_cf_options,
|
||||
std::vector<CompactionInputFiles> _inputs,
|
||||
int _output_level, uint64_t _target_file_size,
|
||||
@ -153,6 +154,7 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
|
||||
output_level_(_output_level),
|
||||
max_output_file_size_(_target_file_size),
|
||||
max_compaction_bytes_(_max_compaction_bytes),
|
||||
immutable_cf_options_(_immutable_cf_options),
|
||||
mutable_cf_options_(_mutable_cf_options),
|
||||
input_version_(nullptr),
|
||||
number_levels_(vstorage->num_levels()),
|
||||
@ -427,7 +429,7 @@ bool Compaction::IsOutputLevelEmpty() const {
|
||||
}
|
||||
|
||||
bool Compaction::ShouldFormSubcompactions() const {
|
||||
if (mutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
|
||||
if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
|
||||
|
||||
@ -35,6 +35,7 @@ class CompactionFilter;
|
||||
class Compaction {
|
||||
public:
|
||||
Compaction(VersionStorageInfo* input_version,
|
||||
const ImmutableCFOptions& immutable_cf_options,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
std::vector<CompactionInputFiles> inputs, int output_level,
|
||||
uint64_t target_file_size, uint64_t max_compaction_bytes,
|
||||
@ -171,6 +172,12 @@ class Compaction {
|
||||
// How many total levels are there?
|
||||
int number_levels() const { return number_levels_; }
|
||||
|
||||
// Return the ImmutableCFOptions that should be used throughout the compaction
|
||||
// procedure
|
||||
const ImmutableCFOptions* immutable_cf_options() const {
|
||||
return &immutable_cf_options_;
|
||||
}
|
||||
|
||||
// Return the MutableCFOptions that should be used throughout the compaction
|
||||
// procedure
|
||||
const MutableCFOptions* mutable_cf_options() const {
|
||||
@ -253,7 +260,8 @@ class Compaction {
|
||||
const int output_level_; // levels to which output files are stored
|
||||
uint64_t max_output_file_size_;
|
||||
uint64_t max_compaction_bytes_;
|
||||
MutableCFOptions mutable_cf_options_;
|
||||
const ImmutableCFOptions immutable_cf_options_;
|
||||
const MutableCFOptions mutable_cf_options_;
|
||||
Version* input_version_;
|
||||
VersionEdit edit_;
|
||||
const int number_levels_;
|
||||
|
||||
@ -68,7 +68,7 @@ class CompactionJobTest : public testing::Test {
|
||||
CompactionJobTest()
|
||||
: env_(Env::Default()),
|
||||
dbname_(test::TmpDir() + "/compaction_job_test"),
|
||||
mutable_cf_options_(Options(), ImmutableCFOptions(Options())),
|
||||
mutable_cf_options_(Options()),
|
||||
table_cache_(NewLRUCache(50000, 16)),
|
||||
write_buffer_manager_(db_options_.db_write_buffer_size),
|
||||
versions_(new VersionSet(dbname_, &db_options_, env_options_,
|
||||
@ -239,7 +239,7 @@ class CompactionJobTest : public testing::Test {
|
||||
num_input_files += level_files.size();
|
||||
}
|
||||
|
||||
Compaction compaction(cfd->current()->storage_info(),
|
||||
Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(),
|
||||
*cfd->GetLatestMutableCFOptions(),
|
||||
compaction_input_files, 1, 1024 * 1024,
|
||||
10 * 1024 * 1024, 0, kNoCompression, {}, true);
|
||||
|
||||
@ -275,8 +275,8 @@ Compaction* CompactionPicker::FormCompaction(
|
||||
return nullptr;
|
||||
}
|
||||
auto c =
|
||||
new Compaction(vstorage, mutable_cf_options, input_files, output_level,
|
||||
compact_options.output_file_size_limit,
|
||||
new Compaction(vstorage, ioptions_, mutable_cf_options, input_files,
|
||||
output_level, compact_options.output_file_size_limit,
|
||||
mutable_cf_options.max_compaction_bytes, output_path_id,
|
||||
compact_options.compression, /* grandparents */ {}, true);
|
||||
|
||||
@ -507,8 +507,8 @@ Compaction* CompactionPicker::CompactRange(
|
||||
}
|
||||
}
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, mutable_cf_options, std::move(inputs), output_level,
|
||||
mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs),
|
||||
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
/* max_compaction_bytes */ LLONG_MAX, output_path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options,
|
||||
output_level, 1),
|
||||
@ -607,8 +607,8 @@ Compaction* CompactionPicker::CompactRange(
|
||||
std::vector<FileMetaData*> grandparents;
|
||||
GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
|
||||
Compaction* compaction = new Compaction(
|
||||
vstorage, mutable_cf_options, std::move(compaction_inputs), output_level,
|
||||
mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
|
||||
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
mutable_cf_options.max_compaction_bytes, output_path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
|
||||
vstorage->base_level()),
|
||||
@ -623,7 +623,7 @@ Compaction* CompactionPicker::CompactRange(
|
||||
// takes running compactions into account (by skipping files that are already
|
||||
// being compacted). Since we just changed compaction score, we recalculate it
|
||||
// here
|
||||
vstorage->ComputeCompactionScore(mutable_cf_options);
|
||||
vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
|
||||
|
||||
return compaction;
|
||||
}
|
||||
@ -1012,8 +1012,8 @@ Compaction* LevelCompactionPicker::PickCompaction(
|
||||
std::vector<FileMetaData*> grandparents;
|
||||
GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
|
||||
auto c = new Compaction(
|
||||
vstorage, mutable_cf_options, std::move(compaction_inputs), output_level,
|
||||
mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
|
||||
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
mutable_cf_options.max_compaction_bytes,
|
||||
GetPathId(ioptions_, mutable_cf_options, output_level),
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
|
||||
@ -1031,7 +1031,7 @@ Compaction* LevelCompactionPicker::PickCompaction(
|
||||
// takes running compactions into account (by skipping files that are already
|
||||
// being compacted). Since we just changed compaction score, we recalculate it
|
||||
// here
|
||||
vstorage->ComputeCompactionScore(mutable_cf_options);
|
||||
vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
|
||||
|
||||
TEST_SYNC_POINT_CALLBACK("LevelCompactionPicker::PickCompaction:Return", c);
|
||||
|
||||
@ -1643,7 +1643,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
|
||||
compaction_reason = CompactionReason::kUniversalSizeRatio;
|
||||
}
|
||||
return new Compaction(
|
||||
vstorage, mutable_cf_options, std::move(inputs), output_level,
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
|
||||
mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
|
||||
1, enable_compression),
|
||||
@ -1767,7 +1767,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
|
||||
}
|
||||
|
||||
return new Compaction(
|
||||
vstorage, mutable_cf_options, std::move(inputs),
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs),
|
||||
vstorage->num_levels() - 1,
|
||||
mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1),
|
||||
/* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
|
||||
@ -1832,7 +1832,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
|
||||
}
|
||||
}
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
|
||||
kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
|
||||
/* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
|
||||
level0_compactions_in_progress_.insert(c);
|
||||
|
||||
@ -48,7 +48,7 @@ class CompactionPickerTest : public testing::Test {
|
||||
: ucmp_(BytewiseComparator()),
|
||||
icmp_(ucmp_),
|
||||
ioptions_(options_),
|
||||
mutable_cf_options_(options_, ioptions_),
|
||||
mutable_cf_options_(options_),
|
||||
level_compaction_picker(ioptions_, &icmp_),
|
||||
cf_name_("dummy"),
|
||||
log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
|
||||
@ -115,11 +115,11 @@ class CompactionPickerTest : public testing::Test {
|
||||
|
||||
void UpdateVersionStorageInfo() {
|
||||
vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
|
||||
vstorage_->UpdateFilesByCompactionPri(mutable_cf_options_);
|
||||
vstorage_->UpdateFilesByCompactionPri(ioptions_.compaction_pri);
|
||||
vstorage_->UpdateNumNonEmptyLevels();
|
||||
vstorage_->GenerateFileIndexer();
|
||||
vstorage_->GenerateLevelFilesBrief();
|
||||
vstorage_->ComputeCompactionScore(mutable_cf_options_);
|
||||
vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_);
|
||||
vstorage_->GenerateLevel0NonOverlapping();
|
||||
vstorage_->SetFinalized();
|
||||
}
|
||||
@ -350,10 +350,10 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic4) {
|
||||
TEST_F(CompactionPickerTest, LevelTriggerDynamic4) {
|
||||
int num_levels = ioptions_.num_levels;
|
||||
ioptions_.level_compaction_dynamic_level_bytes = true;
|
||||
ioptions_.compaction_pri = kMinOverlappingRatio;
|
||||
mutable_cf_options_.level0_file_num_compaction_trigger = 2;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 200;
|
||||
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
||||
mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
|
||||
NewVersionStorage(num_levels, kCompactionStyleLevel);
|
||||
Add(0, 1U, "150", "200");
|
||||
Add(num_levels - 1, 3U, "200", "250", 300U);
|
||||
@ -489,10 +489,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
|
||||
|
||||
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
ioptions_.compaction_pri = kMinOverlappingRatio;
|
||||
mutable_cf_options_.target_file_size_base = 10000000;
|
||||
mutable_cf_options_.target_file_size_multiplier = 10;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
|
||||
mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
|
||||
|
||||
Add(2, 6U, "150", "179", 50000000U);
|
||||
Add(2, 7U, "180", "220", 50000000U);
|
||||
@ -516,10 +516,10 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
|
||||
|
||||
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
ioptions_.compaction_pri = kMinOverlappingRatio;
|
||||
mutable_cf_options_.target_file_size_base = 10000000;
|
||||
mutable_cf_options_.target_file_size_multiplier = 10;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
|
||||
mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
|
||||
|
||||
Add(2, 6U, "150", "175",
|
||||
60000000U); // Overlaps with file 26, 27, total size 521M
|
||||
@ -546,9 +546,9 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
|
||||
|
||||
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
ioptions_.compaction_pri = kMinOverlappingRatio;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10000000;
|
||||
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
||||
mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
|
||||
|
||||
// file 7 and 8 over lap with the same file, but file 8 is smaller so
|
||||
// it will be picked.
|
||||
@ -598,7 +598,7 @@ TEST_F(CompactionPickerTest, ParentIndexResetBug) {
|
||||
// ranges (with different sequence numbers) in the input files.
|
||||
TEST_F(CompactionPickerTest, OverlappingUserKeys) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
mutable_cf_options_.compaction_pri = kByCompensatedSize;
|
||||
ioptions_.compaction_pri = kByCompensatedSize;
|
||||
|
||||
Add(1, 1U, "100", "150", 1U);
|
||||
// Overlapping user keys
|
||||
|
||||
@ -2263,7 +2263,8 @@ Status DBImpl::CompactFilesImpl(
|
||||
// takes running compactions into account (by skipping files that are already
|
||||
// being compacted). Since we just changed compaction score, we recalculate it
|
||||
// here.
|
||||
version->storage_info()->ComputeCompactionScore(*c->mutable_cf_options());
|
||||
version->storage_info()->ComputeCompactionScore(*cfd->ioptions(),
|
||||
*c->mutable_cf_options());
|
||||
|
||||
compaction_job.Prepare();
|
||||
|
||||
@ -4037,9 +4038,8 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
|
||||
write_thread_.EnterUnbatched(&w, &mutex_);
|
||||
// LogAndApply will both write the creation in MANIFEST and create
|
||||
// ColumnFamilyData object
|
||||
s = versions_->LogAndApply(
|
||||
nullptr, MutableCFOptions(opt, ImmutableCFOptions(opt)), &edit,
|
||||
&mutex_, directories_.GetDbDir(), false, &cf_options);
|
||||
s = versions_->LogAndApply(nullptr, MutableCFOptions(opt), &edit, &mutex_,
|
||||
directories_.GetDbDir(), false, &cf_options);
|
||||
|
||||
if (s.ok()) {
|
||||
// If the column family was created successfully, we then persist
|
||||
|
||||
@ -49,7 +49,8 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
|
||||
}
|
||||
// Since we have some more files to compact, we should also recompute
|
||||
// compaction score
|
||||
vstorage->ComputeCompactionScore(*cfd->GetLatestMutableCFOptions());
|
||||
vstorage->ComputeCompactionScore(*cfd->ioptions(),
|
||||
*cfd->GetLatestMutableCFOptions());
|
||||
SchedulePendingCompaction(cfd);
|
||||
MaybeScheduleFlushOrCompaction();
|
||||
}
|
||||
|
||||
@ -127,9 +127,8 @@ TEST_F(MemTableListTest, GetTest) {
|
||||
ImmutableCFOptions ioptions(options);
|
||||
|
||||
WriteBufferManager wb(options.db_write_buffer_size);
|
||||
MemTable* mem =
|
||||
new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
|
||||
kMaxSequenceNumber);
|
||||
MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
|
||||
kMaxSequenceNumber);
|
||||
mem->Ref();
|
||||
|
||||
// Write some keys to this memtable.
|
||||
@ -164,9 +163,8 @@ TEST_F(MemTableListTest, GetTest) {
|
||||
|
||||
// Create another memtable and write some keys to it
|
||||
WriteBufferManager wb2(options.db_write_buffer_size);
|
||||
MemTable* mem2 =
|
||||
new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb2,
|
||||
kMaxSequenceNumber);
|
||||
MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2,
|
||||
kMaxSequenceNumber);
|
||||
mem2->Ref();
|
||||
|
||||
mem2->Add(++seq, kTypeDeletion, "key1", "");
|
||||
@ -229,9 +227,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
|
||||
ImmutableCFOptions ioptions(options);
|
||||
|
||||
WriteBufferManager wb(options.db_write_buffer_size);
|
||||
MemTable* mem =
|
||||
new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
|
||||
kMaxSequenceNumber);
|
||||
MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
|
||||
kMaxSequenceNumber);
|
||||
mem->Ref();
|
||||
|
||||
// Write some keys to this memtable.
|
||||
@ -272,8 +269,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
|
||||
list.PickMemtablesToFlush(&to_flush);
|
||||
ASSERT_EQ(1, to_flush.size());
|
||||
|
||||
s = Mock_InstallMemtableFlushResults(
|
||||
&list, MutableCFOptions(options, ioptions), to_flush, &to_delete);
|
||||
s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options),
|
||||
to_flush, &to_delete);
|
||||
ASSERT_OK(s);
|
||||
ASSERT_EQ(0, list.NumNotFlushed());
|
||||
ASSERT_EQ(1, list.NumFlushed());
|
||||
@ -304,9 +301,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
|
||||
|
||||
// Create another memtable and write some keys to it
|
||||
WriteBufferManager wb2(options.db_write_buffer_size);
|
||||
MemTable* mem2 =
|
||||
new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb2,
|
||||
kMaxSequenceNumber);
|
||||
MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2,
|
||||
kMaxSequenceNumber);
|
||||
mem2->Ref();
|
||||
|
||||
mem2->Add(++seq, kTypeDeletion, "key1", "");
|
||||
@ -321,8 +317,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
|
||||
ASSERT_EQ(1, to_flush.size());
|
||||
|
||||
// Flush second memtable
|
||||
s = Mock_InstallMemtableFlushResults(
|
||||
&list, MutableCFOptions(options, ioptions), to_flush, &to_delete);
|
||||
s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options),
|
||||
to_flush, &to_delete);
|
||||
ASSERT_OK(s);
|
||||
ASSERT_EQ(0, list.NumNotFlushed());
|
||||
ASSERT_EQ(2, list.NumFlushed());
|
||||
@ -330,9 +326,8 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
|
||||
|
||||
// Add a third memtable to push the first memtable out of the history
|
||||
WriteBufferManager wb3(options.db_write_buffer_size);
|
||||
MemTable* mem3 =
|
||||
new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb3,
|
||||
kMaxSequenceNumber);
|
||||
MemTable* mem3 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb3,
|
||||
kMaxSequenceNumber);
|
||||
mem3->Ref();
|
||||
list.Add(mem3, &to_delete);
|
||||
ASSERT_EQ(1, list.NumNotFlushed());
|
||||
@ -401,7 +396,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
||||
|
||||
// Create some MemTables
|
||||
std::vector<MemTable*> tables;
|
||||
MutableCFOptions mutable_cf_options(options, ioptions);
|
||||
MutableCFOptions mutable_cf_options(options);
|
||||
for (int i = 0; i < num_tables; i++) {
|
||||
MemTable* mem = new MemTable(cmp, ioptions, mutable_cf_options, &wb,
|
||||
kMaxSequenceNumber);
|
||||
@ -539,8 +534,8 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
||||
ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
|
||||
|
||||
// Flush the 4 memtables that were picked in to_flush
|
||||
s = Mock_InstallMemtableFlushResults(
|
||||
&list, MutableCFOptions(options, ioptions), to_flush, &to_delete);
|
||||
s = Mock_InstallMemtableFlushResults(&list, MutableCFOptions(options),
|
||||
to_flush, &to_delete);
|
||||
ASSERT_OK(s);
|
||||
|
||||
// Note: now to_flush contains tables[0,1,2,4]. to_flush2 contains
|
||||
@ -560,7 +555,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
||||
|
||||
// Flush the 1 memtable that was picked in to_flush2
|
||||
s = MemTableListTest::Mock_InstallMemtableFlushResults(
|
||||
&list, MutableCFOptions(options, ioptions), to_flush2, &to_delete);
|
||||
&list, MutableCFOptions(options), to_flush2, &to_delete);
|
||||
ASSERT_OK(s);
|
||||
|
||||
// This will actually install 2 tables. The 1 we told it to flush, and also
|
||||
|
||||
@ -141,7 +141,7 @@ class Repairer {
|
||||
cf_name + ", id=" + ToString(cf_id));
|
||||
}
|
||||
Options opts(db_options_, *cf_opts);
|
||||
MutableCFOptions mut_cf_opts(opts, ImmutableCFOptions(opts));
|
||||
MutableCFOptions mut_cf_opts(opts);
|
||||
|
||||
VersionEdit edit;
|
||||
edit.SetComparatorName(opts.comparator->Name());
|
||||
|
||||
@ -29,9 +29,9 @@ class VersionBuilderTest : public testing::Test {
|
||||
: ucmp_(BytewiseComparator()),
|
||||
icmp_(ucmp_),
|
||||
ioptions_(options_),
|
||||
mutable_cf_options_(options_, ioptions_),
|
||||
mutable_cf_options_(options_),
|
||||
vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
|
||||
nullptr),
|
||||
nullptr),
|
||||
file_num_(1) {
|
||||
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
|
||||
size_being_compacted_.resize(options_.num_levels);
|
||||
@ -77,7 +77,7 @@ class VersionBuilderTest : public testing::Test {
|
||||
}
|
||||
|
||||
void UpdateVersionStorageInfo() {
|
||||
vstorage_.UpdateFilesByCompactionPri(mutable_cf_options_);
|
||||
vstorage_.UpdateFilesByCompactionPri(ioptions_.compaction_pri);
|
||||
vstorage_.UpdateNumNonEmptyLevels();
|
||||
vstorage_.GenerateFileIndexer();
|
||||
vstorage_.GenerateLevelFilesBrief();
|
||||
|
||||
@ -1014,7 +1014,7 @@ void Version::PrepareApply(
|
||||
UpdateAccumulatedStats(update_stats);
|
||||
storage_info_.UpdateNumNonEmptyLevels();
|
||||
storage_info_.CalculateBaseBytes(*cfd_->ioptions(), mutable_cf_options);
|
||||
storage_info_.UpdateFilesByCompactionPri(mutable_cf_options);
|
||||
storage_info_.UpdateFilesByCompactionPri(cfd_->ioptions()->compaction_pri);
|
||||
storage_info_.GenerateFileIndexer();
|
||||
storage_info_.GenerateLevelFilesBrief();
|
||||
storage_info_.GenerateLevel0NonOverlapping();
|
||||
@ -1240,6 +1240,7 @@ void VersionStorageInfo::EstimateCompactionBytesNeeded(
|
||||
}
|
||||
|
||||
void VersionStorageInfo::ComputeCompactionScore(
|
||||
const ImmutableCFOptions& immutable_cf_options,
|
||||
const MutableCFOptions& mutable_cf_options) {
|
||||
for (int level = 0; level <= MaxInputLevel(); level++) {
|
||||
double score;
|
||||
@ -1275,8 +1276,9 @@ void VersionStorageInfo::ComputeCompactionScore(
|
||||
}
|
||||
|
||||
if (compaction_style_ == kCompactionStyleFIFO) {
|
||||
score = static_cast<double>(total_size) /
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size;
|
||||
score =
|
||||
static_cast<double>(total_size) /
|
||||
immutable_cf_options.compaction_options_fifo.max_table_files_size;
|
||||
} else {
|
||||
score = static_cast<double>(num_sorted_runs) /
|
||||
mutable_cf_options.level0_file_num_compaction_trigger;
|
||||
@ -1476,7 +1478,7 @@ void SortFileByOverlappingRatio(
|
||||
} // namespace
|
||||
|
||||
void VersionStorageInfo::UpdateFilesByCompactionPri(
|
||||
const MutableCFOptions& mutable_cf_options) {
|
||||
CompactionPri compaction_pri) {
|
||||
if (compaction_style_ == kCompactionStyleFIFO ||
|
||||
compaction_style_ == kCompactionStyleUniversal) {
|
||||
// don't need this
|
||||
@ -1500,7 +1502,7 @@ void VersionStorageInfo::UpdateFilesByCompactionPri(
|
||||
if (num > temp.size()) {
|
||||
num = temp.size();
|
||||
}
|
||||
switch (mutable_cf_options.compaction_pri) {
|
||||
switch (compaction_pri) {
|
||||
case kByCompensatedSize:
|
||||
std::partial_sort(temp.begin(), temp.begin() + num, temp.end(),
|
||||
CompareCompensatedSizeDescending);
|
||||
@ -2132,6 +2134,7 @@ void VersionSet::AppendVersion(ColumnFamilyData* column_family_data,
|
||||
Version* v) {
|
||||
// compute new compaction score
|
||||
v->storage_info()->ComputeCompactionScore(
|
||||
*column_family_data->ioptions(),
|
||||
*column_family_data->GetLatestMutableCFOptions());
|
||||
|
||||
// Mark v finalized
|
||||
@ -2909,7 +2912,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
|
||||
vstorage->files_ = new_files_list;
|
||||
vstorage->num_levels_ = new_levels;
|
||||
|
||||
MutableCFOptions mutable_cf_options(*options, ImmutableCFOptions(*options));
|
||||
MutableCFOptions mutable_cf_options(*options);
|
||||
VersionEdit ve;
|
||||
InstrumentedMutex dummy_mutex;
|
||||
InstrumentedMutexLock l(&dummy_mutex);
|
||||
|
||||
@ -120,7 +120,8 @@ class VersionStorageInfo {
|
||||
// We use compaction scores to figure out which compaction to do next
|
||||
// REQUIRES: db_mutex held!!
|
||||
// TODO find a better way to pass compaction_options_fifo.
|
||||
void ComputeCompactionScore(const MutableCFOptions& mutable_cf_options);
|
||||
void ComputeCompactionScore(const ImmutableCFOptions& immutable_cf_options,
|
||||
const MutableCFOptions& mutable_cf_options);
|
||||
|
||||
// Estimate est_comp_needed_bytes_
|
||||
void EstimateCompactionBytesNeeded(
|
||||
@ -135,7 +136,7 @@ class VersionStorageInfo {
|
||||
// Sort all files for this version based on their file size and
|
||||
// record results in files_by_compaction_pri_. The largest files are listed
|
||||
// first.
|
||||
void UpdateFilesByCompactionPri(const MutableCFOptions& mutable_cf_options);
|
||||
void UpdateFilesByCompactionPri(CompactionPri compaction_pri);
|
||||
|
||||
void GenerateLevel0NonOverlapping();
|
||||
bool level0_non_overlapping() const {
|
||||
|
||||
@ -109,7 +109,7 @@ class VersionStorageInfoTest : public testing::Test {
|
||||
logger_(new CountingLogger()),
|
||||
options_(GetOptionsWithNumLevels(6, logger_)),
|
||||
ioptions_(options_),
|
||||
mutable_cf_options_(options_, ioptions_),
|
||||
mutable_cf_options_(options_),
|
||||
vstorage_(&icmp_, ucmp_, 6, kCompactionStyleLevel, nullptr) {}
|
||||
|
||||
~VersionStorageInfoTest() {
|
||||
|
||||
@ -31,9 +31,8 @@ static std::string PrintContents(WriteBatch* b) {
|
||||
options.memtable_factory = factory;
|
||||
ImmutableCFOptions ioptions(options);
|
||||
WriteBufferManager wb(options.db_write_buffer_size);
|
||||
MemTable* mem =
|
||||
new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
|
||||
kMaxSequenceNumber);
|
||||
MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
|
||||
kMaxSequenceNumber);
|
||||
mem->Ref();
|
||||
std::string state;
|
||||
ColumnFamilyMemTablesDefault cf_mems_default(mem);
|
||||
|
||||
@ -46,8 +46,7 @@ jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(
|
||||
options.memtable_factory = factory;
|
||||
rocksdb::MemTable* mem = new rocksdb::MemTable(
|
||||
cmp, rocksdb::ImmutableCFOptions(options),
|
||||
rocksdb::MutableCFOptions(options, rocksdb::ImmutableCFOptions(options)),
|
||||
&wb, rocksdb::kMaxSequenceNumber);
|
||||
rocksdb::MutableCFOptions(options), &wb, rocksdb::kMaxSequenceNumber);
|
||||
mem->Ref();
|
||||
std::string state;
|
||||
rocksdb::ColumnFamilyMemTablesDefault cf_mems_default(mem);
|
||||
|
||||
@ -75,7 +75,7 @@ struct SstFileWriter::Rep {
|
||||
const Comparator* _user_comparator)
|
||||
: env_options(_env_options),
|
||||
ioptions(options),
|
||||
mutable_cf_options(options, ioptions),
|
||||
mutable_cf_options(options),
|
||||
internal_comparator(_user_comparator) {}
|
||||
|
||||
std::unique_ptr<WritableFileWriter> file_writer;
|
||||
|
||||
@ -415,9 +415,9 @@ class MemTableConstructor: public Constructor {
|
||||
table_factory_(new SkipListFactory) {
|
||||
options_.memtable_factory = table_factory_;
|
||||
ImmutableCFOptions ioptions(options_);
|
||||
memtable_ = new MemTable(internal_comparator_, ioptions,
|
||||
MutableCFOptions(options_, ioptions), wb,
|
||||
kMaxSequenceNumber);
|
||||
memtable_ =
|
||||
new MemTable(internal_comparator_, ioptions, MutableCFOptions(options_),
|
||||
wb, kMaxSequenceNumber);
|
||||
memtable_->Ref();
|
||||
}
|
||||
~MemTableConstructor() {
|
||||
@ -430,8 +430,8 @@ class MemTableConstructor: public Constructor {
|
||||
delete memtable_->Unref();
|
||||
ImmutableCFOptions mem_ioptions(ioptions);
|
||||
memtable_ = new MemTable(internal_comparator_, mem_ioptions,
|
||||
MutableCFOptions(options_, mem_ioptions),
|
||||
write_buffer_manager_, kMaxSequenceNumber);
|
||||
MutableCFOptions(options_), write_buffer_manager_,
|
||||
kMaxSequenceNumber);
|
||||
memtable_->Ref();
|
||||
int seq = 1;
|
||||
for (const auto kv : kv_map) {
|
||||
@ -2410,9 +2410,8 @@ TEST_F(MemTableTest, Simple) {
|
||||
options.memtable_factory = table_factory;
|
||||
ImmutableCFOptions ioptions(options);
|
||||
WriteBufferManager wb(options.db_write_buffer_size);
|
||||
MemTable* memtable =
|
||||
new MemTable(cmp, ioptions, MutableCFOptions(options, ioptions), &wb,
|
||||
kMaxSequenceNumber);
|
||||
MemTable* memtable = new MemTable(cmp, ioptions, MutableCFOptions(options),
|
||||
&wb, kMaxSequenceNumber);
|
||||
memtable->Ref();
|
||||
WriteBatch batch;
|
||||
WriteBatchInternal::SetSequence(&batch, 100);
|
||||
|
||||
@ -10,8 +10,8 @@
|
||||
#endif
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <limits>
|
||||
#include <cassert>
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include "port/port.h"
|
||||
#include "rocksdb/env.h"
|
||||
@ -21,6 +21,7 @@ namespace rocksdb {
|
||||
|
||||
ImmutableCFOptions::ImmutableCFOptions(const Options& options)
|
||||
: compaction_style(options.compaction_style),
|
||||
compaction_pri(options.compaction_pri),
|
||||
compaction_options_universal(options.compaction_options_universal),
|
||||
compaction_options_fifo(options.compaction_options_fifo),
|
||||
prefix_extractor(options.prefix_extractor.get()),
|
||||
@ -58,7 +59,8 @@ ImmutableCFOptions::ImmutableCFOptions(const Options& options)
|
||||
num_levels(options.num_levels),
|
||||
optimize_filters_for_hits(options.optimize_filters_for_hits),
|
||||
listeners(options.listeners),
|
||||
row_cache(options.row_cache) {}
|
||||
row_cache(options.row_cache),
|
||||
max_subcompactions(options.max_subcompactions) {}
|
||||
|
||||
// Multiple two operands. If they overflow, return op1.
|
||||
uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
|
||||
@ -75,11 +77,11 @@ uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
|
||||
return op1 * casted_op2;
|
||||
}
|
||||
|
||||
void MutableCFOptions::RefreshDerivedOptions(
|
||||
const ImmutableCFOptions& ioptions) {
|
||||
max_file_size.resize(ioptions.num_levels);
|
||||
for (int i = 0; i < ioptions.num_levels; ++i) {
|
||||
if (i == 0 && ioptions.compaction_style == kCompactionStyleUniversal) {
|
||||
void MutableCFOptions::RefreshDerivedOptions(int num_levels,
|
||||
CompactionStyle compaction_style) {
|
||||
max_file_size.resize(num_levels);
|
||||
for (int i = 0; i < num_levels; ++i) {
|
||||
if (i == 0 && compaction_style == kCompactionStyleUniversal) {
|
||||
max_file_size[i] = ULLONG_MAX;
|
||||
} else if (i > 1) {
|
||||
max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1],
|
||||
@ -106,10 +108,12 @@ void MutableCFOptions::Dump(Logger* log) const {
|
||||
arena_block_size);
|
||||
Log(log, " memtable_prefix_bloom_ratio: %f",
|
||||
memtable_prefix_bloom_size_ratio);
|
||||
Log(log, " memtable_huge_page_size: %" ROCKSDB_PRIszt,
|
||||
Log(log, " memtable_huge_page_size: %" ROCKSDB_PRIszt,
|
||||
memtable_huge_page_size);
|
||||
Log(log, " max_successive_merges: %" ROCKSDB_PRIszt,
|
||||
max_successive_merges);
|
||||
Log(log, " inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
||||
inplace_update_num_locks);
|
||||
Log(log, " disable_auto_compactions: %d",
|
||||
disable_auto_compactions);
|
||||
Log(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
|
||||
@ -144,6 +148,13 @@ void MutableCFOptions::Dump(Logger* log) const {
|
||||
verify_checksums_in_compaction);
|
||||
Log(log, " max_sequential_skip_in_iterations: %" PRIu64,
|
||||
max_sequential_skip_in_iterations);
|
||||
Log(log, " paranoid_file_checks: %d",
|
||||
paranoid_file_checks);
|
||||
Log(log, " report_bg_io_stats: %d", report_bg_io_stats);
|
||||
Log(log, " compression: %d",
|
||||
static_cast<int>(compression));
|
||||
Log(log, " min_partial_merge_operands: %" PRIu32,
|
||||
min_partial_merge_operands);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
@ -22,6 +22,8 @@ struct ImmutableCFOptions {
|
||||
|
||||
CompactionStyle compaction_style;
|
||||
|
||||
CompactionPri compaction_pri;
|
||||
|
||||
CompactionOptionsUniversal compaction_options_universal;
|
||||
CompactionOptionsFIFO compaction_options_fifo;
|
||||
|
||||
@ -102,10 +104,12 @@ struct ImmutableCFOptions {
|
||||
std::vector<std::shared_ptr<EventListener>> listeners;
|
||||
|
||||
std::shared_ptr<Cache> row_cache;
|
||||
|
||||
uint32_t max_subcompactions;
|
||||
};
|
||||
|
||||
struct MutableCFOptions {
|
||||
MutableCFOptions(const Options& options, const ImmutableCFOptions& ioptions)
|
||||
explicit MutableCFOptions(const ColumnFamilyOptions& options)
|
||||
: write_buffer_size(options.write_buffer_size),
|
||||
max_write_buffer_number(options.max_write_buffer_number),
|
||||
arena_block_size(options.arena_block_size),
|
||||
@ -123,7 +127,6 @@ struct MutableCFOptions {
|
||||
options.level0_file_num_compaction_trigger),
|
||||
level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
|
||||
level0_stop_writes_trigger(options.level0_stop_writes_trigger),
|
||||
compaction_pri(options.compaction_pri),
|
||||
max_compaction_bytes(options.max_compaction_bytes),
|
||||
target_file_size_base(options.target_file_size_base),
|
||||
target_file_size_multiplier(options.target_file_size_multiplier),
|
||||
@ -132,16 +135,15 @@ struct MutableCFOptions {
|
||||
max_bytes_for_level_multiplier_additional(
|
||||
options.max_bytes_for_level_multiplier_additional),
|
||||
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
|
||||
max_subcompactions(options.max_subcompactions),
|
||||
max_sequential_skip_in_iterations(
|
||||
options.max_sequential_skip_in_iterations),
|
||||
paranoid_file_checks(options.paranoid_file_checks),
|
||||
report_bg_io_stats(options.report_bg_io_stats),
|
||||
compression(options.compression),
|
||||
min_partial_merge_operands(options.min_partial_merge_operands),
|
||||
compaction_options_fifo(ioptions.compaction_options_fifo) {
|
||||
RefreshDerivedOptions(ioptions);
|
||||
min_partial_merge_operands(options.min_partial_merge_operands) {
|
||||
RefreshDerivedOptions(options.num_levels, options.compaction_style);
|
||||
}
|
||||
|
||||
MutableCFOptions()
|
||||
: write_buffer_size(0),
|
||||
max_write_buffer_number(0),
|
||||
@ -156,14 +158,12 @@ struct MutableCFOptions {
|
||||
level0_file_num_compaction_trigger(0),
|
||||
level0_slowdown_writes_trigger(0),
|
||||
level0_stop_writes_trigger(0),
|
||||
compaction_pri(kByCompensatedSize),
|
||||
max_compaction_bytes(0),
|
||||
target_file_size_base(0),
|
||||
target_file_size_multiplier(0),
|
||||
max_bytes_for_level_base(0),
|
||||
max_bytes_for_level_multiplier(0),
|
||||
verify_checksums_in_compaction(false),
|
||||
max_subcompactions(1),
|
||||
max_sequential_skip_in_iterations(0),
|
||||
paranoid_file_checks(false),
|
||||
report_bg_io_stats(false),
|
||||
@ -171,7 +171,11 @@ struct MutableCFOptions {
|
||||
min_partial_merge_operands(2) {}
|
||||
|
||||
// Must be called after any change to MutableCFOptions
|
||||
void RefreshDerivedOptions(const ImmutableCFOptions& ioptions);
|
||||
void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style);
|
||||
|
||||
void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) {
|
||||
RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style);
|
||||
}
|
||||
|
||||
// Get the max file size in a given level.
|
||||
uint64_t MaxFileSizeForLevel(int level) const;
|
||||
@ -201,7 +205,6 @@ struct MutableCFOptions {
|
||||
int level0_file_num_compaction_trigger;
|
||||
int level0_slowdown_writes_trigger;
|
||||
int level0_stop_writes_trigger;
|
||||
CompactionPri compaction_pri;
|
||||
uint64_t max_compaction_bytes;
|
||||
uint64_t target_file_size_base;
|
||||
int target_file_size_multiplier;
|
||||
@ -209,7 +212,6 @@ struct MutableCFOptions {
|
||||
int max_bytes_for_level_multiplier;
|
||||
std::vector<int> max_bytes_for_level_multiplier_additional;
|
||||
bool verify_checksums_in_compaction;
|
||||
int max_subcompactions;
|
||||
|
||||
// Misc options
|
||||
uint64_t max_sequential_skip_in_iterations;
|
||||
@ -217,7 +219,6 @@ struct MutableCFOptions {
|
||||
bool report_bg_io_stats;
|
||||
CompressionType compression;
|
||||
uint32_t min_partial_merge_operands;
|
||||
CompactionOptionsFIFO compaction_options_fifo;
|
||||
|
||||
// Derived options
|
||||
// Per-level target file size.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user