3ce3bb3da2
Summary: This diff updates the logic of how we do trivial move, now trivial move can run on any number of files in input level as long as they are not overlapping The conditions for trivial move have been updated Introduced conditions: - Trivial move cannot happen if we have a compaction filter (except if the compaction is not manual) - Input level files cannot be overlapping Removed conditions: - Trivial move only run when the compaction is not manual - Input level should can contain only 1 file More context on what tests failed because of Trivial move ``` DBTest.CompactionsGenerateMultipleFiles This test is expecting compaction on a file in L0 to generate multiple files in L1, this test will fail with trivial move because we end up with one file in L1 ``` ``` DBTest.NoSpaceCompactRange This test expect compaction to fail when we force environment to report running out of space, of course this is not valid in trivial move situation because trivial move does not need any extra space, and did not check for that ``` ``` DBTest.DropWrites Similar to DBTest.NoSpaceCompactRange ``` ``` DBTest.DeleteObsoleteFilesPendingOutputs This test expect that a file in L2 is deleted after it's moved to L3, this is not valid with trivial move because although the file was moved it is now used by L3 ``` ``` CuckooTableDBTest.CompactionIntoMultipleFiles Same as DBTest.CompactionsGenerateMultipleFiles ``` This diff is based on a work by @sdong https://reviews.facebook.net/D34149 Test Plan: make -j64 check Reviewers: rven, sdong, igor Reviewed By: igor Subscribers: yhchiang, ott, march, dhruba, sdong Differential Revision: https://reviews.facebook.net/D34797
306 lines
10 KiB
C++
306 lines
10 KiB
C++
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under the BSD-style license found in the
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
#include <string>
|
|
#include "db/version_edit.h"
|
|
#include "db/version_set.h"
|
|
#include "util/logging.h"
|
|
#include "util/string_util.h"
|
|
#include "util/testharness.h"
|
|
#include "util/testutil.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class VersionBuilderTest : public testing::Test {
|
|
public:
|
|
const Comparator* ucmp_;
|
|
InternalKeyComparator icmp_;
|
|
Options options_;
|
|
ImmutableCFOptions ioptions_;
|
|
MutableCFOptions mutable_cf_options_;
|
|
VersionStorageInfo vstorage_;
|
|
uint32_t file_num_;
|
|
CompactionOptionsFIFO fifo_options_;
|
|
std::vector<uint64_t> size_being_compacted_;
|
|
|
|
VersionBuilderTest()
|
|
: ucmp_(BytewiseComparator()),
|
|
icmp_(ucmp_),
|
|
ioptions_(options_),
|
|
mutable_cf_options_(options_, ioptions_),
|
|
vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
|
|
nullptr),
|
|
file_num_(1) {
|
|
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
|
|
size_being_compacted_.resize(options_.num_levels);
|
|
}
|
|
|
|
~VersionBuilderTest() {
|
|
for (int i = 0; i < vstorage_.num_levels(); i++) {
|
|
for (auto* f : vstorage_.LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
InternalKey GetInternalKey(const char* ukey,
|
|
SequenceNumber smallest_seq = 100) {
|
|
return InternalKey(ukey, smallest_seq, kTypeValue);
|
|
}
|
|
|
|
void Add(int level, uint32_t file_number, const char* smallest,
|
|
const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
|
|
SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100,
|
|
uint64_t num_entries = 0, uint64_t num_deletions = 0,
|
|
bool sampled = false, SequenceNumber smallest_seqno = 0,
|
|
SequenceNumber largest_seqno = 0) {
|
|
assert(level < vstorage_.num_levels());
|
|
FileMetaData* f = new FileMetaData;
|
|
f->fd = FileDescriptor(file_number, path_id, file_size);
|
|
f->smallest = GetInternalKey(smallest, smallest_seq);
|
|
f->largest = GetInternalKey(largest, largest_seq);
|
|
f->smallest_seqno = smallest_seqno;
|
|
f->largest_seqno = largest_seqno;
|
|
f->compensated_file_size = file_size;
|
|
f->refs = 0;
|
|
f->num_entries = num_entries;
|
|
f->num_deletions = num_deletions;
|
|
vstorage_.AddFile(level, f);
|
|
if (sampled) {
|
|
f->init_stats_from_file = true;
|
|
vstorage_.UpdateAccumulatedStats(f);
|
|
}
|
|
}
|
|
|
|
void UpdateVersionStorageInfo() {
|
|
vstorage_.UpdateFilesBySize();
|
|
vstorage_.UpdateNumNonEmptyLevels();
|
|
vstorage_.GenerateFileIndexer();
|
|
vstorage_.GenerateLevelFilesBrief();
|
|
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
|
|
vstorage_.GenerateLevel0NonOverlapping();
|
|
vstorage_.SetFinalized();
|
|
}
|
|
};
|
|
|
|
void UnrefFilesInVersion(VersionStorageInfo* new_vstorage) {
|
|
for (int i = 0; i < new_vstorage->num_levels(); i++) {
|
|
for (auto* f : new_vstorage->LevelFiles(i)) {
|
|
if (--f->refs == 0) {
|
|
delete f;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
|
|
Add(0, 1U, "150", "200", 100U);
|
|
|
|
Add(1, 66U, "150", "200", 100U);
|
|
Add(1, 88U, "201", "300", 100U);
|
|
|
|
Add(2, 6U, "150", "179", 100U);
|
|
Add(2, 7U, "180", "220", 100U);
|
|
Add(2, 8U, "221", "300", 100U);
|
|
|
|
Add(3, 26U, "150", "170", 100U);
|
|
Add(3, 27U, "171", "179", 100U);
|
|
Add(3, 28U, "191", "220", 100U);
|
|
Add(3, 29U, "221", "300", 100U);
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.DeleteFile(3, 27U);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2));
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
|
|
Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
|
|
|
|
Add(4, 6U, "150", "179", 100U);
|
|
Add(4, 7U, "180", "220", 100U);
|
|
Add(4, 8U, "221", "300", 100U);
|
|
|
|
Add(5, 26U, "150", "170", 100U);
|
|
Add(5, 27U, "171", "179", 100U);
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(3, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.DeleteFile(0, 1U);
|
|
version_edit.DeleteFile(0, 88U);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
|
|
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(3));
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(4));
|
|
ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
|
|
ioptions_.level_compaction_dynamic_level_bytes = true;
|
|
|
|
Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
|
|
Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
|
|
|
|
Add(4, 6U, "150", "179", 100U);
|
|
Add(4, 7U, "180", "220", 100U);
|
|
Add(4, 8U, "221", "300", 100U);
|
|
|
|
Add(5, 26U, "150", "170", 100U);
|
|
Add(5, 27U, "171", "179", 100U);
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(4, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.DeleteFile(0, 1U);
|
|
version_edit.DeleteFile(0, 88U);
|
|
version_edit.DeleteFile(4, 6U);
|
|
version_edit.DeleteFile(4, 7U);
|
|
version_edit.DeleteFile(4, 8U);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
|
|
ASSERT_EQ(100U, new_vstorage.NumLevelBytes(4));
|
|
ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
|
|
UpdateVersionStorageInfo();
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
|
|
GetInternalKey("450"), 200, 200);
|
|
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
|
|
GetInternalKey("650"), 200, 200);
|
|
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
|
|
GetInternalKey("550"), 200, 200);
|
|
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
|
|
GetInternalKey("750"), 200, 200);
|
|
|
|
EnvOptions env_options;
|
|
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
version_builder.Apply(&version_edit);
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
|
|
UpdateVersionStorageInfo();
|
|
|
|
EnvOptions env_options;
|
|
VersionBuilder version_builder(env_options, nullptr, &vstorage_);
|
|
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
|
|
kCompactionStyleLevel, nullptr);
|
|
|
|
VersionEdit version_edit;
|
|
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
|
|
GetInternalKey("350"), 200, 200);
|
|
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
|
|
GetInternalKey("450"), 200, 200);
|
|
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
|
|
GetInternalKey("650"), 200, 200);
|
|
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
|
|
GetInternalKey("550"), 200, 200);
|
|
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
|
|
GetInternalKey("750"), 200, 200);
|
|
version_builder.Apply(&version_edit);
|
|
|
|
VersionEdit version_edit2;
|
|
version_edit.AddFile(2, 808, 0, 100U, GetInternalKey("901"),
|
|
GetInternalKey("950"), 200, 200);
|
|
version_edit2.DeleteFile(2, 616);
|
|
version_edit2.DeleteFile(2, 636);
|
|
version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"),
|
|
GetInternalKey("850"), 200, 200);
|
|
version_builder.Apply(&version_edit2);
|
|
|
|
version_builder.SaveTo(&new_vstorage);
|
|
|
|
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2));
|
|
|
|
UnrefFilesInVersion(&new_vstorage);
|
|
}
|
|
|
|
TEST_F(VersionBuilderTest, EstimatedActiveKeys) {
|
|
const uint32_t kTotalSamples = 20;
|
|
const uint32_t kNumLevels = 5;
|
|
const uint32_t kFilesPerLevel = 8;
|
|
const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
|
|
const uint32_t kEntriesPerFile = 1000;
|
|
const uint32_t kDeletionsPerFile = 100;
|
|
for (uint32_t i = 0; i < kNumFiles; ++i) {
|
|
Add(static_cast<int>(i / kFilesPerLevel), i + 1,
|
|
ToString((i + 100) * 1000).c_str(),
|
|
ToString((i + 100) * 1000 + 999).c_str(),
|
|
100U, 0, 100, 100,
|
|
kEntriesPerFile, kDeletionsPerFile,
|
|
(i < kTotalSamples));
|
|
}
|
|
// minus 2X for the number of deletion entries because:
|
|
// 1x for deletion entry does not count as a data entry.
|
|
// 1x for each deletion entry will actually remove one data entry.
|
|
ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
|
|
(kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
|
|
}
|
|
|
|
} // namespace rocksdb
|
|
|
|
int main(int argc, char** argv) {
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
return RUN_ALL_TESTS();
|
|
}
|