Update test to cover a new case in file ingestion (#4614)
Summary: The new case is directIO = true, write_global_seqno = false in which we no longer write global_seqno to the external SST file. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4614 Differential Revision: D12885001 Pulled By: riversand963 fbshipit-source-id: 7541bdc608b3a0c93d3c3c435da1b162b36673d4
This commit is contained in:
Yanqin Jin
Facebook Github Bot
parent
3f8f81cfeb
commit
de18a2d82e
@ -259,6 +259,47 @@ bool DBTestBase::ChangeFilterOptions() {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Switch between different DB options for file ingestion tests.
|
||||
bool DBTestBase::ChangeOptionsForFileIngestionTest() {
|
||||
if (option_config_ == kDefault) {
|
||||
option_config_ = kUniversalCompaction;
|
||||
Destroy(last_options_);
|
||||
auto options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
TryReopen(options);
|
||||
return true;
|
||||
} else if (option_config_ == kUniversalCompaction) {
|
||||
option_config_ = kUniversalCompactionMultiLevel;
|
||||
Destroy(last_options_);
|
||||
auto options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
TryReopen(options);
|
||||
return true;
|
||||
} else if (option_config_ == kUniversalCompactionMultiLevel) {
|
||||
option_config_ = kLevelSubcompactions;
|
||||
Destroy(last_options_);
|
||||
auto options = CurrentOptions();
|
||||
assert(options.max_subcompactions > 1);
|
||||
TryReopen(options);
|
||||
return true;
|
||||
} else if (option_config_ == kLevelSubcompactions) {
|
||||
option_config_ = kUniversalSubcompactions;
|
||||
Destroy(last_options_);
|
||||
auto options = CurrentOptions();
|
||||
assert(options.max_subcompactions > 1);
|
||||
TryReopen(options);
|
||||
return true;
|
||||
} else if (option_config_ == kUniversalSubcompactions) {
|
||||
option_config_ = kDirectIO;
|
||||
Destroy(last_options_);
|
||||
auto options = CurrentOptions();
|
||||
TryReopen(options);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Return the current option configuration.
|
||||
Options DBTestBase::CurrentOptions(
|
||||
const anon::OptionsOverride& options_override) const {
|
||||
|
||||
@ -747,6 +747,9 @@ class DBTestBase : public testing::Test {
|
||||
// Jump from kDefault to kFilter to kFullFilter
|
||||
bool ChangeFilterOptions();
|
||||
|
||||
// Switch between different DB options for file ingestion tests.
|
||||
bool ChangeOptionsForFileIngestionTest();
|
||||
|
||||
// Return the current option configuration.
|
||||
Options CurrentOptions(const anon::OptionsOverride& options_override =
|
||||
anon::OptionsOverride()) const;
|
||||
|
||||
@ -14,7 +14,8 @@
|
||||
namespace rocksdb {
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
class ExternalSSTFileBasicTest : public DBTestBase {
|
||||
class ExternalSSTFileBasicTest : public DBTestBase,
|
||||
public ::testing::WithParamInterface<bool> {
|
||||
public:
|
||||
ExternalSSTFileBasicTest() : DBTestBase("/external_sst_file_test") {
|
||||
sst_files_dir_ = dbname_ + "/sst_files/";
|
||||
@ -41,7 +42,7 @@ class ExternalSSTFileBasicTest : public DBTestBase {
|
||||
const Options options, std::vector<int> keys,
|
||||
const std::vector<ValueType>& value_types,
|
||||
std::vector<std::pair<int, int>> range_deletions, int file_id,
|
||||
std::map<std::string, std::string>* true_data) {
|
||||
bool write_global_seqno, std::map<std::string, std::string>* true_data) {
|
||||
assert(value_types.size() == 1 || keys.size() == value_types.size());
|
||||
std::string file_path = sst_files_dir_ + ToString(file_id);
|
||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||
@ -105,6 +106,7 @@ class ExternalSSTFileBasicTest : public DBTestBase {
|
||||
if (s.ok()) {
|
||||
IngestExternalFileOptions ifo;
|
||||
ifo.allow_global_seqno = true;
|
||||
ifo.write_global_seqno = write_global_seqno;
|
||||
s = db_->IngestExternalFile({file_path}, ifo);
|
||||
}
|
||||
return s;
|
||||
@ -113,17 +115,18 @@ class ExternalSSTFileBasicTest : public DBTestBase {
|
||||
Status GenerateAndAddExternalFile(
|
||||
const Options options, std::vector<int> keys,
|
||||
const std::vector<ValueType>& value_types, int file_id,
|
||||
std::map<std::string, std::string>* true_data) {
|
||||
bool write_global_seqno, std::map<std::string, std::string>* true_data) {
|
||||
return GenerateAndAddExternalFile(options, keys, value_types, {}, file_id,
|
||||
true_data);
|
||||
write_global_seqno, true_data);
|
||||
}
|
||||
|
||||
Status GenerateAndAddExternalFile(
|
||||
const Options options, std::vector<int> keys, const ValueType value_type,
|
||||
int file_id, std::map<std::string, std::string>* true_data) {
|
||||
int file_id, bool write_global_seqno,
|
||||
std::map<std::string, std::string>* true_data) {
|
||||
return GenerateAndAddExternalFile(options, keys,
|
||||
std::vector<ValueType>(1, value_type),
|
||||
file_id, true_data);
|
||||
file_id, write_global_seqno, true_data);
|
||||
}
|
||||
|
||||
~ExternalSSTFileBasicTest() { test::DestroyDir(env_, sst_files_dir_); }
|
||||
@ -245,7 +248,8 @@ TEST_F(ExternalSSTFileBasicTest, NoCopy) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
|
||||
TEST_P(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
|
||||
bool write_global_seqno = GetParam();
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
DestroyAndReopen(options);
|
||||
@ -255,33 +259,37 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1, 4, 6}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 4, 6},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {11, 15, 19}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {11, 15, 19},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {120, 130}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {120, 130},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1, 130}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 130},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
|
||||
|
||||
@ -292,18 +300,21 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
|
||||
}
|
||||
SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {60, 61, 62}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {60, 61, 62},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {40, 41, 42}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {40, 41, 42},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {20, 30, 40}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {20, 30, 40},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
|
||||
|
||||
@ -311,35 +322,39 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
|
||||
|
||||
// We will need a seqno for the file regardless if the file overwrite
|
||||
// keys in the DB or not because we have a snapshot
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1000, 1002}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1000, 1002},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {2000, 3002}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {2000, 3002},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
|
||||
|
||||
db_->ReleaseSnapshot(snapshot);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {5000, 5001}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {5000, 5001},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// No snapshot anymore, no need to assign a seqno
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
|
||||
|
||||
size_t kcnt = 0;
|
||||
VerifyDBFromMap(true_data, &kcnt, false);
|
||||
} while (ChangeCompactOptions());
|
||||
} while (ChangeOptionsForFileIngestionTest());
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) {
|
||||
TEST_P(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) {
|
||||
bool write_global_seqno = GetParam();
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
options.merge_operator.reset(new TestPutOperator());
|
||||
@ -350,52 +365,57 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) {
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1, 4, 6}, ValueType::kTypeMerge, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 4, 6},
|
||||
ValueType::kTypeMerge, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {11, 15, 19},
|
||||
ValueType::kTypeDeletion, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {120, 130}, ValueType::kTypeMerge, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {120, 130},
|
||||
ValueType::kTypeMerge, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1, 130}, ValueType::kTypeDeletion, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 130},
|
||||
ValueType::kTypeDeletion, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {120},
|
||||
{ValueType::kTypeValue}, {{120, 135}},
|
||||
file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {120}, {ValueType::kTypeValue}, {{120, 135}}, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {}, {}, {{110, 120}},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno,
|
||||
&true_data));
|
||||
// The range deletion ends on a key, but it doesn't actually delete
|
||||
// this key because the largest key in the range is exclusive. Still,
|
||||
// it counts as an overlap so a new seqno will be assigned.
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {}, {}, {{100, 109}},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno,
|
||||
&true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5);
|
||||
|
||||
@ -406,19 +426,21 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) {
|
||||
}
|
||||
SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {60, 61, 62}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {60, 61, 62},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {40, 41, 42}, ValueType::kTypeMerge, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {40, 41, 42},
|
||||
ValueType::kTypeMerge, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {20, 30, 40},
|
||||
ValueType::kTypeDeletion, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
|
||||
|
||||
@ -426,35 +448,39 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) {
|
||||
|
||||
// We will need a seqno for the file regardless if the file overwrite
|
||||
// keys in the DB or not because we have a snapshot
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1000, 1002}, ValueType::kTypeMerge, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1000, 1002},
|
||||
ValueType::kTypeMerge, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {2000, 3002}, ValueType::kTypeMerge, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {2000, 3002},
|
||||
ValueType::kTypeMerge, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150},
|
||||
ValueType::kTypeMerge, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
|
||||
|
||||
db_->ReleaseSnapshot(snapshot);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {5000, 5001}, ValueType::kTypeValue, file_id++, &true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {5000, 5001},
|
||||
ValueType::kTypeValue, file_id++,
|
||||
write_global_seqno, &true_data));
|
||||
// No snapshot anymore, no need to assign a seqno
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
|
||||
|
||||
size_t kcnt = 0;
|
||||
VerifyDBFromMap(true_data, &kcnt, false);
|
||||
} while (ChangeCompactOptions());
|
||||
} while (ChangeOptionsForFileIngestionTest());
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
TEST_P(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
bool write_global_seqno = GetParam();
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
options.merge_operator.reset(new TestPutOperator());
|
||||
@ -467,7 +493,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
options, {1, 2, 3, 4, 5, 6},
|
||||
{ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue,
|
||||
ValueType::kTypeMerge, ValueType::kTypeValue, ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
|
||||
|
||||
@ -475,33 +501,35 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
options, {10, 11, 12, 13},
|
||||
{ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue,
|
||||
ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1, 4, 6}, {ValueType::kTypeDeletion, ValueType::kTypeValue,
|
||||
options, {1, 4, 6},
|
||||
{ValueType::kTypeDeletion, ValueType::kTypeValue,
|
||||
ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {11, 15, 19}, {ValueType::kTypeDeletion, ValueType::kTypeMerge,
|
||||
options, {11, 15, 19},
|
||||
{ValueType::kTypeDeletion, ValueType::kTypeMerge,
|
||||
ValueType::kTypeValue},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {120, 130}, {ValueType::kTypeValue, ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1, 130}, {ValueType::kTypeMerge, ValueType::kTypeDeletion},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
|
||||
|
||||
@ -509,14 +537,14 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
options, {150, 151, 152},
|
||||
{ValueType::kTypeValue, ValueType::kTypeMerge,
|
||||
ValueType::kTypeDeletion},
|
||||
{{150, 160}, {180, 190}}, file_id++, &true_data));
|
||||
{{150, 160}, {180, 190}}, file_id++, write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {150, 151, 152},
|
||||
{ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue},
|
||||
{{200, 250}}, file_id++, &true_data));
|
||||
{{200, 250}}, file_id++, write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4);
|
||||
|
||||
@ -524,7 +552,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
options, {300, 301, 302},
|
||||
{ValueType::kTypeValue, ValueType::kTypeMerge,
|
||||
ValueType::kTypeDeletion},
|
||||
{{1, 2}, {152, 154}}, file_id++, &true_data));
|
||||
{{1, 2}, {152, 154}}, file_id++, write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5);
|
||||
|
||||
@ -538,7 +566,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {60, 61, 62},
|
||||
{ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File doesn't overwrite any keys, no seqno needed
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
|
||||
|
||||
@ -546,7 +574,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
options, {40, 41, 42},
|
||||
{ValueType::kTypeValue, ValueType::kTypeDeletion,
|
||||
ValueType::kTypeDeletion},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
|
||||
|
||||
@ -554,7 +582,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
options, {20, 30, 40},
|
||||
{ValueType::kTypeDeletion, ValueType::kTypeDeletion,
|
||||
ValueType::kTypeDeletion},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// File overwrites some keys, a seqno will be assigned
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
|
||||
|
||||
@ -564,13 +592,13 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
// keys in the DB or not because we have a snapshot
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {1000, 1002}, {ValueType::kTypeValue, ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {2000, 3002}, {ValueType::kTypeValue, ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
|
||||
|
||||
@ -578,7 +606,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
options, {1, 20, 40, 100, 150},
|
||||
{ValueType::kTypeDeletion, ValueType::kTypeDeletion,
|
||||
ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// A global seqno will be assigned anyway because of the snapshot
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
|
||||
|
||||
@ -586,13 +614,13 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {5000, 5001}, {ValueType::kTypeValue, ValueType::kTypeMerge},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
// No snapshot anymore, no need to assign a seqno
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
|
||||
|
||||
size_t kcnt = 0;
|
||||
VerifyDBFromMap(true_data, &kcnt, false);
|
||||
} while (ChangeCompactOptions());
|
||||
} while (ChangeOptionsForFileIngestionTest());
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) {
|
||||
@ -635,7 +663,7 @@ TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) {
|
||||
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
|
||||
TEST_P(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
|
||||
int kNumLevels = 7;
|
||||
Options options = CurrentOptions();
|
||||
options.disable_auto_compactions = true;
|
||||
@ -662,12 +690,13 @@ TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
|
||||
ASSERT_EQ(0, NumTableFilesAtLevel(kNumLevels - 2));
|
||||
ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 1));
|
||||
|
||||
bool write_global_seqno = GetParam();
|
||||
// overlaps with L0 file but not memtable, so flush is skipped and file is
|
||||
// ingested into L0
|
||||
SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {60, 90}, {ValueType::kTypeValue, ValueType::kTypeValue},
|
||||
{{65, 70}, {70, 85}}, file_id++, &true_data));
|
||||
{{65, 70}, {70, 85}}, file_id++, write_global_seqno, &true_data));
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno);
|
||||
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
||||
ASSERT_EQ(0, NumTableFilesAtLevel(kNumLevels - 2));
|
||||
@ -677,7 +706,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
|
||||
// file is ingested into L5
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {10, 40}, {ValueType::kTypeValue, ValueType::kTypeValue},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno);
|
||||
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
||||
ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2));
|
||||
@ -686,7 +715,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
|
||||
// overlaps with L5 file but not memtable or L0 file, so flush is skipped and
|
||||
// file is ingested into L4
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {}, {}, {{5, 15}}, file_id++,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno);
|
||||
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
||||
ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2));
|
||||
@ -698,7 +727,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
|
||||
// count increases by two.
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {100, 140}, {ValueType::kTypeValue, ValueType::kTypeValue},
|
||||
file_id++, &true_data));
|
||||
file_id++, write_global_seqno, &true_data));
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno);
|
||||
ASSERT_EQ(4, NumTableFilesAtLevel(0));
|
||||
ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2));
|
||||
@ -711,13 +740,16 @@ TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
|
||||
// seqnum.
|
||||
ASSERT_OK(GenerateAndAddExternalFile(
|
||||
options, {151, 175}, {ValueType::kTypeValue, ValueType::kTypeValue},
|
||||
{{160, 200}}, file_id++, &true_data));
|
||||
{{160, 200}}, file_id++, write_global_seqno, &true_data));
|
||||
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
|
||||
ASSERT_EQ(4, NumTableFilesAtLevel(0));
|
||||
ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2));
|
||||
ASSERT_EQ(2, NumTableFilesAtLevel(options.num_levels - 1));
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(ExternalSSTFileBasicTest, ExternalSSTFileBasicTest,
|
||||
testing::Bool());
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
@ -15,7 +15,8 @@
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class ExternalSSTFileTest : public DBTestBase {
|
||||
class ExternalSSTFileTest : public DBTestBase,
|
||||
public ::testing::WithParamInterface<bool> {
|
||||
public:
|
||||
ExternalSSTFileTest() : DBTestBase("/external_sst_file_test") {
|
||||
sst_files_dir_ = dbname_ + "/sst_files/";
|
||||
@ -30,7 +31,8 @@ class ExternalSSTFileTest : public DBTestBase {
|
||||
Status GenerateAndAddExternalFile(
|
||||
const Options options,
|
||||
std::vector<std::pair<std::string, std::string>> data, int file_id = -1,
|
||||
bool allow_global_seqno = false, bool sort_data = false,
|
||||
bool allow_global_seqno = false, bool write_global_seqno = false,
|
||||
bool sort_data = false,
|
||||
std::map<std::string, std::string>* true_data = nullptr,
|
||||
ColumnFamilyHandle* cfh = nullptr) {
|
||||
// Generate a file id if not provided
|
||||
@ -73,6 +75,7 @@ class ExternalSSTFileTest : public DBTestBase {
|
||||
if (s.ok()) {
|
||||
IngestExternalFileOptions ifo;
|
||||
ifo.allow_global_seqno = allow_global_seqno;
|
||||
ifo.write_global_seqno = allow_global_seqno ? write_global_seqno : false;
|
||||
if (cfh) {
|
||||
s = db_->IngestExternalFile(cfh, {file_path}, ifo);
|
||||
} else {
|
||||
@ -149,11 +152,10 @@ class ExternalSSTFileTest : public DBTestBase {
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
|
||||
Status GenerateAndAddExternalFile(
|
||||
const Options options, std::vector<std::pair<int, std::string>> data,
|
||||
int file_id = -1, bool allow_global_seqno = false, bool sort_data = false,
|
||||
int file_id = -1, bool allow_global_seqno = false,
|
||||
bool write_global_seqno = false, bool sort_data = false,
|
||||
std::map<std::string, std::string>* true_data = nullptr,
|
||||
ColumnFamilyHandle* cfh = nullptr) {
|
||||
std::vector<std::pair<std::string, std::string>> file_data;
|
||||
@ -161,13 +163,14 @@ class ExternalSSTFileTest : public DBTestBase {
|
||||
file_data.emplace_back(Key(entry.first), entry.second);
|
||||
}
|
||||
return GenerateAndAddExternalFile(options, file_data, file_id,
|
||||
allow_global_seqno, sort_data, true_data,
|
||||
cfh);
|
||||
allow_global_seqno, write_global_seqno,
|
||||
sort_data, true_data, cfh);
|
||||
}
|
||||
|
||||
Status GenerateAndAddExternalFile(
|
||||
const Options options, std::vector<int> keys, int file_id = -1,
|
||||
bool allow_global_seqno = false, bool sort_data = false,
|
||||
bool allow_global_seqno = false, bool write_global_seqno = false,
|
||||
bool sort_data = false,
|
||||
std::map<std::string, std::string>* true_data = nullptr,
|
||||
ColumnFamilyHandle* cfh = nullptr) {
|
||||
std::vector<std::pair<std::string, std::string>> file_data;
|
||||
@ -175,18 +178,20 @@ class ExternalSSTFileTest : public DBTestBase {
|
||||
file_data.emplace_back(Key(k), Key(k) + ToString(file_id));
|
||||
}
|
||||
return GenerateAndAddExternalFile(options, file_data, file_id,
|
||||
allow_global_seqno, sort_data, true_data,
|
||||
cfh);
|
||||
allow_global_seqno, write_global_seqno,
|
||||
sort_data, true_data, cfh);
|
||||
}
|
||||
|
||||
Status DeprecatedAddFile(const std::vector<std::string>& files,
|
||||
bool move_files = false,
|
||||
bool skip_snapshot_check = false) {
|
||||
bool skip_snapshot_check = false,
|
||||
bool skip_write_global_seqno = false) {
|
||||
IngestExternalFileOptions opts;
|
||||
opts.move_files = move_files;
|
||||
opts.snapshot_consistency = !skip_snapshot_check;
|
||||
opts.allow_global_seqno = false;
|
||||
opts.allow_blocking_flush = false;
|
||||
opts.write_global_seqno = !skip_write_global_seqno;
|
||||
return db_->IngestExternalFile(files, opts);
|
||||
}
|
||||
|
||||
@ -453,6 +458,7 @@ TEST_F(ExternalSSTFileTest, Basic) {
|
||||
} while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction |
|
||||
kRangeDelSkipConfigs));
|
||||
}
|
||||
|
||||
class SstFileWriterCollector : public TablePropertiesCollector {
|
||||
public:
|
||||
explicit SstFileWriterCollector(const std::string prefix) : prefix_(prefix) {
|
||||
@ -1141,7 +1147,7 @@ TEST_F(ExternalSSTFileTest, OverlappingRanges) {
|
||||
} while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction));
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileTest, PickedLevel) {
|
||||
TEST_P(ExternalSSTFileTest, PickedLevel) {
|
||||
Options options = CurrentOptions();
|
||||
options.disable_auto_compactions = false;
|
||||
options.level0_file_num_compaction_trigger = 4;
|
||||
@ -1152,11 +1158,11 @@ TEST_F(ExternalSSTFileTest, PickedLevel) {
|
||||
|
||||
// File 0 will go to last level (L3)
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 10}, -1, false, false,
|
||||
&true_data));
|
||||
false, &true_data));
|
||||
EXPECT_EQ(FilesPerLevel(), "0,0,0,1");
|
||||
|
||||
// File 1 will go to level L2 (since it overlap with file 0 in L3)
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {2, 9}, -1, false, false,
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {2, 9}, -1, false, false, false,
|
||||
&true_data));
|
||||
EXPECT_EQ(FilesPerLevel(), "0,0,1,1");
|
||||
|
||||
@ -1186,13 +1192,13 @@ TEST_F(ExternalSSTFileTest, PickedLevel) {
|
||||
|
||||
// This file overlaps with file 0 (L3), file 1 (L2) and the
|
||||
// output of compaction going to L1
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {4, 7}, -1, false, false,
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {4, 7}, -1, false, false, false,
|
||||
&true_data));
|
||||
EXPECT_EQ(FilesPerLevel(), "5,0,1,1");
|
||||
|
||||
// This file does not overlap with any file or with the running compaction
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {9000, 9001}, -1, false, false,
|
||||
&true_data));
|
||||
false, &true_data));
|
||||
EXPECT_EQ(FilesPerLevel(), "5,0,1,2");
|
||||
|
||||
// Hold compaction from finishing
|
||||
@ -1422,12 +1428,12 @@ TEST_F(ExternalSSTFileTest, PickedLevelDynamic) {
|
||||
// This file overlaps with the output of the compaction (going to L3)
|
||||
// so the file will be added to L0 since L3 is the base level
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {31, 32, 33, 34}, -1, false,
|
||||
false, &true_data));
|
||||
false, false, &true_data));
|
||||
EXPECT_EQ(FilesPerLevel(), "5");
|
||||
|
||||
// This file does not overlap with the current running compactiong
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {9000, 9001}, -1, false, false,
|
||||
&true_data));
|
||||
false, &true_data));
|
||||
EXPECT_EQ(FilesPerLevel(), "5,0,0,1");
|
||||
|
||||
// Hold compaction from finishing
|
||||
@ -1442,24 +1448,24 @@ TEST_F(ExternalSSTFileTest, PickedLevelDynamic) {
|
||||
Reopen(options);
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 15, 19}, -1, false, false,
|
||||
&true_data));
|
||||
false, &true_data));
|
||||
ASSERT_EQ(FilesPerLevel(), "1,0,0,3");
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1000, 1001, 1002}, -1, false,
|
||||
false, &true_data));
|
||||
false, false, &true_data));
|
||||
ASSERT_EQ(FilesPerLevel(), "1,0,0,4");
|
||||
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {500, 600, 700}, -1, false,
|
||||
false, &true_data));
|
||||
false, false, &true_data));
|
||||
ASSERT_EQ(FilesPerLevel(), "1,0,0,5");
|
||||
|
||||
// File 5 overlaps with file 2 (L3 / base level)
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {2, 10}, -1, false, false,
|
||||
&true_data));
|
||||
false, &true_data));
|
||||
ASSERT_EQ(FilesPerLevel(), "2,0,0,5");
|
||||
|
||||
// File 6 overlaps with file 2 (L3 / base level) and file 5 (L0)
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {3, 9}, -1, false, false,
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {3, 9}, -1, false, false, false,
|
||||
&true_data));
|
||||
ASSERT_EQ(FilesPerLevel(), "3,0,0,5");
|
||||
|
||||
@ -1479,7 +1485,7 @@ TEST_F(ExternalSSTFileTest, PickedLevelDynamic) {
|
||||
|
||||
// File 7 overlaps with file 4 (L3)
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {650, 651, 652}, -1, false,
|
||||
false, &true_data));
|
||||
false, false, &true_data));
|
||||
ASSERT_EQ(FilesPerLevel(), "5,0,0,5");
|
||||
|
||||
VerifyDBFromMap(true_data, &kcnt, false);
|
||||
@ -1613,12 +1619,13 @@ TEST_F(ExternalSSTFileTest, SstFileWriterNonSharedKeys) {
|
||||
ASSERT_OK(DeprecatedAddFile({file_path}));
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoRandomized) {
|
||||
TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoRandomized) {
|
||||
Options options = CurrentOptions();
|
||||
options.IncreaseParallelism(20);
|
||||
options.level0_slowdown_writes_trigger = 256;
|
||||
options.level0_stop_writes_trigger = 256;
|
||||
|
||||
bool write_global_seqno = GetParam();
|
||||
for (int iter = 0; iter < 2; iter++) {
|
||||
bool write_to_memtable = (iter == 0);
|
||||
DestroyAndReopen(options);
|
||||
@ -1643,7 +1650,8 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoRandomized) {
|
||||
}
|
||||
} else {
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, random_data, -1, true,
|
||||
true, &true_data));
|
||||
write_global_seqno, true,
|
||||
&true_data));
|
||||
}
|
||||
}
|
||||
size_t kcnt = 0;
|
||||
@ -1653,7 +1661,7 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoRandomized) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
|
||||
TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
|
||||
Options options = CurrentOptions();
|
||||
options.num_levels = 5;
|
||||
options.disable_auto_compactions = true;
|
||||
@ -1672,8 +1680,9 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
|
||||
for (int i = 0; i <= 20; i++) {
|
||||
file_data.emplace_back(Key(i), "L4");
|
||||
}
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true, false,
|
||||
&true_data));
|
||||
bool write_global_seqno = GetParam();
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true,
|
||||
write_global_seqno, false, &true_data));
|
||||
|
||||
// This file dont overlap with anything in the DB, will go to L4
|
||||
ASSERT_EQ("0,0,0,0,1", FilesPerLevel());
|
||||
@ -1683,8 +1692,8 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
|
||||
for (int i = 80; i <= 130; i++) {
|
||||
file_data.emplace_back(Key(i), "L0");
|
||||
}
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true, false,
|
||||
&true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true,
|
||||
write_global_seqno, false, &true_data));
|
||||
|
||||
// This file overlap with the memtable, so it will flush it and add
|
||||
// it self to L0
|
||||
@ -1695,8 +1704,8 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
|
||||
for (int i = 30; i <= 50; i++) {
|
||||
file_data.emplace_back(Key(i), "L4");
|
||||
}
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true, false,
|
||||
&true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true,
|
||||
write_global_seqno, false, &true_data));
|
||||
|
||||
// This file dont overlap with anything in the DB and fit in L4 as well
|
||||
ASSERT_EQ("2,0,0,0,2", FilesPerLevel());
|
||||
@ -1706,8 +1715,8 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
|
||||
for (int i = 10; i <= 40; i++) {
|
||||
file_data.emplace_back(Key(i), "L3");
|
||||
}
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true, false,
|
||||
&true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, file_data, -1, true,
|
||||
write_global_seqno, false, &true_data));
|
||||
|
||||
// This file overlap with files in L4, we will ingest it in L3
|
||||
ASSERT_EQ("2,0,0,1,2", FilesPerLevel());
|
||||
@ -1716,7 +1725,7 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
|
||||
VerifyDBFromMap(true_data, &kcnt, false);
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) {
|
||||
TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) {
|
||||
Options options = CurrentOptions();
|
||||
DestroyAndReopen(options);
|
||||
uint64_t entries_in_memtable;
|
||||
@ -1730,16 +1739,17 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) {
|
||||
&entries_in_memtable);
|
||||
ASSERT_GE(entries_in_memtable, 1);
|
||||
|
||||
bool write_global_seqno = GetParam();
|
||||
// No need for flush
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {90, 100, 110}, -1, true, false,
|
||||
&true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {90, 100, 110}, -1, true,
|
||||
write_global_seqno, false, &true_data));
|
||||
db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable,
|
||||
&entries_in_memtable);
|
||||
ASSERT_GE(entries_in_memtable, 1);
|
||||
|
||||
// This file will flush the memtable
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {19, 20, 21}, -1, true, false,
|
||||
&true_data));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {19, 20, 21}, -1, true,
|
||||
write_global_seqno, false, &true_data));
|
||||
db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable,
|
||||
&entries_in_memtable);
|
||||
ASSERT_EQ(entries_in_memtable, 0);
|
||||
@ -1754,14 +1764,14 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) {
|
||||
|
||||
// No need for flush, this file keys fit between the memtable keys
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {202, 203, 204}, -1, true,
|
||||
false, &true_data));
|
||||
write_global_seqno, false, &true_data));
|
||||
db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable,
|
||||
&entries_in_memtable);
|
||||
ASSERT_GE(entries_in_memtable, 1);
|
||||
|
||||
// This file will flush the memtable
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {206, 207}, -1, true, false,
|
||||
&true_data));
|
||||
write_global_seqno, &true_data));
|
||||
db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable,
|
||||
&entries_in_memtable);
|
||||
ASSERT_EQ(entries_in_memtable, 0);
|
||||
@ -1770,7 +1780,7 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) {
|
||||
VerifyDBFromMap(true_data, &kcnt, false);
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileTest, L0SortingIssue) {
|
||||
TEST_P(ExternalSSTFileTest, L0SortingIssue) {
|
||||
Options options = CurrentOptions();
|
||||
options.num_levels = 2;
|
||||
DestroyAndReopen(options);
|
||||
@ -1779,10 +1789,13 @@ TEST_F(ExternalSSTFileTest, L0SortingIssue) {
|
||||
ASSERT_OK(Put(Key(1), "memtable"));
|
||||
ASSERT_OK(Put(Key(10), "memtable"));
|
||||
|
||||
bool write_global_seqno = GetParam();
|
||||
// No Flush needed, No global seqno needed, Ingest in L1
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {7, 8}, -1, true, false));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {7, 8}, -1, true,
|
||||
write_global_seqno, false));
|
||||
// No Flush needed, but need a global seqno, Ingest in L0
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {7, 8}, -1, true, false));
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {7, 8}, -1, true,
|
||||
write_global_seqno, false));
|
||||
printf("%s\n", FilesPerLevel().c_str());
|
||||
|
||||
// Overwrite what we added using external files
|
||||
@ -2011,15 +2024,17 @@ class TestIngestExternalFileListener : public EventListener {
|
||||
std::vector<ExternalFileIngestionInfo> ingested_files;
|
||||
};
|
||||
|
||||
TEST_F(ExternalSSTFileTest, IngestionListener) {
|
||||
TEST_P(ExternalSSTFileTest, IngestionListener) {
|
||||
Options options = CurrentOptions();
|
||||
TestIngestExternalFileListener* listener =
|
||||
new TestIngestExternalFileListener();
|
||||
options.listeners.emplace_back(listener);
|
||||
CreateAndReopenWithCF({"koko", "toto"}, options);
|
||||
|
||||
bool write_global_seqno = GetParam();
|
||||
// Ingest into default cf
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2}, -1, true, true, nullptr,
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2}, -1, true,
|
||||
write_global_seqno, true, nullptr,
|
||||
handles_[0]));
|
||||
ASSERT_EQ(listener->ingested_files.size(), 1);
|
||||
ASSERT_EQ(listener->ingested_files.back().cf_name, "default");
|
||||
@ -2030,7 +2045,8 @@ TEST_F(ExternalSSTFileTest, IngestionListener) {
|
||||
"default");
|
||||
|
||||
// Ingest into cf1
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2}, -1, true, true, nullptr,
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2}, -1, true,
|
||||
write_global_seqno, true, nullptr,
|
||||
handles_[1]));
|
||||
ASSERT_EQ(listener->ingested_files.size(), 2);
|
||||
ASSERT_EQ(listener->ingested_files.back().cf_name, "koko");
|
||||
@ -2041,7 +2057,8 @@ TEST_F(ExternalSSTFileTest, IngestionListener) {
|
||||
"koko");
|
||||
|
||||
// Ingest into cf2
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2}, -1, true, true, nullptr,
|
||||
ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2}, -1, true,
|
||||
write_global_seqno, true, nullptr,
|
||||
handles_[2]));
|
||||
ASSERT_EQ(listener->ingested_files.size(), 3);
|
||||
ASSERT_EQ(listener->ingested_files.back().cf_name, "toto");
|
||||
@ -2084,7 +2101,7 @@ TEST_F(ExternalSSTFileTest, SnapshotInconsistencyBug) {
|
||||
db_->ReleaseSnapshot(snap);
|
||||
}
|
||||
|
||||
TEST_F(ExternalSSTFileTest, IngestBehind) {
|
||||
TEST_P(ExternalSSTFileTest, IngestBehind) {
|
||||
Options options = CurrentOptions();
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.num_levels = 3;
|
||||
@ -2108,6 +2125,7 @@ TEST_F(ExternalSSTFileTest, IngestBehind) {
|
||||
IngestExternalFileOptions ifo;
|
||||
ifo.allow_global_seqno = true;
|
||||
ifo.ingest_behind = true;
|
||||
ifo.write_global_seqno = GetParam();
|
||||
|
||||
// Can't ingest behind since allow_ingest_behind isn't set to true
|
||||
ASSERT_NOK(GenerateAndAddExternalFileIngestBehind(options, ifo,
|
||||
@ -2195,6 +2213,9 @@ TEST_F(ExternalSSTFileTest, SkipBloomFilter) {
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(ExternalSSTFileTest, ExternalSSTFileTest,
|
||||
testing::Bool());
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user