andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Enable ASSERT_STATUS_CHECKED for db_universal_compaction_test (#7460)

Browse Source 
Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7460

Reviewed By: riversand963

Differential Revision: D24057636

Pulled By: anand1976

fbshipit-source-id: bfb13da6993a5e407be20073e4d6751dfb38e442
This commit is contained in:
anand76 2020-10-06 14:40:37 -07:00 committed by Facebook GitHub Bot
parent 1bcef3d83c
commit a242a58301
4 changed files with 121 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -661,6 +661,7 @@ ifdef ASSERT_STATUS_CHECKED
column_family_test \
file_reader_writer_test \
corruption_test \
db_universal_compaction_test \
ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
TESTS_PASSING_ASC += folly_synchronization_distributed_mutex_test

4
db/db_impl/db_impl_compaction_flush.cc
View File

@ -1167,6 +1167,10 @@ Status DBImpl::CompactFilesImpl(
&job_context->superversion_contexts[0],
*c->mutable_cf_options());
}
// status above captures any error during compaction_job.Install, so its ok
// not check compaction_job.io_status() explicitly if we're not calling
// SetBGError
compaction_job.io_status().PermitUncheckedError();
c->ReleaseCompactionFiles(s);
#ifndef ROCKSDB_LITE
// Need to make sure SstFileManager does its bookkeeping

3
db/db_test_util.cc
View File

@ -684,7 +684,8 @@ void DBTestBase::Destroy(const Options& options, bool delete_cf_paths) {
if (delete_cf_paths) {
for (size_t i = 0; i < handles_.size(); ++i) {
ColumnFamilyDescriptor cfdescriptor;
handles_[i]->GetDescriptor(&cfdescriptor);
// GetDescriptor is not implemented for ROCKSDB_LITE
handles_[i]->GetDescriptor(&cfdescriptor).PermitUncheckedError();
column_families.push_back(cfdescriptor);
}
}

215
db/db_universal_compaction_test.cc
View File

@ -126,11 +126,11 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
for (int num = 0; num < 16; num++) {
// Write 100KB file. And immediately it should be compacted to one file.
GenerateNewFile(&rnd, &key_idx);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumSortedRuns(0), 1);
}
ASSERT_OK(Put(Key(key_idx), ""));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumSortedRuns(0), 1);
}
@ -162,15 +162,15 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
Env::Priority::LOW);
for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
Put(Key(num * 10), "val");
ASSERT_OK(Put(Key(num * 10), "val"));
if (num) {
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
}
Put(Key(30 + num * 10), "val");
Put(Key(60 + num * 10), "val");
ASSERT_OK(Put(Key(30 + num * 10), "val"));
ASSERT_OK(Put(Key(60 + num * 10), "val"));
}
Put("", "");
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_OK(Put("", ""));
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
// Query set of non existing keys
for (int i = 5; i < 90; i += 10) {
@ -190,7 +190,7 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
// Unblock compaction and wait it for happening.
sleeping_task_low.WakeUp();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// The same queries will not trigger bloom filter
for (int i = 5; i < 90; i += 10) {
@ -294,7 +294,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
// Now we have 3 files at level 0, with size 4, 2.4, 2. Let's generate a
// new file of size 1.
GenerateNewFile(1, &rnd, &key_idx);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Level-0 compaction is triggered, but no file will be picked up.
ASSERT_EQ(NumSortedRuns(1), 4);
@ -303,7 +303,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
// a new file of size 1.
filter->expect_full_compaction_.store(true);
GenerateNewFile(1, &rnd, &key_idx);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// All files at level 0 will be compacted into a single one.
ASSERT_EQ(NumSortedRuns(1), 1);
@ -336,7 +336,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_EQ(NumSortedRuns(1), num + 1);
}
ASSERT_EQ(NumSortedRuns(1), 2);
@ -346,7 +346,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
// but will instead trigger size amplification.
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Verify that size amplification did occur
ASSERT_EQ(NumSortedRuns(1), 1);
@ -394,7 +394,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) {
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_EQ(NumSortedRuns(1), num + 1);
}
ASSERT_EQ(NumSortedRuns(1), 2);
@ -404,7 +404,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) {
// but could instead trigger size amplification if it's set
// to 110.
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Verify compaction did not happen
ASSERT_EQ(NumSortedRuns(1), 3);
@ -425,7 +425,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) {
ASSERT_EQ(110u, mutable_cf_options.compaction_options_universal
.max_size_amplification_percent);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Verify that size amplification did happen
ASSERT_EQ(NumSortedRuns(1), 1);
ASSERT_EQ(total_picked_compactions, 1);
@ -473,7 +473,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionReadAmplification) {
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_EQ(NumSortedRuns(1), num + 1);
}
ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger);
@ -481,7 +481,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionReadAmplification) {
// Flush whatever is remaining in memtable. This is typically small, about
// 30KB.
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Verify compaction did not happen
ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger + 1);
ASSERT_EQ(total_picked_compactions, 0);
@ -510,7 +510,7 @@ TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionReadAmplification) {
ASSERT_EQ(mutable_cf_options.compaction_options_universal.max_merge_width,
2u);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Files in L0 are approx: 0.3 (30KB), 1, 1, 1.
// On compaction: the files are below the size amp threshold, so we
@ -550,8 +550,8 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) {
ASSERT_OK(Put(1, ToString(key), rnd.RandomString(kTestValueSize)));
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ColumnFamilyMetaData cf_meta;
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
std::vector<std::string> compaction_input_file_names;
@ -631,7 +631,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
compact_options.change_level = true;
compact_options.target_level = 4;
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_;
db_->CompactRange(compact_options, nullptr, nullptr);
ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
ASSERT_EQ("0,0,0,0,1", FilesPerLevel(0));
}
@ -665,7 +665,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
ASSERT_OK(Put(1, Key(i % num_keys), Key(i)));
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
for (int i = num_keys; i < num_keys * 2; i++) {
ASSERT_EQ(Get(1, Key(i % num_keys)), Key(i));
@ -712,7 +712,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
std::vector<std::string> values;
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_GT(trivial_move, 0);
ASSERT_GT(non_trivial_move, 0);
@ -775,7 +775,7 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
for (int i = 0; i < num_keys * 2; i++) {
ASSERT_OK(Put(1, Key(i % num_keys), Key(i)));
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
ASSERT_EQ(num_compactions_running.load(), 0);
@ -853,7 +853,7 @@ TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) {
// Hold the 1st compaction from finishing
TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2");
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// There should only be one picked compaction as the score drops below one
// after the first one is picked.
@ -901,7 +901,7 @@ TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) {
// Hold the 1st and 2nd compaction from finishing
TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2");
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// This time we will trigger a compaction because of size ratio and
// another compaction because of number of files that are not compacted
@ -935,14 +935,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(990)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
if (num < options.level0_file_num_compaction_trigger - 1) {
ASSERT_EQ(NumSortedRuns(1), num + 1);
}
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumSortedRuns(1), 1);
}
@ -973,7 +973,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
ASSERT_EQ(NumSortedRuns(), num + 1);
}
@ -983,7 +983,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990)));
key_idx++;
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Suppose each file flushed from mem table has size 1. Now we compact
// (level0_file_num_compaction_trigger+1)=4 files and should have a big
// file of size 4.
@ -1004,7 +1004,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
ASSERT_EQ(NumSortedRuns(), num + 3);
}
@ -1014,7 +1014,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990)));
key_idx++;
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1.
// After compaction, we should have 3 files, with size 4, 0.4, 2.
ASSERT_EQ(NumSortedRuns(), 3);
@ -1025,7 +1025,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990)));
key_idx++;
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Level-0 compaction is triggered, but no file will be picked up.
ASSERT_EQ(NumSortedRuns(), 4);
}
@ -1054,8 +1054,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_LT(TotalSize(), 110000U * 2 * 0.9);
@ -1066,8 +1066,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_LT(TotalSize(), 110000 * 4 * 0.9);
@ -1079,8 +1079,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_LT(TotalSize(), 110000 * 6 * 0.9);
@ -1092,8 +1092,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_GT(TotalSize(), 110000 * 11 * 0.8 + 110000 * 2);
}
@ -1122,8 +1122,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForFlushMemTable();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_LT(TotalSize(), 120000U * 12 * 0.82 + 120000 * 2);
}
@ -1169,7 +1169,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
std::vector<std::string> values;
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_GT(trivial_move, 0);
ASSERT_GT(non_trivial_move, 0);
@ -1215,7 +1215,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
std::vector<std::string> values;
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_GT(trivial_move, 0);
@ -1239,12 +1239,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
options.num_levels = 1;
std::vector<std::string> filenames;
env_->GetChildren(options.db_paths[1].path, &filenames);
if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) {
// Delete archival files.
for (size_t i = 0; i < filenames.size(); ++i) {
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i]);
ASSERT_OK(
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i]));
}
ASSERT_OK(env_->DeleteDir(options.db_paths[1].path));
}
env_->DeleteDir(options.db_paths[1].path);
Reopen(options);
Random rnd(301);
@ -1503,11 +1505,11 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
for (int i = 0; i <= max_key1; i++) {
// each value is 10K
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000)));
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Stage 2: reopen with universal compaction, num_levels=4
options.compaction_style = kCompactionStyleUniversal;
@ -1521,11 +1523,11 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
for (int i = max_key1 + 1; i <= max_key2; i++) {
// each value is 10K
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000)));
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
verify_func(max_key2);
// Compaction to non-L0 has happened.
@ -1540,7 +1542,8 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
compact_options.change_level = true;
compact_options.target_level = 0;
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_;
dbfull()->CompactRange(compact_options, handles_[1], nullptr, nullptr);
ASSERT_OK(
dbfull()->CompactRange(compact_options, handles_[1], nullptr, nullptr));
// Need to restart it once to remove higher level records in manifest.
ReopenWithColumnFamilies({"default", "pikachu"}, options);
// Final reopen
@ -1553,11 +1556,11 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
for (int i = max_key2 + 1; i <= max_key3; i++) {
// each value is 10K
ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000)));
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
verify_func(max_key3);
}
@ -1579,12 +1582,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
std::vector<std::string> filenames;
env_->GetChildren(options.db_paths[1].path, &filenames);
if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) {
// Delete archival files.
for (size_t i = 0; i < filenames.size(); ++i) {
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i]);
ASSERT_OK(
env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i]));
}
ASSERT_OK(env_->DeleteDir(options.db_paths[1].path));
}
env_->DeleteDir(options.db_paths[1].path);
Reopen(options);
Random rnd(301);
@ -1699,14 +1704,14 @@ TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
// use no_wait above because that one waits for flush and compaction. We
// don't want to wait for compaction because the full compaction is
// intentionally blocked while more files are flushed.
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
}
if (i == 0) {
TEST_SYNC_POINT(
"DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0");
}
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// First compaction should output to bottom level. Second should output to L0
// since older L0 files pending compaction prevent it from being placed lower.
@ -1745,7 +1750,7 @@ TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) {
int key_idx = 0;
GenerateNewFile(&rnd, &key_idx);
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Compacting the first four files was enough to bring the score below one so
// there's no need to schedule any more compactions.
ASSERT_EQ(1, num_compactions_attempted);
@ -1775,9 +1780,9 @@ TEST_P(DBTestUniversalCompaction, FinalSortedRunCompactFilesConflict) {
auto stop_token =
dbfull()->TEST_write_controler().GetCompactionPressureToken();
Put("key", "val");
ASSERT_OK(Put("key", "val"));
Flush();
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
ASSERT_EQ(NumTableFilesAtLevel(num_levels_ - 1), 1);
ColumnFamilyMetaData cf_meta;
ColumnFamilyHandle* default_cfh = db_->DefaultColumnFamily();
@ -1801,10 +1806,10 @@ TEST_P(DBTestUniversalCompaction, FinalSortedRunCompactFilesConflict) {
TEST_SYNC_POINT(
"DBTestUniversalCompaction:FinalSortedRunCompactFilesConflict:0");
for (int i = 0; i < 2; ++i) {
Put("key", "val");
ASSERT_OK(Put("key", "val"));
Flush();
}
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
compact_files_thread.join();
}
@ -1835,7 +1840,7 @@ TEST_P(DBTestUniversalManualCompactionOutputPathId,
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
MakeTables(3, "p", "q", 1);
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(2, TotalLiveFiles(1));
ASSERT_EQ(2, GetSstFileCount(options.db_paths[0].path));
ASSERT_EQ(0, GetSstFileCount(options.db_paths[1].path));
@ -1844,7 +1849,7 @@ TEST_P(DBTestUniversalManualCompactionOutputPathId,
CompactRangeOptions compact_options;
compact_options.target_path_id = 1;
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_;
db_->CompactRange(compact_options, handles_[1], nullptr, nullptr);
ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr));
ASSERT_EQ(1, TotalLiveFiles(1));
ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path));
ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path));
@ -1867,7 +1872,7 @@ TEST_P(DBTestUniversalManualCompactionOutputPathId,
// Full compaction to DB path 0
compact_options.target_path_id = 0;
compact_options.exclusive_manual_compaction = exclusive_manual_compaction_;
db_->CompactRange(compact_options, handles_[1], nullptr, nullptr);
ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr));
ASSERT_EQ(1, TotalLiveFiles(1));
ASSERT_EQ(1, GetSstFileCount(options.db_paths[0].path));
ASSERT_EQ(0, GetSstFileCount(options.db_paths[1].path));
@ -1904,23 +1909,23 @@ TEST_F(DBTestUniversalCompaction2, BasicL0toL1) {
// during flush
int i;
for (i = 0; i < 2000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
// MoveFilesToLevel(6);
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
for (i = 1999; i < kNumKeys; ++i) {
if (i >= kNumKeys - kWindowSize &&
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
Delete(Key(i));
ASSERT_OK(Delete(Key(i)));
} else {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
}
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
ASSERT_GT(NumTableFilesAtLevel(6), 0);
}
@ -1947,21 +1952,21 @@ TEST_F(DBTestUniversalCompaction2, SingleLevel) {
// during flush
int i;
for (i = 0; i < 2000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 1999; i < kNumKeys; ++i) {
if (i >= kNumKeys - kWindowSize &&
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
Delete(Key(i));
ASSERT_OK(Delete(Key(i)));
} else {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
}
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(1, NumTableFilesAtLevel(0));
}
#endif // ENABLE_SINGLE_LEVEL_DTC
@ -1985,50 +1990,50 @@ TEST_F(DBTestUniversalCompaction2, MultipleLevels) {
// during flush
int i;
for (i = 0; i < 500; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 500; i < 1000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 1000; i < 1500; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 1500; i < 2000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
ASSERT_GT(NumTableFilesAtLevel(6), 0);
for (i = 1999; i < 2333; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 2333; i < 2666; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 2666; i < 2999; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
ASSERT_GT(NumTableFilesAtLevel(6), 0);
ASSERT_GT(NumTableFilesAtLevel(5), 0);
for (i = 1900; i < 2100; ++i) {
Delete(Key(i));
ASSERT_OK(Delete(Key(i)));
}
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
ASSERT_EQ(0, NumTableFilesAtLevel(1));
ASSERT_EQ(0, NumTableFilesAtLevel(2));
@ -2057,23 +2062,23 @@ TEST_F(DBTestUniversalCompaction2, OverlappingL0) {
// during flush
int i;
for (i = 0; i < 2000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 2000; i < 3000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 3500; i < 4000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
for (i = 2900; i < 3100; ++i) {
Delete(Key(i));
ASSERT_OK(Delete(Key(i)));
}
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(2, NumTableFilesAtLevel(0));
ASSERT_GT(NumTableFilesAtLevel(6), 0);
}
@ -2099,23 +2104,23 @@ TEST_F(DBTestUniversalCompaction2, IngestBehind) {
// during flush
int i;
for (i = 0; i < 2000; ++i) {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
Flush();
// MoveFilesToLevel(6);
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
for (i = 1999; i < kNumKeys; ++i) {
if (i >= kNumKeys - kWindowSize &&
i < kNumKeys - kWindowSize + kNumDelsTrigger) {
Delete(Key(i));
ASSERT_OK(Delete(Key(i)));
} else {
Put(Key(i), "val");
ASSERT_OK(Put(Key(i), "val"));
}
}
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, NumTableFilesAtLevel(0));
ASSERT_EQ(0, NumTableFilesAtLevel(6));
ASSERT_GT(NumTableFilesAtLevel(5), 0);
@ -2188,7 +2193,7 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) {
// Another flush would trigger compaction the oldest file.
ASSERT_OK(Put("foo", "bar2"));
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(1, periodic_compactions);
ASSERT_EQ(0, start_level);
@ -2199,7 +2204,7 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) {
// A flush doesn't trigger a periodic compaction when threshold not hit
ASSERT_OK(Put("foo", "bar2"));
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(0, periodic_compactions);
// After periodic compaction threshold hits, a flush will trigger
@ -2207,7 +2212,7 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) {
ASSERT_OK(Put("foo", "bar2"));
env_->MockSleepForSeconds(48 * 60 * 60 + 100);
Flush();
dbfull()->TEST_WaitForCompact();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(1, periodic_compactions);
ASSERT_EQ(0, start_level);
ASSERT_EQ(4, output_level);