andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Simplify DBWithMaxSpaceAllowedRandomized (#4235)

Browse Source 
Summary:
The test has become complicated over the years and hard to reason about the corner cases that makes the test flaky. The patch simplifies the test and also fixes some probable synchronization issues.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4235

Differential Revision: D9187995

Pulled By: maysamyabandeh

fbshipit-source-id: 53c7b060f14367e5a9e361014578c26debfe3d27
This commit is contained in:
Maysam Yabandeh 2018-08-08 07:23:10 -07:00 committed by Facebook Github Bot
parent badfd70a3e
commit d8d66c937e
1 changed files with 8 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
db/db_sst_test.cc
View File

@ -703,26 +703,19 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
// When bg_error_ is set we will verify that the DB size is greater // When bg_error_ is set we will verify that the DB size is greater
// than the limit. // than the limit.
std::vector<int> max_space_limits_mbs = {1, 2, 4, 8, 10}; std::vector<int> max_space_limits_mbs = {1, 10};
decltype(max_space_limits_mbs)::value_type limit_mb_cb; std::atomic<bool> bg_error_set(false);
bool bg_error_set = false;
uint64_t total_sst_files_size = 0;
std::atomic<int> estimate_multiplier(1); std::atomic<int> reached_max_space_on_flush(0);
int reached_max_space_on_flush = 0; std::atomic<int> reached_max_space_on_compaction(0);
int reached_max_space_on_compaction = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached", "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
[&](void* arg) { [&](void* arg) {
Status* bg_error = static_cast<Status*>(arg); Status* bg_error = static_cast<Status*>(arg);
bg_error_set = true; bg_error_set = true;
GetAllSSTFiles(&total_sst_files_size);
reached_max_space_on_flush++; reached_max_space_on_flush++;
// low limit for size calculated using sst files
ASSERT_GE(total_sst_files_size, limit_mb_cb * 1024 * 1024);
// clear error to ensure compaction callback is called // clear error to ensure compaction callback is called
*bg_error = Status::OK(); *bg_error = Status::OK();
estimate_multiplier++; // used in the main loop assert
}); });
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
@ -735,15 +728,11 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
"CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached", "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached",
[&](void* /*arg*/) { [&](void* /*arg*/) {
bg_error_set = true; bg_error_set = true;
GetAllSSTFiles(&total_sst_files_size);
reached_max_space_on_compaction++; reached_max_space_on_compaction++;
}); });
for (auto limit_mb : max_space_limits_mbs) { for (auto limit_mb : max_space_limits_mbs) {
bg_error_set = false; bg_error_set = false;
total_sst_files_size = 0;
estimate_multiplier = 1;
limit_mb_cb = limit_mb;
rocksdb::SyncPoint::GetInstance()->ClearTrace(); rocksdb::SyncPoint::GetInstance()->ClearTrace();
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_)); std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
@ -757,21 +746,17 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
sfm->SetMaxAllowedSpaceUsage(limit_mb * 1024 * 1024); sfm->SetMaxAllowedSpaceUsage(limit_mb * 1024 * 1024);
int keys_written = 0; // It is easy to detect if the test is stuck in a loop. No need for
uint64_t estimated_db_size = 0; // complex termination logic.
while (true) { while (true) {
auto s = Put(RandomString(&rnd, 10), RandomString(&rnd, 50)); auto s = Put(RandomString(&rnd, 10), RandomString(&rnd, 50));
if (!s.ok()) { if (!s.ok()) {
break; break;
} }
keys_written++;
// Check the estimated db size vs the db limit just to make sure we
// dont run into an infinite loop
estimated_db_size = keys_written * 60; // ~60 bytes per key
ASSERT_LT(estimated_db_size,
estimate_multiplier * limit_mb * 1024 * 1024 * 2);
} }
ASSERT_TRUE(bg_error_set); ASSERT_TRUE(bg_error_set);
uint64_t total_sst_files_size = 0;
GetAllSSTFiles(&total_sst_files_size);
ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024); ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024);
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); rocksdb::SyncPoint::GetInstance()->DisableProcessing();
} }