andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix FIFO Compaction with TTL tests

Browse Source 
Summary:
- FIFOCompactionWithTTLTest was flaky when run in parallel earlier, and hence it was disabled. Fixed it now.
- Also, faking sleep now instead of really sleeping to make tests more realistic by using TTLs like 1 hour and 1 day.
Closes https://github.com/facebook/rocksdb/pull/2650

Differential Revision: D5506038

Pulled By: sagar0

fbshipit-source-id: deb429a527f045e3e2c5138b547c3e8ac8586aa2
This commit is contained in:
Sagar Vemuri 2017-07-28 14:28:09 -07:00 committed by Facebook Github Bot
parent aaf42fe775
commit ac748c57ed
1 changed files with 35 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
db/db_test.cc
View File

@ -2847,20 +2847,23 @@ TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) {
ASSERT_TRUE(TryReopen(options).IsNotSupported());
}
TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
TEST_F(DBTest, FIFOCompactionWithTTLTest) {
Options options;
options.compaction_style = kCompactionStyleFIFO;
options.write_buffer_size = 10 << 10; // 10KB
options.arena_block_size = 4096;
options.compression = kNoCompression;
options.create_if_missing = true;
env_->time_elapse_only_sleep_ = false;
options.env = env_;
// Test to make sure that all files with expired ttl are deleted on next
// manual compaction.
{
env_->addon_time_.store(0);
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 600; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60 ; // 1 hour
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2871,19 +2874,21 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
// sleep for 5 seconds
env_->SleepForMicroseconds(5 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
// Note: Couldn't use SleepForMicroseconds because it takes an int instead
// of uint64_t. Hence used addon_time_ directly.
// env_->SleepForMicroseconds(2 * 60 * 60 * 1000 * 1000);
env_->addon_time_.fetch_add(2 * 60 * 60);
// Since no flushes and compactions have run, the db should still be in
// the same state even after considerable time has passed.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
// change ttl to 1 sec. So all files should be deleted on next compaction.
options.compaction_options_fifo.ttl = 1;
Reopen(options);
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
}
@ -2893,7 +2898,7 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
{
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 5; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2904,11 +2909,13 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
env_->SleepForMicroseconds(6 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
env_->addon_time_.fetch_add(2 * 60 * 60);
// Just to make sure that we are in the same state even after sleeping.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
@ -2930,10 +2937,10 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
// Test that shows the fall back to size-based FIFO compaction if TTL-based
// deletion doesn't move the total size to be less than max_table_files_size.
{
options.write_buffer_size = 110 << 10; // 10KB
options.write_buffer_size = 10 << 10; // 10KB
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = false;
options.compaction_options_fifo.ttl = 5; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2944,11 +2951,13 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 3);
env_->SleepForMicroseconds(6 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
env_->addon_time_.fetch_add(2 * 60 * 60);
// Just to make sure that we are in the same state even after sleeping.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 3);
@ -2957,8 +2966,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Size limit is still guaranteed.
ASSERT_LE(SizeAtLevel(0),
options.compaction_options_fifo.max_table_files_size);
@ -2968,7 +2977,7 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
{
options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB
options.compaction_options_fifo.allow_compaction = true;
options.compaction_options_fifo.ttl = 5; // seconds
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options.level0_file_num_compaction_trigger = 6;
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -2980,15 +2989,16 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
// With Intra-L0 compaction, out of 10 files, 6 files will be compacted to 1
// (due to level0_file_num_compaction_trigger = 6).
// So total files = 1 + remaining 4 = 5.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
// Sleep for a little over ttl time.
env_->SleepForMicroseconds(6 * 1000 * 1000);
// Sleep for 2 hours -- which is much greater than TTL.
env_->addon_time_.fetch_add(2 * 60 * 60);
// Just to make sure that we are in the same state even after sleeping.
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
@ -2998,8 +3008,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
ASSERT_LE(SizeAtLevel(0),
options.compaction_options_fifo.max_table_files_size);
@ -3011,7 +3021,7 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
options.write_buffer_size = 20 << 10; // 20K
options.compaction_options_fifo.max_table_files_size = 1500 << 10; // 1.5MB
options.compaction_options_fifo.allow_compaction = true;
options.compaction_options_fifo.ttl = 60 * 60; // 1 hour
options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour
options.level0_file_num_compaction_trigger = 6;
options = CurrentOptions(options);
DestroyAndReopen(options);
@ -3023,8 +3033,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// It should be compacted to 10 files.
ASSERT_EQ(NumTableFilesAtLevel(0), 10);
@ -3034,8 +3044,8 @@ TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
ASSERT_OK(Put(ToString(i * 20 + j + 2000), RandomString(&rnd, 980)));
}
Flush();
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// It should be compacted to no more than 20 files.
ASSERT_GT(NumTableFilesAtLevel(0), 10);