andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make DBTest.OptimizeFiltersForHits more deterministic

Browse Source 
Summary:
This commit makes DBTest.OptimizeFiltersForHits more deterministic by:
(1) make key inserts more random
(2) make sure L0 has one file
(3) make file size smaller compared to level target so L1 will cover more range.

Test Plan: Run the test many times.

Reviewers: rven, IslamAbdelRahman, kradhakrishnan, igor, anthony

Reviewed By: anthony

Subscribers: leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D46461
This commit is contained in:
sdong 2015-09-08 18:31:36 -07:00
parent e17e92ea19
commit 342ba80895
1 changed files with 35 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
db/db_test.cc
View File

@ -7613,8 +7613,9 @@ TEST_F(DBTest, DontDeleteMovedFile) {
TEST_F(DBTest, OptimizeFiltersForHits) { TEST_F(DBTest, OptimizeFiltersForHits) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.write_buffer_size = 256 * 1024; options.write_buffer_size = 64 * 1024;
options.target_file_size_base = 256 * 1024; options.arena_block_size = 4 * 1024;
options.target_file_size_base = 64 * 1024;
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.level0_slowdown_writes_trigger = 2; options.level0_slowdown_writes_trigger = 2;
options.level0_stop_writes_trigger = 4; options.level0_stop_writes_trigger = 4;
@ -7622,7 +7623,9 @@ TEST_F(DBTest, OptimizeFiltersForHits) {
options.max_write_buffer_number = 2; options.max_write_buffer_number = 2;
options.max_background_compactions = 8; options.max_background_compactions = 8;
options.max_background_flushes = 8; options.max_background_flushes = 8;
options.compression = kNoCompression;
options.compaction_style = kCompactionStyleLevel; options.compaction_style = kCompactionStyleLevel;
options.level_compaction_dynamic_level_bytes = true;
BlockBasedTableOptions bbto; BlockBasedTableOptions bbto;
bbto.filter_policy.reset(NewBloomFilterPolicy(10, true)); bbto.filter_policy.reset(NewBloomFilterPolicy(10, true));
bbto.whole_key_filtering = true; bbto.whole_key_filtering = true;
@ -7632,16 +7635,37 @@ TEST_F(DBTest, OptimizeFiltersForHits) {
CreateAndReopenWithCF({"mypikachu"}, options); CreateAndReopenWithCF({"mypikachu"}, options);
int numkeys = 200000; int numkeys = 200000;
for (int i = 0; i < 20; i += 2) {
for (int j = i; j < numkeys; j += 20) { // Generate randomly shuffled keys, so the updates are almost
ASSERT_OK(Put(1, Key(j), "val")); // random.
std::vector<int> keys;
keys.reserve(numkeys);
for (int i = 0; i < numkeys; i += 2) {
keys.push_back(i);
}
std::random_shuffle(std::begin(keys), std::end(keys));
int num_inserted = 0;
for (int key : keys) {
ASSERT_OK(Put(1, Key(key), "val"));
if (++num_inserted % 1000 == 0) {
dbfull()->TEST_WaitForFlushMemTable();
dbfull()->TEST_WaitForCompact();
} }
} }
ASSERT_OK(Put(1, Key(0), "val"));
ASSERT_OK(Put(1, Key(numkeys), "val"));
ASSERT_OK(Flush(1)); ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact(); dbfull()->TEST_WaitForCompact();
if (NumTableFilesAtLevel(0, 1) == 0) {
// No Level 0 file. Create one.
ASSERT_OK(Put(1, Key(0), "val"));
ASSERT_OK(Put(1, Key(numkeys), "val"));
ASSERT_OK(Flush(1));
dbfull()->TEST_WaitForCompact();
}
for (int i = 1; i < numkeys; i += 2) { for (int i = 1; i < numkeys; i += 2) {
ASSERT_EQ(Get(1, Key(i)), "NOT_FOUND"); ASSERT_EQ(Get(1, Key(i)), "NOT_FOUND");
} }
@ -7650,13 +7674,10 @@ TEST_F(DBTest, OptimizeFiltersForHits) {
ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L1)); ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L1));
ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L2_AND_UP)); ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L2_AND_UP));
// When the skip_filters_on_last_level is ON, the last level which has // Now we have three sorted run, L0, L5 and L6 with most files in L6 have
// most of the keys does not use bloom filters. We end up using // no blooom filter. Most keys be checked bloom filters twice.
// bloom filters in a very small number of cases. Without the flag. ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 65000 * 2);
// this number would be close to 150000 (all the key at the last level) + ASSERT_LT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 120000 * 2);
// some use in the upper levels
//
ASSERT_GT(90000, TestGetTickerCount(options, BLOOM_FILTER_USEFUL));
for (int i = 0; i < numkeys; i += 2) { for (int i = 0; i < numkeys; i += 2) {
ASSERT_EQ(Get(1, Key(i)), "val"); ASSERT_EQ(Get(1, Key(i)), "val");