andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Streamline persistent_cache_test for testing efficiency (#6601)

Browse Source 
Summary:
This test was written like a stress test, using up to 3x26GB
RSS memory during parallel 'make check'. Now, while this code is mostly
dormant, I've made the "for Travis" versions of the expensive tests the
canonical versions and disabled the expensive versions. This has the
side benefit of removing some arbitrary conditional compilation.

For unknown reason, the super expensive tests were gated on
Snappy_Supported, which appears to be irrelevant, so I removed it.

The tests can be fixed / improved / migrated to stress test if/when they
are deemed important again.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6601

Test Plan:
make check + CI

./persistent_cache_test Before:
...
[==========] 10 tests from 2 test cases ran. (114541 ms total)
[  PASSED  ] 10 tests.
YOU HAVE 1 DISABLED TEST

After:
...
[==========] 3 tests from 2 test cases ran. (1714 ms total)
[  PASSED  ] 3 tests.
YOU HAVE 10 DISABLED TESTS

Reviewed By: siying

Differential Revision: D20680983

Pulled By: pdillinger

fbshipit-source-id: 2be0fde13eeb0a71110ac7f5477cfe63996a509e
This commit is contained in:
Peter Dillinger 2020-03-26 19:34:06 -07:00 committed by Facebook GitHub Bot
parent 6f62322fe4
commit e91d1a21a6
1 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
utilities/persistent_cache/persistent_cache_test.cc
View File

@ -160,7 +160,6 @@ TEST_F(PersistentCacheTierTest, DISABLED_BlockCacheInsertWithFileCreateError) {
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
}
#if defined(TRAVIS) || defined(ROCKSDB_VALGRIND_RUN)
// Travis is unable to handle the normal version of the tests running out of
// fds, out of space and timeouts. This is an easier version of the test
// specifically written for Travis
@ -177,9 +176,10 @@ TEST_F(PersistentCacheTierTest, BasicTest) {
/*memory_size=*/static_cast<size_t>(1 * 1024 * 1024));
RunInsertTest(/*nthreads=*/1, /*max_keys=*/1024);
}
#else
// Volatile cache tests
TEST_F(PersistentCacheTierTest, VolatileCacheInsert) {
// DISABLED for now (somewhat expensive)
TEST_F(PersistentCacheTierTest, DISABLED_VolatileCacheInsert) {
for (auto nthreads : {1, 5}) {
for (auto max_keys :
{10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) {
@ -189,7 +189,8 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsert) {
}
}
TEST_F(PersistentCacheTierTest, VolatileCacheInsertWithEviction) {
// DISABLED for now (somewhat expensive)
TEST_F(PersistentCacheTierTest, DISABLED_VolatileCacheInsertWithEviction) {
for (auto nthreads : {1, 5}) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = std::make_shared<VolatileCacheTier>(
@ -200,7 +201,8 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsertWithEviction) {
}
// Block cache tests
TEST_F(PersistentCacheTierTest, BlockCacheInsert) {
// DISABLED for now (expensive)
TEST_F(PersistentCacheTierTest, DISABLED_BlockCacheInsert) {
for (auto direct_writes : {true, false}) {
for (auto nthreads : {1, 5}) {
for (auto max_keys :
@ -214,7 +216,8 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsert) {
}
}
TEST_F(PersistentCacheTierTest, BlockCacheInsertWithEviction) {
// DISABLED for now (somewhat expensive)
TEST_F(PersistentCacheTierTest, DISABLED_BlockCacheInsertWithEviction) {
for (auto nthreads : {1, 5}) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewBlockCache(Env::Default(), path_,
@ -225,7 +228,8 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsertWithEviction) {
}
// Tiered cache tests
TEST_F(PersistentCacheTierTest, TieredCacheInsert) {
// DISABLED for now (expensive)
TEST_F(PersistentCacheTierTest, DISABLED_TieredCacheInsert) {
for (auto nthreads : {1, 5}) {
for (auto max_keys :
{10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) {
@ -238,7 +242,8 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsert) {
// the tests causes a lot of file deletions which Travis limited testing
// environment cannot handle
TEST_F(PersistentCacheTierTest, TieredCacheInsertWithEviction) {
// DISABLED for now (somewhat expensive)
TEST_F(PersistentCacheTierTest, DISABLED_TieredCacheInsertWithEviction) {
for (auto nthreads : {1, 5}) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewTieredCache(
@ -249,7 +254,6 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsertWithEviction) {
}
}
}
#endif
std::shared_ptr<PersistentCacheTier> MakeVolatileCache(
const std::string& /*dbname*/) {
@ -308,9 +312,6 @@ PersistentCacheDBTest::PersistentCacheDBTest() : DBTestBase("/cache_test") {
void PersistentCacheDBTest::RunTest(
const std::function<std::shared_ptr<PersistentCacheTier>(bool)>& new_pcache,
const size_t max_keys = 100 * 1024, const size_t max_usecase = 5) {
if (!Snappy_Supported()) {
return;
}
// number of insertion interations
int num_iter = static_cast<int>(max_keys * kStressFactor);
@ -438,30 +439,32 @@ void PersistentCacheDBTest::RunTest(
}
}
#if defined(TRAVIS) || defined(ROCKSDB_VALGRIND_RUN)
// Travis is unable to handle the normal version of the tests running out of
// fds, out of space and timeouts. This is an easier version of the test
// specifically written for Travis
// specifically written for Travis.
// Now used generally because main tests are too expensive as unit tests.
TEST_F(PersistentCacheDBTest, BasicTest) {
RunTest(std::bind(&MakeBlockCache, dbname_), /*max_keys=*/1024,
/*max_usecase=*/1);
}
#else
// test table with block page cache
TEST_F(PersistentCacheDBTest, BlockCacheTest) {
// DISABLED for now (very expensive, especially memory)
TEST_F(PersistentCacheDBTest, DISABLED_BlockCacheTest) {
RunTest(std::bind(&MakeBlockCache, dbname_));
}
// test table with volatile page cache
TEST_F(PersistentCacheDBTest, VolatileCacheTest) {
// DISABLED for now (very expensive, especially memory)
TEST_F(PersistentCacheDBTest, DISABLED_VolatileCacheTest) {
RunTest(std::bind(&MakeVolatileCache, dbname_));
}
// test table with tiered page cache
TEST_F(PersistentCacheDBTest, TieredCacheTest) {
// DISABLED for now (very expensive, especially memory)
TEST_F(PersistentCacheDBTest, DISABLED_TieredCacheTest) {
RunTest(std::bind(&MakeTieredCache, dbname_));
}
#endif
} // namespace ROCKSDB_NAMESPACE