From 4f1534bdb028b5e37f2f282433906ae493c54163 Mon Sep 17 00:00:00 2001 From: sdong Date: Wed, 1 Jul 2020 19:36:15 -0700 Subject: [PATCH] Whole DBTest to skip fsync (#7049) Summary: After https://github.com/facebook/rocksdb/pull/7036, we still see extra DBTest that can timeout when running 10 or 20 in parallel. Expand skip-fsync mode in whole DBTest. Still preserve other tests from doing this mode to be conservative. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7049 Test Plan: Run all existing files. Reviewed By: pdillinger Differential Revision: D22301700 fbshipit-source-id: f9a9e3b3b26ce640665a47cb8bff33ba0c89b565 --- db/blob/db_blob_index_test.cc | 3 ++- db/db_basic_test.cc | 4 ++-- db/db_block_cache_test.cc | 3 ++- db/db_bloom_filter_test.cc | 8 +++++--- db/db_compaction_filter_test.cc | 3 ++- db/db_compaction_test.cc | 9 ++++++--- db/db_dynamic_level_test.cc | 3 ++- db/db_encryption_test.cc | 3 ++- db/db_flush_test.cc | 2 +- db/db_impl/db_secondary_test.cc | 2 +- db/db_inplace_update_test.cc | 3 ++- db/db_io_failure_test.cc | 3 ++- db/db_iterator_test.cc | 2 +- db/db_log_iter_test.cc | 3 ++- db/db_memtable_test.cc | 2 +- db/db_merge_operand_test.cc | 3 ++- db/db_merge_operator_test.cc | 3 ++- db/db_options_test.cc | 2 +- db/db_properties_test.cc | 3 ++- db/db_range_del_test.cc | 2 +- db/db_sst_test.cc | 2 +- db/db_statistics_test.cc | 3 ++- db/db_table_properties_test.cc | 3 ++- db/db_tailing_iter_test.cc | 3 ++- db/db_test.cc | 9 ++++----- db/db_test2.cc | 12 +++++++----- db/db_test_util.cc | 3 ++- db/db_test_util.h | 5 ++++- db/db_universal_compaction_test.cc | 7 ++++--- db/db_wal_test.cc | 5 +++-- db/db_with_timestamp_basic_test.cc | 2 +- db/db_with_timestamp_compaction_test.cc | 2 +- db/db_write_test.cc | 2 +- db/deletefile_test.cc | 2 +- db/error_handler_fs_test.cc | 3 ++- db/external_sst_file_basic_test.cc | 3 ++- db/external_sst_file_test.cc | 5 +++-- db/import_column_family_test.cc | 3 ++- db/listener_test.cc | 2 +- db/obsolete_files_test.cc | 3 ++- db/repair_test.cc | 2 +- monitoring/stats_history_test.cc | 3 ++- .../option_change_migration_test.cc | 5 +++-- utilities/persistent_cache/persistent_cache_test.cc | 3 ++- utilities/simulator_cache/sim_cache_test.cc | 2 +- 45 files changed, 97 insertions(+), 63 deletions(-) diff --git a/db/blob/db_blob_index_test.cc b/db/blob/db_blob_index_test.cc index 24862f771..b235bbfa9 100644 --- a/db/blob/db_blob_index_test.cc +++ b/db/blob/db_blob_index_test.cc @@ -41,7 +41,8 @@ class DBBlobIndexTest : public DBTestBase { Tier::kImmutableMemtables, Tier::kL0SstFile, Tier::kLnSstFile}; - DBBlobIndexTest() : DBTestBase("/db_blob_index_test") {} + DBBlobIndexTest() + : DBTestBase("/db_blob_index_test", /*env_do_fsync=*/true) {} ColumnFamilyHandle* cfh() { return dbfull()->DefaultColumnFamily(); } diff --git a/db/db_basic_test.cc b/db/db_basic_test.cc index e9584b568..9f96ae9f7 100644 --- a/db/db_basic_test.cc +++ b/db/db_basic_test.cc @@ -27,7 +27,7 @@ namespace ROCKSDB_NAMESPACE { class DBBasicTest : public DBTestBase { public: - DBBasicTest() : DBTestBase("/db_basic_test") {} + DBBasicTest() : DBTestBase("/db_basic_test", /*env_do_fsync=*/true) {} }; TEST_F(DBBasicTest, OpenWhenOpen) { @@ -2352,7 +2352,7 @@ class DBBasicTestMultiGet : public DBTestBase { DBBasicTestMultiGet(std::string test_dir, int num_cfs, bool compressed_cache, bool uncompressed_cache, bool _compression_enabled, bool _fill_cache, uint32_t compression_parallel_threads) - : DBTestBase(test_dir) { + : DBTestBase(test_dir, /*env_do_fsync=*/true) { compression_enabled_ = _compression_enabled; fill_cache_ = _fill_cache; diff --git a/db/db_block_cache_test.cc b/db/db_block_cache_test.cc index 0841a655e..426a1930b 100644 --- a/db/db_block_cache_test.cc +++ b/db/db_block_cache_test.cc @@ -32,7 +32,8 @@ class DBBlockCacheTest : public DBTestBase { const size_t kNumBlocks = 10; const size_t kValueSize = 100; - DBBlockCacheTest() : DBTestBase("/db_block_cache_test") {} + DBBlockCacheTest() + : DBTestBase("/db_block_cache_test", /*env_do_fsync=*/true) {} BlockBasedTableOptions GetTableOptions() { BlockBasedTableOptions table_options; diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index 79f2bcbae..2231af6da 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -23,7 +23,8 @@ using BFP = BloomFilterPolicy; class DBBloomFilterTest : public DBTestBase { public: - DBBloomFilterTest() : DBTestBase("/db_bloom_filter_test") {} + DBBloomFilterTest() + : DBTestBase("/db_bloom_filter_test", /*env_do_fsync=*/true) {} }; class DBBloomFilterTestWithParam : public DBTestBase, @@ -36,7 +37,8 @@ class DBBloomFilterTestWithParam : public DBTestBase, uint32_t format_version_; public: - DBBloomFilterTestWithParam() : DBTestBase("/db_bloom_filter_tests") {} + DBBloomFilterTestWithParam() + : DBTestBase("/db_bloom_filter_tests", /*env_do_fsync=*/true) {} ~DBBloomFilterTestWithParam() override {} @@ -1039,7 +1041,7 @@ class DBBloomFilterTestVaryPrefixAndFormatVer public: DBBloomFilterTestVaryPrefixAndFormatVer() - : DBTestBase("/db_bloom_filter_tests") {} + : DBTestBase("/db_bloom_filter_tests", /*env_do_fsync=*/true) {} ~DBBloomFilterTestVaryPrefixAndFormatVer() override {} diff --git a/db/db_compaction_filter_test.cc b/db/db_compaction_filter_test.cc index a708c0b1a..d357e634f 100644 --- a/db/db_compaction_filter_test.cc +++ b/db/db_compaction_filter_test.cc @@ -21,7 +21,8 @@ static std::string NEW_VALUE = "NewValue"; class DBTestCompactionFilter : public DBTestBase { public: - DBTestCompactionFilter() : DBTestBase("/db_compaction_filter_test") {} + DBTestCompactionFilter() + : DBTestBase("/db_compaction_filter_test", /*env_do_fsync=*/true) {} }; // Param variant of DBTestBase::ChangeCompactOptions diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index eb86ec2dc..3946d36b5 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -25,14 +25,16 @@ namespace ROCKSDB_NAMESPACE { class DBCompactionTest : public DBTestBase { public: - DBCompactionTest() : DBTestBase("/db_compaction_test") {} + DBCompactionTest() + : DBTestBase("/db_compaction_test", /*env_do_fsync=*/true) {} }; class DBCompactionTestWithParam : public DBTestBase, public testing::WithParamInterface> { public: - DBCompactionTestWithParam() : DBTestBase("/db_compaction_test") { + DBCompactionTestWithParam() + : DBTestBase("/db_compaction_test", /*env_do_fsync=*/true) { max_subcompactions_ = std::get<0>(GetParam()); exclusive_manual_compaction_ = std::get<1>(GetParam()); } @@ -4753,7 +4755,8 @@ INSTANTIATE_TEST_CASE_P(DBCompactionDirectIOTest, DBCompactionDirectIOTest, class CompactionPriTest : public DBTestBase, public testing::WithParamInterface { public: - CompactionPriTest() : DBTestBase("/compaction_pri_test") { + CompactionPriTest() + : DBTestBase("/compaction_pri_test", /*env_do_fsync=*/true) { compaction_pri_ = GetParam(); } diff --git a/db/db_dynamic_level_test.cc b/db/db_dynamic_level_test.cc index ec059feb9..cba997eb5 100644 --- a/db/db_dynamic_level_test.cc +++ b/db/db_dynamic_level_test.cc @@ -19,7 +19,8 @@ namespace ROCKSDB_NAMESPACE { class DBTestDynamicLevel : public DBTestBase { public: - DBTestDynamicLevel() : DBTestBase("/db_dynamic_level_test") {} + DBTestDynamicLevel() + : DBTestBase("/db_dynamic_level_test", /*env_do_fsync=*/true) {} }; TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) { diff --git a/db/db_encryption_test.cc b/db/db_encryption_test.cc index b1f3ce23f..dddc83307 100644 --- a/db/db_encryption_test.cc +++ b/db/db_encryption_test.cc @@ -16,7 +16,8 @@ namespace ROCKSDB_NAMESPACE { class DBEncryptionTest : public DBTestBase { public: - DBEncryptionTest() : DBTestBase("/db_encryption_test") {} + DBEncryptionTest() + : DBTestBase("/db_encryption_test", /*env_do_fsync=*/true) {} }; #ifndef ROCKSDB_LITE diff --git a/db/db_flush_test.cc b/db/db_flush_test.cc index 9485d5f22..ca8e69d29 100644 --- a/db/db_flush_test.cc +++ b/db/db_flush_test.cc @@ -22,7 +22,7 @@ namespace ROCKSDB_NAMESPACE { class DBFlushTest : public DBTestBase { public: - DBFlushTest() : DBTestBase("/db_flush_test") {} + DBFlushTest() : DBTestBase("/db_flush_test", /*env_do_fsync=*/true) {} }; class DBFlushDirectIOTest : public DBFlushTest, diff --git a/db/db_impl/db_secondary_test.cc b/db/db_impl/db_secondary_test.cc index d54a52493..7a412aedc 100644 --- a/db/db_impl/db_secondary_test.cc +++ b/db/db_impl/db_secondary_test.cc @@ -19,7 +19,7 @@ namespace ROCKSDB_NAMESPACE { class DBSecondaryTest : public DBTestBase { public: DBSecondaryTest() - : DBTestBase("/db_secondary_test"), + : DBTestBase("/db_secondary_test", /*env_do_fsync=*/true), secondary_path_(), handles_secondary_(), db_secondary_(nullptr) { diff --git a/db/db_inplace_update_test.cc b/db/db_inplace_update_test.cc index 26405864e..944ca24a7 100644 --- a/db/db_inplace_update_test.cc +++ b/db/db_inplace_update_test.cc @@ -13,7 +13,8 @@ namespace ROCKSDB_NAMESPACE { class DBTestInPlaceUpdate : public DBTestBase { public: - DBTestInPlaceUpdate() : DBTestBase("/db_inplace_update_test") {} + DBTestInPlaceUpdate() + : DBTestBase("/db_inplace_update_test", /*env_do_fsync=*/true) {} }; TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) { diff --git a/db/db_io_failure_test.cc b/db/db_io_failure_test.cc index f8d562447..fa8700347 100644 --- a/db/db_io_failure_test.cc +++ b/db/db_io_failure_test.cc @@ -14,7 +14,8 @@ namespace ROCKSDB_NAMESPACE { class DBIOFailureTest : public DBTestBase { public: - DBIOFailureTest() : DBTestBase("/db_io_failure_test") {} + DBIOFailureTest() + : DBTestBase("/db_io_failure_test", /*env_do_fsync=*/true) {} }; #ifndef ROCKSDB_LITE diff --git a/db/db_iterator_test.cc b/db/db_iterator_test.cc index dcef1f897..c66ca2a9a 100644 --- a/db/db_iterator_test.cc +++ b/db/db_iterator_test.cc @@ -33,7 +33,7 @@ class DummyReadCallback : public ReadCallback { class DBIteratorTest : public DBTestBase, public testing::WithParamInterface { public: - DBIteratorTest() : DBTestBase("/db_iterator_test") {} + DBIteratorTest() : DBTestBase("/db_iterator_test", /*env_do_fsync=*/true) {} Iterator* NewIterator(const ReadOptions& read_options, ColumnFamilyHandle* column_family = nullptr) { diff --git a/db/db_log_iter_test.cc b/db/db_log_iter_test.cc index 1f9ff0d45..51d232a6a 100644 --- a/db/db_log_iter_test.cc +++ b/db/db_log_iter_test.cc @@ -19,7 +19,8 @@ namespace ROCKSDB_NAMESPACE { class DBTestXactLogIterator : public DBTestBase { public: - DBTestXactLogIterator() : DBTestBase("/db_log_iter_test") {} + DBTestXactLogIterator() + : DBTestBase("/db_log_iter_test", /*env_do_fsync=*/true) {} std::unique_ptr OpenTransactionLogIter( const SequenceNumber seq) { diff --git a/db/db_memtable_test.cc b/db/db_memtable_test.cc index 5b8a9b7b8..99763e351 100644 --- a/db/db_memtable_test.cc +++ b/db/db_memtable_test.cc @@ -17,7 +17,7 @@ namespace ROCKSDB_NAMESPACE { class DBMemTableTest : public DBTestBase { public: - DBMemTableTest() : DBTestBase("/db_memtable_test") {} + DBMemTableTest() : DBTestBase("/db_memtable_test", /*env_do_fsync=*/true) {} }; class MockMemTableRep : public MemTableRep { diff --git a/db/db_merge_operand_test.cc b/db/db_merge_operand_test.cc index a0ab34e01..2f1e8ead8 100644 --- a/db/db_merge_operand_test.cc +++ b/db/db_merge_operand_test.cc @@ -21,7 +21,8 @@ namespace ROCKSDB_NAMESPACE { class DBMergeOperandTest : public DBTestBase { public: - DBMergeOperandTest() : DBTestBase("/db_merge_operand_test") {} + DBMergeOperandTest() + : DBTestBase("/db_merge_operand_test", /*env_do_fsync=*/true) {} }; TEST_F(DBMergeOperandTest, GetMergeOperandsBasic) { diff --git a/db/db_merge_operator_test.cc b/db/db_merge_operator_test.cc index 4f762468d..9fb527c08 100644 --- a/db/db_merge_operator_test.cc +++ b/db/db_merge_operator_test.cc @@ -35,7 +35,8 @@ class TestReadCallback : public ReadCallback { // Test merge operator functionality. class DBMergeOperatorTest : public DBTestBase { public: - DBMergeOperatorTest() : DBTestBase("/db_merge_operator_test") {} + DBMergeOperatorTest() + : DBTestBase("/db_merge_operator_test", /*env_do_fsync=*/true) {} std::string GetWithReadCallback(SnapshotChecker* snapshot_checker, const Slice& key, diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 8cc09ec54..add2e72a2 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -27,7 +27,7 @@ namespace ROCKSDB_NAMESPACE { class DBOptionsTest : public DBTestBase { public: - DBOptionsTest() : DBTestBase("/db_options_test") {} + DBOptionsTest() : DBTestBase("/db_options_test", /*env_do_fsync=*/true) {} #ifndef ROCKSDB_LITE std::unordered_map GetMutableDBOptionsMap( diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc index 50dc3efef..afb503285 100644 --- a/db/db_properties_test.cc +++ b/db/db_properties_test.cc @@ -26,7 +26,8 @@ namespace ROCKSDB_NAMESPACE { class DBPropertiesTest : public DBTestBase { public: - DBPropertiesTest() : DBTestBase("/db_properties_test") {} + DBPropertiesTest() + : DBTestBase("/db_properties_test", /*env_do_fsync=*/true) {} }; #ifndef ROCKSDB_LITE diff --git a/db/db_range_del_test.cc b/db/db_range_del_test.cc index 83fb27ff3..3932f4185 100644 --- a/db/db_range_del_test.cc +++ b/db/db_range_del_test.cc @@ -13,7 +13,7 @@ namespace ROCKSDB_NAMESPACE { class DBRangeDelTest : public DBTestBase { public: - DBRangeDelTest() : DBTestBase("/db_range_del_test") {} + DBRangeDelTest() : DBTestBase("/db_range_del_test", /*env_do_fsync=*/true) {} std::string GetNumericStr(int key) { uint64_t uint64_key = static_cast(key); diff --git a/db/db_sst_test.cc b/db/db_sst_test.cc index 687d3b852..27a7f7636 100644 --- a/db/db_sst_test.cc +++ b/db/db_sst_test.cc @@ -17,7 +17,7 @@ namespace ROCKSDB_NAMESPACE { class DBSSTTest : public DBTestBase { public: - DBSSTTest() : DBTestBase("/db_sst_test") {} + DBSSTTest() : DBTestBase("/db_sst_test", /*env_do_fsync=*/true) {} }; #ifndef ROCKSDB_LITE diff --git a/db/db_statistics_test.cc b/db/db_statistics_test.cc index 8fbbb96d5..5a7c8ea66 100644 --- a/db/db_statistics_test.cc +++ b/db/db_statistics_test.cc @@ -14,7 +14,8 @@ namespace ROCKSDB_NAMESPACE { class DBStatisticsTest : public DBTestBase { public: - DBStatisticsTest() : DBTestBase("/db_statistics_test") {} + DBStatisticsTest() + : DBTestBase("/db_statistics_test", /*env_do_fsync=*/true) {} }; TEST_F(DBStatisticsTest, CompressionStatsTest) { diff --git a/db/db_table_properties_test.cc b/db/db_table_properties_test.cc index 98512e22f..17bdf8268 100644 --- a/db/db_table_properties_test.cc +++ b/db/db_table_properties_test.cc @@ -48,7 +48,8 @@ void VerifyTableProperties(DB* db, uint64_t expected_entries_size) { class DBTablePropertiesTest : public DBTestBase, public testing::WithParamInterface { public: - DBTablePropertiesTest() : DBTestBase("/db_table_properties_test") {} + DBTablePropertiesTest() + : DBTestBase("/db_table_properties_test", /*env_do_fsync=*/true) {} TablePropertiesCollection TestGetPropertiesOfTablesInRange( std::vector ranges, std::size_t* num_properties = nullptr, std::size_t* num_files = nullptr); diff --git a/db/db_tailing_iter_test.cc b/db/db_tailing_iter_test.cc index 39988638b..f33b7cb13 100644 --- a/db/db_tailing_iter_test.cc +++ b/db/db_tailing_iter_test.cc @@ -20,7 +20,8 @@ namespace ROCKSDB_NAMESPACE { class DBTestTailingIterator : public DBTestBase { public: - DBTestTailingIterator() : DBTestBase("/db_tailing_iterator_test") {} + DBTestTailingIterator() + : DBTestBase("/db_tailing_iterator_test", /*env_do_fsync=*/true) {} }; TEST_F(DBTestTailingIterator, TailingIteratorSingle) { diff --git a/db/db_test.cc b/db/db_test.cc index 4af91c990..d14f400fe 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -70,9 +70,12 @@ namespace ROCKSDB_NAMESPACE { +// Note that whole DBTest and its child classes disable fsync on files +// and directories for speed. +// If fsync needs to be covered in a test, put it in other places. class DBTest : public DBTestBase { public: - DBTest() : DBTestBase("/db_test") {} + DBTest() : DBTestBase("/db_test", /*env_do_fsync=*/false) {} }; class DBTestWithParam @@ -1790,7 +1793,6 @@ TEST_F(DBTest, Snapshot) { TEST_F(DBTest, HiddenValuesAreRemoved) { anon::OptionsOverride options_override; options_override.skip_policy = kSkipNoSnapshot; - env_->skip_fsync_ = true; do { Options options = CurrentOptions(options_override); CreateAndReopenWithCF({"pikachu"}, options); @@ -3995,7 +3997,6 @@ TEST_F(DBTest, DynamicMemtableOptions) { const uint64_t k128KB = 1 << 17; const uint64_t k5KB = 5 * 1024; Options options; - env_->skip_fsync_ = true; options.env = env_; options.create_if_missing = true; options.compression = kNoCompression; @@ -5142,7 +5143,6 @@ TEST_F(DBTest, DynamicUniversalCompactionOptions) { TEST_F(DBTest, FileCreationRandomFailure) { Options options; - env_->skip_fsync_ = true; options.env = env_; options.create_if_missing = true; options.write_buffer_size = 100000; // Small write buffer @@ -5502,7 +5502,6 @@ TEST_F(DBTest, MergeTestTime) { #ifndef ROCKSDB_LITE TEST_P(DBTestWithParam, MergeCompactionTimeTest) { SetPerfLevel(kEnableTime); - env_->skip_fsync_ = true; Options options = CurrentOptions(); options.compaction_filter_factory = std::make_shared(); options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); diff --git a/db/db_test2.cc b/db/db_test2.cc index fd2933b8c..a48e66fde 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -22,7 +22,7 @@ namespace ROCKSDB_NAMESPACE { class DBTest2 : public DBTestBase { public: - DBTest2() : DBTestBase("/db_test2") {} + DBTest2() : DBTestBase("/db_test2", /*env_do_fsync=*/true) {} }; #ifndef ROCKSDB_LITE @@ -92,7 +92,8 @@ class TestReadOnlyWithCompressedCache public testing::WithParamInterface> { public: TestReadOnlyWithCompressedCache() - : DBTestBase("/test_readonly_with_compressed_cache") { + : DBTestBase("/test_readonly_with_compressed_cache", + /*env_do_fsync=*/true) { max_open_files_ = std::get<0>(GetParam()); use_mmap_ = std::get<1>(GetParam()); } @@ -183,7 +184,7 @@ class PrefixFullBloomWithReverseComparator public ::testing::WithParamInterface { public: PrefixFullBloomWithReverseComparator() - : DBTestBase("/prefix_bloom_reverse") {} + : DBTestBase("/prefix_bloom_reverse", /*env_do_fsync=*/true) {} void SetUp() override { if_cache_filter_ = GetParam(); } bool if_cache_filter_; }; @@ -323,7 +324,7 @@ class DBTestSharedWriteBufferAcrossCFs public testing::WithParamInterface> { public: DBTestSharedWriteBufferAcrossCFs() - : DBTestBase("/db_test_shared_write_buffer") {} + : DBTestBase("/db_test_shared_write_buffer", /*env_do_fsync=*/true) {} void SetUp() override { use_old_interface_ = std::get<0>(GetParam()); cost_cache_ = std::get<1>(GetParam()); @@ -1790,7 +1791,8 @@ class PinL0IndexAndFilterBlocksTest : public DBTestBase, public testing::WithParamInterface> { public: - PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {} + PinL0IndexAndFilterBlocksTest() + : DBTestBase("/db_pin_l0_index_bloom_test", /*env_do_fsync=*/true) {} void SetUp() override { infinite_max_files_ = std::get<0>(GetParam()); disallow_preload_ = std::get<1>(GetParam()); diff --git a/db/db_test_util.cc b/db/db_test_util.cc index c2c1d44ea..0a61d56ad 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -56,7 +56,7 @@ SpecialEnv::SpecialEnv(Env* base) ROT13BlockCipher rot13Cipher_(16); #endif // ROCKSDB_LITE -DBTestBase::DBTestBase(const std::string path) +DBTestBase::DBTestBase(const std::string path, bool env_do_fsync) : mem_env_(nullptr), encrypted_env_(nullptr), option_config_(kDefault) { Env* base_env = Env::Default(); #ifndef ROCKSDB_LITE @@ -83,6 +83,7 @@ DBTestBase::DBTestBase(const std::string path) : (mem_env_ ? mem_env_ : base_env)); env_->SetBackgroundThreads(1, Env::LOW); env_->SetBackgroundThreads(1, Env::HIGH); + env_->skip_fsync_ = !env_do_fsync; dbname_ = test::PerThreadDBPath(env_, path); alternative_wal_dir_ = dbname_ + "/wal"; alternative_db_log_dir_ = dbname_ + "/db_log_dir"; diff --git a/db/db_test_util.h b/db/db_test_util.h index 7a40a1346..4330b9cfb 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -872,7 +872,10 @@ class DBTestBase : public testing::Test { // requires. kSkipMmapReads; - explicit DBTestBase(const std::string path); + // `env_do_fsync` decides whether the special Env would do real + // fsync for files and directories. Skipping fsync can speed up + // tests, but won't cover the exact fsync logic. + DBTestBase(const std::string path, bool env_do_fsync); ~DBTestBase(); diff --git a/db/db_universal_compaction_test.cc b/db/db_universal_compaction_test.cc index 889e380dd..40b975280 100644 --- a/db/db_universal_compaction_test.cc +++ b/db/db_universal_compaction_test.cc @@ -25,8 +25,8 @@ class DBTestUniversalCompactionBase : public DBTestBase, public ::testing::WithParamInterface> { public: - explicit DBTestUniversalCompactionBase( - const std::string& path) : DBTestBase(path) {} + explicit DBTestUniversalCompactionBase(const std::string& path) + : DBTestBase(path, /*env_do_fsync=*/true) {} void SetUp() override { num_levels_ = std::get<0>(GetParam()); exclusive_manual_compaction_ = std::get<1>(GetParam()); @@ -43,7 +43,8 @@ class DBTestUniversalCompaction : public DBTestUniversalCompactionBase { class DBTestUniversalCompaction2 : public DBTestBase { public: - DBTestUniversalCompaction2() : DBTestBase("/db_universal_compaction_test2") {} + DBTestUniversalCompaction2() + : DBTestBase("/db_universal_compaction_test2", /*env_do_fsync=*/true) {} }; namespace { diff --git a/db/db_wal_test.cc b/db/db_wal_test.cc index f2f661c30..9fad7f6bc 100644 --- a/db/db_wal_test.cc +++ b/db/db_wal_test.cc @@ -18,7 +18,7 @@ namespace ROCKSDB_NAMESPACE { class DBWALTest : public DBTestBase { public: - DBWALTest() : DBTestBase("/db_wal_test") {} + DBWALTest() : DBTestBase("/db_wal_test", /*env_do_fsync=*/true) {} #if defined(ROCKSDB_PLATFORM_POSIX) uint64_t GetAllocatedFileSize(std::string file_name) { @@ -86,7 +86,8 @@ class EnrichedSpecialEnv : public SpecialEnv { class DBWALTestWithEnrichedEnv : public DBTestBase { public: - DBWALTestWithEnrichedEnv() : DBTestBase("/db_wal_test") { + DBWALTestWithEnrichedEnv() + : DBTestBase("/db_wal_test", /*env_do_fsync=*/true) { enriched_env_ = new EnrichedSpecialEnv(env_->target()); auto options = CurrentOptions(); options.env = enriched_env_; diff --git a/db/db_with_timestamp_basic_test.cc b/db/db_with_timestamp_basic_test.cc index a2a7313af..c431fc673 100644 --- a/db/db_with_timestamp_basic_test.cc +++ b/db/db_with_timestamp_basic_test.cc @@ -22,7 +22,7 @@ namespace ROCKSDB_NAMESPACE { class DBBasicTestWithTimestampBase : public DBTestBase { public: explicit DBBasicTestWithTimestampBase(const std::string& dbname) - : DBTestBase(dbname) {} + : DBTestBase(dbname, /*env_do_fsync=*/true) {} protected: static std::string Key1(uint64_t k) { diff --git a/db/db_with_timestamp_compaction_test.cc b/db/db_with_timestamp_compaction_test.cc index f5cc54df1..6b74fa21f 100644 --- a/db/db_with_timestamp_compaction_test.cc +++ b/db/db_with_timestamp_compaction_test.cc @@ -31,7 +31,7 @@ std::string Timestamp(uint64_t ts) { class TimestampCompatibleCompactionTest : public DBTestBase { public: TimestampCompatibleCompactionTest() - : DBTestBase("/ts_compatible_compaction_test") {} + : DBTestBase("/ts_compatible_compaction_test", /*env_do_fsync=*/true) {} std::string Get(const std::string& key, uint64_t ts) { ReadOptions read_opts; diff --git a/db/db_write_test.cc b/db/db_write_test.cc index cc1aaac08..7f05ad7ac 100644 --- a/db/db_write_test.cc +++ b/db/db_write_test.cc @@ -22,7 +22,7 @@ namespace ROCKSDB_NAMESPACE { // Test variations of WriteImpl. class DBWriteTest : public DBTestBase, public testing::WithParamInterface { public: - DBWriteTest() : DBTestBase("/db_write_test") {} + DBWriteTest() : DBTestBase("/db_write_test", /*env_do_fsync=*/true) {} Options GetOptions() { return DBTestBase::GetOptions(GetParam()); } diff --git a/db/deletefile_test.cc b/db/deletefile_test.cc index f202388c0..635951a1e 100644 --- a/db/deletefile_test.cc +++ b/db/deletefile_test.cc @@ -35,7 +35,7 @@ class DeleteFileTest : public DBTestBase { const std::string wal_dir_; DeleteFileTest() - : DBTestBase("/deletefile_test"), + : DBTestBase("/deletefile_test", /*env_do_fsync=*/true), numlevels_(7), wal_dir_(dbname_ + "/wal_files") {} diff --git a/db/error_handler_fs_test.cc b/db/error_handler_fs_test.cc index 912baa1ba..5ad0a5629 100644 --- a/db/error_handler_fs_test.cc +++ b/db/error_handler_fs_test.cc @@ -23,7 +23,8 @@ namespace ROCKSDB_NAMESPACE { class DBErrorHandlingFSTest : public DBTestBase { public: - DBErrorHandlingFSTest() : DBTestBase("/db_error_handling_fs_test") {} + DBErrorHandlingFSTest() + : DBTestBase("/db_error_handling_fs_test", /*env_do_fsync=*/true) {} std::string GetManifestNameFromLiveFiles() { std::vector live_files; diff --git a/db/external_sst_file_basic_test.cc b/db/external_sst_file_basic_test.cc index 7e62963ec..1745f57ad 100644 --- a/db/external_sst_file_basic_test.cc +++ b/db/external_sst_file_basic_test.cc @@ -20,7 +20,8 @@ class ExternalSSTFileBasicTest : public DBTestBase, public ::testing::WithParamInterface> { public: - ExternalSSTFileBasicTest() : DBTestBase("/external_sst_file_basic_test") { + ExternalSSTFileBasicTest() + : DBTestBase("/external_sst_file_basic_test", /*env_do_fsync=*/true) { sst_files_dir_ = dbname_ + "/sst_files/"; fault_injection_test_env_.reset(new FaultInjectionTestEnv(Env::Default())); DestroyAndRecreateExternalSSTFilesDir(); diff --git a/db/external_sst_file_test.cc b/db/external_sst_file_test.cc index 649bf7ac2..d64bebbe6 100644 --- a/db/external_sst_file_test.cc +++ b/db/external_sst_file_test.cc @@ -43,7 +43,7 @@ class ExternSSTFileLinkFailFallbackTest public ::testing::WithParamInterface> { public: ExternSSTFileLinkFailFallbackTest() - : DBTestBase("/external_sst_file_test"), + : DBTestBase("/external_sst_file_test", /*env_do_fsync=*/true), test_env_(new ExternalSSTTestEnv(env_, true)) { sst_files_dir_ = dbname_ + "/sst_files/"; test::DestroyDir(env_, sst_files_dir_); @@ -71,7 +71,8 @@ class ExternalSSTFileTest : public DBTestBase, public ::testing::WithParamInterface> { public: - ExternalSSTFileTest() : DBTestBase("/external_sst_file_test") { + ExternalSSTFileTest() + : DBTestBase("/external_sst_file_test", /*env_do_fsync=*/true) { sst_files_dir_ = dbname_ + "/sst_files/"; DestroyAndRecreateExternalSSTFilesDir(); } diff --git a/db/import_column_family_test.cc b/db/import_column_family_test.cc index e17895b46..37167034d 100644 --- a/db/import_column_family_test.cc +++ b/db/import_column_family_test.cc @@ -11,7 +11,8 @@ namespace ROCKSDB_NAMESPACE { class ImportColumnFamilyTest : public DBTestBase { public: - ImportColumnFamilyTest() : DBTestBase("/import_column_family_test") { + ImportColumnFamilyTest() + : DBTestBase("/import_column_family_test", /*env_do_fsync=*/true) { sst_files_dir_ = dbname_ + "/sst_files/"; DestroyAndRecreateExternalSSTFilesDir(); export_files_dir_ = test::PerThreadDBPath(env_, "export"); diff --git a/db/listener_test.cc b/db/listener_test.cc index de5eb58c6..2845f778b 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -41,7 +41,7 @@ namespace ROCKSDB_NAMESPACE { class EventListenerTest : public DBTestBase { public: - EventListenerTest() : DBTestBase("/listener_test") {} + EventListenerTest() : DBTestBase("/listener_test", /*env_do_fsync=*/true) {} static std::string BlobStr(uint64_t blob_file_number, uint64_t offset, uint64_t size) { diff --git a/db/obsolete_files_test.cc b/db/obsolete_files_test.cc index 42269a56c..98d98eae2 100644 --- a/db/obsolete_files_test.cc +++ b/db/obsolete_files_test.cc @@ -38,7 +38,8 @@ namespace ROCKSDB_NAMESPACE { class ObsoleteFilesTest : public DBTestBase { public: ObsoleteFilesTest() - : DBTestBase("/obsolete_files_test"), wal_dir_(dbname_ + "/wal_files") {} + : DBTestBase("/obsolete_files_test", /*env_do_fsync=*/true), + wal_dir_(dbname_ + "/wal_files") {} void AddKeys(int numkeys, int startkey) { WriteOptions options; diff --git a/db/repair_test.cc b/db/repair_test.cc index ba2bae3d0..d1b8fc700 100644 --- a/db/repair_test.cc +++ b/db/repair_test.cc @@ -22,7 +22,7 @@ namespace ROCKSDB_NAMESPACE { #ifndef ROCKSDB_LITE class RepairTest : public DBTestBase { public: - RepairTest() : DBTestBase("/repair_test") {} + RepairTest() : DBTestBase("/repair_test", /*env_do_fsync=*/true) {} std::string GetFirstSstPath() { uint64_t manifest_size; diff --git a/monitoring/stats_history_test.cc b/monitoring/stats_history_test.cc index 843e7d65d..ffa1f5e28 100644 --- a/monitoring/stats_history_test.cc +++ b/monitoring/stats_history_test.cc @@ -28,7 +28,8 @@ namespace ROCKSDB_NAMESPACE { class StatsHistoryTest : public DBTestBase { public: - StatsHistoryTest() : DBTestBase("/stats_history_test") {} + StatsHistoryTest() + : DBTestBase("/stats_history_test", /*env_do_fsync=*/true) {} }; #ifndef ROCKSDB_LITE diff --git a/utilities/option_change_migration/option_change_migration_test.cc b/utilities/option_change_migration/option_change_migration_test.cc index 5bc883ff7..8fb9f1296 100644 --- a/utilities/option_change_migration/option_change_migration_test.cc +++ b/utilities/option_change_migration/option_change_migration_test.cc @@ -19,7 +19,7 @@ class DBOptionChangeMigrationTests std::tuple> { public: DBOptionChangeMigrationTests() - : DBTestBase("/db_option_change_migration_test") { + : DBTestBase("/db_option_change_migration_test", /*env_do_fsync=*/true) { level1_ = std::get<0>(GetParam()); compaction_style1_ = std::get<1>(GetParam()); is_dynamic1_ = std::get<2>(GetParam()); @@ -350,7 +350,8 @@ INSTANTIATE_TEST_CASE_P( class DBOptionChangeMigrationTest : public DBTestBase { public: DBOptionChangeMigrationTest() - : DBTestBase("/db_option_change_migration_test2") {} + : DBTestBase("/db_option_change_migration_test2", /*env_do_fsync=*/true) { + } }; TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) { diff --git a/utilities/persistent_cache/persistent_cache_test.cc b/utilities/persistent_cache/persistent_cache_test.cc index 92b39307d..52fb66af5 100644 --- a/utilities/persistent_cache/persistent_cache_test.cc +++ b/utilities/persistent_cache/persistent_cache_test.cc @@ -295,7 +295,8 @@ TEST_F(PersistentCacheTierTest, FactoryTest) { } } -PersistentCacheDBTest::PersistentCacheDBTest() : DBTestBase("/cache_test") { +PersistentCacheDBTest::PersistentCacheDBTest() + : DBTestBase("/cache_test", /*env_do_fsync=*/true) { #ifdef OS_LINUX ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( diff --git a/utilities/simulator_cache/sim_cache_test.cc b/utilities/simulator_cache/sim_cache_test.cc index 996ea90f6..ac2e24867 100644 --- a/utilities/simulator_cache/sim_cache_test.cc +++ b/utilities/simulator_cache/sim_cache_test.cc @@ -21,7 +21,7 @@ class SimCacheTest : public DBTestBase { const size_t kNumBlocks = 5; const size_t kValueSize = 1000; - SimCacheTest() : DBTestBase("/sim_cache_test") {} + SimCacheTest() : DBTestBase("/sim_cache_test", /*env_do_fsync=*/true) {} BlockBasedTableOptions GetTableOptions() { BlockBasedTableOptions table_options;