From 731f022dc8f5efea048465793ca429f81a22dfb5 Mon Sep 17 00:00:00 2001 From: Zhichao Cao Date: Sat, 19 Sep 2020 17:56:39 -0700 Subject: [PATCH] fix the flaky test failure (#7415) Summary: Fix the flaky test failure in error_handler_fs_test. Add the sync point, solve the dependency. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7415 Test Plan: make asan_check, ~/gtest-parallel/gtest-parallel -r 100 ./error_handler_fs_test Reviewed By: siying Differential Revision: D23804330 Pulled By: zhichao-cao fbshipit-source-id: 5175108651f7652e47e15978f2a9c1669ef59d80 --- db/builder.cc | 2 + db/error_handler_fs_test.cc | 109 ++++++++++++++++++++++++++---------- 2 files changed, 82 insertions(+), 29 deletions(-) diff --git a/db/builder.cc b/db/builder.cc index 9dfdf4217..86253260a 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -226,10 +226,12 @@ Status BuildTable( delete builder; // Finish and check for file errors + TEST_SYNC_POINT("BuildTable:BeforeSyncTable"); if (s.ok() && !empty) { StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS); *io_status = file_writer->Sync(ioptions.use_fsync); } + TEST_SYNC_POINT("BuildTable:BeforeCloseTableFile"); if (s.ok() && io_status->ok() && !empty) { *io_status = file_writer->Close(); } diff --git a/db/error_handler_fs_test.cc b/db/error_handler_fs_test.cc index 24291cca1..4336043af 100644 --- a/db/error_handler_fs_test.cc +++ b/db/error_handler_fs_test.cc @@ -247,7 +247,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError) { +TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { std::shared_ptr fault_fs( new FaultInjectionTestFS(FileSystem::Default())); std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); @@ -289,47 +289,98 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError) { ASSERT_OK(s); ASSERT_EQ("val3", Get(Key(3))); - Put(Key(4), "val4", wo); - SyncPoint::GetInstance()->SetCallBack( - "BuildTable:BeforeCloseTableFile", - [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); - SyncPoint::GetInstance()->EnableProcessing(); - s = Flush(); - Put(Key(5), "val5", wo); - ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); - ASSERT_EQ("val5", Get(Key(5))); - SyncPoint::GetInstance()->DisableProcessing(); - fault_fs->SetFilesystemActive(true); - s = dbfull()->Resume(); - ASSERT_EQ(s, Status::OK()); - ASSERT_EQ("val4", Get(Key(4))); - ASSERT_EQ("val5", Get(Key(5))); - Put(Key(6), "val6", wo); - ASSERT_EQ("val6", Get(Key(6))); - s = Flush(); - ASSERT_OK(s); - ASSERT_EQ("val6", Get(Key(6))); + Destroy(options); +} - Put(Key(7), "val7", wo); +TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError2) { + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_fs_env.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + + Put(Key(1), "val1", wo); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeSyncTable", [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); - Put(Key(8), "val8", wo); + Put(Key(2), "val2", wo); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); - ASSERT_EQ("val8", Get(Key(8))); + ASSERT_EQ("val2", Get(Key(2))); SyncPoint::GetInstance()->DisableProcessing(); fault_fs->SetFilesystemActive(true); s = dbfull()->Resume(); ASSERT_EQ(s, Status::OK()); - ASSERT_EQ("val7", Get(Key(7))); - ASSERT_EQ("val8", Get(Key(8))); - Put(Key(9), "val9", wo); - ASSERT_EQ("val9", Get(Key(9))); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ("val2", Get(Key(2))); + Put(Key(3), "val3", wo); + ASSERT_EQ("val3", Get(Key(3))); s = Flush(); ASSERT_OK(s); - ASSERT_EQ("val9", Get(Key(9))); + ASSERT_EQ("val3", Get(Key(3))); + + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError3) { + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_fs_env.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + + Put(Key(1), "val1", wo); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeCloseTableFile", + [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + Put(Key(2), "val2", wo); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_EQ("val2", Get(Key(2))); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ("val2", Get(Key(2))); + Put(Key(3), "val3", wo); + ASSERT_EQ("val3", Get(Key(3))); + s = Flush(); + ASSERT_OK(s); + ASSERT_EQ("val3", Get(Key(3))); + Destroy(options); }