andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Silence false alarms in db_stress fault injection (#6741)

Browse Source 
Summary:
False alarms are caused by codepaths that intentionally swallow IO
errors.

Tests:
make crash_test
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6741

Reviewed By: ltamasi

Differential Revision: D21181138

Pulled By: anand1976

fbshipit-source-id: 5ccfbc68eb192033488de6269e59c00f2c65ce00
This commit is contained in:
anand76 2020-04-24 13:03:08 -07:00 committed by Facebook GitHub Bot
parent e04f3bce4f
commit 9e7b7e2c08
12 changed files with 73 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
db_stress_tool/db_stress_shared_state.cc
View File

@ -16,12 +16,12 @@ const uint32_t SharedState::UNKNOWN_SENTINEL = 0xfffffffe;
const uint32_t SharedState::DELETION_SENTINEL = 0xffffffff;
#if defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
#if defined(OS_SOLARIS)
__thread bool SharedState::filter_read_error;
__thread bool SharedState::ignore_read_error;
#else
thread_local bool SharedState::filter_read_error;
thread_local bool SharedState::ignore_read_error;
#endif // OS_SOLARIS
#else
bool SharedState::filter_read_error;
bool SharedState::ignore_read_error;
#endif // ROCKSDB_SUPPORT_THREAD_LOCAL
} // namespace ROCKSDB_NAMESPACE
#endif // GFLAGS

14
db_stress_tool/db_stress_shared_state.h
View File

@ -48,12 +48,12 @@ class SharedState {
// for those calls
#if defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
#if defined(OS_SOLARIS)
static __thread bool filter_read_error;
static __thread bool ignore_read_error;
#else
static thread_local bool filter_read_error;
static thread_local bool ignore_read_error;
#endif // OS_SOLARIS
#else
static bool filter_read_error;
static bool ignore_read_error;
#endif // ROCKSDB_SUPPORT_THREAD_LOCAL
SharedState(Env* env, StressTest* stress_test)
@ -192,8 +192,8 @@ class SharedState {
}
#ifndef NDEBUG
if (FLAGS_read_fault_one_in) {
SyncPoint::GetInstance()->SetCallBack("FilterReadError",
FilterReadErrorCallback);
SyncPoint::GetInstance()->SetCallBack("FaultInjectionIgnoreError",
IgnoreReadErrorCallback);
SyncPoint::GetInstance()->EnableProcessing();
}
#endif // NDEBUG
@ -362,8 +362,8 @@ class SharedState {
}
private:
static void FilterReadErrorCallback(void*) {
filter_read_error = true;
static void IgnoreReadErrorCallback(void*) {
ignore_read_error = true;
}
port::Mutex mu_;

8
db_stress_tool/no_batched_ops_stress.cc
View File

@ -152,7 +152,7 @@ class NonBatchedOpsStressTest : public StressTest {
#ifndef NDEBUG
if (fault_fs_guard) {
fault_fs_guard->EnableErrorInjection();
SharedState::filter_read_error = false;
SharedState::ignore_read_error = false;
}
#endif // NDEBUG
Status s = db_->Get(read_opts, cfh, key, &from_db);
@ -164,7 +164,7 @@ class NonBatchedOpsStressTest : public StressTest {
if (s.ok()) {
#ifndef NDEBUG
if (fault_fs_guard) {
if (error_count && !SharedState::filter_read_error) {
if (error_count && !SharedState::ignore_read_error) {
// Grab mutex so multiple thread don't try to print the
// stack trace at the same time
MutexLock l(thread->shared->GetMutex());
@ -272,7 +272,7 @@ class NonBatchedOpsStressTest : public StressTest {
#ifndef NDEBUG
if (fault_fs_guard) {
fault_fs_guard->EnableErrorInjection();
SharedState::filter_read_error = false;
SharedState::ignore_read_error = false;
}
#endif // NDEBUG
db_->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
@ -291,7 +291,7 @@ class NonBatchedOpsStressTest : public StressTest {
}
#ifndef NDEBUG
if (fault_fs_guard && error_count && !SharedState::filter_read_error) {
if (fault_fs_guard && error_count && !SharedState::ignore_read_error) {
int stat_nok = 0;
for (const auto& s : statuses) {
if (!s.ok() && !s.IsNotFound()) {

11
file/file_prefetch_buffer.cc
View File

@ -17,6 +17,7 @@
#include "monitoring/iostats_context_imp.h"
#include "port/port.h"
#include "test_util/sync_point.h"
#include "test_util/testharness.h"
#include "util/random.h"
#include "util/rate_limiter.h"
@ -86,9 +87,17 @@ Status FilePrefetchBuffer::Prefetch(RandomAccessFileReader* reader,
}
Slice result;
size_t read_len = static_cast<size_t>(roundup_len - chunk_len);
s = reader->Read(rounddown_offset + chunk_len,
static_cast<size_t>(roundup_len - chunk_len), &result,
read_len, &result,
buffer_.BufferStart() + chunk_len, nullptr, for_compaction);
#ifndef NDEBUG
if (!s.ok() || result.size() < read_len) {
// Fake an IO error to force db_stress fault injection to ignore
// truncated read errors
IGNORE_STATUS_IF_ERROR(Status::IOError());
}
#endif
if (s.ok()) {
buffer_offset_ = rounddown_offset;
buffer_.Size(static_cast<size_t>(chunk_len) + result.size());

2
table/block_based/block_based_filter_block.cc
View File

@ -14,6 +14,7 @@
#include "monitoring/perf_context_imp.h"
#include "rocksdb/filter_policy.h"
#include "table/block_based/block_based_table_reader.h"
#include "test_util/testharness.h"
#include "util/coding.h"
#include "util/string_util.h"
@ -184,6 +185,7 @@ std::unique_ptr<FilterBlockReader> BlockBasedFilterBlockReader::Create(
use_cache, nullptr /* get_context */,
lookup_context, &filter_block);
if (!s.ok()) {
IGNORE_STATUS_IF_ERROR(s);
return std::unique_ptr<FilterBlockReader>();
}

10
table/block_based/block_based_table_reader.cc
View File

@ -52,6 +52,7 @@
#include "table/sst_file_writer_collectors.h"
#include "table/two_level_iterator.h"
#include "test_util/testharness.h"
#include "monitoring/perf_context_imp.h"
#include "port/lang.h"
#include "test_util/sync_point.h"
@ -728,6 +729,7 @@ Status BlockBasedTable::PrefetchTail(
nullptr, 0, 0, true /* enable */, true /* track_min_offset */));
s = (*prefetch_buffer)->Prefetch(file, prefetch_off, prefetch_len);
}
return s;
}
@ -788,10 +790,12 @@ Status BlockBasedTable::ReadPropertiesBlock(
nullptr /* ret_block_handle */, nullptr /* ret_block_contents */,
false /* compression_type_missing */, nullptr /* memory_allocator */);
}
IGNORE_STATUS_IF_ERROR(s);
if (s.IsCorruption()) {
s = TryReadPropertiesWithGlobalSeqno(prefetch_buffer, meta_iter->value(),
&table_properties);
IGNORE_STATUS_IF_ERROR(s);
}
std::unique_ptr<TableProperties> props_guard;
if (table_properties != nullptr) {
@ -890,6 +894,7 @@ Status BlockBasedTable::ReadRangeDelBlock(
rep_->ioptions.info_log,
"Encountered error while reading data from range del block %s",
s.ToString().c_str());
IGNORE_STATUS_IF_ERROR(s);
} else {
rep_->fragmented_range_dels =
std::make_shared<FragmentedRangeTombstoneList>(std::move(iter),
@ -994,11 +999,6 @@ Status BlockBasedTable::PrefetchIndexAndFilterBlocks(
auto filter = new_table->CreateFilterBlockReader(
prefetch_buffer, use_cache, prefetch_filter, pin_filter,
lookup_context);
#ifndef NDEBUG
if (rep_->filter_type != Rep::FilterType::kNoFilter && !filter) {
TEST_SYNC_POINT("FilterReadError");
}
#endif
if (filter) {
// Refer to the comment above about paritioned indexes always being cached
if (prefetch_all) {

6
table/block_based/full_filter_block.cc
View File

@ -11,6 +11,7 @@
#include "port/port.h"
#include "rocksdb/filter_policy.h"
#include "table/block_based/block_based_table_reader.h"
#include "test_util/testharness.h"
#include "util/coding.h"
namespace ROCKSDB_NAMESPACE {
@ -132,6 +133,7 @@ std::unique_ptr<FilterBlockReader> FullFilterBlockReader::Create(
use_cache, nullptr /* get_context */,
lookup_context, &filter_block);
if (!s.ok()) {
IGNORE_STATUS_IF_ERROR(s);
return std::unique_ptr<FilterBlockReader>();
}
@ -164,7 +166,7 @@ bool FullFilterBlockReader::MayMatch(
const Status s =
GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block);
if (!s.ok()) {
TEST_SYNC_POINT("FilterReadError");
IGNORE_STATUS_IF_ERROR(s);
return true;
}
@ -222,7 +224,7 @@ void FullFilterBlockReader::MayMatch(
const Status s = GetOrReadFilterBlock(no_io, range->begin()->get_context,
lookup_context, &filter_block);
if (!s.ok()) {
TEST_SYNC_POINT("FilterReadError");
IGNORE_STATUS_IF_ERROR(s);
return;
}

19
table/block_based/partitioned_filter_block.cc
View File

@ -13,6 +13,7 @@
#include "rocksdb/filter_policy.h"
#include "table/block_based/block.h"
#include "table/block_based/block_based_table_reader.h"
#include "test_util/testharness.h"
#include "util/coding.h"
namespace ROCKSDB_NAMESPACE {
@ -143,6 +144,7 @@ std::unique_ptr<FilterBlockReader> PartitionedFilterBlockReader::Create(
use_cache, nullptr /* get_context */,
lookup_context, &filter_block);
if (!s.ok()) {
IGNORE_STATUS_IF_ERROR(s);
return std::unique_ptr<FilterBlockReader>();
}
@ -254,7 +256,7 @@ bool PartitionedFilterBlockReader::MayMatch(
Status s =
GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block);
if (UNLIKELY(!s.ok())) {
TEST_SYNC_POINT("FilterReadError");
IGNORE_STATUS_IF_ERROR(s);
return true;
}
@ -272,7 +274,7 @@ bool PartitionedFilterBlockReader::MayMatch(
no_io, get_context, lookup_context,
&filter_partition_block);
if (UNLIKELY(!s.ok())) {
TEST_SYNC_POINT("FilterReadError");
IGNORE_STATUS_IF_ERROR(s);
return true;
}
@ -312,7 +314,7 @@ void PartitionedFilterBlockReader::CacheDependencies(bool pin) {
"Error retrieving top-level filter block while trying to "
"cache filter partitions: %s",
s.ToString().c_str());
TEST_SYNC_POINT("FilterReadError");
IGNORE_STATUS_IF_ERROR(s);
return;
}
@ -343,11 +345,6 @@ void PartitionedFilterBlockReader::CacheDependencies(bool pin) {
prefetch_buffer.reset(new FilePrefetchBuffer());
s = prefetch_buffer->Prefetch(rep->file.get(), prefetch_off,
static_cast<size_t>(prefetch_len));
#ifndef NDEBUG
if (!s.ok()) {
TEST_SYNC_POINT("FilterReadError");
}
#endif
// After prefetch, read the partitions one by one
ReadOptions read_options;
@ -370,11 +367,7 @@ void PartitionedFilterBlockReader::CacheDependencies(bool pin) {
}
}
}
#ifndef NDEBUG
if (!s.ok()) {
TEST_SYNC_POINT("FilterReadError");
}
#endif
IGNORE_STATUS_IF_ERROR(s);
}
}

4
table/block_based/partitioned_index_reader.cc
View File

@ -8,6 +8,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "table/block_based/partitioned_index_reader.h"
#include "table/block_based/partitioned_index_iterator.h"
#include "test_util/testharness.h"
namespace ROCKSDB_NAMESPACE {
Status PartitionIndexReader::Create(
@ -116,6 +117,7 @@ void PartitionIndexReader::CacheDependencies(bool pin) {
"Error retrieving top-level index block while trying to "
"cache index partitions: %s",
s.ToString().c_str());
IGNORE_STATUS_IF_ERROR(s);
return;
}
@ -162,6 +164,8 @@ void PartitionIndexReader::CacheDependencies(bool pin) {
&block, BlockType::kIndex, /*get_context=*/nullptr, &lookup_context,
/*contents=*/nullptr);
IGNORE_STATUS_IF_ERROR(s);
assert(s.ok() || block.GetValue() == nullptr);
if (s.ok() && block.GetValue() != nullptr) {
if (block.IsCached()) {

10
test_util/fault_injection_test_fs.cc
View File

@ -473,13 +473,14 @@ IOStatus FaultInjectionTestFS::InjectError(ErrorOperation op,
switch (op) {
case kRead:
{
uint32_t type = ctx->rand.Uniform(3);
ErrorType type =
static_cast<ErrorType>(ctx->rand.Uniform(ErrorType::kErrorTypeMax));
switch (type) {
// Inject IO error
case 0:
case ErrorType::kErrorTypeStatus:
return IOStatus::IOError();
// Inject random corruption
case 1:
case ErrorType::kErrorTypeCorruption:
{
if (result->data() == scratch) {
uint64_t offset = ctx->rand.Uniform((uint32_t)result->size());
@ -496,7 +497,7 @@ IOStatus FaultInjectionTestFS::InjectError(ErrorOperation op,
}
}
// Truncate the result
case 2:
case ErrorType::kErrorTypeTruncated:
{
assert(result->size() > 0);
uint64_t offset = ctx->rand.Uniform((uint32_t)result->size());
@ -525,6 +526,7 @@ void FaultInjectionTestFS::PrintFaultBacktrace() {
if (ctx == nullptr) {
return;
}
fprintf(stderr, "Injected error type = %d\n", ctx->type);
port::PrintAndFreeStack(ctx->callstack, ctx->frames);
ctx->callstack = nullptr;
#endif

8
test_util/fault_injection_test_fs.h
View File

@ -355,6 +355,13 @@ class FaultInjectionTestFS : public FileSystemWrapper {
// to underlying FS for writable files
IOStatus error_;
enum ErrorType : int {
kErrorTypeStatus = 0,
kErrorTypeCorruption,
kErrorTypeTruncated,
kErrorTypeMax
};
struct ErrorContext {
Random rand;
int one_in;
@ -362,6 +369,7 @@ class FaultInjectionTestFS : public FileSystemWrapper {
bool enable_error_injection;
void* callstack;
int frames;
ErrorType type;
explicit ErrorContext(uint32_t seed)
: rand(seed),

15
test_util/testharness.h
View File

@ -42,6 +42,19 @@ int RandomSeed();
#define EXPECT_OK(s) \
EXPECT_PRED_FORMAT1(ROCKSDB_NAMESPACE::test::AssertStatus, s)
#define EXPECT_NOK(s) EXPECT_FALSE((s).ok())
} // namespace test
// Callback sync point for any read IO errors that should be ignored by
// the fault injection framework
#ifdef NDEBUG
// Disable in release mode
#define IGNORE_STATUS_IF_ERROR(_status_)
#else
#define IGNORE_STATUS_IF_ERROR(_status_) \
{ \
if (!_status_.ok()) { \
TEST_SYNC_POINT("FaultInjectionIgnoreError"); \
} \
}
#endif // NDEBUG
} // namespace ROCKSDB_NAMESPACE