andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Separate crash test with atomic flush (#4945)

Browse Source 
Summary:
Currently crash test covers cases with and without atomic flush, but takes too
long to finish. Therefore it may be a better idea to put crash test with atomic
flush in a separate set of tests.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4945

Differential Revision: D13947548

Pulled By: riversand963

fbshipit-source-id: 177c6de865290fd650b0103408339eaa3f801d8c
This commit is contained in:
Yanqin Jin 2019-02-19 14:02:30 -08:00 committed by Facebook Github Bot
parent 3c5d1b16b1
commit 7d23210226
3 changed files with 131 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Makefile
View File

@ -686,7 +686,8 @@ endif # PLATFORM_SHARED_EXT
.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \ .PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \
release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \ release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \
dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \ dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
analyze tools tools_lib analyze tools tools_lib \
blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush
all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS) all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
@ -894,23 +895,29 @@ ldb_tests: ldb
crash_test: whitebox_crash_test blackbox_crash_test crash_test: whitebox_crash_test blackbox_crash_test
crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_crash_test_with_atomic_flush
blackbox_crash_test: db_stress blackbox_crash_test: db_stress
python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS) python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py --enable_atomic_flush blackbox $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS) python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --enable_atomic_flush blackbox $(CRASH_TEST_EXT_ARGS)
ifeq ($(CRASH_TEST_KILL_ODD),) ifeq ($(CRASH_TEST_KILL_ODD),)
CRASH_TEST_KILL_ODD=888887 CRASH_TEST_KILL_ODD=888887
endif endif
whitebox_crash_test: db_stress whitebox_crash_test: db_stress
python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \ python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py --enable_atomic_flush whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py whitebox --random_kill_odd \ python -u tools/db_crashtest.py whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --enable_atomic_flush whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
asan_check: asan_check:
$(MAKE) clean $(MAKE) clean
COMPILE_WITH_ASAN=1 $(MAKE) check -j32 COMPILE_WITH_ASAN=1 $(MAKE) check -j32
@ -921,6 +928,11 @@ asan_crash_test:
COMPILE_WITH_ASAN=1 $(MAKE) crash_test COMPILE_WITH_ASAN=1 $(MAKE) crash_test
$(MAKE) clean $(MAKE) clean
asan_crash_test_with_atomic_flush:
$(MAKE) clean
COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_atomic_flush
$(MAKE) clean
ubsan_check: ubsan_check:
$(MAKE) clean $(MAKE) clean
COMPILE_WITH_UBSAN=1 $(MAKE) check -j32 COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
@ -931,6 +943,11 @@ ubsan_crash_test:
COMPILE_WITH_UBSAN=1 $(MAKE) crash_test COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
$(MAKE) clean $(MAKE) clean
ubsan_crash_test_with_atomic_flush:
$(MAKE) clean
COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_atomic_flush
$(MAKE) clean
valgrind_test: valgrind_test:
ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check

3
build_tools/error_filter.py
View File

@ -132,11 +132,14 @@ _TEST_NAME_TO_PARSERS = {
'lite': [CompilerErrorParser], 'lite': [CompilerErrorParser],
'lite_test': [CompilerErrorParser, GTestErrorParser], 'lite_test': [CompilerErrorParser, GTestErrorParser],
'stress_crash': [CompilerErrorParser, DbCrashErrorParser], 'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
'stress_crash_with_atomic_flush': [CompilerErrorParser, DbCrashErrorParser],
'write_stress': [CompilerErrorParser, WriteStressErrorParser], 'write_stress': [CompilerErrorParser, WriteStressErrorParser],
'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser], 'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser], 'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
'asan_crash_with_atomic_flush': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser], 'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser], 'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
'ubsan_crash_with_atomic_flush': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser], 'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser], 'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
'format_compatible': [CompilerErrorParser, CompatErrorParser], 'format_compatible': [CompilerErrorParser, CompatErrorParser],

107
build_tools/rocksdb-lego-determinator
View File

@ -400,6 +400,35 @@ STRESS_CRASH_TEST_COMMANDS="[
} }
]" ]"
#
# RocksDB stress/crash test with atomic flush
#
STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
{
'name':'Rocksdb Stress/Crash Test (atomic flush)',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug stress tests',
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
{
'name':'Build and run RocksDB debug crash tests with atomic flush',
'timeout': 86400,
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL',
'user':'root',
$PARSER
}
],
$ARTIFACTS,
$REPORT
}
]"
# RocksDB write stress test. # RocksDB write stress test.
# We run on disk device on purpose (i.e. no $SHM) # We run on disk device on purpose (i.e. no $SHM)
# because we want to add some randomness to fsync commands # because we want to add some randomness to fsync commands
@ -464,6 +493,28 @@ ASAN_CRASH_TEST_COMMANDS="[
} }
]" ]"
#
# RocksDB crash testing with atomic flush under address sanitizer
#
ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
{
'name':'Rocksdb crash test (atomic flush) under ASAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush',
'timeout': 86400,
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
# #
# RocksDB test under undefined behavior sanitizer # RocksDB test under undefined behavior sanitizer
# #
@ -506,6 +557,28 @@ UBSAN_CRASH_TEST_COMMANDS="[
} }
]" ]"
#
# RocksDB crash testing with atomic flush under undefined behavior sanitizer
#
UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
{
'name':'Rocksdb crash test (atomic flush) under UBSAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush',
'timeout': 86400,
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
# #
# RocksDB unit test under valgrind # RocksDB unit test under valgrind
# #
@ -572,6 +645,28 @@ TSAN_CRASH_TEST_COMMANDS="[
} }
]" ]"
#
# RocksDB crash test with atomic flush under TSAN
#
TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
{
'name':'Rocksdb Crash Test with atomic flush under TSAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Compile and run',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
# #
# RocksDB format compatible # RocksDB format compatible
# #
@ -753,6 +848,9 @@ case $1 in
stress_crash) stress_crash)
echo $STRESS_CRASH_TEST_COMMANDS echo $STRESS_CRASH_TEST_COMMANDS
;; ;;
stress_crash_with_atomic_flush)
echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
write_stress) write_stress)
echo $WRITE_STRESS_COMMANDS echo $WRITE_STRESS_COMMANDS
;; ;;
@ -762,12 +860,18 @@ case $1 in
asan_crash) asan_crash)
echo $ASAN_CRASH_TEST_COMMANDS echo $ASAN_CRASH_TEST_COMMANDS
;; ;;
asan_crash_with_atomic_flush)
echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
ubsan) ubsan)
echo $UBSAN_TEST_COMMANDS echo $UBSAN_TEST_COMMANDS
;; ;;
ubsan_crash) ubsan_crash)
echo $UBSAN_CRASH_TEST_COMMANDS echo $UBSAN_CRASH_TEST_COMMANDS
;; ;;
ubsan_crash_with_atomic_flush)
echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
valgrind) valgrind)
echo $VALGRIND_TEST_COMMANDS echo $VALGRIND_TEST_COMMANDS
;; ;;
@ -777,6 +881,9 @@ case $1 in
tsan_crash) tsan_crash)
echo $TSAN_CRASH_TEST_COMMANDS echo $TSAN_CRASH_TEST_COMMANDS
;; ;;
tsan_crash_with_atomic_flush)
echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
format_compatible) format_compatible)
echo $FORMAT_COMPATIBLE_COMMANDS echo $FORMAT_COMPATIBLE_COMMANDS
;; ;;