andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Enable txn in crash tests (#6155)

Browse Source 
Summary:
Start daily crash tests with use_txn flag.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6155

Differential Revision: D18943630

Pulled By: maysamyabandeh

fbshipit-source-id: eea99a6ffd5f57fb9651f6ca7dab8fbf70379c87
This commit is contained in:
Maysam Yabandeh 2019-12-11 15:59:45 -08:00 committed by Facebook Github Bot
parent d0ad3c59d8
commit 1ad6fa9cc7
4 changed files with 155 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Makefile
View File

@ -743,7 +743,8 @@ endif # PLATFORM_SHARED_EXT
release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \
dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
analyze tools tools_lib \
blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush
blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush \
blackbox_crash_test_with_txn whitebox_crash_test_with_txn
all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
@ -955,6 +956,8 @@ crash_test: whitebox_crash_test blackbox_crash_test
crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_crash_test_with_atomic_flush
crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn
blackbox_crash_test: db_stress
python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
@ -962,6 +965,9 @@ blackbox_crash_test: db_stress
blackbox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_txn: db_stress
python -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS)
ifeq ($(CRASH_TEST_KILL_ODD),)
CRASH_TEST_KILL_ODD=888887
endif
@ -976,6 +982,10 @@ whitebox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --cf_consistency whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_txn: db_stress
python -u tools/db_crashtest.py --txn whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
asan_check:
$(MAKE) clean
COMPILE_WITH_ASAN=1 $(MAKE) check -j32
@ -991,6 +1001,11 @@ asan_crash_test_with_atomic_flush:
COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_atomic_flush
$(MAKE) clean
asan_crash_test_with_txn:
$(MAKE) clean
COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_txn
$(MAKE) clean
ubsan_check:
$(MAKE) clean
COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
@ -1006,6 +1021,11 @@ ubsan_crash_test_with_atomic_flush:
COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_atomic_flush
$(MAKE) clean
ubsan_crash_test_with_txn:
$(MAKE) clean
COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_txn
$(MAKE) clean
valgrind_test:
ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check

3
build_tools/error_filter.py
View File

@ -133,13 +133,16 @@ _TEST_NAME_TO_PARSERS = {
'lite_test': [CompilerErrorParser, GTestErrorParser],
'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
'stress_crash_with_atomic_flush': [CompilerErrorParser, DbCrashErrorParser],
'stress_crash_with_txn': [CompilerErrorParser, DbCrashErrorParser],
'write_stress': [CompilerErrorParser, WriteStressErrorParser],
'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
'asan_crash_with_atomic_flush': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
'asan_crash_with_txn': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
'ubsan_crash_with_atomic_flush': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
'ubsan_crash_with_txn': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
'format_compatible': [CompilerErrorParser, CompatErrorParser],

114
build_tools/rocksdb-lego-determinator
View File

@ -450,6 +450,36 @@ STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
}
]"
#
# RocksDB stress/crash test with txn
#
STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[
{
'name':'Rocksdb Stress and Crash Test with txn',
'oncall':'$ONCALL',
'executeLocal': 'true',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug stress tests',
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
{
'name':'Build and run RocksDB debug crash tests with txn',
'timeout': 86400,
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
$UPLOAD_DB_DIR,
],
$REPORT
}
]"
# RocksDB write stress test.
# We run on disk device on purpose (i.e. no $SHM)
# because we want to add some randomness to fsync commands
@ -541,6 +571,30 @@ ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
}
]"
#
# RocksDB crash testing with txn under address sanitizer
#
ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[
{
'name':'Rocksdb crash test with txn under ASAN',
'oncall':'$ONCALL',
'executeLocal': 'true',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug asan_crash_test_with_txn',
'timeout': 86400,
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
$UPLOAD_DB_DIR,
],
$REPORT
}
]"
#
# RocksDB test under undefined behavior sanitizer
#
@ -609,6 +663,30 @@ UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
}
]"
#
# RocksDB crash testing with txn under undefined behavior sanitizer
#
UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
{
'name':'Rocksdb crash test with txn under UBSAN',
'oncall':'$ONCALL',
'executeLocal': 'true',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug ubsan_crash_test_with_txn',
'timeout': 86400,
'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
$UPLOAD_DB_DIR,
],
$REPORT
}
]"
#
# RocksDB unit test under valgrind
#
@ -702,6 +780,30 @@ TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
}
]"
#
# RocksDB crash test with txn under TSAN
#
TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
{
'name':'Rocksdb Crash Test with txn under TSAN',
'oncall':'$ONCALL',
'executeLocal': 'true',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Compile and run',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL',
'user':'root',
$PARSER
},
$UPLOAD_DB_DIR,
],
$REPORT
}
]"
#
# RocksDB format compatible
#
@ -889,6 +991,9 @@ case $1 in
stress_crash_with_atomic_flush)
echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
stress_crash_with_txn)
echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS
;;
write_stress)
echo $WRITE_STRESS_COMMANDS
;;
@ -901,6 +1006,9 @@ case $1 in
asan_crash_with_atomic_flush)
echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
asan_crash_with_txn)
echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS
;;
ubsan)
echo $UBSAN_TEST_COMMANDS
;;
@ -910,6 +1018,9 @@ case $1 in
ubsan_crash_with_atomic_flush)
echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
ubsan_crash_with_txn)
echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS
;;
valgrind)
echo $VALGRIND_TEST_COMMANDS
;;
@ -922,6 +1033,9 @@ case $1 in
tsan_crash_with_atomic_flush)
echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
;;
tsan_crash_with_txn)
echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS
;;
format_compatible)
echo $FORMAT_COMPATIBLE_COMMANDS
;;

19
tools/db_crashtest.py
View File

@ -19,6 +19,8 @@ import argparse
# for cf_consistency:
# default_params < {blackbox,whitebox}_default_params <
# cf_consistency_params < args
# for txn:
# default_params < {blackbox,whitebox}_default_params < txn_params < args
expected_values_file = tempfile.NamedTemporaryFile()
@ -155,6 +157,14 @@ cf_consistency_params = {
"enable_pipelined_write": lambda: random.randint(0, 1),
}
txn_params = {
"use_txn" : 1,
"disable_wal": 0,
# OpenReadOnly after checkpoint is not currnetly compatible with WritePrepared txns
"checkpoint_one_in": 0,
# pipeline write is not currnetly compatible with WritePrepared txns
"enable_pipelined_write": 0,
}
def finalize_and_sanitize(src_params):
dest_params = dict([(k, v() if callable(v) else v)
@ -168,7 +178,9 @@ def finalize_and_sanitize(src_params):
dest_params["db"]):
dest_params["use_direct_io_for_flush_and_compaction"] = 0
dest_params["use_direct_reads"] = 0
if dest_params.get("test_batches_snapshots") == 1:
# DeleteRange is not currnetly compatible with Txns
if dest_params.get("test_batches_snapshots") == 1 or \
dest_params.get("use_txn") == 1:
dest_params["delpercent"] += dest_params["delrangepercent"]
dest_params["delrangepercent"] = 0
if dest_params.get("disable_wal", 0) == 1:
@ -209,6 +221,8 @@ def gen_cmd_params(args):
params.update(whitebox_simple_default_params)
if args.cf_consistency:
params.update(cf_consistency_params)
if args.txn:
params.update(txn_params)
for k, v in vars(args).items():
if v is not None:
@ -221,7 +235,7 @@ def gen_cmd(params, unknown_params):
'--{0}={1}'.format(k, v)
for k, v in finalize_and_sanitize(params).items()
if k not in set(['test_type', 'simple', 'duration', 'interval',
'random_kill_odd', 'cf_consistency'])
'random_kill_odd', 'cf_consistency', 'txn'])
and v is not None] + unknown_params
return cmd
@ -419,6 +433,7 @@ def main():
parser.add_argument("test_type", choices=["blackbox", "whitebox"])
parser.add_argument("--simple", action="store_true")
parser.add_argument("--cf_consistency", action='store_true')
parser.add_argument("--txn", action='store_true')
all_params = dict(default_params.items()
+ blackbox_default_params.items()