andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
0d590d9991
rocksdb/build_tools/rocksdb-lego-determinator
Andrew Kryczka ec84bef24a New legocastle output parsing 
Summary:
Added a python script to parse combined stdout/stderr of legocastle
steps. Previously we just matched words like 'Failure', which didn't work since
even our test names matched that pattern.

I went through all the legocastle steps to come up with strict failure regexes
for the common failure cases. There is also some more complex logic to present
gtest failures, since the test name and failure message are not on the same
line.

There will definitely be error cases that don't match any of these patterns, so
we can iterate on it over time.

Test Plan:
no end-to-end test. I ran the legocastle steps locally and piped to
my script, then verified output, e.g.,

  $ set -o pipefail && TEST_TMPDIR=/dev/shm/rocksdb COMPILE_WITH_ASAN=1 OPT=-g make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d |& python build_tools/error_filter.py asan
  ==2058029==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x60200000a414 at pc 0x4c12f6 bp 0x7ffcfb7a0520 sp 0x7ffcfb7a0518

Reviewers: kradhakrishnan

Reviewed By: kradhakrishnan

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D56691
2016-04-18 13:01:10 -07:00

741 lines
17 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# This script is executed by Sandcastle
# to determine next steps to run
# Usage:
# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name>
#
# Input Value
# -------------------------------------------------------------------------
# EMAIL Email address to report on trigger conditions
# ONCAL Email address to raise a task on failure
# TRIGGER Trigger conditions for email. Valid values are fail, warn, all
# SUBSCRIBER Email addresss to add as subscriber for task
#
#
# Report configuration
#
REPORT_EMAIL=
if [ ! -z $EMAIL ]; then
if [ -z $TRIGGER ]; then
TRIGGER="fail"
fi
REPORT_EMAIL="
{
'type':'email',
'triggers': [ '$TRIGGER' ],
'emails':['$EMAIL']
},"
fi
CREATE_TASK=
if [ ! -z $ONCALL ]; then
CREATE_TASK="
{
'type':'task',
'triggers':[ 'fail' ],
'priority':0,
'subscribers':[ '$SUBSCRIBER' ],
'tags':[ 'rocksdb', 'ci' ],
},"
fi
REPORT=
if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
REPORT="'report': [
$REPORT_EMAIL
$CREATE_TASK
]"
fi
#
# Helper variables
#
CLEANUP_ENV="
{
'name':'Cleanup environment',
'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean',
'user':'root'
}"
DEBUG="OPT=-g"
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
ASAN="COMPILE_WITH_ASAN=1"
CLANG="USE_CLANG=1"
LITE="OPT=-DROCKSDB_LITE"
TSAN="COMPILE_WITH_TSAN=1"
UBSAN="COMPILE_WITH_UBSAN=1"
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-7u10-64/; export PATH=\$PATH:\$JAVA_HOME/bin"
PARSER="'parser':'python build_tools/error_filter.py $1'"
ARTIFACTS=" 'artifacts': [
{
'name':'database',
'paths':[ '/dev/shm/rocksdb' ],
}
]"
#
# A mechanism to disable tests temporarily
#
DISABLE_COMMANDS="[
{
'name':'Disable test',
'oncall':'$ONCALL',
'steps': [
{
'name':'Job disabled. Please contact test owner',
'shell':'exit 1',
'user':'root'
},
],
}
]"
#
# RocksDB unit test in parallel
# Currently we always have noise in our parallel runs. This job is to help
# manage the noise
#
PARALLEL_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Parallel Unit Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'user':'root',
$PARSER
},
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version under gcc-4.8.1',
'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test
#
UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $DEBUG make J=1 check',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test on gcc-4.8.1
#
UNIT_TEST_COMMANDS_481="[
{
'name':'Rocksdb Unit Test on GCC 4.8.1',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug version',
'shell':'$SHM $GCC_481 $DEBUG make J=1 check',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test with CLANG
#
CLANG_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and test RocksDB debug',
'shell':'$CLANG $SHM $DEBUG make J=1 check',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB analyze
#
CLANG_ANALYZE_COMMANDS="[
{
'name':'Rocksdb analyze',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'RocksDB build and analyze',
'shell':'$CLANG $SHM $DEBUG make J=1 analyze',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB code coverage
#
CODE_COV_COMMANDS="[
{
'name':'Rocksdb Unit Test Code Coverage',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build, test and collect code coverage info',
'shell':'$SHM $DEBUG make J=1 coverage',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unity
#
UNITY_COMMANDS="[
{
'name':'Rocksdb Unity',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build, test unity test',
'shell':'$SHM $DEBUG V=1 make J=1 unity_test',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# Build RocksDB lite
#
LITE_BUILD_COMMANDS="[
{
'name':'Rocksdb Lite build',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$LITE $DEBUG make J=1 static_lib',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB lite tests
#
LITE_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Lite Unit Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB debug version',
'shell':'$SHM $LITE $DEBUG make J=1 check',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB stress/crash test
#
STRESS_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb Stress/Crash Test',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug stress tests',
'shell':'$SHM $DEBUG make J=1 db_stress',
'user':'root',
$PARSER
},
{
'name':'Build and run RocksDB debug crash tests',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 crash_test',
'user':'root',
$PARSER
}
],
$ARTIFACTS,
$REPORT
}
]"
# RocksDB write stress test.
# We run on disk device on purpose (i.e. no $SHM)
# because we want to add some randomness to fsync commands
WRITE_STRESS_COMMANDS="[
{
'name':'Rocksdb Write Stress Test',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB write stress tests',
'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress',
'user':'root',
$PARSER
}
],
'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}],
$REPORT
}
]"
#
# RocksDB test under address sanitizer
#
ASAN_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under ASAN',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under ASAN',
'shell':'set -o pipefail && $SHM $ASAN $DEBUG make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'user':'root',
$PARSER
}
],
$REPORT
}
]"
#
# RocksDB crash testing under address sanitizer
#
ASAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb crash test under ASAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug asan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 asan_crash_test',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB test under undefined behavior sanitizer
#
UBSAN_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under UBSAN',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Test RocksDB debug under UBSAN',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check',
'user':'root',
$PARSER
}
],
$REPORT
}
]"
#
# RocksDB crash testing under udnefined behavior sanitizer
#
UBSAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb crash test under UBSAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Build and run RocksDB debug ubsan_crash_test',
'timeout': 86400,
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB unit test under valgrind
#
VALGRIND_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under valgrind',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit tests',
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make valgrind_check',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB test under TSAN
#
TSAN_UNIT_TEST_COMMANDS="[
{
'name':'Rocksdb Unit Test under TSAN',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 check',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB crash test under TSAN
#
TSAN_CRASH_TEST_COMMANDS="[
{
'name':'Rocksdb Crash Test under TSAN',
'oncall':'$ONCALL',
'timeout': 86400,
'steps': [
$CLEANUP_ENV,
{
'name':'Compile and run',
'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 crash_test',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
TSAN_CRASH_TEST_COMMANDS=$DISABLE_COMMANDS
#
# RocksDB format compatible
#
run_format_compatible()
{
export TEST_TMPDIR=/dev/shm/rocksdb
rm -rf /dev/shm/rocksdb
mkdir /dev/shm/rocksdb
echo '
if [ -e "build_tools/build_detect_platform" ]
then
sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
rm -rf build_tools/build_detect_platform
cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
chmod +x build_tools/build_detect_platform
fi
if [ -e "build_detect_platform" ]
then
sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
rm -rf build_detect_platform
cp $TEST_TMPDIR/temp_build_file build_detect_platform
chmod +x build_detect_platform
fi
make ldb -j32
if [ -e "build_detect_platform" ]
then
git checkout -- build_detect_platform
fi
if [ -e "build_tools/build_detect_platform" ]
then
git checkout -- build_tools/build_detect_platform
fi
' > temp_build_ldb.sh
sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
chmod +x tools/temp_check_format_compatible.sh
tools/temp_check_format_compatible.sh
}
FORMAT_COMPATIBLE_COMMANDS="[
{
'name':'Rocksdb Format Compatible tests',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB no compression
#
run_no_compression()
{
export TEST_TMPDIR=/dev/shm/rocksdb
rm -rf /dev/shm/rocksdb
mkdir /dev/shm/rocksdb
make clean
cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
mv .tmp.Makefile Makefile
make $DEBUG J=1 check
}
NO_COMPRESSION_COMMANDS="[
{
'name':'Rocksdb No Compression tests',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Run RocksDB debug unit test',
'shell':'build_tools/rocksdb-lego-determinator run_no_compression',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB regression
#
run_regression()
{
time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
# ======= report size to ODS ========
# parameters: $1 -- key, $2 -- value
function send_size_to_ods {
curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
--connect-timeout 60
}
# === normal build ===
make clean
make -j$(nproc) static_lib
send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
strip librocksdb.a
send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
make -j$(nproc) shared_lib
send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
strip `readlink -f librocksdb.so`
send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
# === lite build ===
make clean
OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
strip librocksdb.a
send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
strip `readlink -f librocksdb.so`
send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
}
REGRESSION_COMMANDS="[
{
'name':'Rocksdb regression commands',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Make and run script',
'shell':'build_tools/rocksdb-lego-determinator run_regression',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
#
# RocksDB Java build
#
JAVA_BUILD_TEST_COMMANDS="[
{
'name':'Rocksdb Java Build',
'oncall':'$ONCALL',
'steps': [
$CLEANUP_ENV,
{
'name':'Build RocksDB for Java',
'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava',
'user':'root',
$PARSER
},
],
$REPORT
}
]"
case $1 in
punit)
echo $PARALLEL_UNIT_TEST_COMMANDS
;;
unit)
echo $UNIT_TEST_COMMANDS
;;
unit_481)
echo $UNIT_TEST_COMMANDS_481
;;
clang_unit)
echo $CLANG_UNIT_TEST_COMMANDS
;;
clang_analyze)
echo $CLANG_ANALYZE_COMMANDS
;;
code_cov)
echo $CODE_COV_COMMANDS
;;
unity)
echo $UNITY_COMMANDS
;;
lite)
echo $LITE_BUILD_COMMANDS
;;
lite_test)
echo $LITE_UNIT_TEST_COMMANDS
;;
stress_crash)
echo $STRESS_CRASH_TEST_COMMANDS
;;
write_stress)
echo $WRITE_STRESS_COMMANDS
;;
asan)
echo $ASAN_TEST_COMMANDS
;;
asan_crash)
echo $ASAN_CRASH_TEST_COMMANDS
;;
ubsan)
echo $UBSAN_TEST_COMMANDS
;;
ubsan_crash)
echo $UBSAN_CRASH_TEST_COMMANDS
;;
valgrind)
echo $VALGRIND_TEST_COMMANDS
;;
tsan)
echo $TSAN_UNIT_TEST_COMMANDS
;;
tsan_crash)
echo $TSAN_CRASH_TEST_COMMANDS
;;
format_compatible)
echo $FORMAT_COMPATIBLE_COMMANDS
;;
run_format_compatible)
run_format_compatible
;;
no_compression)
echo $NO_COMPRESSION_COMMANDS
;;
run_no_compression)
run_no_compression
;;
regression)
echo $REGRESSION_COMMANDS
;;
run_regression)
run_regression
;;
java_build)
echo $JAVA_BUILD_TEST_COMMANDS
;;
*)
echo "Invalid determinator command"
;;
esac
Reference in New Issue View Git Blame Copy Permalink