628216fc1f
Summary: Our valgrind testing is buggy and cumbersome in terms of locating the error. It originates from the fact we accumulate all output for the tests. It is extremely hard to locate the point of error. The communication between valgrind and the script is not sturdy. We are experiencing bugs. Simplifying to stop on first error. Test Plan: Run manually Reviewers: sdong igor CC: leveldb@ Task ID: #6968635 Blame Rev:
548 lines
12 KiB
Bash
Executable File
548 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# This script is executed by Sandcastle
|
|
# to determine next steps to run
|
|
|
|
#
|
|
# Helper variables
|
|
#
|
|
REPORT_EMAIL="'report': [
|
|
{
|
|
'type':'email',
|
|
'triggers':['fail'],
|
|
'emails':['leveldb@fb.com']
|
|
}
|
|
]"
|
|
|
|
CLEANUP_ENV="
|
|
{
|
|
'name':'Cleanup environment',
|
|
'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean',
|
|
'user':'root'
|
|
}"
|
|
|
|
ONCALL="'oncall':'leveldb@fb.com'"
|
|
DEBUG="OPT=-g"
|
|
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
|
|
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
|
|
ASAN="COMPILE_WITH_ASAN=1"
|
|
CLANG="USE_CLANG=1"
|
|
LITE="OPT=-DROCKSDB_LITE"
|
|
TSAN="COMPILE_WITH_TSAN=1"
|
|
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
|
|
|
|
#
|
|
# A mechanism to disable tests temporarily
|
|
#
|
|
DISABLE_COMMANDS="[
|
|
{
|
|
'name':'Disable test',
|
|
$ONCALL,
|
|
'steps': [
|
|
{
|
|
'name':'Raise disabled notification',
|
|
'shell':'exit 1',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test in parallel
|
|
# Currently we always have noise in our parallel runs. This job is to help
|
|
# manage the noise
|
|
#
|
|
PARALLEL_UNIT_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Parallel Unit Test',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug version',
|
|
'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
|
|
'user':'root'
|
|
},
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug version under gcc-4.8.1',
|
|
'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test
|
|
#
|
|
UNIT_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug version',
|
|
'shell':'$SHM $DEBUG make J=1 check',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test on gcc-4.8.1
|
|
#
|
|
UNIT_TEST_COMMANDS_481="[
|
|
{
|
|
'name':'Rocksdb Unit Test on GCC 4.8.1',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug version',
|
|
'shell':'$SHM $GCC_481 $DEBUG make J=1 check',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test with CLANG
|
|
#
|
|
CLANG_UNIT_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug',
|
|
'shell':'$CLANG $SHM $DEBUG make J=1 check',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB analyze
|
|
#
|
|
CLANG_ANALYZE_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb analyze',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'RocksDB build and analyze',
|
|
'shell':'$CLANG $SHM $DEBUG make J=1 analyze',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB code coverage
|
|
#
|
|
CODE_COV_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test Code Coverage',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build, test and collect code coverage info',
|
|
'shell':'$SHM $DEBUG make J=1 coverage',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unity
|
|
#
|
|
UNITY_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unity',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build, test unity test',
|
|
'shell':'$SHM $DEBUG V=1 make J=1 unity',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
#
|
|
# Build RocksDB lite
|
|
#
|
|
LITE_BUILD_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Lite build',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build RocksDB debug version',
|
|
'shell':'$LITE $DEBUG make J=1 static_lib',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB stress/crash test
|
|
#
|
|
STRESS_CRASH_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Stress/Crash Test',
|
|
$ONCALL,
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
'shell':'$SHM $DEBUG make J=1 db_stress',
|
|
'user':'root'
|
|
},
|
|
{
|
|
'name':'Build and run RocksDB debug crash tests',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG make J=1 crash_test',
|
|
'user':'root'
|
|
}
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
STRESS_CRASH_TEST_COMMANDS=$DISABLE_COMMANDS
|
|
|
|
#
|
|
# RocksDB test under address sanitizer
|
|
#
|
|
ASAN_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test under ASAN',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Test RocksDB debug under ASAN',
|
|
'shell':'set -o pipefail && $SHM $ASAN $DEBUG make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
|
|
'user':'root'
|
|
}
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash testing under address sanitizer
|
|
#
|
|
ASAN_CRASH_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb crash test under ASAN',
|
|
$ONCALL,
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug asan_crash_test',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG make J=1 asan_crash_test',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
ASAN_CRASH_TEST_COMMANDS=$DISABLE_COMMANDS
|
|
|
|
#
|
|
# RocksDB unit test under valgrind
|
|
#
|
|
VALGRIND_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test under valgrind',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit tests',
|
|
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make valgrind_check',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB test under TSAN
|
|
#
|
|
TSAN_UNIT_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test under TSAN',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit test',
|
|
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 check',
|
|
'user':'root'
|
|
},
|
|
],
|
|
$REPORT_EMAIL
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash test under TSAN
|
|
#
|
|
TSAN_CRASH_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Crash Test under TSAN',
|
|
$ONCALL,
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Compile and run',
|
|
'timeout': 86400,
|
|
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 crash_test',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
TSAN_CRASH_TEST_COMMANDS=$DISABLE_COMMANDS
|
|
|
|
#
|
|
# RocksDB format compatible
|
|
#
|
|
|
|
run_format_compatible()
|
|
{
|
|
export TEST_TMPDIR=/dev/shm/rocksdb
|
|
rm -rf /dev/shm/rocksdb
|
|
mkdir /dev/shm/rocksdb
|
|
|
|
echo '
|
|
if [ -e "build_tools/build_detect_platform" ]
|
|
then
|
|
sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
|
|
rm -rf build_tools/build_detect_platform
|
|
cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
|
|
chmod +x build_tools/build_detect_platform
|
|
fi
|
|
|
|
if [ -e "build_detect_platform" ]
|
|
then
|
|
sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
|
|
rm -rf build_detect_platform
|
|
cp $TEST_TMPDIR/temp_build_file build_detect_platform
|
|
chmod +x build_detect_platform
|
|
fi
|
|
|
|
make ldb -j32
|
|
|
|
if [ -e "build_detect_platform" ]
|
|
then
|
|
git checkout -- build_detect_platform
|
|
fi
|
|
|
|
if [ -e "build_tools/build_detect_platform" ]
|
|
then
|
|
git checkout -- build_tools/build_detect_platform
|
|
fi
|
|
' > temp_build_ldb.sh
|
|
|
|
sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
|
|
chmod +x tools/temp_check_format_compatible.sh
|
|
tools/temp_check_format_compatible.sh
|
|
}
|
|
|
|
FORMAT_COMPATIBLE_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Format Compatible tests',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit test',
|
|
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB no compression
|
|
#
|
|
run_no_compression()
|
|
{
|
|
export TEST_TMPDIR=/dev/shm/rocksdb
|
|
rm -rf /dev/shm/rocksdb
|
|
mkdir /dev/shm/rocksdb
|
|
make clean
|
|
cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv
|
|
dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
|
|
mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
|
|
cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
|
|
mv .tmp.Makefile Makefile
|
|
make $DEBUG J=1 check
|
|
}
|
|
|
|
NO_COMPRESSION_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb No Compression tests',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit test',
|
|
'shell':'build_tools/rocksdb-lego-determinator run_no_compression',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB regression
|
|
#
|
|
run_regression()
|
|
{
|
|
time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
|
|
|
|
# ======= report size to ODS ========
|
|
|
|
# parameters: $1 -- key, $2 -- value
|
|
function send_size_to_ods {
|
|
curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
|
|
--connect-timeout 60
|
|
}
|
|
|
|
# === normal build ===
|
|
make clean
|
|
make -j$(nproc) static_lib
|
|
send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
|
|
strip librocksdb.a
|
|
send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
|
|
|
|
make -j$(nproc) shared_lib
|
|
send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
strip `readlink -f librocksdb.so`
|
|
send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
|
|
# === lite build ===
|
|
make clean
|
|
OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
|
|
send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
|
|
strip librocksdb.a
|
|
send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
|
|
|
|
OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
|
|
send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
strip `readlink -f librocksdb.so`
|
|
send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
}
|
|
|
|
REGRESSION_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb regression commands',
|
|
$ONCALL,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Make and run script',
|
|
'shell':'build_tools/rocksdb-lego-determinator run_regression',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
case $1 in
|
|
punit)
|
|
echo $PARALLEL_UNIT_TEST_COMMANDS
|
|
;;
|
|
unit)
|
|
echo $UNIT_TEST_COMMANDS
|
|
;;
|
|
unit_481)
|
|
echo $UNIT_TEST_COMMANDS_481
|
|
;;
|
|
clang_unit)
|
|
echo $CLANG_UNIT_TEST_COMMANDS
|
|
;;
|
|
clang_analyze)
|
|
echo $CLANG_ANALYZE_COMMANDS
|
|
;;
|
|
code_cov)
|
|
echo $CODE_COV_COMMANDS
|
|
;;
|
|
unity)
|
|
echo $UNITY_COMMANDS
|
|
;;
|
|
lite)
|
|
echo $LITE_BUILD_COMMANDS
|
|
;;
|
|
stress_crash)
|
|
echo $STRESS_CRASH_TEST_COMMANDS
|
|
;;
|
|
asan)
|
|
echo $ASAN_TEST_COMMANDS
|
|
;;
|
|
asan_crash)
|
|
echo $ASAN_CRASH_TEST_COMMANDS
|
|
;;
|
|
valgrind)
|
|
echo $VALGRIND_TEST_COMMANDS
|
|
;;
|
|
tsan)
|
|
echo $TSAN_UNIT_TEST_COMMANDS
|
|
;;
|
|
tsan_crash)
|
|
echo $TSAN_CRASH_TEST_COMMANDS
|
|
;;
|
|
format_compatible)
|
|
echo $FORMAT_COMPATIBLE_COMMANDS
|
|
;;
|
|
run_format_compatible)
|
|
run_format_compatible
|
|
;;
|
|
no_compression)
|
|
echo $NO_COMPRESSION_COMMANDS
|
|
;;
|
|
run_no_compression)
|
|
run_no_compression
|
|
;;
|
|
regression)
|
|
echo $REGRESSION_COMMANDS
|
|
;;
|
|
run_regression)
|
|
run_regression
|
|
;;
|
|
*)
|
|
echo "Invalid determinator command"
|
|
;;
|
|
esac
|