90c71aa5d9
Summary: I set up a mirror of our Java deps on github so we can download them through github URLs rather than maven.org, which is proving terribly unreliable from Travis builds. Also sanitized calls to curl, so they are easier to read and appropriately fail on download failure. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6348 Test Plan: CI Differential Revision: D19633621 Pulled By: pdillinger fbshipit-source-id: 7eb3f730953db2ead758dc94039c040f406790f3
1069 lines
28 KiB
Bash
Executable File
1069 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# This script is executed by Sandcastle
|
|
# to determine next steps to run
|
|
|
|
# Usage:
|
|
# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name>
|
|
#
|
|
# Input Value
|
|
# -------------------------------------------------------------------------
|
|
# EMAIL Email address to report on trigger conditions
|
|
# ONCALL Email address to raise a task on failure
|
|
# TRIGGER Trigger conditions for email. Valid values are fail, warn, all
|
|
# SUBSCRIBER Email addresss to add as subscriber for task
|
|
#
|
|
|
|
#
|
|
# Report configuration
|
|
#
|
|
REPORT_EMAIL=
|
|
if [ ! -z $EMAIL ]; then
|
|
if [ -z $TRIGGER ]; then
|
|
TRIGGER="fail"
|
|
fi
|
|
|
|
REPORT_EMAIL="
|
|
{
|
|
'type':'email',
|
|
'triggers': [ '$TRIGGER' ],
|
|
'emails':['$EMAIL']
|
|
},"
|
|
fi
|
|
|
|
CREATE_TASK=
|
|
if [ ! -z $ONCALL ]; then
|
|
CREATE_TASK="
|
|
{
|
|
'type':'task',
|
|
'triggers':[ 'fail' ],
|
|
'priority':0,
|
|
'subscribers':[ '$SUBSCRIBER' ],
|
|
'tags':[ 'rocksdb', 'ci' ],
|
|
},"
|
|
fi
|
|
|
|
# For now, create the tasks using only the dedicated task creation tool.
|
|
CREATE_TASK=
|
|
|
|
REPORT=
|
|
if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
|
|
REPORT="'report': [
|
|
$REPORT_EMAIL
|
|
$CREATE_TASK
|
|
]"
|
|
fi
|
|
|
|
#
|
|
# Helper variables
|
|
#
|
|
CLEANUP_ENV="
|
|
{
|
|
'name':'Cleanup environment',
|
|
'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean',
|
|
'user':'root'
|
|
}"
|
|
|
|
UPLOAD_DB_DIR="
|
|
{
|
|
'name':'Upload database directory',
|
|
'shell':'tar -cvzf rocksdb_db.tar.gz /dev/shm/rocksdb/',
|
|
'user':'root',
|
|
'cleanup':true,
|
|
'provide_artifacts': [
|
|
{
|
|
'name':'rocksdb_db_dir',
|
|
'paths': ['rocksdb_db.tar.gz'],
|
|
'bundle': false,
|
|
},
|
|
],
|
|
}"
|
|
|
|
# We will eventually set the RATIO to 1, but we want do this
|
|
# in steps. RATIO=$(nproc) will make it work as J=1
|
|
if [ -z $RATIO ]; then
|
|
RATIO=$(nproc)
|
|
fi
|
|
|
|
if [ -z $PARALLEL_J ]; then
|
|
PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
|
|
fi
|
|
|
|
if [ -z $PARALLEL_j ]; then
|
|
PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
|
|
fi
|
|
|
|
PARALLELISM="$PARALLEL_J $PARALLEL_j"
|
|
|
|
DEBUG="OPT=-g"
|
|
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
|
|
NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
|
|
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
|
|
ASAN="COMPILE_WITH_ASAN=1"
|
|
CLANG="USE_CLANG=1"
|
|
# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
|
|
# using platform007 gives us gcc-8 or higher which has that bug fixed.
|
|
TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
|
|
UBSAN="COMPILE_WITH_UBSAN=1"
|
|
TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"'
|
|
NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd"
|
|
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
|
|
HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
|
|
SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
|
|
PARSER="'parser':'python build_tools/error_filter.py $1'"
|
|
|
|
CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
|
|
|
|
# This code is getting called under various scenarios. What we care about is to
|
|
# understand when it's called from nightly contruns because in that case we'll
|
|
# create tasks for any failures. To follow the existing pattern, we'll check
|
|
# the value of $ONCALL. If it's a diff then just call `false` to make sure
|
|
# that errors will be properly propagated to the caller.
|
|
if [ ! -z $ONCALL ]; then
|
|
TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL"
|
|
else
|
|
TASK_CREATION_TOOL="false"
|
|
fi
|
|
|
|
#
|
|
# A mechanism to disable tests temporarily
|
|
#
|
|
DISABLE_COMMANDS="[
|
|
{
|
|
'name':'Disable test',
|
|
'oncall':'$ONCALL',
|
|
'steps': [
|
|
{
|
|
'name':'Job disabled. Please contact test owner',
|
|
'shell':'exit 1',
|
|
'user':'root'
|
|
},
|
|
],
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test
|
|
#
|
|
UNIT_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug version',
|
|
'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test not under /dev/shm
|
|
#
|
|
UNIT_TEST_NON_SHM_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug version',
|
|
'timeout': 86400,
|
|
'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB release build and unit tests
|
|
#
|
|
RELEASE_BUILD_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Release Build',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build RocksDB release',
|
|
'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test on gcc-4.8.1
|
|
#
|
|
UNIT_TEST_COMMANDS_481="[
|
|
{
|
|
'name':'Rocksdb Unit Test on GCC 4.8.1',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug version',
|
|
'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB release build and unit tests
|
|
#
|
|
RELEASE_BUILD_COMMANDS_481="[
|
|
{
|
|
'name':'Rocksdb Release on GCC 4.8.1',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build RocksDB release on GCC 4.8.1',
|
|
'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test with CLANG
|
|
#
|
|
CLANG_UNIT_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and test RocksDB debug',
|
|
'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB release build with CLANG
|
|
#
|
|
CLANG_RELEASE_BUILD_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb CLANG Release Build',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build RocksDB release',
|
|
'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB analyze
|
|
#
|
|
CLANG_ANALYZE_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb analyze',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'RocksDB build and analyze',
|
|
'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB code coverage
|
|
#
|
|
CODE_COV_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test Code Coverage',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build, test and collect code coverage info',
|
|
'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unity
|
|
#
|
|
UNITY_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unity',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build, test unity test',
|
|
'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# Build RocksDB lite
|
|
#
|
|
LITE_BUILD_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Lite build',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build RocksDB debug version',
|
|
'shell':'make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# Report RocksDB lite binary size to scuba
|
|
REPORT_LITE_BINARY_SIZE_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Lite Binary Size',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Report RocksDB Lite binary size to scuba',
|
|
'shell':'tools/report_lite_binary_size.sh',
|
|
'user':'root',
|
|
},
|
|
],
|
|
]"
|
|
|
|
#
|
|
# RocksDB stress/crash test
|
|
#
|
|
STRESS_CRASH_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Stress and Crash Test',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
{
|
|
'name':'Build and run RocksDB debug crash tests',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB stress/crash test with atomic flush
|
|
#
|
|
STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Stress and Crash Test with atomic flush',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
{
|
|
'name':'Build and run RocksDB debug crash tests with atomic flush',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB stress/crash test with txn
|
|
#
|
|
STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Stress and Crash Test with txn',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug stress tests',
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
{
|
|
'name':'Build and run RocksDB debug crash tests with txn',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
# RocksDB write stress test.
|
|
# We run on disk device on purpose (i.e. no $SHM)
|
|
# because we want to add some randomness to fsync commands
|
|
WRITE_STRESS_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Write Stress Test',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB write stress tests',
|
|
'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
}
|
|
],
|
|
'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
|
|
#
|
|
# RocksDB test under address sanitizer
|
|
#
|
|
ASAN_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test under ASAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Test RocksDB debug under ASAN',
|
|
'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
|
|
'user':'root',
|
|
$PARSER
|
|
}
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash testing under address sanitizer
|
|
#
|
|
ASAN_CRASH_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb crash test under ASAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug asan_crash_test',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash testing with atomic flush under address sanitizer
|
|
#
|
|
ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb crash test with atomic flush under ASAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash testing with txn under address sanitizer
|
|
#
|
|
ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb crash test with txn under ASAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug asan_crash_test_with_txn',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB test under undefined behavior sanitizer
|
|
#
|
|
UBSAN_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test under UBSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Test RocksDB debug under UBSAN',
|
|
'shell':'set -o pipefail && $SHM $UBSAN $CLANG $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
}
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash testing under udnefined behavior sanitizer
|
|
#
|
|
UBSAN_CRASH_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb crash test under UBSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug ubsan_crash_test',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash testing with atomic flush under undefined behavior sanitizer
|
|
#
|
|
UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb crash test with atomic flush under UBSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash testing with txn under undefined behavior sanitizer
|
|
#
|
|
UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb crash test with txn under UBSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build and run RocksDB debug ubsan_crash_test_with_txn',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB unit test under valgrind
|
|
#
|
|
VALGRIND_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test under valgrind',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit tests',
|
|
'timeout': 86400,
|
|
'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB test under TSAN
|
|
#
|
|
TSAN_UNIT_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Unit Test under TSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit test',
|
|
'timeout': 86400,
|
|
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash test under TSAN
|
|
#
|
|
TSAN_CRASH_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Crash Test under TSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Compile and run',
|
|
'timeout': 86400,
|
|
'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash test with atomic flush under TSAN
|
|
#
|
|
TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Crash Test with atomic flush under TSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Compile and run',
|
|
'timeout': 86400,
|
|
'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB crash test with txn under TSAN
|
|
#
|
|
TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Crash Test with txn under TSAN',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'timeout': 86400,
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Compile and run',
|
|
'timeout': 86400,
|
|
'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
$UPLOAD_DB_DIR,
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB format compatible
|
|
#
|
|
|
|
run_format_compatible()
|
|
{
|
|
export TEST_TMPDIR=/dev/shm/rocksdb
|
|
rm -rf /dev/shm/rocksdb
|
|
mkdir /dev/shm/rocksdb
|
|
|
|
tools/check_format_compatible.sh
|
|
}
|
|
|
|
FORMAT_COMPATIBLE_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Format Compatible tests',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit test',
|
|
'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB no compression
|
|
#
|
|
run_no_compression()
|
|
{
|
|
export TEST_TMPDIR=/dev/shm/rocksdb
|
|
rm -rf /dev/shm/rocksdb
|
|
mkdir /dev/shm/rocksdb
|
|
make clean
|
|
cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
|
|
mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
|
|
cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
|
|
mv .tmp.Makefile Makefile
|
|
make $DEBUG J=1 check
|
|
}
|
|
|
|
NO_COMPRESSION_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb No Compression tests',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Run RocksDB debug unit test',
|
|
'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB regression
|
|
#
|
|
run_regression()
|
|
{
|
|
time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
|
|
|
|
# ======= report size to ODS ========
|
|
|
|
# parameters: $1 -- key, $2 -- value
|
|
function send_size_to_ods {
|
|
curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
|
|
--connect-timeout 60
|
|
}
|
|
|
|
# === normal build ===
|
|
make clean
|
|
make -j$(nproc) static_lib
|
|
send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
|
|
strip librocksdb.a
|
|
send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
|
|
|
|
make -j$(nproc) shared_lib
|
|
send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
strip `readlink -f librocksdb.so`
|
|
send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
|
|
# === lite build ===
|
|
make clean
|
|
make LITE=1 -j$(nproc) static_lib
|
|
send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
|
|
strip librocksdb.a
|
|
send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
|
|
|
|
make LITE=1 -j$(nproc) shared_lib
|
|
send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
strip `readlink -f librocksdb.so`
|
|
send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
}
|
|
|
|
REGRESSION_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb regression commands',
|
|
'oncall':'$ONCALL',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Make and run script',
|
|
'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
#
|
|
# RocksDB Java build
|
|
#
|
|
JAVA_BUILD_TEST_COMMANDS="[
|
|
{
|
|
'name':'Rocksdb Java Build',
|
|
'oncall':'$ONCALL',
|
|
'executeLocal': 'true',
|
|
'steps': [
|
|
$CLEANUP_ENV,
|
|
{
|
|
'name':'Build RocksDB for Java',
|
|
'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
|
|
'user':'root',
|
|
$PARSER
|
|
},
|
|
],
|
|
$REPORT
|
|
}
|
|
]"
|
|
|
|
|
|
case $1 in
|
|
unit)
|
|
echo $UNIT_TEST_COMMANDS
|
|
;;
|
|
unit_non_shm)
|
|
echo $UNIT_TEST_NON_SHM_COMMANDS
|
|
;;
|
|
release)
|
|
echo $RELEASE_BUILD_COMMANDS
|
|
;;
|
|
unit_481)
|
|
echo $UNIT_TEST_COMMANDS_481
|
|
;;
|
|
release_481)
|
|
echo $RELEASE_BUILD_COMMANDS_481
|
|
;;
|
|
clang_unit)
|
|
echo $CLANG_UNIT_TEST_COMMANDS
|
|
;;
|
|
clang_release)
|
|
echo $CLANG_RELEASE_BUILD_COMMANDS
|
|
;;
|
|
clang_analyze)
|
|
echo $CLANG_ANALYZE_COMMANDS
|
|
;;
|
|
code_cov)
|
|
echo $CODE_COV_COMMANDS
|
|
;;
|
|
unity)
|
|
echo $UNITY_COMMANDS
|
|
;;
|
|
lite)
|
|
echo $LITE_BUILD_COMMANDS
|
|
;;
|
|
report_lite_binary_size)
|
|
echo $REPORT_LITE_BINARY_SIZE_COMMANDS
|
|
;;
|
|
stress_crash)
|
|
echo $STRESS_CRASH_TEST_COMMANDS
|
|
;;
|
|
stress_crash_with_atomic_flush)
|
|
echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
;;
|
|
stress_crash_with_txn)
|
|
echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS
|
|
;;
|
|
write_stress)
|
|
echo $WRITE_STRESS_COMMANDS
|
|
;;
|
|
asan)
|
|
echo $ASAN_TEST_COMMANDS
|
|
;;
|
|
asan_crash)
|
|
echo $ASAN_CRASH_TEST_COMMANDS
|
|
;;
|
|
asan_crash_with_atomic_flush)
|
|
echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
;;
|
|
asan_crash_with_txn)
|
|
echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS
|
|
;;
|
|
ubsan)
|
|
echo $UBSAN_TEST_COMMANDS
|
|
;;
|
|
ubsan_crash)
|
|
echo $UBSAN_CRASH_TEST_COMMANDS
|
|
;;
|
|
ubsan_crash_with_atomic_flush)
|
|
echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
;;
|
|
ubsan_crash_with_txn)
|
|
echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS
|
|
;;
|
|
valgrind)
|
|
echo $VALGRIND_TEST_COMMANDS
|
|
;;
|
|
tsan)
|
|
echo $TSAN_UNIT_TEST_COMMANDS
|
|
;;
|
|
tsan_crash)
|
|
echo $TSAN_CRASH_TEST_COMMANDS
|
|
;;
|
|
tsan_crash_with_atomic_flush)
|
|
echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
|
;;
|
|
tsan_crash_with_txn)
|
|
echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS
|
|
;;
|
|
format_compatible)
|
|
echo $FORMAT_COMPATIBLE_COMMANDS
|
|
;;
|
|
run_format_compatible)
|
|
run_format_compatible
|
|
;;
|
|
no_compression)
|
|
echo $NO_COMPRESSION_COMMANDS
|
|
;;
|
|
run_no_compression)
|
|
run_no_compression
|
|
;;
|
|
regression)
|
|
echo $REGRESSION_COMMANDS
|
|
;;
|
|
run_regression)
|
|
run_regression
|
|
;;
|
|
java_build)
|
|
echo $JAVA_BUILD_TEST_COMMANDS
|
|
;;
|
|
*)
|
|
echo "Invalid determinator command"
|
|
exit 1
|
|
;;
|
|
esac
|