andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Testing out parallel sandcastle changes

Browse Source 
Summary:
Removing moreutils from sandcastle and adding gnu parallel.
Then passing in J= nproc command

Test Plan: Testing on sandcastle

Reviewers: sdong, kradhakrishnan

Reviewed By: kradhakrishnan

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D61017
This commit is contained in:
Anirban Rahut 2016-07-27 11:58:57 -07:00
parent 7efd9c25c4
commit d3bfd33972
5 changed files with 7978 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Makefile
View File

@ -621,7 +621,7 @@ check_0:
} \ } \
| $(prioritize_long_running_tests) \ | $(prioritize_long_running_tests) \
| grep -E '$(tests-regexp)' \ | grep -E '$(tests-regexp)' \
| parallel -j$(J) --joblog=LOG $$eta --gnu '{} >& t/log-{/}' | build_tools/gnu_parallel -j$(J) --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
.PHONY: valgrind_check_0 .PHONY: valgrind_check_0
valgrind_check_0: valgrind_check_0:
@ -636,7 +636,7 @@ valgrind_check_0:
} \ } \
| $(prioritize_long_running_tests) \ | $(prioritize_long_running_tests) \
| grep -E '$(tests-regexp)' \ | grep -E '$(tests-regexp)' \
| parallel -j$(J) --joblog=LOG $$eta --gnu \ | build_tools/gnu_parallel -j$(J) --joblog=LOG $$eta --gnu \
'if [[ "{}" == "./"* ]] ; then $(DRIVER) {} >& t/valgrind_log-{/}; ' \ 'if [[ "{}" == "./"* ]] ; then $(DRIVER) {} >& t/valgrind_log-{/}; ' \
'else {} >& t/valgrind_log-{/}; fi' 'else {} >& t/valgrind_log-{/}; fi'
@ -657,7 +657,7 @@ watch-log:
check: all check: all
$(MAKE) gen_parallel_tests $(MAKE) gen_parallel_tests
$(AM_V_GEN)if test "$(J)" != 1 \ $(AM_V_GEN)if test "$(J)" != 1 \
&& (parallel --gnu --help 2>/dev/null) | \ && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
grep -q 'GNU Parallel'; \ grep -q 'GNU Parallel'; \
then \ then \
$(MAKE) T="$$t" TMPD=$(TMPD) check_0; \ $(MAKE) T="$$t" TMPD=$(TMPD) check_0; \
@ -717,7 +717,7 @@ ubsan_crash_test:
valgrind_check: $(TESTS) valgrind_check: $(TESTS)
$(MAKE) gen_parallel_tests $(MAKE) gen_parallel_tests
$(AM_V_GEN)if test "$(J)" != 1 \ $(AM_V_GEN)if test "$(J)" != 1 \
&& (parallel --gnu --help 2>/dev/null) | \ && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
grep -q 'GNU Parallel'; \ grep -q 'GNU Parallel'; \
then \ then \
$(MAKE) TMPD=$(TMPD) \ $(MAKE) TMPD=$(TMPD) \
@ -739,10 +739,10 @@ parloop:
for t in $(PAR_TEST); do \ for t in $(PAR_TEST); do \
echo "===== Running $$t in parallel $(NUM_PAR)";\ echo "===== Running $$t in parallel $(NUM_PAR)";\
if [ $(db_test) -eq 1 ]; then \ if [ $(db_test) -eq 1 ]; then \
seq $(J) | v="$$t" parallel --gnu 's=$(TMPD)/rdb-{}; export TEST_TMPDIR=$$s;' \ seq $(J) | v="$$t" build_tools/gnu_parallel --gnu 's=$(TMPD)/rdb-{}; export TEST_TMPDIR=$$s;' \
'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \ 'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \
else\ else\
seq $(J) | v="./$$t" parallel --gnu 's=$(TMPD)/rdb-{};' \ seq $(J) | v="./$$t" build_tools/gnu_parallel --gnu 's=$(TMPD)/rdb-{};' \
'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \ 'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \
fi; \ fi; \
ret_code=$$?; \ ret_code=$$?; \
@ -763,7 +763,7 @@ test_names = \
parallel_check: $(TESTS) parallel_check: $(TESTS)
$(AM_V_GEN)if test "$(J)" > 1 \ $(AM_V_GEN)if test "$(J)" > 1 \
&& (parallel --gnu --help 2>/dev/null) | \ && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
grep -q 'GNU Parallel'; \ grep -q 'GNU Parallel'; \
then \ then \
echo Running in parallel $(J); \ echo Running in parallel $(J); \
@ -773,7 +773,7 @@ parallel_check: $(TESTS)
ret_bad=0; \ ret_bad=0; \
echo $(J);\ echo $(J);\
echo Test Dir: $(TMPD); \ echo Test Dir: $(TMPD); \
seq $(J) | parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \ seq $(J) | build_tools/gnu_parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
$(MAKE) PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \ $(MAKE) PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \
J=$(J) db_test=1 parloop; \ J=$(J) db_test=1 parloop; \
$(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \ $(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \

5
arcanist_util/config/RocksDBCommonHelper.php
View File

@ -75,9 +75,10 @@ function getSteps($applyDiff, $diffID, $username, $test) {
); );
// arc demands certain permission on its config. // arc demands certain permission on its config.
// also fix the sticky bit issue in sandcastle
$fix_permission = array( $fix_permission = array(
"name" => "Fix environment", "name" => "Fix environment",
"shell" => "chmod 600 ~/.arcrc", "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
"user" => "root" "user" => "root"
); );
@ -115,7 +116,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
} }
// Run the actual command. // Run the actual command.
$cmd = $cmd . "./build_tools/precommit_checker.py " . $test $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
. "; exit_code=$?; "; . "; exit_code=$?; ";
if ($applyDiff) { if ($applyDiff) {

7936
build_tools/gnu_parallel Executable file
View File

File diff suppressed because it is too large Load Diff

2
build_tools/precommit_checker.py
View File

@ -94,7 +94,7 @@ class PreCommitChecker(Env):
# #
def get_commands(self, test): def get_commands(self, test):
status, out = self.GetOutput( status, out = self.GetOutput(
"build_tools/rocksdb-lego-determinator %s" % test, ".") "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
return status, out return status, out
# #

44
build_tools/rocksdb-lego-determinator
View File

@ -56,10 +56,26 @@ fi
CLEANUP_ENV=" CLEANUP_ENV="
{ {
'name':'Cleanup environment', 'name':'Cleanup environment',
'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean; chmod -t /dev/shm', 'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && chmod +t /dev/shm && make clean',
'user':'root' 'user':'root'
}" }"
# We will eventually set the RATIO to 1, but we want do this
# in steps. RATIO=$(nproc) will make it work as J=1
if [ -z $RATIO ]; then
RATIO=$(nproc)
fi
if [ -z $PARALLEL_J ]; then
PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
fi
if [ -z $PARALLEL_j ]; then
PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
fi
PARALLELISM="$PARALLEL_J $PARALLEL_j"
DEBUG="OPT=-g" DEBUG="OPT=-g"
SHM="TEST_TMPDIR=/dev/shm/rocksdb" SHM="TEST_TMPDIR=/dev/shm/rocksdb"
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
@ -137,7 +153,7 @@ UNIT_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build and test RocksDB debug version', 'name':'Build and test RocksDB debug version',
'shell':'$SHM $DEBUG make J=1 check', 'shell':'$SHM $DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -159,7 +175,7 @@ UNIT_TEST_NON_SHM_COMMANDS="[
{ {
'name':'Build and test RocksDB debug version', 'name':'Build and test RocksDB debug version',
'timeout': 86400, 'timeout': 86400,
'shell':'$DEBUG make J=1 check', 'shell':'$DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -179,7 +195,7 @@ RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build RocksDB release', 'name':'Build RocksDB release',
'shell':'make J=1 release', 'shell':'make $PARALLEL_j release',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -199,7 +215,7 @@ UNIT_TEST_COMMANDS_481="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build and test RocksDB debug version', 'name':'Build and test RocksDB debug version',
'shell':'$SHM $GCC_481 $DEBUG make J=1 check', 'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -219,7 +235,7 @@ RELEASE_BUILD_COMMANDS_481="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build RocksDB release on GCC 4.8.1', 'name':'Build RocksDB release on GCC 4.8.1',
'shell':'$GCC_481 make J=1 release', 'shell':'$GCC_481 make $PARALLEL_j release',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -239,7 +255,7 @@ CLANG_UNIT_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build and test RocksDB debug', 'name':'Build and test RocksDB debug',
'shell':'$CLANG $SHM $DEBUG make J=1 check', 'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -259,7 +275,7 @@ CLANG_RELEASE_BUILD_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build RocksDB release', 'name':'Build RocksDB release',
'shell':'$CLANG make J=1 release', 'shell':'$CLANG make $PARALLEL_j release',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -279,7 +295,7 @@ CLANG_ANALYZE_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'RocksDB build and analyze', 'name':'RocksDB build and analyze',
'shell':'$CLANG $SHM $DEBUG make J=1 analyze', 'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -299,7 +315,7 @@ CODE_COV_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Build, test and collect code coverage info', 'name':'Build, test and collect code coverage info',
'shell':'$SHM $DEBUG make J=1 coverage', 'shell':'$SHM $DEBUG make $PARALLELISM coverage',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -430,7 +446,7 @@ ASAN_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Test RocksDB debug under ASAN', 'name':'Test RocksDB debug under ASAN',
'shell':'set -o pipefail && $SHM $ASAN $DEBUG make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d', 'shell':'set -o pipefail && $SHM $ASAN $DEBUG make $PARALLELISM asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
'user':'root', 'user':'root',
$PARSER $PARSER
} }
@ -472,7 +488,7 @@ UBSAN_TEST_COMMANDS="[
$CLEANUP_ENV, $CLEANUP_ENV,
{ {
'name':'Test RocksDB debug under UBSAN', 'name':'Test RocksDB debug under UBSAN',
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check', 'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check',
'user':'root', 'user':'root',
$PARSER $PARSER
} }
@ -516,7 +532,7 @@ VALGRIND_TEST_COMMANDS="[
{ {
'name':'Run RocksDB debug unit tests', 'name':'Run RocksDB debug unit tests',
'timeout': 86400, 'timeout': 86400,
'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make valgrind_check', 'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make $PARALLELISM valgrind_check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },
@ -538,7 +554,7 @@ TSAN_UNIT_TEST_COMMANDS="[
{ {
'name':'Run RocksDB debug unit test', 'name':'Run RocksDB debug unit test',
'timeout': 86400, 'timeout': 86400,
'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 check', 'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check',
'user':'root', 'user':'root',
$PARSER $PARSER
}, },