Fix parallel test updates in CI; fbcode LIB_MODE=shared (#9553)
Summary: * Fix LIB_MODE=shared for Meta-internal builds (use PIC libraries appropriately) * Fix gnu_parallel to recognize CircleCI and Travis builds as not connected to a terminal (was previously relying on the `| cat_ignore_eagain` stuff for Ubuntu 16). This problem could cause timeouts that should be 10m to balloon to 5h. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9553 Test Plan: manual and CI Reviewed By: jay-zhuang Differential Revision: D34182886 Pulled By: pdillinger fbshipit-source-id: e95fd8002d94c8dc414bae1975e4fd348589f2b5
This commit is contained in:
parent
eed71dfa82
commit
5cdc8af66c
1
Makefile
1
Makefile
@ -229,6 +229,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
|
|||||||
export PORTABLE="$(PORTABLE)"; \
|
export PORTABLE="$(PORTABLE)"; \
|
||||||
export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \
|
export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \
|
||||||
export USE_CLANG="$(USE_CLANG)"; \
|
export USE_CLANG="$(USE_CLANG)"; \
|
||||||
|
export LIB_MODE="$(LIB_MODE)"; \
|
||||||
"$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
|
"$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
|
||||||
# this file is generated by the previous line to set build flags and sources
|
# this file is generated by the previous line to set build flags and sources
|
||||||
include make_config.mk
|
include make_config.mk
|
||||||
|
@ -58,8 +58,11 @@ COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
|
|||||||
# Default to fbcode gcc on internal fb machines
|
# Default to fbcode gcc on internal fb machines
|
||||||
if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
||||||
FBCODE_BUILD="true"
|
FBCODE_BUILD="true"
|
||||||
# If we're compiling with TSAN we need pic build
|
# If we're compiling with TSAN or shared lib, we need pic build
|
||||||
PIC_BUILD=$COMPILE_WITH_TSAN
|
PIC_BUILD=$COMPILE_WITH_TSAN
|
||||||
|
if [ "$LIB_MODE" == "shared" ]; then
|
||||||
|
PIC_BUILD=1
|
||||||
|
fi
|
||||||
if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
||||||
# we need this to build with MySQL. Don't use for other purposes.
|
# we need this to build with MySQL. Don't use for other purposes.
|
||||||
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
||||||
|
@ -21,85 +21,63 @@ LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
|||||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
||||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
||||||
|
|
||||||
|
if test -z $PIC_BUILD; then
|
||||||
|
MAYBE_PIC=
|
||||||
|
else
|
||||||
|
MAYBE_PIC=_pic
|
||||||
|
fi
|
||||||
|
|
||||||
# snappy
|
# snappy
|
||||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
||||||
if test -z $PIC_BUILD; then
|
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a"
|
||||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
|
|
||||||
else
|
|
||||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DSNAPPY"
|
CFLAGS+=" -DSNAPPY"
|
||||||
|
|
||||||
if test -z $PIC_BUILD; then
|
# location of zlib headers and libraries
|
||||||
# location of zlib headers and libraries
|
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
|
||||||
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
|
ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a"
|
||||||
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
|
CFLAGS+=" -DZLIB"
|
||||||
CFLAGS+=" -DZLIB"
|
|
||||||
|
|
||||||
# location of bzip headers and libraries
|
# location of bzip headers and libraries
|
||||||
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
|
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
|
||||||
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
|
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a"
|
||||||
CFLAGS+=" -DBZIP2"
|
CFLAGS+=" -DBZIP2"
|
||||||
|
|
||||||
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
||||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a"
|
||||||
CFLAGS+=" -DLZ4"
|
CFLAGS+=" -DLZ4"
|
||||||
fi
|
|
||||||
|
|
||||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||||
if test -z $PIC_BUILD; then
|
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a"
|
||||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
|
||||||
else
|
|
||||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DZSTD"
|
CFLAGS+=" -DZSTD"
|
||||||
|
|
||||||
# location of gflags headers and libraries
|
# location of gflags headers and libraries
|
||||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
||||||
if test -z $PIC_BUILD; then
|
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags${MAYBE_PIC}.a"
|
||||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
|
|
||||||
else
|
|
||||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DGFLAGS=gflags"
|
CFLAGS+=" -DGFLAGS=gflags"
|
||||||
|
|
||||||
BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/"
|
BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/"
|
||||||
if test -z $PIC_BUILD; then
|
BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark${MAYBE_PIC}.a"
|
||||||
BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark.a"
|
|
||||||
else
|
|
||||||
BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark_pic.a"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# location of jemalloc
|
# location of jemalloc
|
||||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
||||||
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
|
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc${MAYBE_PIC}.a"
|
||||||
|
|
||||||
if test -z $PIC_BUILD; then
|
# location of numa
|
||||||
# location of numa
|
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
||||||
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
NUMA_LIB=" $NUMA_BASE/lib/libnuma${MAYBE_PIC}.a"
|
||||||
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
|
CFLAGS+=" -DNUMA"
|
||||||
CFLAGS+=" -DNUMA"
|
|
||||||
|
|
||||||
# location of libunwind
|
# location of libunwind
|
||||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind${MAYBE_PIC}.a"
|
||||||
fi
|
|
||||||
|
|
||||||
# location of TBB
|
# location of TBB
|
||||||
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
||||||
if test -z $PIC_BUILD; then
|
TBB_LIBS="$TBB_BASE/lib/libtbb${MAYBE_PIC}.a"
|
||||||
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
|
|
||||||
else
|
|
||||||
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DTBB"
|
CFLAGS+=" -DTBB"
|
||||||
|
|
||||||
# location of LIBURING
|
# location of LIBURING
|
||||||
LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
|
LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
|
||||||
if test -z $PIC_BUILD; then
|
LIBURING_LIBS="$LIBURING_BASE/lib/liburing${MAYBE_PIC}.a"
|
||||||
LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
|
|
||||||
else
|
|
||||||
LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DLIBURING"
|
CFLAGS+=" -DLIBURING"
|
||||||
|
|
||||||
test "$USE_SSE" || USE_SSE=1
|
test "$USE_SSE" || USE_SSE=1
|
||||||
|
@ -1561,7 +1561,7 @@ sub save_stdin_stdout_stderr {
|
|||||||
::die_bug("Can't dup STDERR: $!");
|
::die_bug("Can't dup STDERR: $!");
|
||||||
open $Global::original_stdin, "<&", "STDIN" or
|
open $Global::original_stdin, "<&", "STDIN" or
|
||||||
::die_bug("Can't dup STDIN: $!");
|
::die_bug("Can't dup STDIN: $!");
|
||||||
$Global::is_terminal = (-t $Global::original_stderr);
|
$Global::is_terminal = (-t $Global::original_stderr) && !$ENV{'CIRCLECI'} && !$ENV{'TRAVIS'};
|
||||||
}
|
}
|
||||||
|
|
||||||
sub enough_file_handles {
|
sub enough_file_handles {
|
||||||
|
Loading…
Reference in New Issue
Block a user