From 5cdc8af66cd4e6c766a446bafee7cdc991498517 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Mon, 14 Feb 2022 08:53:29 -0800 Subject: [PATCH] Fix parallel test updates in CI; fbcode LIB_MODE=shared (#9553) Summary: * Fix LIB_MODE=shared for Meta-internal builds (use PIC libraries appropriately) * Fix gnu_parallel to recognize CircleCI and Travis builds as not connected to a terminal (was previously relying on the `| cat_ignore_eagain` stuff for Ubuntu 16). This problem could cause timeouts that should be 10m to balloon to 5h. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9553 Test Plan: manual and CI Reviewed By: jay-zhuang Differential Revision: D34182886 Pulled By: pdillinger fbshipit-source-id: e95fd8002d94c8dc414bae1975e4fd348589f2b5 --- Makefile | 1 + build_tools/build_detect_platform | 5 +- build_tools/fbcode_config_platform009.sh | 82 +++++++++--------------- build_tools/gnu_parallel | 2 +- 4 files changed, 36 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index ca6a48984..fc11b90cf 100644 --- a/Makefile +++ b/Makefile @@ -229,6 +229,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ export PORTABLE="$(PORTABLE)"; \ export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \ export USE_CLANG="$(USE_CLANG)"; \ + export LIB_MODE="$(LIB_MODE)"; \ "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) # this file is generated by the previous line to set build flags and sources include make_config.mk diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index b1f8f1aed..a99f3a368 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -58,8 +58,11 @@ COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX" # Default to fbcode gcc on internal fb machines if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then FBCODE_BUILD="true" - # If we're compiling with TSAN we need pic build + # If we're compiling with TSAN or shared lib, we need pic build PIC_BUILD=$COMPILE_WITH_TSAN + if [ "$LIB_MODE" == "shared" ]; then + PIC_BUILD=1 + fi if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then # we need this to build with MySQL. Don't use for other purposes. source "$PWD/build_tools/fbcode_config4.8.1.sh" diff --git a/build_tools/fbcode_config_platform009.sh b/build_tools/fbcode_config_platform009.sh index 634e45891..e9cae8684 100644 --- a/build_tools/fbcode_config_platform009.sh +++ b/build_tools/fbcode_config_platform009.sh @@ -21,85 +21,63 @@ LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" GLIBC_INCLUDE="$GLIBC_BASE/include" GLIBC_LIBS=" -L $GLIBC_BASE/lib" +if test -z $PIC_BUILD; then + MAYBE_PIC= +else + MAYBE_PIC=_pic +fi + # snappy SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" -if test -z $PIC_BUILD; then - SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" -else - SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" -fi +SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a" CFLAGS+=" -DSNAPPY" -if test -z $PIC_BUILD; then - # location of zlib headers and libraries - ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" - ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" - CFLAGS+=" -DZLIB" +# location of zlib headers and libraries +ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" +ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a" +CFLAGS+=" -DZLIB" - # location of bzip headers and libraries - BZIP_INCLUDE=" -I $BZIP2_BASE/include/" - BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" - CFLAGS+=" -DBZIP2" +# location of bzip headers and libraries +BZIP_INCLUDE=" -I $BZIP2_BASE/include/" +BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a" +CFLAGS+=" -DBZIP2" - LZ4_INCLUDE=" -I $LZ4_BASE/include/" - LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" - CFLAGS+=" -DLZ4" -fi +LZ4_INCLUDE=" -I $LZ4_BASE/include/" +LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a" +CFLAGS+=" -DLZ4" ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" -if test -z $PIC_BUILD; then - ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" -else - ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a" -fi +ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a" CFLAGS+=" -DZSTD" # location of gflags headers and libraries GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" -if test -z $PIC_BUILD; then - GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" -else - GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a" -fi +GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags${MAYBE_PIC}.a" CFLAGS+=" -DGFLAGS=gflags" BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/" -if test -z $PIC_BUILD; then - BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark.a" -else - BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark_pic.a" -fi +BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark${MAYBE_PIC}.a" # location of jemalloc JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" -JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a" +JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc${MAYBE_PIC}.a" -if test -z $PIC_BUILD; then - # location of numa - NUMA_INCLUDE=" -I $NUMA_BASE/include/" - NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" - CFLAGS+=" -DNUMA" +# location of numa +NUMA_INCLUDE=" -I $NUMA_BASE/include/" +NUMA_LIB=" $NUMA_BASE/lib/libnuma${MAYBE_PIC}.a" +CFLAGS+=" -DNUMA" - # location of libunwind - LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" -fi +# location of libunwind +LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind${MAYBE_PIC}.a" # location of TBB TBB_INCLUDE=" -isystem $TBB_BASE/include/" -if test -z $PIC_BUILD; then - TBB_LIBS="$TBB_BASE/lib/libtbb.a" -else - TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a" -fi +TBB_LIBS="$TBB_BASE/lib/libtbb${MAYBE_PIC}.a" CFLAGS+=" -DTBB" # location of LIBURING LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/" -if test -z $PIC_BUILD; then - LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a" -else - LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a" -fi +LIBURING_LIBS="$LIBURING_BASE/lib/liburing${MAYBE_PIC}.a" CFLAGS+=" -DLIBURING" test "$USE_SSE" || USE_SSE=1 diff --git a/build_tools/gnu_parallel b/build_tools/gnu_parallel index 6b7849b0e..757b25f11 100755 --- a/build_tools/gnu_parallel +++ b/build_tools/gnu_parallel @@ -1561,7 +1561,7 @@ sub save_stdin_stdout_stderr { ::die_bug("Can't dup STDERR: $!"); open $Global::original_stdin, "<&", "STDIN" or ::die_bug("Can't dup STDIN: $!"); - $Global::is_terminal = (-t $Global::original_stderr); + $Global::is_terminal = (-t $Global::original_stderr) && !$ENV{'CIRCLECI'} && !$ENV{'TRAVIS'}; } sub enough_file_handles {