From ec77a928827af090d09e61e59b70feb50fa671dd Mon Sep 17 00:00:00 2001 From: Jay Zhuang Date: Mon, 4 Apr 2022 09:58:18 -0700 Subject: [PATCH] Fix commit_prereq and other targets (#9797) Summary: Make `commit_prereq` work and a few other improvements: * Remove gcc 481 and gcc5xx which are no longer supported * Remove platform007 which is gone * `make clean` work for both mac and linux * `precommit_checker.py` to python3 Pull Request resolved: https://github.com/facebook/rocksdb/pull/9797 Test Plan: `make commit_prereq` Reviewed By: ajkr Differential Revision: D35338536 Pulled By: jay-zhuang fbshipit-source-id: 1e159962ab9d31c43c4b85de7d0f582d3e881ffe --- Makefile | 8 +- build_tools/build_detect_platform | 13 +- build_tools/dependencies.sh | 19 --- build_tools/dependencies_4.8.1.sh | 20 --- build_tools/dependencies_platform007.sh | 20 --- build_tools/fbcode_config4.8.1.sh | 120 ---------------- build_tools/fbcode_config_platform007.sh | 170 ----------------------- build_tools/precommit_checker.py | 38 ++--- build_tools/rocksdb-lego-determinator | 49 ------- coverage/coverage_test.sh | 2 +- 10 files changed, 18 insertions(+), 441 deletions(-) delete mode 100644 build_tools/dependencies.sh delete mode 100644 build_tools/dependencies_4.8.1.sh delete mode 100644 build_tools/dependencies_platform007.sh delete mode 100644 build_tools/fbcode_config4.8.1.sh delete mode 100644 build_tools/fbcode_config_platform007.sh diff --git a/Makefile b/Makefile index ab7fe48bf..c216c5458 100644 --- a/Makefile +++ b/Makefile @@ -826,8 +826,8 @@ release: clean coverage: clean COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check cd coverage && ./coverage_test.sh - # Delete intermediate files - $(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm -f {} \; + # Delete intermediate files + $(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \; ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),) # Use /dev/shm if it has the sticky bit set (otherwise, /tmp), @@ -1214,7 +1214,7 @@ clean-rocks: rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(ALL_STATIC_LIBS) $(ALL_SHARED_LIBS) $(MICROBENCHS) rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report $(FIND) . -name "*.[oda]" -exec rm -f {} \; - $(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm -f {} \; + $(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \; clean-rocksjava: clean-rocks rm -rf jl jls @@ -2374,7 +2374,7 @@ jdb_bench: commit_prereq: build_tools/rocksdb-lego-determinator \ build_tools/precommit_checker.py - J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit release release_481 clang_release tsan asan ubsan lite unit_non_shm + J=$(J) build_tools/precommit_checker.py unit clang_unit release clang_release tsan asan ubsan lite unit_non_shm $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava; # --------------------------------------------------------------------------- diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 3b5c817ea..da31570c5 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -63,18 +63,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then if [ "$LIB_MODE" == "shared" ]; then PIC_BUILD=1 fi - if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then - # we need this to build with MySQL. Don't use for other purposes. - source "$PWD/build_tools/fbcode_config4.8.1.sh" - elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then - source "$PWD/build_tools/fbcode_config.sh" - elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then - source "$PWD/build_tools/fbcode_config_platform007.sh" - elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then - source "$PWD/build_tools/fbcode_config_platform009.sh" - else - source "$PWD/build_tools/fbcode_config_platform009.sh" - fi + source "$PWD/build_tools/fbcode_config_platform009.sh" fi # Delete existing output, if it exists diff --git a/build_tools/dependencies.sh b/build_tools/dependencies.sh deleted file mode 100644 index 22454c76f..000000000 --- a/build_tools/dependencies.sh +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/5.x/centos7-native/c447969 -CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/1bd23f9917738974ad0ff305aa23eb5f93f18305/9.0.0/centos7-native/c9f9104 -LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/5.x/gcc-5-glibc-2.23/339d858 -GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.23/gcc-5-glibc-2.23/ca1d1c0 -SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/gcc-5-glibc-2.23/9bc6787 -ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/gcc-5-glibc-2.23/9bc6787 -BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/gcc-5-glibc-2.23/9bc6787 -LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/gcc-5-glibc-2.23/9bc6787 -ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/gcc-5-glibc-2.23/03859b5 -GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/gcc-5-glibc-2.23/9bc6787 -JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/gcc-5-glibc-2.23/0c8f76d -NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/gcc-5-glibc-2.23/9bc6787 -LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/gcc-5-glibc-2.23/b443de1 -TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/gcc-5-glibc-2.23/9bc6787 -KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e -BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/2e3cb7d119b3cea5f1e738cc13a1ac69f49eb875/2.29.1/centos7-native/da39a3e -VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/gcc-5-glibc-2.23/9bc6787 -LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.2.3/gcc-5-glibc-2.23/65372bd diff --git a/build_tools/dependencies_4.8.1.sh b/build_tools/dependencies_4.8.1.sh deleted file mode 100644 index 5c8f7e4f8..000000000 --- a/build_tools/dependencies_4.8.1.sh +++ /dev/null @@ -1,20 +0,0 @@ -# shellcheck disable=SC2148 -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc -CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd -LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc -GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc -SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a -ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a -BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a -LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a -ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a -GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a -JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51 -NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a -LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945 -TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a -KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e -BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e -VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a -LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e diff --git a/build_tools/dependencies_platform007.sh b/build_tools/dependencies_platform007.sh deleted file mode 100644 index e95f2979e..000000000 --- a/build_tools/dependencies_platform007.sh +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6 -CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/ -LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc -GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413 -SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d -ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d -BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d -LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d -ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614 -GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d -JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002 -NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d -LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9 -TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d -LIBURING_BASE=/mnt/gvfs/third-party2/liburing/79427253fd0d42677255aacfe6d13bfe63f752eb/20190828/platform007/ca4da3d -KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e -BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614 -VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d -LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832 diff --git a/build_tools/fbcode_config4.8.1.sh b/build_tools/fbcode_config4.8.1.sh deleted file mode 100644 index c5ba8ddf4..000000000 --- a/build_tools/fbcode_config4.8.1.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/sh -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# Set environment variables so that we can compile rocksdb using -# fbcode settings. It uses the latest g++ compiler and also -# uses jemalloc - -BASEDIR=`dirname $BASH_SOURCE` -source "$BASEDIR/dependencies_4.8.1.sh" - -# location of libgcc -LIBGCC_INCLUDE="$LIBGCC_BASE/include" -LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" - -# location of glibc -GLIBC_INCLUDE="$GLIBC_BASE/include" -GLIBC_LIBS=" -L $GLIBC_BASE/lib" - -# location of snappy headers and libraries -SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include" -SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" - -# location of zlib headers and libraries -ZLIB_INCLUDE=" -I $ZLIB_BASE/include" -ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" - -# location of bzip headers and libraries -BZIP2_INCLUDE=" -I $BZIP2_BASE/include/" -BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a" - -LZ4_INCLUDE=" -I $LZ4_BASE/include" -LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" - -ZSTD_INCLUDE=" -I $ZSTD_BASE/include" -ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" - -# location of gflags headers and libraries -GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" -GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" - -# location of jemalloc -JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include" -JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a" - -# location of numa -NUMA_INCLUDE=" -I $NUMA_BASE/include/" -NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" - -# location of libunwind -LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" - -# location of tbb -TBB_INCLUDE=" -isystem $TBB_BASE/include/" -TBB_LIBS="$TBB_BASE/lib/libtbb.a" - -test "$USE_SSE" || USE_SSE=1 -export USE_SSE -test "$PORTABLE" || PORTABLE=1 -export PORTABLE - -BINUTILS="$BINUTILS_BASE/bin" -AR="$BINUTILS/ar" - -DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE" - -STDLIBS="-L $GCC_BASE/lib64" - -if [ -z "$USE_CLANG" ]; then - # gcc - CC="$GCC_BASE/bin/gcc" - CXX="$GCC_BASE/bin/g++" - CXX="$GCC_BASE/bin/gcc-ar" - - CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic" - CFLAGS+=" -isystem $GLIBC_INCLUDE" - CFLAGS+=" -isystem $LIBGCC_INCLUDE" - JEMALLOC=1 -else - # clang - CLANG_BIN="$CLANG_BASE/bin" - CLANG_LIB="$CLANG_BASE/lib" - CLANG_INCLUDE="$CLANG_LIB/clang/*/include" - CC="$CLANG_BIN/clang" - CXX="$CLANG_BIN/clang++" - AR="$CLANG_BIN/llvm-ar" - - KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/" - - CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib" - CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 " - CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux " - CFLAGS+=" -isystem $GLIBC_INCLUDE" - CFLAGS+=" -isystem $LIBGCC_INCLUDE" - CFLAGS+=" -isystem $CLANG_INCLUDE" - CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " - CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " - CXXFLAGS="-nostdinc++" -fi - -CFLAGS+=" $DEPS_INCLUDE" -CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42" -CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB" -CXXFLAGS+=" $CFLAGS" - -EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS" -EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so" -EXEC_LDFLAGS+=" $LIBUNWIND" -EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib" -# required by libtbb -EXEC_LDFLAGS+=" -ldl" - -PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" - -EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS" - -VALGRIND_VER="$VALGRIND_BASE/bin/" - -LUA_PATH="$LUA_BASE" - -export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH diff --git a/build_tools/fbcode_config_platform007.sh b/build_tools/fbcode_config_platform007.sh deleted file mode 100644 index 9c8e382e6..000000000 --- a/build_tools/fbcode_config_platform007.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/sh -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# Set environment variables so that we can compile rocksdb using -# fbcode settings. It uses the latest g++ and clang compilers and also -# uses jemalloc -# Environment variables that change the behavior of this script: -# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included - - -BASEDIR=`dirname $BASH_SOURCE` -source "$BASEDIR/dependencies_platform007.sh" - -CFLAGS="" - -# libgcc -LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0" -LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" - -# glibc -GLIBC_INCLUDE="$GLIBC_BASE/include" -GLIBC_LIBS=" -L $GLIBC_BASE/lib" - -# snappy -SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" -if test -z $PIC_BUILD; then - SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" -else - SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" -fi -CFLAGS+=" -DSNAPPY" - -if test -z $PIC_BUILD; then - # location of zlib headers and libraries - ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" - ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" - CFLAGS+=" -DZLIB" - - # location of bzip headers and libraries - BZIP_INCLUDE=" -I $BZIP2_BASE/include/" - BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" - CFLAGS+=" -DBZIP2" - - LZ4_INCLUDE=" -I $LZ4_BASE/include/" - LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" - CFLAGS+=" -DLZ4" -fi - -ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" -if test -z $PIC_BUILD; then - ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" -else - ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a" -fi -CFLAGS+=" -DZSTD" - -# location of gflags headers and libraries -GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" -if test -z $PIC_BUILD; then - GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" -else - GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a" -fi -CFLAGS+=" -DGFLAGS=gflags" - -# location of jemalloc -JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" -JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a" - -if test -z $PIC_BUILD; then - # location of numa - NUMA_INCLUDE=" -I $NUMA_BASE/include/" - NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" - CFLAGS+=" -DNUMA" - - # location of libunwind - LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" -fi - -# location of TBB -TBB_INCLUDE=" -isystem $TBB_BASE/include/" -if test -z $PIC_BUILD; then - TBB_LIBS="$TBB_BASE/lib/libtbb.a" -else - TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a" -fi -CFLAGS+=" -DTBB" - -# location of LIBURING -LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/" -if test -z $PIC_BUILD; then - LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a" -else - LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a" -fi -CFLAGS+=" -DLIBURING" - -test "$USE_SSE" || USE_SSE=1 -export USE_SSE -test "$PORTABLE" || PORTABLE=1 -export PORTABLE - -BINUTILS="$BINUTILS_BASE/bin" -AR="$BINUTILS/ar" - -DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE" - -STDLIBS="-L $GCC_BASE/lib64" - -CLANG_BIN="$CLANG_BASE/bin" -CLANG_LIB="$CLANG_BASE/lib" -CLANG_SRC="$CLANG_BASE/../../src" - -CLANG_ANALYZER="$CLANG_BIN/clang++" -CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build" - -if [ -z "$USE_CLANG" ]; then - # gcc - CC="$GCC_BASE/bin/gcc" - CXX="$GCC_BASE/bin/g++" - AR="$GCC_BASE/bin/gcc-ar" - - CFLAGS+=" -B$BINUTILS/gold" - CFLAGS+=" -isystem $LIBGCC_INCLUDE" - CFLAGS+=" -isystem $GLIBC_INCLUDE" - JEMALLOC=1 -else - # clang - CLANG_INCLUDE="$CLANG_LIB/clang/stable/include" - CC="$CLANG_BIN/clang" - CXX="$CLANG_BIN/clang++" - AR="$CLANG_BIN/llvm-ar" - - KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include" - - CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib" - CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x " - CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux " - CFLAGS+=" -isystem $GLIBC_INCLUDE" - CFLAGS+=" -isystem $LIBGCC_INCLUDE" - CFLAGS+=" -isystem $CLANG_INCLUDE" - CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " - CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " - CFLAGS+=" -Wno-expansion-to-defined " - CXXFLAGS="-nostdinc++" -fi - -CFLAGS+=" $DEPS_INCLUDE" -CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT" -CXXFLAGS+=" $CFLAGS" - -EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS" -EXEC_LDFLAGS+=" -B$BINUTILS/gold" -EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so" -EXEC_LDFLAGS+=" $LIBUNWIND" -EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib" -# required by libtbb -EXEC_LDFLAGS+=" -ldl" - -PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" - -EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS" - -VALGRIND_VER="$VALGRIND_BASE/bin/" - -# lua not supported because it's on track for deprecation, I think -LUA_PATH= -LUA_LIB= - -export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB diff --git a/build_tools/precommit_checker.py b/build_tools/precommit_checker.py index 7d5eeecad..212bb5227 100755 --- a/build_tools/precommit_checker.py +++ b/build_tools/precommit_checker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. from __future__ import absolute_import @@ -6,7 +6,6 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals import argparse -import commands import subprocess import sys import re @@ -22,7 +21,7 @@ class Log: def __init__(self, filename): self.filename = filename - self.f = open(self.filename, 'w+', 0) + self.f = open(self.filename, 'w+') def caption(self, str): line = "\n##### %s #####\n" % str @@ -61,23 +60,14 @@ class Env(object): self.log.log("==== shell session ===========================") self.log.log("%s> %s" % (path, cmd)) - status = subprocess.call("cd %s; %s" % (path, cmd), shell=True, - stdout=self.log.f, stderr=self.log.f) - self.log.log("status = %s" % status) + p = subprocess.run("cd %s; %s" % (path, cmd), shell=True, + capture_output=True, text=True) + self.log.log("status = %s" % p.returncode) + self.log.log("stdout = %s" % p.stdout) + if p.stderr: + self.log.log("stderr = %s" % p.stderr) self.log.log("============================================== \n\n") - return status - - def GetOutput(self, cmd, path=os.getcwd()): - if path: - os.chdir(path) - - self.log.log("==== shell session ===========================") - self.log.log("%s> %s" % (path, cmd)) - status, out = commands.getstatusoutput(cmd) - self.log.log("status = %s" % status) - self.log.log("out = %s" % out) - self.log.log("============================================== \n\n") - return status, out + return p.returncode, p.stdout # # Pre-commit checker @@ -94,7 +84,7 @@ class PreCommitChecker(Env): # Get commands for a given job from the determinator file # def get_commands(self, test): - status, out = self.GetOutput( + status, out = self.shell( "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".") return status, out @@ -111,7 +101,7 @@ class PreCommitChecker(Env): return False # Parse the JSON to extract the commands to run - cmds = re.findall("'shell':'([^\']*)'", cmds) + cmds = re.findall('"shell":"([^\"]*)', cmds) if len(cmds) == 0: self.log.log("No commands found") @@ -119,12 +109,8 @@ class PreCommitChecker(Env): # Run commands for cmd in cmds: - # Replace J=<..> with the local environment variable - if "J" in os.environ: - cmd = cmd.replace("J=1", "J=%s" % os.environ["J"]) - cmd = cmd.replace("make ", "make -j%s " % os.environ["J"]) # Run the command - status = self.shell(cmd, ".") + status, out = self.shell(cmd, ".") if status != 0: self.log.error("Error running command %s for test %s" % (cmd, test)) diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator index 162ebbe0e..ab01d1d88 100755 --- a/build_tools/rocksdb-lego-determinator +++ b/build_tools/rocksdb-lego-determinator @@ -104,7 +104,6 @@ PARALLELISM="$PARALLEL_J $PARALLEL_j" DEBUG="OPT=-g" SHM="TEST_TMPDIR=/dev/shm/rocksdb" NON_SHM="TMPD=/tmp/rocksdb_test_tmp" -GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" ASAN="COMPILE_WITH_ASAN=1" ASAN_TEST_OPTIONS="ASAN_OPTIONS=detect_stack_use_after_return=1" CLANG="USE_CLANG=1" @@ -215,48 +214,6 @@ RELEASE_BUILD_COMMANDS="[ } ]" -# -# RocksDB unit test on gcc-4.8.1 -# -UNIT_TEST_COMMANDS_481="[ - { - \"name\":\"Rocksdb Unit Test on GCC 4.8.1\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build and test RocksDB debug version\", - \"shell\":\"cd $WORKING_DIR; $SHM $GCC_481 $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - -# -# RocksDB release build and unit tests -# -RELEASE_BUILD_COMMANDS_481="[ - { - \"name\":\"Rocksdb Release on GCC 4.8.1\", - \"oncall\":\"$ONCALL\", - \"executeLocal\": \"true\", - \"steps\": [ - $CLEANUP_ENV, - { - \"name\":\"Build RocksDB release on GCC 4.8.1\", - \"shell\":\"cd $WORKING_DIR; $GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL\", - \"user\":\"root\", - $PARSER - } - ] - $REPORT - } -]" - # # RocksDB unit test with CLANG # @@ -1288,12 +1245,6 @@ case $1 in release) echo $RELEASE_BUILD_COMMANDS ;; - unit_481) - echo $UNIT_TEST_COMMANDS_481 - ;; - release_481) - echo $RELEASE_BUILD_COMMANDS_481 - ;; clang_unit) echo $CLANG_UNIT_TEST_COMMANDS ;; diff --git a/coverage/coverage_test.sh b/coverage/coverage_test.sh index afe77f663..aa5f68c77 100755 --- a/coverage/coverage_test.sh +++ b/coverage/coverage_test.sh @@ -12,7 +12,7 @@ fi ROOT=".." # Fetch right version of gcov if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then - source $ROOT/build_tools/fbcode_config_platform007.sh + source $ROOT/build_tools/fbcode_config_platform009.sh GCOV=$GCC_BASE/bin/gcov else GCOV=$(which gcov)