Fix commit_prereq and other targets (#9797)
Summary: Make `commit_prereq` work and a few other improvements: * Remove gcc 481 and gcc5xx which are no longer supported * Remove platform007 which is gone * `make clean` work for both mac and linux * `precommit_checker.py` to python3 Pull Request resolved: https://github.com/facebook/rocksdb/pull/9797 Test Plan: `make commit_prereq` Reviewed By: ajkr Differential Revision: D35338536 Pulled By: jay-zhuang fbshipit-source-id: 1e159962ab9d31c43c4b85de7d0f582d3e881ffe
This commit is contained in:
Jay Zhuang
Facebook GitHub Bot
parent
f68706409d
commit
ec77a92882
8
Makefile
8
Makefile
@ -826,8 +826,8 @@ release: clean
|
|||||||
coverage: clean
|
coverage: clean
|
||||||
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check
|
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check
|
||||||
cd coverage && ./coverage_test.sh
|
cd coverage && ./coverage_test.sh
|
||||||
# Delete intermediate files
|
# Delete intermediate files
|
||||||
$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm -f {} \;
|
$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;
|
||||||
|
|
||||||
ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
|
ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
|
||||||
# Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
|
# Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
|
||||||
@ -1214,7 +1214,7 @@ clean-rocks:
|
|||||||
rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(ALL_STATIC_LIBS) $(ALL_SHARED_LIBS) $(MICROBENCHS)
|
rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(ALL_STATIC_LIBS) $(ALL_SHARED_LIBS) $(MICROBENCHS)
|
||||||
rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
|
rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
|
||||||
$(FIND) . -name "*.[oda]" -exec rm -f {} \;
|
$(FIND) . -name "*.[oda]" -exec rm -f {} \;
|
||||||
$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm -f {} \;
|
$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;
|
||||||
|
|
||||||
clean-rocksjava: clean-rocks
|
clean-rocksjava: clean-rocks
|
||||||
rm -rf jl jls
|
rm -rf jl jls
|
||||||
@ -2374,7 +2374,7 @@ jdb_bench:
|
|||||||
|
|
||||||
commit_prereq: build_tools/rocksdb-lego-determinator \
|
commit_prereq: build_tools/rocksdb-lego-determinator \
|
||||||
build_tools/precommit_checker.py
|
build_tools/precommit_checker.py
|
||||||
J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit release release_481 clang_release tsan asan ubsan lite unit_non_shm
|
J=$(J) build_tools/precommit_checker.py unit clang_unit release clang_release tsan asan ubsan lite unit_non_shm
|
||||||
$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
|
$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@ -63,18 +63,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
|||||||
if [ "$LIB_MODE" == "shared" ]; then
|
if [ "$LIB_MODE" == "shared" ]; then
|
||||||
PIC_BUILD=1
|
PIC_BUILD=1
|
||||||
fi
|
fi
|
||||||
if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
source "$PWD/build_tools/fbcode_config_platform009.sh"
|
||||||
# we need this to build with MySQL. Don't use for other purposes.
|
|
||||||
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
|
||||||
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
|
|
||||||
source "$PWD/build_tools/fbcode_config.sh"
|
|
||||||
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then
|
|
||||||
source "$PWD/build_tools/fbcode_config_platform007.sh"
|
|
||||||
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then
|
|
||||||
source "$PWD/build_tools/fbcode_config_platform009.sh"
|
|
||||||
else
|
|
||||||
source "$PWD/build_tools/fbcode_config_platform009.sh"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Delete existing output, if it exists
|
# Delete existing output, if it exists
|
||||||
|
|||||||
@ -1,19 +0,0 @@
|
|||||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
||||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/5.x/centos7-native/c447969
|
|
||||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/1bd23f9917738974ad0ff305aa23eb5f93f18305/9.0.0/centos7-native/c9f9104
|
|
||||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/5.x/gcc-5-glibc-2.23/339d858
|
|
||||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.23/gcc-5-glibc-2.23/ca1d1c0
|
|
||||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/gcc-5-glibc-2.23/9bc6787
|
|
||||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/gcc-5-glibc-2.23/9bc6787
|
|
||||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/gcc-5-glibc-2.23/9bc6787
|
|
||||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/gcc-5-glibc-2.23/9bc6787
|
|
||||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/gcc-5-glibc-2.23/03859b5
|
|
||||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/gcc-5-glibc-2.23/9bc6787
|
|
||||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/gcc-5-glibc-2.23/0c8f76d
|
|
||||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/gcc-5-glibc-2.23/9bc6787
|
|
||||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/gcc-5-glibc-2.23/b443de1
|
|
||||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/gcc-5-glibc-2.23/9bc6787
|
|
||||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
|
|
||||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/2e3cb7d119b3cea5f1e738cc13a1ac69f49eb875/2.29.1/centos7-native/da39a3e
|
|
||||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/gcc-5-glibc-2.23/9bc6787
|
|
||||||
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.2.3/gcc-5-glibc-2.23/65372bd
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
# shellcheck disable=SC2148
|
|
||||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
||||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
|
|
||||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
|
|
||||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
|
|
||||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc
|
|
||||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
|
|
||||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945
|
|
||||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e
|
|
||||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
|
|
||||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
|
|
||||||
LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
||||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
|
|
||||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
|
|
||||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
|
|
||||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
|
|
||||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
|
|
||||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
|
|
||||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
|
|
||||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
|
|
||||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
|
|
||||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
|
|
||||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
|
|
||||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
|
|
||||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
|
|
||||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
|
|
||||||
LIBURING_BASE=/mnt/gvfs/third-party2/liburing/79427253fd0d42677255aacfe6d13bfe63f752eb/20190828/platform007/ca4da3d
|
|
||||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
|
|
||||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
|
|
||||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
|
|
||||||
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
|
|
||||||
@ -1,120 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Set environment variables so that we can compile rocksdb using
|
|
||||||
# fbcode settings. It uses the latest g++ compiler and also
|
|
||||||
# uses jemalloc
|
|
||||||
|
|
||||||
BASEDIR=`dirname $BASH_SOURCE`
|
|
||||||
source "$BASEDIR/dependencies_4.8.1.sh"
|
|
||||||
|
|
||||||
# location of libgcc
|
|
||||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
|
|
||||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
|
||||||
|
|
||||||
# location of glibc
|
|
||||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
|
||||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
|
||||||
|
|
||||||
# location of snappy headers and libraries
|
|
||||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include"
|
|
||||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
|
|
||||||
|
|
||||||
# location of zlib headers and libraries
|
|
||||||
ZLIB_INCLUDE=" -I $ZLIB_BASE/include"
|
|
||||||
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
|
|
||||||
|
|
||||||
# location of bzip headers and libraries
|
|
||||||
BZIP2_INCLUDE=" -I $BZIP2_BASE/include/"
|
|
||||||
BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a"
|
|
||||||
|
|
||||||
LZ4_INCLUDE=" -I $LZ4_BASE/include"
|
|
||||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
|
||||||
|
|
||||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include"
|
|
||||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
|
||||||
|
|
||||||
# location of gflags headers and libraries
|
|
||||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
|
||||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
|
|
||||||
|
|
||||||
# location of jemalloc
|
|
||||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include"
|
|
||||||
JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a"
|
|
||||||
|
|
||||||
# location of numa
|
|
||||||
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
|
||||||
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
|
|
||||||
|
|
||||||
# location of libunwind
|
|
||||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
|
||||||
|
|
||||||
# location of tbb
|
|
||||||
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
|
||||||
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
|
|
||||||
|
|
||||||
test "$USE_SSE" || USE_SSE=1
|
|
||||||
export USE_SSE
|
|
||||||
test "$PORTABLE" || PORTABLE=1
|
|
||||||
export PORTABLE
|
|
||||||
|
|
||||||
BINUTILS="$BINUTILS_BASE/bin"
|
|
||||||
AR="$BINUTILS/ar"
|
|
||||||
|
|
||||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
|
|
||||||
|
|
||||||
STDLIBS="-L $GCC_BASE/lib64"
|
|
||||||
|
|
||||||
if [ -z "$USE_CLANG" ]; then
|
|
||||||
# gcc
|
|
||||||
CC="$GCC_BASE/bin/gcc"
|
|
||||||
CXX="$GCC_BASE/bin/g++"
|
|
||||||
CXX="$GCC_BASE/bin/gcc-ar"
|
|
||||||
|
|
||||||
CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
|
|
||||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
|
||||||
JEMALLOC=1
|
|
||||||
else
|
|
||||||
# clang
|
|
||||||
CLANG_BIN="$CLANG_BASE/bin"
|
|
||||||
CLANG_LIB="$CLANG_BASE/lib"
|
|
||||||
CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
|
|
||||||
CC="$CLANG_BIN/clang"
|
|
||||||
CXX="$CLANG_BIN/clang++"
|
|
||||||
AR="$CLANG_BIN/llvm-ar"
|
|
||||||
|
|
||||||
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
|
|
||||||
|
|
||||||
CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux "
|
|
||||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $CLANG_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
|
|
||||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
|
|
||||||
CXXFLAGS="-nostdinc++"
|
|
||||||
fi
|
|
||||||
|
|
||||||
CFLAGS+=" $DEPS_INCLUDE"
|
|
||||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
|
|
||||||
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
|
|
||||||
CXXFLAGS+=" $CFLAGS"
|
|
||||||
|
|
||||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
|
|
||||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
|
|
||||||
EXEC_LDFLAGS+=" $LIBUNWIND"
|
|
||||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
|
|
||||||
# required by libtbb
|
|
||||||
EXEC_LDFLAGS+=" -ldl"
|
|
||||||
|
|
||||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
|
||||||
|
|
||||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
|
|
||||||
|
|
||||||
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
|
||||||
|
|
||||||
LUA_PATH="$LUA_BASE"
|
|
||||||
|
|
||||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH
|
|
||||||
@ -1,170 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Set environment variables so that we can compile rocksdb using
|
|
||||||
# fbcode settings. It uses the latest g++ and clang compilers and also
|
|
||||||
# uses jemalloc
|
|
||||||
# Environment variables that change the behavior of this script:
|
|
||||||
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
|
|
||||||
|
|
||||||
|
|
||||||
BASEDIR=`dirname $BASH_SOURCE`
|
|
||||||
source "$BASEDIR/dependencies_platform007.sh"
|
|
||||||
|
|
||||||
CFLAGS=""
|
|
||||||
|
|
||||||
# libgcc
|
|
||||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
|
|
||||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
|
||||||
|
|
||||||
# glibc
|
|
||||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
|
||||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
|
||||||
|
|
||||||
# snappy
|
|
||||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
|
||||||
if test -z $PIC_BUILD; then
|
|
||||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
|
|
||||||
else
|
|
||||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DSNAPPY"
|
|
||||||
|
|
||||||
if test -z $PIC_BUILD; then
|
|
||||||
# location of zlib headers and libraries
|
|
||||||
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
|
|
||||||
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
|
|
||||||
CFLAGS+=" -DZLIB"
|
|
||||||
|
|
||||||
# location of bzip headers and libraries
|
|
||||||
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
|
|
||||||
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
|
|
||||||
CFLAGS+=" -DBZIP2"
|
|
||||||
|
|
||||||
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
|
||||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
|
||||||
CFLAGS+=" -DLZ4"
|
|
||||||
fi
|
|
||||||
|
|
||||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
|
||||||
if test -z $PIC_BUILD; then
|
|
||||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
|
||||||
else
|
|
||||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DZSTD"
|
|
||||||
|
|
||||||
# location of gflags headers and libraries
|
|
||||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
|
||||||
if test -z $PIC_BUILD; then
|
|
||||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
|
|
||||||
else
|
|
||||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DGFLAGS=gflags"
|
|
||||||
|
|
||||||
# location of jemalloc
|
|
||||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
|
||||||
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
|
|
||||||
|
|
||||||
if test -z $PIC_BUILD; then
|
|
||||||
# location of numa
|
|
||||||
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
|
||||||
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
|
|
||||||
CFLAGS+=" -DNUMA"
|
|
||||||
|
|
||||||
# location of libunwind
|
|
||||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# location of TBB
|
|
||||||
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
|
||||||
if test -z $PIC_BUILD; then
|
|
||||||
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
|
|
||||||
else
|
|
||||||
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DTBB"
|
|
||||||
|
|
||||||
# location of LIBURING
|
|
||||||
LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
|
|
||||||
if test -z $PIC_BUILD; then
|
|
||||||
LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
|
|
||||||
else
|
|
||||||
LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
|
|
||||||
fi
|
|
||||||
CFLAGS+=" -DLIBURING"
|
|
||||||
|
|
||||||
test "$USE_SSE" || USE_SSE=1
|
|
||||||
export USE_SSE
|
|
||||||
test "$PORTABLE" || PORTABLE=1
|
|
||||||
export PORTABLE
|
|
||||||
|
|
||||||
BINUTILS="$BINUTILS_BASE/bin"
|
|
||||||
AR="$BINUTILS/ar"
|
|
||||||
|
|
||||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
|
|
||||||
|
|
||||||
STDLIBS="-L $GCC_BASE/lib64"
|
|
||||||
|
|
||||||
CLANG_BIN="$CLANG_BASE/bin"
|
|
||||||
CLANG_LIB="$CLANG_BASE/lib"
|
|
||||||
CLANG_SRC="$CLANG_BASE/../../src"
|
|
||||||
|
|
||||||
CLANG_ANALYZER="$CLANG_BIN/clang++"
|
|
||||||
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
|
|
||||||
|
|
||||||
if [ -z "$USE_CLANG" ]; then
|
|
||||||
# gcc
|
|
||||||
CC="$GCC_BASE/bin/gcc"
|
|
||||||
CXX="$GCC_BASE/bin/g++"
|
|
||||||
AR="$GCC_BASE/bin/gcc-ar"
|
|
||||||
|
|
||||||
CFLAGS+=" -B$BINUTILS/gold"
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
|
||||||
JEMALLOC=1
|
|
||||||
else
|
|
||||||
# clang
|
|
||||||
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
|
|
||||||
CC="$CLANG_BIN/clang"
|
|
||||||
CXX="$CLANG_BIN/clang++"
|
|
||||||
AR="$CLANG_BIN/llvm-ar"
|
|
||||||
|
|
||||||
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
|
|
||||||
|
|
||||||
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
|
|
||||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $CLANG_INCLUDE"
|
|
||||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
|
|
||||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
|
|
||||||
CFLAGS+=" -Wno-expansion-to-defined "
|
|
||||||
CXXFLAGS="-nostdinc++"
|
|
||||||
fi
|
|
||||||
|
|
||||||
CFLAGS+=" $DEPS_INCLUDE"
|
|
||||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
|
|
||||||
CXXFLAGS+=" $CFLAGS"
|
|
||||||
|
|
||||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
|
|
||||||
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
|
|
||||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
|
|
||||||
EXEC_LDFLAGS+=" $LIBUNWIND"
|
|
||||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
|
|
||||||
# required by libtbb
|
|
||||||
EXEC_LDFLAGS+=" -ldl"
|
|
||||||
|
|
||||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
|
||||||
|
|
||||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
|
|
||||||
|
|
||||||
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
|
||||||
|
|
||||||
# lua not supported because it's on track for deprecation, I think
|
|
||||||
LUA_PATH=
|
|
||||||
LUA_LIB=
|
|
||||||
|
|
||||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
|
|
||||||
@ -1,4 +1,4 @@
|
|||||||
#!/usr/bin/env python2.7
|
#!/usr/bin/env python3
|
||||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||||
|
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
@ -6,7 +6,6 @@ from __future__ import division
|
|||||||
from __future__ import print_function
|
from __future__ import print_function
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
import argparse
|
import argparse
|
||||||
import commands
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import re
|
import re
|
||||||
@ -22,7 +21,7 @@ class Log:
|
|||||||
|
|
||||||
def __init__(self, filename):
|
def __init__(self, filename):
|
||||||
self.filename = filename
|
self.filename = filename
|
||||||
self.f = open(self.filename, 'w+', 0)
|
self.f = open(self.filename, 'w+')
|
||||||
|
|
||||||
def caption(self, str):
|
def caption(self, str):
|
||||||
line = "\n##### %s #####\n" % str
|
line = "\n##### %s #####\n" % str
|
||||||
@ -61,23 +60,14 @@ class Env(object):
|
|||||||
|
|
||||||
self.log.log("==== shell session ===========================")
|
self.log.log("==== shell session ===========================")
|
||||||
self.log.log("%s> %s" % (path, cmd))
|
self.log.log("%s> %s" % (path, cmd))
|
||||||
status = subprocess.call("cd %s; %s" % (path, cmd), shell=True,
|
p = subprocess.run("cd %s; %s" % (path, cmd), shell=True,
|
||||||
stdout=self.log.f, stderr=self.log.f)
|
capture_output=True, text=True)
|
||||||
self.log.log("status = %s" % status)
|
self.log.log("status = %s" % p.returncode)
|
||||||
|
self.log.log("stdout = %s" % p.stdout)
|
||||||
|
if p.stderr:
|
||||||
|
self.log.log("stderr = %s" % p.stderr)
|
||||||
self.log.log("============================================== \n\n")
|
self.log.log("============================================== \n\n")
|
||||||
return status
|
return p.returncode, p.stdout
|
||||||
|
|
||||||
def GetOutput(self, cmd, path=os.getcwd()):
|
|
||||||
if path:
|
|
||||||
os.chdir(path)
|
|
||||||
|
|
||||||
self.log.log("==== shell session ===========================")
|
|
||||||
self.log.log("%s> %s" % (path, cmd))
|
|
||||||
status, out = commands.getstatusoutput(cmd)
|
|
||||||
self.log.log("status = %s" % status)
|
|
||||||
self.log.log("out = %s" % out)
|
|
||||||
self.log.log("============================================== \n\n")
|
|
||||||
return status, out
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# Pre-commit checker
|
# Pre-commit checker
|
||||||
@ -94,7 +84,7 @@ class PreCommitChecker(Env):
|
|||||||
# Get commands for a given job from the determinator file
|
# Get commands for a given job from the determinator file
|
||||||
#
|
#
|
||||||
def get_commands(self, test):
|
def get_commands(self, test):
|
||||||
status, out = self.GetOutput(
|
status, out = self.shell(
|
||||||
"RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
|
"RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
|
||||||
return status, out
|
return status, out
|
||||||
|
|
||||||
@ -111,7 +101,7 @@ class PreCommitChecker(Env):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
# Parse the JSON to extract the commands to run
|
# Parse the JSON to extract the commands to run
|
||||||
cmds = re.findall("'shell':'([^\']*)'", cmds)
|
cmds = re.findall('"shell":"([^\"]*)', cmds)
|
||||||
|
|
||||||
if len(cmds) == 0:
|
if len(cmds) == 0:
|
||||||
self.log.log("No commands found")
|
self.log.log("No commands found")
|
||||||
@ -119,12 +109,8 @@ class PreCommitChecker(Env):
|
|||||||
|
|
||||||
# Run commands
|
# Run commands
|
||||||
for cmd in cmds:
|
for cmd in cmds:
|
||||||
# Replace J=<..> with the local environment variable
|
|
||||||
if "J" in os.environ:
|
|
||||||
cmd = cmd.replace("J=1", "J=%s" % os.environ["J"])
|
|
||||||
cmd = cmd.replace("make ", "make -j%s " % os.environ["J"])
|
|
||||||
# Run the command
|
# Run the command
|
||||||
status = self.shell(cmd, ".")
|
status, out = self.shell(cmd, ".")
|
||||||
if status != 0:
|
if status != 0:
|
||||||
self.log.error("Error running command %s for test %s"
|
self.log.error("Error running command %s for test %s"
|
||||||
% (cmd, test))
|
% (cmd, test))
|
||||||
|
|||||||
@ -104,7 +104,6 @@ PARALLELISM="$PARALLEL_J $PARALLEL_j"
|
|||||||
DEBUG="OPT=-g"
|
DEBUG="OPT=-g"
|
||||||
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
|
SHM="TEST_TMPDIR=/dev/shm/rocksdb"
|
||||||
NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
|
NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
|
||||||
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
|
|
||||||
ASAN="COMPILE_WITH_ASAN=1"
|
ASAN="COMPILE_WITH_ASAN=1"
|
||||||
ASAN_TEST_OPTIONS="ASAN_OPTIONS=detect_stack_use_after_return=1"
|
ASAN_TEST_OPTIONS="ASAN_OPTIONS=detect_stack_use_after_return=1"
|
||||||
CLANG="USE_CLANG=1"
|
CLANG="USE_CLANG=1"
|
||||||
@ -215,48 +214,6 @@ RELEASE_BUILD_COMMANDS="[
|
|||||||
}
|
}
|
||||||
]"
|
]"
|
||||||
|
|
||||||
#
|
|
||||||
# RocksDB unit test on gcc-4.8.1
|
|
||||||
#
|
|
||||||
UNIT_TEST_COMMANDS_481="[
|
|
||||||
{
|
|
||||||
\"name\":\"Rocksdb Unit Test on GCC 4.8.1\",
|
|
||||||
\"oncall\":\"$ONCALL\",
|
|
||||||
\"executeLocal\": \"true\",
|
|
||||||
\"steps\": [
|
|
||||||
$CLEANUP_ENV,
|
|
||||||
{
|
|
||||||
\"name\":\"Build and test RocksDB debug version\",
|
|
||||||
\"shell\":\"cd $WORKING_DIR; $SHM $GCC_481 $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL\",
|
|
||||||
\"user\":\"root\",
|
|
||||||
$PARSER
|
|
||||||
}
|
|
||||||
]
|
|
||||||
$REPORT
|
|
||||||
}
|
|
||||||
]"
|
|
||||||
|
|
||||||
#
|
|
||||||
# RocksDB release build and unit tests
|
|
||||||
#
|
|
||||||
RELEASE_BUILD_COMMANDS_481="[
|
|
||||||
{
|
|
||||||
\"name\":\"Rocksdb Release on GCC 4.8.1\",
|
|
||||||
\"oncall\":\"$ONCALL\",
|
|
||||||
\"executeLocal\": \"true\",
|
|
||||||
\"steps\": [
|
|
||||||
$CLEANUP_ENV,
|
|
||||||
{
|
|
||||||
\"name\":\"Build RocksDB release on GCC 4.8.1\",
|
|
||||||
\"shell\":\"cd $WORKING_DIR; $GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL\",
|
|
||||||
\"user\":\"root\",
|
|
||||||
$PARSER
|
|
||||||
}
|
|
||||||
]
|
|
||||||
$REPORT
|
|
||||||
}
|
|
||||||
]"
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# RocksDB unit test with CLANG
|
# RocksDB unit test with CLANG
|
||||||
#
|
#
|
||||||
@ -1288,12 +1245,6 @@ case $1 in
|
|||||||
release)
|
release)
|
||||||
echo $RELEASE_BUILD_COMMANDS
|
echo $RELEASE_BUILD_COMMANDS
|
||||||
;;
|
;;
|
||||||
unit_481)
|
|
||||||
echo $UNIT_TEST_COMMANDS_481
|
|
||||||
;;
|
|
||||||
release_481)
|
|
||||||
echo $RELEASE_BUILD_COMMANDS_481
|
|
||||||
;;
|
|
||||||
clang_unit)
|
clang_unit)
|
||||||
echo $CLANG_UNIT_TEST_COMMANDS
|
echo $CLANG_UNIT_TEST_COMMANDS
|
||||||
;;
|
;;
|
||||||
|
|||||||
@ -12,7 +12,7 @@ fi
|
|||||||
ROOT=".."
|
ROOT=".."
|
||||||
# Fetch right version of gcov
|
# Fetch right version of gcov
|
||||||
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
|
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
|
||||||
source $ROOT/build_tools/fbcode_config_platform007.sh
|
source $ROOT/build_tools/fbcode_config_platform009.sh
|
||||||
GCOV=$GCC_BASE/bin/gcov
|
GCOV=$GCC_BASE/bin/gcov
|
||||||
else
|
else
|
||||||
GCOV=$(which gcov)
|
GCOV=$(which gcov)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user