From 6980fff4450a850bf2844b5b9a2abf48bab7db64 Mon Sep 17 00:00:00 2001 From: Jay Zhuang Date: Wed, 23 Sep 2020 14:40:53 -0700 Subject: [PATCH] CircleCI build improvements (#7392) Summary: * add slack integration * parallelize `check_some` tests with gtest-parallel * upgrade build image to the latest one: ubuntu-1604:202007-01 * clean up the config Pull Request resolved: https://github.com/facebook/rocksdb/pull/7392 Reviewed By: pdillinger Differential Revision: D23845379 Pulled By: jay-zhuang fbshipit-source-id: f3af82dc2daeba6441f61c858e08d3936c6ccd10 --- .circleci/config.yml | 250 ++++++++++++++++++++++++------------------- Makefile | 3 + 2 files changed, 143 insertions(+), 110 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45c9e65e5..f52e5631a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,59 @@ version: 2.1 orbs: win: circleci/windows@2.4.0 + slack: circleci/slack@3.4.2 + +aliases: + - ¬ify-on-master-failure + fail_only: true + only_for_branches: master + +commands: + pre-steps: + steps: + - checkout + - run: pyenv global 3.5.9 + - run: + name: Setup Environment Variables + command: | + echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV + echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV + echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV + echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV + + post-steps: + steps: + - slack/status: *notify-on-master-failure + - store_test_results: # store test result if there's any + path: /tmp/test-results + - store_artifacts: # store LOG for debugging if there's any + path: LOG + + install-clang-10: + steps: + - run: + name: Install Clang 10 + command: | + echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list + echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list + echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable + sudo apt-get update -y && sudo apt-get install -y clang-10 + + install-gflags: + steps: + - run: + name: Install gflags + command: | + sudo apt-get update -y && sudo apt-get install -y libgflags-dev + + install-gtest-parallel: + steps: + - run: + name: Install gtest-parallel + command: | + echo "Currently using a forked version to print task status in circleCI output, pending: https://github.com/google/gtest-parallel/pull/79" + git clone --single-branch --branch master --depth 1 https://github.com/jay-zhuang/gtest-parallel.git ~/gtest-parallel + echo "export PATH=$HOME/gtest-parallel:$PATH" >> $BASH_ENV executors: windows-2xlarge: @@ -13,39 +66,36 @@ executors: jobs: build-linux: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev - - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain - - store_test_results: - path: /tmp/test-results + - pre-steps + - install-gflags + - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain + - post-steps build-linux-shared_lib-alt_namespace-status_checked: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev - - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain - - store_test_results: - path: /tmp/test-results + - pre-steps + - install-gflags + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain + - post-steps build-linux-release: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - checkout # check out the code in the project directory - run: make V=1 -j32 release | .circleci/cat_ignore_eagain - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - install-gflags - run: make V=1 -j32 release | .circleci/cat_ignore_eagain - run: ./db_stress --version # ensure with gflags + - post-steps build-linux-release-rtti: machine: @@ -63,112 +113,90 @@ jobs: build-linux-lite: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev - - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 LITE=1 GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain - - store_test_results: - path: /tmp/test-results + - pre-steps + - install-gflags + - run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain + - post-steps build-linux-lite-release: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: large steps: - checkout # check out the code in the project directory - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - install-gflags - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain - run: ./db_stress --version # ensure with gflags + - post-steps build-linux-clang-no-test: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - checkout # check out the code in the project directory - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain - - build-linux-clang10-no-test: - machine: - image: ubuntu-1604:201903-01 - resource_class: 2xlarge - steps: - - checkout # check out the code in the project directory - - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable - - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev - - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 all | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out + - post-steps build-linux-clang10-asan: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable - - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev - - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out - - store_test_results: - path: /tmp/test-results + - pre-steps + - install-gflags + - install-clang-10 + - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out + - post-steps build-linux-clang10-mini-tsan: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable - - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev - - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 EXCLUDE_TESTS_REGEX="TransactionStressTest|SnapshotConcurrentAccess|SeqAdvanceConcurrent|DeadlockStress|MultiThreadedDBTest.MultiThreaded|WriteUnpreparedStressTest.ReadYourOwnWriteStress|DBAsBaseDB/TransactionStressTest|FlushCloseWALFiles|BackgroundPurgeCFDropTest" GTEST_THROW_ON_FAILURE=0 GTEST_OUTPUT="xml:/tmp/test-results/" make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. Exclude FlushCloseWALFiles and BackgroundPurgeCFDropTest for occasional failures. - - store_test_results: - path: /tmp/test-results + - pre-steps + - install-gflags + - install-clang-10 + - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 EXCLUDE_TESTS_REGEX="TransactionStressTest|SnapshotConcurrentAccess|SeqAdvanceConcurrent|DeadlockStress|MultiThreadedDBTest.MultiThreaded|WriteUnpreparedStressTest.ReadYourOwnWriteStress|DBAsBaseDB/TransactionStressTest|FlushCloseWALFiles|BackgroundPurgeCFDropTest" make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. Exclude FlushCloseWALFiles and BackgroundPurgeCFDropTest for occasional failures. + - post-steps build-linux-clang10-ubsan: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable - - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev - - run: SKIP_FORMAT_BUCK_CHECKS=1 COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out + - pre-steps + - install-gflags + - install-clang-10 + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out + - post-steps build-linux-clang10-clang-analyze: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list - - run: echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable - - run: sudo apt-get update -y && sudo apt-get install -y clang-10 libgflags-dev clang-tools-10 - - run: SKIP_FORMAT_BUCK_CHECKS=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. + - pre-steps + - install-gflags + - install-clang-10 + - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10 + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. + - post-steps build-linux-cmake: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - checkout # check out the code in the project directory - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain + - post-steps build-linux-unity: docker: # executor type @@ -178,16 +206,17 @@ jobs: - checkout # check out the code in the project directory - run: apt-get update -y && apt-get install -y libgflags-dev - run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain + - post-steps build-linux-gcc-4-8: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: large steps: - - checkout - - run: pyenv global 3.5.2 + - pre-steps - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI + - post-steps build-windows: executor: windows-2xlarge @@ -252,38 +281,38 @@ jobs: build-linux-java: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge steps: - - checkout - - run: pyenv global 3.5.2 - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - pre-steps + - install-gflags - run: name: "Build RocksDBJava" command: | - export JAVA_HOME=/usr/lib/jvm/jdk1.8.0 + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 export PATH=$JAVA_HOME/bin:$PATH echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 make V=1 J=32 -j32 rocksdbjava jtest | .circleci/cat_ignore_eagain + make V=1 J=32 -j32 rocksdbjava jtest | .circleci/cat_ignore_eagain + - post-steps build-examples: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: medium steps: - - checkout # check out the code in the project directory - - run: pyenv global 3.5.2 - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - pre-steps + - install-gflags - run: name: "Build examples" command: | OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain + - post-steps build-linux-non-shm: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-1604:202007-01 resource_class: 2xlarge parameters: start_test: @@ -293,13 +322,23 @@ jobs: default: "" type: string steps: - - checkout # checkout the code in the project directory - - run: pyenv global 3.5.2 - - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + - pre-steps + - install-gflags + - install-gtest-parallel - run: - name: "Build and unit tests on non-shm" + name: "Build unit tests" command: | - TMPD=/tmp/rocksdb_test_tmp ROCKSDBTESTS_START=<> ROCKSDBTESTS_END=<> make V=1 J=32 -j32 check_some | .circleci/cat_ignore_eagain + echo "env: $(env)" + echo "** done env" + ROCKSDBTESTS_START=<> ROCKSDBTESTS_END=<> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests + - run: + name: "Run unit tests in parallel" + command: | + sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list + cat /tmp/test_list + export TEST_TMPDIR=/tmp/rocksdb_test_tmp + /usr/bin/python ../gtest-parallel/gtest-parallel $( "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}"; else echo "$(ROCKSDBTESTS_SUBSET)"; fi + # if the make goal is either "clean" or "format", we shouldn't # try to import the *.d files. # TODO(kailiu) The unfamiliarity of Make's conditions leads to the ugly