3e745053b7
Summary: This PR addresses some build and functional issues on MSVC targets, as a step towards an eventual goal of having RocksDB build successfully for Windows on ARM64. Addressed issues include: - BitsSetToOne and CountTrailingZeroBits do not compile on non-x64 MSVC targets. A fallback implementation of BitsSetToOne when Intel intrinsics are not available is added, based on the C++20 `<bit>` popcount implementation in Microsoft's STL. - The implementation of FloorLog2 for MSVC targets (including x64) gives incorrect results. The unit test easily detects this, but CircleCI is currently configured to only run a specific set of tests for Windows CMake builds, so this seems to have been unnoticed. - AsmVolatilePause does not use YieldProcessor on Windows ARM64 targets, even though it is available. - When CondVar::TimedWait calls Microsoft STL's condition_variable::wait_for, it can potentially trigger a bug (just recently fixed in the upcoming VS 16.8's STL) that deadlocks various tests that wait for a timer to execute, since `Timer::Run` doesn't get a chance to execute before being blocked by the test function acquiring the mutex. - In c_test, `GetTempDir` assumes a POSIX-style temp path. - `NormalizePath` did not eliminate consecutive POSIX-style path separators on Windows, resulting in test failures in e.g., wal_manager_test. - Various other test failures. In a followup PR I hope to modify CircleCI's config.yml to invoke all RocksDB unit tests in Windows CMake builds with CTest, instead of the current use of `run_ci_db_test.ps1` which requires individual tests to be specified and is missing many of the existing tests. Notes from peterd: FloorLog2 is not yet used in production code (it's for something in progress). I also added a few more inexpensive platform-dependent tests to Windows CircleCI runs. And included facebook/folly#1461 as requested Pull Request resolved: https://github.com/facebook/rocksdb/pull/7439 Reviewed By: jay-zhuang Differential Revision: D24021563 Pulled By: pdillinger fbshipit-source-id: 0ec2027c0d6a494d8a0fe38d9667fc2f7e29f7e7
422 lines
14 KiB
YAML
422 lines
14 KiB
YAML
version: 2.1
|
|
|
|
orbs:
|
|
win: circleci/windows@2.4.0
|
|
slack: circleci/slack@3.4.2
|
|
|
|
aliases:
|
|
- ¬ify-on-master-failure
|
|
fail_only: true
|
|
only_for_branches: master
|
|
|
|
commands:
|
|
pre-steps:
|
|
steps:
|
|
- checkout
|
|
- run: pyenv global 3.5.9
|
|
- run:
|
|
name: Setup Environment Variables
|
|
command: |
|
|
echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
|
|
echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
|
|
echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
|
|
echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
|
|
|
|
post-steps:
|
|
steps:
|
|
- slack/status: *notify-on-master-failure
|
|
- store_test_results: # store test result if there's any
|
|
path: /tmp/test-results
|
|
- store_artifacts: # store LOG for debugging if there's any
|
|
path: LOG
|
|
|
|
install-clang-10:
|
|
steps:
|
|
- run:
|
|
name: Install Clang 10
|
|
command: |
|
|
echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
|
|
echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
|
|
echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
|
|
sudo apt-get update -y && sudo apt-get install -y clang-10
|
|
|
|
install-gflags:
|
|
steps:
|
|
- run:
|
|
name: Install gflags
|
|
command: |
|
|
sudo apt-get update -y && sudo apt-get install -y libgflags-dev
|
|
|
|
install-gtest-parallel:
|
|
steps:
|
|
- run:
|
|
name: Install gtest-parallel
|
|
command: |
|
|
git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
|
|
echo "export PATH=$HOME/gtest-parallel:$PATH" >> $BASH_ENV
|
|
|
|
executors:
|
|
windows-2xlarge:
|
|
machine:
|
|
image: 'windows-server-2019-vs2019:stable'
|
|
resource_class: windows.2xlarge
|
|
shell: bash.exe
|
|
|
|
jobs:
|
|
build-linux:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-linux-shared_lib-alt_namespace-status_checked:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-linux-release:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: make V=1 -j32 release | .circleci/cat_ignore_eagain
|
|
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
|
|
- install-gflags
|
|
- run: make V=1 -j32 release | .circleci/cat_ignore_eagain
|
|
- run: ./db_stress --version # ensure with gflags
|
|
- post-steps
|
|
|
|
build-linux-release-rtti:
|
|
machine:
|
|
image: ubuntu-1604:201903-01
|
|
resource_class: large
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: make clean
|
|
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
|
|
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
|
|
- run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
|
|
- run: make clean
|
|
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
|
|
- run: ./db_stress --version # ensure with gflags
|
|
|
|
build-linux-lite:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-linux-lite-release:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: large
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
|
|
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
|
|
- install-gflags
|
|
- run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
|
|
- run: ./db_stress --version # ensure with gflags
|
|
- post-steps
|
|
|
|
build-linux-clang-no-test:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
|
|
- run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-linux-clang10-asan:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- install-clang-10
|
|
- run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
|
|
- post-steps
|
|
|
|
build-linux-clang10-mini-tsan:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- install-clang-10
|
|
- run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
|
|
- post-steps
|
|
|
|
build-linux-clang10-ubsan:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- install-clang-10
|
|
- run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
|
|
- post-steps
|
|
|
|
build-linux-clang10-clang-analyze:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- install-clang-10
|
|
- run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
|
|
- run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
|
|
- post-steps
|
|
|
|
build-linux-cmake:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-linux-unity:
|
|
docker: # executor type
|
|
- image: gcc:latest
|
|
resource_class: xlarge
|
|
steps:
|
|
- checkout # check out the code in the project directory
|
|
- run: apt-get update -y && apt-get install -y libgflags-dev
|
|
- run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-linux-gcc-4-8:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: large
|
|
steps:
|
|
- pre-steps
|
|
- run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
|
|
- run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
|
|
- post-steps
|
|
|
|
build-windows:
|
|
executor: windows-2xlarge
|
|
parameters:
|
|
extra_cmake_opt:
|
|
default: ""
|
|
type: string
|
|
vs_year:
|
|
default: "2019"
|
|
type: string
|
|
cmake_generator:
|
|
default: "Visual Studio 16 2019"
|
|
type: string
|
|
environment:
|
|
THIRDPARTY_HOME: C:/Users/circleci/thirdparty
|
|
CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
|
|
CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
|
|
SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
|
|
SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
|
|
SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
|
|
VS_YEAR: <<parameters.vs_year>>
|
|
CMAKE_GENERATOR: <<parameters.cmake_generator>>
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: "Setup VS"
|
|
command: |
|
|
if [[ "${VS_YEAR}" == "2017" ]]; then
|
|
powershell .circleci/vs2017_install.ps1
|
|
elif [[ "${VS_YEAR}" == "2015" ]]; then
|
|
powershell .circleci/vs2015_install.ps1
|
|
fi
|
|
- run:
|
|
name: "Install thirdparty dependencies"
|
|
command: |
|
|
mkdir ${THIRDPARTY_HOME}
|
|
cd ${THIRDPARTY_HOME}
|
|
echo "Installing CMake..."
|
|
curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
|
|
unzip -q cmake-3.16.4-win64-x64.zip
|
|
echo "Building Snappy dependency..."
|
|
curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
|
|
unzip -q snappy-1.1.7.zip
|
|
cd snappy-1.1.7
|
|
mkdir build
|
|
cd build
|
|
${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
|
|
msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
|
|
- run:
|
|
name: "Build RocksDB"
|
|
command: |
|
|
mkdir build
|
|
cd build
|
|
${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
|
|
cd ..
|
|
msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
|
|
- run:
|
|
name: "Test RocksDB"
|
|
shell: powershell.exe
|
|
command: |
|
|
build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
|
|
|
|
build-linux-java:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run:
|
|
name: "Build RocksDBJava"
|
|
command: |
|
|
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
echo "JAVA_HOME=${JAVA_HOME}"
|
|
which java && java -version
|
|
which javac && javac -version
|
|
make V=1 J=32 -j32 rocksdbjava jtest | .circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-examples:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: medium
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- run:
|
|
name: "Build examples"
|
|
command: |
|
|
OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain
|
|
- post-steps
|
|
|
|
build-linux-non-shm:
|
|
machine:
|
|
image: ubuntu-1604:202007-01
|
|
resource_class: 2xlarge
|
|
parameters:
|
|
start_test:
|
|
default: ""
|
|
type: string
|
|
end_test:
|
|
default: ""
|
|
type: string
|
|
steps:
|
|
- pre-steps
|
|
- install-gflags
|
|
- install-gtest-parallel
|
|
- run:
|
|
name: "Build unit tests"
|
|
command: |
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
|
|
- run:
|
|
name: "Run unit tests in parallel"
|
|
command: |
|
|
sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
|
|
cat /tmp/test_list
|
|
export TEST_TMPDIR=/tmp/rocksdb_test_tmp
|
|
/usr/bin/python ../gtest-parallel/gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
|
|
- post-steps
|
|
|
|
workflows:
|
|
build-linux:
|
|
jobs:
|
|
- build-linux
|
|
build-linux-shared_lib-alt_namespace-status_checked:
|
|
jobs:
|
|
- build-linux-shared_lib-alt_namespace-status_checked
|
|
build-linux-lite:
|
|
jobs:
|
|
- build-linux-lite
|
|
build-linux-release:
|
|
jobs:
|
|
- build-linux-release
|
|
build-linux-release-rtti:
|
|
jobs:
|
|
- build-linux-release-rtti
|
|
build-linux-lite-release:
|
|
jobs:
|
|
- build-linux-lite-release
|
|
build-linux-clang-no-test:
|
|
jobs:
|
|
- build-linux-clang-no-test
|
|
build-linux-clang10-asan:
|
|
jobs:
|
|
- build-linux-clang10-asan
|
|
build-linux-clang10-mini-tsan:
|
|
jobs:
|
|
- build-linux-clang10-mini-tsan
|
|
build-linux-clang10-ubsan:
|
|
jobs:
|
|
- build-linux-clang10-ubsan
|
|
build-linux-clang10-clang-analyze:
|
|
jobs:
|
|
- build-linux-clang10-clang-analyze
|
|
build-linux-cmake:
|
|
jobs:
|
|
- build-linux-cmake
|
|
build-linux-unity:
|
|
jobs:
|
|
- build-linux-unity
|
|
build-windows:
|
|
jobs:
|
|
- build-windows
|
|
build-windows-vs2017:
|
|
jobs:
|
|
- build-windows:
|
|
vs_year: "2017"
|
|
cmake_generator: "Visual Studio 15 Win64"
|
|
build-windows-vs2015:
|
|
jobs:
|
|
- build-windows:
|
|
vs_year: "2015"
|
|
cmake_generator: "Visual Studio 14 Win64"
|
|
build-windows-cxx20:
|
|
jobs:
|
|
- build-windows:
|
|
extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
|
|
build-java:
|
|
jobs:
|
|
- build-linux-java
|
|
build-examples:
|
|
jobs:
|
|
- build-examples
|
|
build-linux-non-shm:
|
|
jobs:
|
|
- build-linux-non-shm:
|
|
start_test: ""
|
|
end_test: "db_options_test" # make sure unique in src.mk
|
|
- build-linux-non-shm:
|
|
start_test: "db_options_test" # make sure unique in src.mk
|
|
end_test: "filename_test" # make sure unique in src.mk
|
|
- build-linux-non-shm:
|
|
start_test: "filename_test" # make sure unique in src.mk
|
|
end_test: "statistics_test" # make sure unique in src.mk
|
|
- build-linux-non-shm:
|
|
start_test: "statistics_test" # make sure unique in src.mk
|
|
end_test: ""
|
|
build-linux-gcc-4-8:
|
|
jobs:
|
|
- build-linux-gcc-4-8
|