2020-03-14 05:51:26 +01:00
version : 2.1
orbs :
win : circleci/windows@2.4.0
2020-09-23 23:40:53 +02:00
slack : circleci/slack@3.4.2
aliases :
- ¬ify-on-master-failure
fail_only : true
only_for_branches : master
commands :
2020-10-12 19:45:21 +02:00
install-pyenv-on-macos :
steps :
- run :
name : Install pyenv on macos
command : |
HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv
2020-12-05 08:22:18 +01:00
install-cmake-on-macos :
steps :
- run :
name : Install cmake on macos
command : |
HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
2020-10-12 19:45:21 +02:00
increase-max-open-files-on-macos :
steps :
- run :
name : Increase max open files
command : |
sudo sysctl -w kern.maxfiles=1048576
sudo sysctl -w kern.maxfilesperproc=1048576
sudo launchctl limit maxfiles 1048576
2020-09-23 23:40:53 +02:00
pre-steps :
2020-12-17 00:58:56 +01:00
parameters :
python-version :
default : "3.5.9"
type : string
2020-09-23 23:40:53 +02:00
steps :
- checkout
2020-12-17 00:58:56 +01:00
- run : pyenv install --skip-existing <<parameters.python-version>>
- run : pyenv global <<parameters.python-version>>
2020-09-23 23:40:53 +02:00
- run :
name : Setup Environment Variables
command : |
echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
2020-12-17 00:58:56 +01:00
pre-steps-macos :
steps :
- pre-steps :
python-version : "3.6.0"
2020-09-23 23:40:53 +02:00
post-steps :
steps :
- slack/status : *notify-on-master-failure
- store_test_results : # store test result if there's any
path : /tmp/test-results
- store_artifacts : # store LOG for debugging if there's any
path : LOG
install-clang-10 :
steps :
- run :
name : Install Clang 10
command : |
echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
sudo apt-get update -y && sudo apt-get install -y clang-10
install-gflags :
steps :
- run :
name : Install gflags
command : |
sudo apt-get update -y && sudo apt-get install -y libgflags-dev
2020-12-05 08:22:18 +01:00
2020-10-12 19:45:21 +02:00
install-gflags-on-macos :
steps :
- run :
name : Install gflags on macos
command : |
HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
2020-09-23 23:40:53 +02:00
install-gtest-parallel :
steps :
- run :
name : Install gtest-parallel
command : |
2020-10-01 01:06:44 +02:00
git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
2020-12-11 05:36:44 +01:00
echo 'export PATH=$HOME/gtest-parallel:$PATH' >> $BASH_ENV
2020-03-14 05:51:26 +01:00
executors :
windows-2xlarge :
machine :
2020-09-15 01:59:00 +02:00
image : 'windows-server-2019-vs2019:stable'
2020-03-14 05:51:26 +01:00
resource_class : windows.2xlarge
shell : bash.exe
jobs :
2020-10-12 19:45:21 +02:00
build-macos :
macos :
2020-12-17 00:58:56 +01:00
xcode : 9.4 .1
2020-10-12 19:45:21 +02:00
steps :
- increase-max-open-files-on-macos
- install-pyenv-on-macos
- install-gflags-on-macos
2020-12-17 00:58:56 +01:00
- pre-steps-macos
2020-10-12 19:45:21 +02:00
- run : ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
2020-12-05 08:22:18 +01:00
build-macos-cmake :
macos :
2020-12-17 00:58:56 +01:00
xcode : 9.4 .1
2020-12-05 08:22:18 +01:00
steps :
- increase-max-open-files-on-macos
- install-pyenv-on-macos
- install-cmake-on-macos
- install-gflags-on-macos
2020-12-17 00:58:56 +01:00
- pre-steps-macos
2020-12-05 08:22:18 +01:00
- run : ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
- post-steps
2020-06-09 04:32:18 +02:00
build-linux :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-06-09 04:32:18 +02:00
resource_class : 2xlarge
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
- run : make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
build-linux-mem-env :
machine :
image : ubuntu-1604:202007-01
resource_class : 2xlarge
steps :
- pre-steps
- install-gflags
- run : MEM_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
build-linux-encrypted-env :
machine :
image : ubuntu-1604:202007-01
resource_class : 2xlarge
steps :
- pre-steps
- install-gflags
- run : ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
build-linux-shared_lib-alt_namespace-status_checked :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
resource_class : 2xlarge
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
2020-11-11 08:41:05 +01:00
- run : ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain
2020-09-23 23:40:53 +02:00
- post-steps
2020-06-09 04:32:18 +02:00
build-linux-release :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-06-09 04:32:18 +02:00
resource_class : 2xlarge
steps :
- checkout # check out the code in the project directory
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
- run : make V=1 -j32 release | .circleci/cat_ignore_eagain
- run : if ./db_stress --version; then false; else true; fi # ensure without gflags
2020-09-23 23:40:53 +02:00
- install-gflags
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
- run : make V=1 -j32 release | .circleci/cat_ignore_eagain
- run : ./db_stress --version # ensure with gflags
2020-09-23 23:40:53 +02:00
- post-steps
2020-06-09 04:32:18 +02:00
2020-09-22 19:26:34 +02:00
build-linux-release-rtti :
machine :
image : ubuntu-1604:201903-01
resource_class : large
steps :
- checkout # check out the code in the project directory
- run : make clean
- run : USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
- run : if ./db_stress --version; then false; else true; fi # ensure without gflags
- run : sudo apt-get update -y && sudo apt-get install -y libgflags-dev
- run : make clean
- run : USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
- run : ./db_stress --version # ensure with gflags
2020-06-09 04:32:18 +02:00
build-linux-lite :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-06-09 04:32:18 +02:00
resource_class : 2xlarge
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
- run : LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
2020-06-09 04:32:18 +02:00
build-linux-lite-release :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-06-09 04:32:18 +02:00
resource_class : large
steps :
- checkout # check out the code in the project directory
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
- run : LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
- run : if ./db_stress --version; then false; else true; fi # ensure without gflags
2020-09-23 23:40:53 +02:00
- install-gflags
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
- run : LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
- run : ./db_stress --version # ensure with gflags
2020-09-23 23:40:53 +02:00
- post-steps
2020-06-09 04:32:18 +02:00
build-linux-clang-no-test :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-06-09 04:32:18 +02:00
resource_class : 2xlarge
steps :
- checkout # check out the code in the project directory
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
- run : sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
- run : CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain
2020-09-23 23:40:53 +02:00
- post-steps
2020-06-25 01:20:55 +02:00
2020-06-26 02:40:08 +02:00
build-linux-clang10-asan :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-06-26 02:40:08 +02:00
resource_class : 2xlarge
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
- install-clang-10
- run : COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
- post-steps
2020-06-26 02:40:08 +02:00
2020-07-15 00:31:57 +02:00
build-linux-clang10-mini-tsan :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-07-15 00:31:57 +02:00
resource_class : 2xlarge
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
- install-clang-10
Fix TSAN build and re-enable the tests (#7386)
Summary:
Resolve TSAN build warnings and re-enable disabled TSAN tests.
Not sure if it's a compiler issue or TSAN check issue. Switching from
conditional operator to if-else mitigated the problem.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7386
Test Plan:
run TSAN check 10 times in circleci.
```
WARNING: ThreadSanitizer: data race (pid=27735)
Atomic write of size 8 at 0x7b54000005e8 by thread T32:
#0 __tsan_atomic64_store <null> (db_test+0x4cee95)
https://github.com/facebook/rocksdb/issues/1 std::__atomic_base<unsigned long>::store(unsigned long, std::memory_order) /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/atomic_base.h:374:2 (db_test+0x78460e)
https://github.com/facebook/rocksdb/issues/2 rocksdb::VersionSet::SetLastSequence(unsigned long) /home/circleci/project/./db/version_set.h:1058:20 (db_test+0x78460e)
...
Previous read of size 8 at 0x7b54000005e8 by thread T31:
#0 bool rocksdb::DBImpl::MultiCFSnapshot<std::unordered_map<unsigned int, rocksdb::DBImpl::MultiGetColumnFamilyData, std::hash<unsigned int>, std::equal_to<unsigned int>, std::allocator<std::pair<unsigned int const, rocksdb::DBImpl::MultiGetColumnFamilyData> > > >(rocksdb::ReadOptions const&, rocksdb::ReadCallback*, std::function<rocksdb::DBImpl::MultiGetColumnFamilyData* (std::unordered_map<unsigned int, rocksdb::DBImpl::MultiGetColumnFamilyData, std::hash<unsigned int>, std::equal_to<unsigned int>, std::allocator<std::pair<unsigned int const, rocksdb::DBImpl::MultiGetColumnFamilyData> > >::iterator&)>&, std::unordered_map<unsigned int, rocksdb::DBImpl::MultiGetColumnFamilyData, std::hash<unsigned int>, std::equal_to<unsigned int>, std::allocator<std::pair<unsigned int const, rocksdb::DBImpl::MultiGetColumnFamilyData> > >*, unsigned long*) /home/circleci/project/db/db_impl/db_impl.cc (db_test+0x715087)
```
Reviewed By: ltamasi
Differential Revision: D23725226
Pulled By: jay-zhuang
fbshipit-source-id: a6d662a5ea68111246cd32ec95f3411a25f76bc6
2020-09-25 23:44:58 +02:00
- run : COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
2020-09-23 23:40:53 +02:00
- post-steps
2020-07-15 00:31:57 +02:00
2020-07-11 05:02:16 +02:00
build-linux-clang10-ubsan :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-07-11 05:02:16 +02:00
resource_class : 2xlarge
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
- install-clang-10
- run : COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
- post-steps
2020-07-11 05:02:16 +02:00
2020-07-13 21:31:41 +02:00
build-linux-clang10-clang-analyze :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-07-13 21:31:41 +02:00
resource_class : 2xlarge
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
- install-clang-10
- run : sudo apt-get update -y && sudo apt-get install -y clang-tools-10
- run : CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
- post-steps
2020-07-13 21:31:41 +02:00
2020-06-09 04:32:18 +02:00
build-linux-cmake :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-06-09 04:32:18 +02:00
resource_class : 2xlarge
steps :
- checkout # check out the code in the project directory
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
- run : (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
2020-09-23 23:40:53 +02:00
- post-steps
2020-06-09 04:32:18 +02:00
2020-06-26 20:12:06 +02:00
build-linux-unity :
docker : # executor type
- image : gcc:latest
resource_class : xlarge
steps :
- checkout # check out the code in the project directory
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
- run : apt-get update -y && apt-get install -y libgflags-dev
- run : TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain
2020-09-23 23:40:53 +02:00
- post-steps
2020-06-26 20:12:06 +02:00
2020-09-08 21:08:05 +02:00
build-linux-gcc-4-8 :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-09-08 21:08:05 +02:00
resource_class : large
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
2020-09-08 21:08:05 +02:00
- run : sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
- run : CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
2020-09-23 23:40:53 +02:00
- post-steps
2020-09-08 21:08:05 +02:00
2020-06-09 04:32:18 +02:00
build-windows :
2020-03-14 05:51:26 +01:00
executor : windows-2xlarge
2020-06-29 23:29:15 +02:00
parameters :
extra_cmake_opt :
default : ""
type : string
2020-07-14 03:08:08 +02:00
vs_year :
default : "2019"
type : string
cmake_generator :
default : "Visual Studio 16 2019"
type : string
2020-03-14 05:51:26 +01:00
environment :
THIRDPARTY_HOME : C:/Users/circleci/thirdparty
CMAKE_HOME : C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
CMAKE_BIN : C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
SNAPPY_HOME : C:/Users/circleci/thirdparty/snappy-1.1.7
SNAPPY_INCLUDE : C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
SNAPPY_LIB_DEBUG : C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
2020-07-14 03:08:08 +02:00
VS_YEAR : <<parameters.vs_year>>
CMAKE_GENERATOR : <<parameters.cmake_generator>>
2020-03-14 05:51:26 +01:00
steps :
- checkout
2020-07-14 03:08:08 +02:00
- run :
name : "Setup VS"
command : |
if [[ "${VS_YEAR}" == "2017" ]]; then
powershell .circleci/vs2017_install.ps1
elif [[ "${VS_YEAR}" == "2015" ]]; then
powershell .circleci/vs2015_install.ps1
fi
2020-03-14 05:51:26 +01:00
- run :
name : "Install thirdparty dependencies"
command : |
mkdir ${THIRDPARTY_HOME}
cd ${THIRDPARTY_HOME}
echo "Installing CMake..."
curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
unzip -q cmake-3.16.4-win64-x64.zip
echo "Building Snappy dependency..."
curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
unzip -q snappy-1.1.7.zip
cd snappy-1.1.7
mkdir build
cd build
${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
- run :
name : "Build RocksDB"
command : |
mkdir build
cd build
2020-06-29 23:29:15 +02:00
${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
2020-03-14 05:51:26 +01:00
cd ..
msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
- run :
name : "Test RocksDB"
shell : powershell.exe
command : |
Fix MSVC-related build issues (#7439)
Summary:
This PR addresses some build and functional issues on MSVC targets, as a step towards an eventual goal of having RocksDB build successfully for Windows on ARM64.
Addressed issues include:
- BitsSetToOne and CountTrailingZeroBits do not compile on non-x64 MSVC targets. A fallback implementation of BitsSetToOne when Intel intrinsics are not available is added, based on the C++20 `<bit>` popcount implementation in Microsoft's STL.
- The implementation of FloorLog2 for MSVC targets (including x64) gives incorrect results. The unit test easily detects this, but CircleCI is currently configured to only run a specific set of tests for Windows CMake builds, so this seems to have been unnoticed.
- AsmVolatilePause does not use YieldProcessor on Windows ARM64 targets, even though it is available.
- When CondVar::TimedWait calls Microsoft STL's condition_variable::wait_for, it can potentially trigger a bug (just recently fixed in the upcoming VS 16.8's STL) that deadlocks various tests that wait for a timer to execute, since `Timer::Run` doesn't get a chance to execute before being blocked by the test function acquiring the mutex.
- In c_test, `GetTempDir` assumes a POSIX-style temp path.
- `NormalizePath` did not eliminate consecutive POSIX-style path separators on Windows, resulting in test failures in e.g., wal_manager_test.
- Various other test failures.
In a followup PR I hope to modify CircleCI's config.yml to invoke all RocksDB unit tests in Windows CMake builds with CTest, instead of the current use of `run_ci_db_test.ps1` which requires individual tests to be specified and is missing many of the existing tests.
Notes from peterd: FloorLog2 is not yet used in production code (it's for something in progress). I also added a few more inexpensive platform-dependent tests to Windows CircleCI runs. And included facebook/folly#1461 as requested
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7439
Reviewed By: jay-zhuang
Differential Revision: D24021563
Pulled By: pdillinger
fbshipit-source-id: 0ec2027c0d6a494d8a0fe38d9667fc2f7e29f7e7
2020-10-01 18:21:30 +02:00
build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
2020-06-09 04:32:18 +02:00
2020-07-14 00:04:27 +02:00
build-linux-java :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-12-17 00:58:56 +01:00
resource_class : large
environment :
JAVA_HOME : /usr/lib/jvm/java-1.8.0-openjdk-amd64
2020-07-14 00:04:27 +02:00
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
2020-07-14 00:04:27 +02:00
- run :
2020-12-17 00:58:56 +01:00
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Build RocksDBJava Shared Library"
command : make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
- run :
name : "Test RocksDBJava"
command : make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
- post-steps
build-linux-java-static :
machine :
image : ubuntu-1604:202007-01
resource_class : large
environment :
JAVA_HOME : /usr/lib/jvm/java-1.8.0-openjdk-amd64
steps :
- pre-steps
- install-gflags
- run :
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Build RocksDBJava Static Library"
command : make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
- post-steps
build-macos-java :
macos :
xcode : 9.4 .1
resource_class : medium
environment :
JAVA_HOME : /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
steps :
- increase-max-open-files-on-macos
- install-pyenv-on-macos
- install-gflags-on-macos
- pre-steps-macos
- run :
name : "Set Java Environment"
command : |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run :
name : "Build RocksDBJava Shared Library"
command : make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
- run :
name : "Test RocksDBJava"
command : make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
- post-steps
build-macos-java-static :
macos :
xcode : 9.4 .1
resource_class : medium
environment :
JAVA_HOME : /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
steps :
- increase-max-open-files-on-macos
- install-pyenv-on-macos
- install-gflags-on-macos
- install-cmake-on-macos
- pre-steps-macos
- run :
name : "Set Java Environment"
2020-07-14 00:04:27 +02:00
command : |
echo "JAVA_HOME=${JAVA_HOME}"
2020-12-17 00:58:56 +01:00
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
2020-07-14 00:04:27 +02:00
which java && java -version
which javac && javac -version
2020-12-17 00:58:56 +01:00
- run :
name : "Build RocksDBJava Static Library"
command : make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
2020-09-23 23:40:53 +02:00
- post-steps
2020-07-14 00:04:27 +02:00
2020-07-16 02:46:18 +02:00
build-examples :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-11-18 03:18:13 +01:00
resource_class : large
2020-07-16 02:46:18 +02:00
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
2020-07-16 02:46:18 +02:00
- run :
name : "Build examples"
command : |
OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain
2020-09-23 23:40:53 +02:00
- post-steps
2020-07-16 02:46:18 +02:00
2020-11-18 03:18:13 +01:00
build-cmake-mingw :
machine :
image : ubuntu-1604:202007-01
resource_class : 2xlarge
steps :
- pre-steps
- install-gflags
- run : sudo apt-get update -y && sudo apt-get install -y mingw-w64
- run : sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
- run :
name : "Build cmake-mingw"
command : |
sudo apt-get install snapd && sudo snap install cmake --beta --classic
export PATH=/snap/bin:$PATH
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export PATH=$JAVA_HOME/bin:$PATH
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
- post-steps
2020-08-12 03:29:15 +02:00
build-linux-non-shm :
machine :
2020-09-23 23:40:53 +02:00
image : ubuntu-1604:202007-01
2020-08-12 03:29:15 +02:00
resource_class : 2xlarge
parameters :
start_test :
default : ""
type : string
end_test :
default : ""
type : string
steps :
2020-09-23 23:40:53 +02:00
- pre-steps
- install-gflags
- install-gtest-parallel
2020-08-12 03:29:15 +02:00
- run :
2020-09-23 23:40:53 +02:00
name : "Build unit tests"
2020-08-12 03:29:15 +02:00
command : |
2020-09-23 23:40:53 +02:00
echo "env: $(env)"
echo "** done env"
ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
- run :
name : "Run unit tests in parallel"
command : |
sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
cat /tmp/test_list
export TEST_TMPDIR=/tmp/rocksdb_test_tmp
2020-12-11 05:36:44 +01:00
gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
2020-09-23 23:40:53 +02:00
- post-steps
2020-08-12 03:29:15 +02:00
2020-06-09 04:32:18 +02:00
workflows :
build-linux :
jobs :
- build-linux
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
build-linux-mem-env :
jobs :
- build-linux-mem-env
build-linux-encrypted-env :
jobs :
- build-linux-encrypted-env
Major CircleCI/Linux fixes / tweaks / enhancements (#7078)
Summary:
Primarily, this change adds a way to work around a bug limiting the effective output (and therefore debugability) of the Linux builds using parallel make. We would get
make[1]: write error: stdout
probably due to a kernel bug, apparently affecting both available ubuntu 16 machine images (maybe not affecting docker images, less horsepower). https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
Now in the CircleCI config, make output on Ubuntu is piped through a custom 'cat' that ignores EAGAIN errors, which seems to fix the problem.
Significant other changes:
* Add another linux build that combines
* LIB_MODE=shared, to ensure this works with compile and unit test execution
* Alternative rocksdb namespace, to ensure this works (not rely on Travis)
* ASSERT_STATUS_CHECKED=1, but with building all unit tests and running those expected to pass with it
* Run release build with and without gflags. (Was running only without, ignore large swaths of code in a normal release build! Two regressions in this build, only with gflags, in the last week not caught by CI!)
* Use gflags with unity and LITE build, as typical case.
Debugability improvements:
* Use V=1 to show commands being executed (thanks to EAGAIN work-around)
* Print kernel version and compiler versions as part of V=1 output from Makefile
Cosmetic other changes:
* Put more commands on one line, for less clutter in CircleCI output pages
* Remove redundant "all" in "make all check" and put make command options before targets
* Change some recursive "make clean" into dependency on "clean," toward minimizing unnecessary overhead (detect platform, build version, etc.) of extra recursive makes
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7078
Reviewed By: siying
Differential Revision: D22391647
Pulled By: pdillinger
fbshipit-source-id: d446fccf5a8c568b37dc8748621c8a5c546fe135
2020-07-07 20:24:00 +02:00
build-linux-shared_lib-alt_namespace-status_checked :
jobs :
- build-linux-shared_lib-alt_namespace-status_checked
2020-06-09 04:32:18 +02:00
build-linux-lite :
jobs :
- build-linux-lite
build-linux-release :
jobs :
- build-linux-release
2020-09-22 19:26:34 +02:00
build-linux-release-rtti :
jobs :
- build-linux-release-rtti
2020-06-09 04:32:18 +02:00
build-linux-lite-release :
jobs :
- build-linux-lite-release
build-linux-clang-no-test :
jobs :
- build-linux-clang-no-test
2020-06-26 02:40:08 +02:00
build-linux-clang10-asan :
jobs :
- build-linux-clang10-asan
2020-07-15 00:31:57 +02:00
build-linux-clang10-mini-tsan :
jobs :
- build-linux-clang10-mini-tsan
2020-07-11 05:02:16 +02:00
build-linux-clang10-ubsan :
jobs :
- build-linux-clang10-ubsan
2020-07-13 21:31:41 +02:00
build-linux-clang10-clang-analyze :
jobs :
- build-linux-clang10-clang-analyze
2020-06-09 04:32:18 +02:00
build-linux-cmake :
jobs :
- build-linux-cmake
2020-06-26 20:12:06 +02:00
build-linux-unity :
jobs :
- build-linux-unity
2020-06-09 04:32:18 +02:00
build-windows :
jobs :
- build-windows
2020-07-14 03:08:08 +02:00
build-windows-vs2017 :
jobs :
- build-windows :
vs_year : "2017"
cmake_generator : "Visual Studio 15 Win64"
build-windows-vs2015 :
jobs :
- build-windows :
vs_year : "2015"
cmake_generator : "Visual Studio 14 Win64"
2020-06-29 23:29:15 +02:00
build-windows-cxx20 :
jobs :
- build-windows :
extra_cmake_opt : -DCMAKE_CXX_STANDARD=20
2020-07-14 00:04:27 +02:00
build-java :
jobs :
- build-linux-java
2020-12-17 00:58:56 +01:00
- build-linux-java-static
- build-macos-java
- build-macos-java-static
2020-07-16 02:46:18 +02:00
build-examples :
jobs :
- build-examples
2020-09-23 23:40:53 +02:00
build-linux-non-shm :
2020-08-12 03:29:15 +02:00
jobs :
- build-linux-non-shm :
start_test : ""
2020-09-23 23:40:53 +02:00
end_test : "db_options_test" # make sure unique in src.mk
2020-08-12 03:29:15 +02:00
- build-linux-non-shm :
2020-09-23 23:40:53 +02:00
start_test : "db_options_test" # make sure unique in src.mk
end_test : "filename_test" # make sure unique in src.mk
2020-08-12 03:29:15 +02:00
- build-linux-non-shm :
2020-09-23 23:40:53 +02:00
start_test : "filename_test" # make sure unique in src.mk
end_test : "statistics_test" # make sure unique in src.mk
2020-08-12 03:29:15 +02:00
- build-linux-non-shm :
2020-09-23 23:40:53 +02:00
start_test : "statistics_test" # make sure unique in src.mk
2020-08-12 03:29:15 +02:00
end_test : ""
2020-09-08 21:08:05 +02:00
build-linux-gcc-4-8 :
jobs :
- build-linux-gcc-4-8
2020-10-12 19:45:21 +02:00
build-macos :
jobs :
- build-macos
2020-12-05 08:22:18 +01:00
build-macos-cmake :
jobs :
- build-macos-cmake
2020-11-18 03:18:13 +01:00
build-cmake-mingw :
jobs :
- build-cmake-mingw
