andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate away from Travis+Linux+amd64 (#7791)

Browse Source 
Summary:
This disables Linux/amd64 builds in Travis for PRs, and adds a
gcc-10+c++20 build in CircleCI, which should fill out sufficient coverage
vs. what we had in Travis

Fixed a use of std::is_pod, which is deprecated in c++20

Fixed ++ on a volatile in db_repl_stress.cc, with bigger refactoring.
Although ++ on this volatile was probably ok with one thread writer and
one thread reader, the code was still overly complex. There was a
deadcode check for error
`if (replThread.no_read < dataPump.no_records)` which can be proven
never to happen based on the structure of the code. It infinite loops
instead for the case intended to be checked. I just simplified the code
for what should be the same checking power.

Also most configurations seem to be using make parallelism = 2 * vcores,
so fixing / using that.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7791

Test Plan:
CI
and `while ./db_repl_stress; do echo again; done` for a while

Reviewed By: siying

Differential Revision: D25669834

Pulled By: pdillinger

fbshipit-source-id: b2c688053d0b1d52c989903449d3cd27a04130d6
This commit is contained in:
Peter Dillinger 2020-12-22 00:19:44 -08:00 committed by Facebook GitHub Bot
parent 861b0d1a49
commit 4d897e51df
6 changed files with 107 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
.circleci/config.yml
View File

@ -169,13 +169,13 @@ jobs:
build-linux-release:
machine:
image: ubuntu-1604:202007-01
resource_class: 2xlarge
resource_class: large
steps:
- checkout # check out the code in the project directory
- run: make V=1 -j32 release | .circleci/cat_ignore_eagain
- run: make V=1 -j8 release | .circleci/cat_ignore_eagain
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
- install-gflags
- run: make V=1 -j32 release | .circleci/cat_ignore_eagain
- run: make V=1 -j8 release | .circleci/cat_ignore_eagain
- run: ./db_stress --version # ensure with gflags
- post-steps
@ -186,11 +186,11 @@ jobs:
steps:
- checkout # check out the code in the project directory
- run: make clean
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
- run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
- run: make clean
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
- run: ./db_stress --version # ensure with gflags
build-linux-lite:
@ -209,21 +209,21 @@ jobs:
resource_class: large
steps:
- checkout # check out the code in the project directory
- run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
- run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
- install-gflags
- run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
- run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
- run: ./db_stress --version # ensure with gflags
- post-steps
build-linux-clang-no-test:
build-linux-clang-no_test_run:
machine:
image: ubuntu-1604:202007-01
resource_class: 2xlarge
resource_class: xlarge
steps:
- checkout # check out the code in the project directory
- run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
- run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain
- run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain
- post-steps
build-linux-clang10-asan:
@ -271,33 +271,63 @@ jobs:
- run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
- post-steps
build-linux-cmake:
build-linux-cmake-no_test_run:
machine:
image: ubuntu-1604:202007-01
resource_class: 2xlarge
resource_class: large
steps:
- checkout # check out the code in the project directory
- run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
- run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain
- post-steps
build-linux-unity:
docker: # executor type
- image: gcc:latest
resource_class: xlarge
resource_class: large
steps:
- checkout # check out the code in the project directory
- run: apt-get update -y && apt-get install -y libgflags-dev
- run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain
- run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain
- post-steps
build-linux-gcc-4-8:
build-linux-gcc-4_8-no_test_run:
machine:
image: ubuntu-1604:202007-01
resource_class: large
steps:
- pre-steps
- run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
- run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
- run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
- post-steps
build-linux-gcc-8-no_test_run:
machine:
image: ubuntu-2004:202010-01
resource_class: large
steps:
- pre-steps
- run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
- run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
- post-steps
build-linux-gcc-9-no_test_run:
machine:
image: ubuntu-2004:202010-01
resource_class: large
steps:
- pre-steps
- run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev
- run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
- post-steps
build-linux-gcc-10-cxx20-no_test_run:
machine:
image: ubuntu-2004:202010-01
resource_class: xlarge
steps:
- pre-steps
- run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
- run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
- post-steps
build-windows:
@ -547,9 +577,6 @@ workflows:
build-linux-lite-release:
jobs:
- build-linux-lite-release
build-linux-clang-no-test:
jobs:
- build-linux-clang-no-test
build-linux-clang10-asan:
jobs:
- build-linux-clang10-asan
@ -562,9 +589,6 @@ workflows:
build-linux-clang10-clang-analyze:
jobs:
- build-linux-clang10-clang-analyze
build-linux-cmake:
jobs:
- build-linux-cmake
build-linux-unity:
jobs:
- build-linux-unity
@ -608,9 +632,14 @@ workflows:
- build-linux-non-shm:
start_test: "statistics_test" # make sure unique in src.mk
end_test: ""
build-linux-gcc-4-8:
build-linux-compilers-no_test_run:
jobs:
- build-linux-gcc-4-8
- build-linux-clang-no_test_run
- build-linux-cmake-no_test_run
- build-linux-gcc-4_8-no_test_run
- build-linux-gcc-8-no_test_run
- build-linux-gcc-9-no_test_run
- build-linux-gcc-10-cxx20-no_test_run
build-macos:
jobs:
- build-macos

12
.travis.yml
View File

@ -86,19 +86,11 @@ matrix:
env: JOB_NAME=make-gcc4.8
- os: linux
compiler: clang
# Exclude all but most unique cmake variants for pull requests, but build all in branches
# With migration to CircleCI, exclude Linux/amd64 for pull requests
# (but build in branches for now)
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: amd64
env: JOB_NAME=cmake
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: amd64
env: JOB_NAME=cmake-gcc8
- if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux
arch: amd64
env: JOB_NAME=cmake-gcc9
# Exclude most osx, arm64 and ppc64le tests for pull requests, but build in branches
# Temporarily disable ppc64le cmake test while snapd is broken
- if: type = pull_request AND commit_message !~ /FULL_CI/

9
build_tools/build_detect_platform
View File

@ -45,8 +45,13 @@ if test -z "$OUTPUT"; then
exit 1
fi
# we depend on C++11
PLATFORM_CXXFLAGS="-std=c++11"
# we depend on C++11, but should be compatible with newer standards
if [ "$ROCKSDB_CXX_STANDARD" ]; then
PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD"
else
PLATFORM_CXXFLAGS="-std=c++11"
fi
# we currently depend on POSIX platform
COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"

2
table/block_based/filter_policy.cc
View File

@ -63,6 +63,8 @@ class XXH3pFilterBitsBuilder : public BuiltinFilterBitsBuilder {
}
protected:
static constexpr uint32_t kMetadataLen = 5;
// For delegating between XXH3pFilterBitsBuilders
void SwapEntriesWith(XXH3pFilterBitsBuilder* other) {
std::swap(hash_entries_, other->hash_entries_);

99
tools/db_repl_stress.cc
View File

@ -27,13 +27,20 @@ int main() {
// --num_inserts = the num of inserts the first thread should perform.
// --wal_ttl = the wal ttl for the run.
DEFINE_uint64(num_inserts, 1000,
"the num of inserts the first thread should"
" perform.");
DEFINE_uint64(wal_ttl_seconds, 1000, "the wal ttl for the run(in seconds)");
DEFINE_uint64(wal_size_limit_MB, 10,
"the wal size limit for the run"
"(in MB)");
using namespace ROCKSDB_NAMESPACE;
using GFLAGS_NAMESPACE::ParseCommandLineFlags;
using GFLAGS_NAMESPACE::SetUsageMessage;
struct DataPumpThread {
size_t no_records;
DB* db; // Assumption DB is Open'ed already.
};
@ -41,8 +48,8 @@ static void DataPumpThreadBody(void* arg) {
DataPumpThread* t = reinterpret_cast<DataPumpThread*>(arg);
DB* db = t->db;
Random rnd(301);
size_t i = 0;
while (i++ < t->no_records) {
uint64_t i = 0;
while (i++ < FLAGS_num_inserts) {
if (!db->Put(WriteOptions(), Slice(rnd.RandomString(500)),
Slice(rnd.RandomString(500)))
.ok()) {
@ -52,45 +59,6 @@ static void DataPumpThreadBody(void* arg) {
}
}
struct ReplicationThread {
std::atomic<bool> stop;
DB* db;
volatile size_t no_read;
};
static void ReplicationThreadBody(void* arg) {
ReplicationThread* t = reinterpret_cast<ReplicationThread*>(arg);
DB* db = t->db;
std::unique_ptr<TransactionLogIterator> iter;
SequenceNumber currentSeqNum = 1;
while (!t->stop.load(std::memory_order_acquire)) {
iter.reset();
Status s;
while (!db->GetUpdatesSince(currentSeqNum, &iter).ok()) {
if (t->stop.load(std::memory_order_acquire)) {
return;
}
}
fprintf(stderr, "Refreshing iterator\n");
for (; iter->Valid(); iter->Next(), t->no_read++, currentSeqNum++) {
BatchResult res = iter->GetBatch();
if (res.sequence != currentSeqNum) {
fprintf(stderr, "Missed a seq no. b/w %ld and %ld\n",
(long)currentSeqNum, (long)res.sequence);
exit(1);
}
}
}
}
DEFINE_uint64(num_inserts, 1000,
"the num of inserts the first thread should"
" perform.");
DEFINE_uint64(wal_ttl_seconds, 1000, "the wal ttl for the run(in seconds)");
DEFINE_uint64(wal_size_limit_MB, 10,
"the wal size limit for the run"
"(in MB)");
int main(int argc, const char** argv) {
SetUsageMessage(
std::string("\nUSAGE:\n") + std::string(argv[0]) +
@ -117,29 +85,38 @@ int main(int argc, const char** argv) {
}
DataPumpThread dataPump;
dataPump.no_records = FLAGS_num_inserts;
dataPump.db = db;
env->StartThread(DataPumpThreadBody, &dataPump);
ReplicationThread replThread;
replThread.db = db;
replThread.no_read = 0;
replThread.stop.store(false, std::memory_order_release);
env->StartThread(ReplicationThreadBody, &replThread);
while (replThread.no_read < FLAGS_num_inserts)
;
replThread.stop.store(true, std::memory_order_release);
if (replThread.no_read < dataPump.no_records) {
// no. read should be => than inserted.
fprintf(stderr,
"No. of Record's written and read not same\nRead : %" ROCKSDB_PRIszt
" Written : %" ROCKSDB_PRIszt "\n",
replThread.no_read, dataPump.no_records);
exit(1);
std::unique_ptr<TransactionLogIterator> iter;
SequenceNumber currentSeqNum = 1;
uint64_t num_read = 0;
for (;;) {
iter.reset();
// Continue to probe a bit more after all received
size_t probes = 0;
while (!db->GetUpdatesSince(currentSeqNum, &iter).ok()) {
probes++;
if (probes > 100 && num_read >= FLAGS_num_inserts) {
if (num_read > FLAGS_num_inserts) {
fprintf(stderr, "Too many updates read: %ld expected: %ld\n",
(long)num_read, (long)FLAGS_num_inserts);
exit(1);
}
fprintf(stderr, "Successful!\n");
return 0;
}
}
fprintf(stderr, "Refreshing iterator\n");
for (; iter->Valid(); iter->Next(), num_read++, currentSeqNum++) {
BatchResult res = iter->GetBatch();
if (res.sequence != currentSeqNum) {
fprintf(stderr, "Missed a seq no. b/w %ld and %ld\n",
(long)currentSeqNum, (long)res.sequence);
exit(1);
}
}
}
fprintf(stderr, "Successful!\n");
exit(0);
}
#endif // GFLAGS

6
utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h
View File

@ -28,8 +28,10 @@
#define paranoid_invariant_notnull(a) assert(a)
#define paranoid_invariant(a) assert(a)
#define ENSURE_POD(type) \
static_assert(std::is_pod<type>::value, #type "isn't POD")
#define ENSURE_POD(type) \
static_assert( \
std::is_standard_layout<type>::value && std::is_trivial<type>::value, \
#type "isn't POD")
inline int get_error_errno(void) {
invariant(errno);