From 6a8ddd374dfb86c4512bde58b76baf150db6e7b4 Mon Sep 17 00:00:00 2001
From: sdong <siying.d@fb.com>
Date: Mon, 8 Jun 2020 19:32:18 -0700
Subject: [PATCH] Introduce some Linux build to CircleCI (#6937)

Summary:
Moving towards the long term goal of moving most CI build to CircleCI when possible, add some Linux tests in CircleCI. This is not all what we can include to CircleCI. For example, Java builds are not includ
ed.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6937

Test Plan: Watch CI build results.

Reviewed By: pdillinger

Differential Revision: D21941605

fbshipit-source-id: db6aead3c45f523386d4fb30d224cfde573cccad
---
 .circleci/config.yml | 77 +++++++++++++++++++++++++++++++++++++++++++-
 Makefile             | 15 +++++++--
 2 files changed, 89 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8dbbc193e..04851cab3 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -11,7 +11,59 @@ executors:
       shell: bash.exe
 
 jobs:
-  build:
+  build-linux:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: pyenv global 3.5.2
+      - run: sudo apt-get update -y
+      - run: sudo apt-get install -y libgflags-dev
+      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 make J=32 all check -j32
+
+  build-linux-release:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: make release -j32
+
+  build-linux-lite:
+    machine:
+      image: ubuntu-1604:201903-01 
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: pyenv global 3.5.2
+      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 LITE=1 make J=32 all check -j32
+
+  build-linux-lite-release:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: large
+    steps:
+      - checkout # check out the code in the project directory
+      - run: make release -j32
+
+  build-linux-clang-no-test:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: USE_CLANG=1 make all -j32
+
+  build-linux-cmake:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make -j32
+
+  build-windows:
     executor: windows-2xlarge
     
     environment:
@@ -54,3 +106,26 @@ jobs:
           shell: powershell.exe
           command: |
             build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,env_basic_test,env_test,db_merge_operand_test -Concurrency 16
+
+workflows:
+  build-linux:
+    jobs:
+      - build-linux
+  build-linux-lite:
+    jobs:
+      - build-linux-lite
+  build-linux-release:
+    jobs:
+      - build-linux-release
+  build-linux-lite-release:
+    jobs:
+      - build-linux-lite-release
+  build-linux-clang-no-test:
+    jobs:
+      - build-linux-clang-no-test
+  build-linux-cmake:
+    jobs:
+      - build-linux-cmake
+  build-windows:
+    jobs:
+      - build-windows
diff --git a/Makefile b/Makefile
index 874bf5993..77066fbe8 100644
--- a/Makefile
+++ b/Makefile
@@ -976,6 +976,12 @@ J ?= 100%
 # Use this regexp to select the subset of tests whose names match.
 tests-regexp = .
 
+ifeq ($(PRINT_PARALLEL_OUTPUTS), 1)	
+	parallel_com = '{}'
+else
+	parallel_com = '{} >& t/log-{/}'
+endif
+
 .PHONY: check_0
 check_0:
 	$(AM_V_GEN)export TEST_TMPDIR=$(TMPD); \
@@ -989,7 +995,10 @@ check_0:
 	} \
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
-	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
+	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu  $(parallel_com) ; \
+	parallel_retcode=$$? ; \
+	awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \
+	if [ $$parallel_retcode -ne 0 ] ; then exit 1 ; fi
 
 valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest
 
@@ -1048,9 +1057,11 @@ ifndef ASSERT_STATUS_CHECKED # not yet working with these tests
 	sh tools/rocksdb_dump_test.sh
 endif
 endif
-endif
+endif	
+ifndef SKIP_FORMAT_BUCK_CHECKS
 	$(MAKE) check-format
 	$(MAKE) check-buck-targets
+endif
 
 # TODO add ldb_tests
 check_some: $(SUBSET)