From 90c71aa5d95adbd65db3071207f91fe80cc7c879 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Thu, 30 Jan 2020 11:00:08 -0800 Subject: [PATCH] Don't download from (unreliable) maven.org (#6348) Summary: I set up a mirror of our Java deps on github so we can download them through github URLs rather than maven.org, which is proving terribly unreliable from Travis builds. Also sanitized calls to curl, so they are easier to read and appropriately fail on download failure. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6348 Test Plan: CI Differential Revision: D19633621 Pulled By: pdillinger fbshipit-source-id: 7eb3f730953db2ead758dc94039c040f406790f3 --- .travis.yml | 2 +- Makefile | 10 +++++----- appveyor.yml | 6 +++--- build_tools/regression_build_test.sh | 2 +- build_tools/rocksdb-lego-determinator | 2 +- java/CMakeLists.txt | 22 ++++++++++++---------- java/Makefile | 17 ++++++++++------- 7 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d91a4e7f..6f8f77ee4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ install: sudo apt-get install -y mingw-w64 ; fi - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then - mkdir cmake-dist && curl -sfSL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH; + mkdir cmake-dist && curl --silent --fail --show-error --location https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH; fi - if [[ "${JOB_NAME}" == java_test ]]; then java -version && echo "JAVA_HOME=${JAVA_HOME}"; diff --git a/Makefile b/Makefile index 8322b325a..bad05c1b5 100644 --- a/Makefile +++ b/Makefile @@ -1839,7 +1839,7 @@ endif libz.a: -rm -rf zlib-$(ZLIB_VER) ifeq (,$(wildcard ./zlib-$(ZLIB_VER).tar.gz)) - curl --output zlib-$(ZLIB_VER).tar.gz -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz + curl --fail --output zlib-$(ZLIB_VER).tar.gz --location ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz endif ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \ @@ -1853,7 +1853,7 @@ endif libbz2.a: -rm -rf bzip2-$(BZIP2_VER) ifeq (,$(wildcard ./bzip2-$(BZIP2_VER).tar.gz)) - curl --output bzip2-$(BZIP2_VER).tar.gz -L ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz + curl --fail --output bzip2-$(BZIP2_VER).tar.gz --location ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz endif BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \ @@ -1867,7 +1867,7 @@ endif libsnappy.a: -rm -rf snappy-$(SNAPPY_VER) ifeq (,$(wildcard ./snappy-$(SNAPPY_VER).tar.gz)) - curl --output snappy-$(SNAPPY_VER).tar.gz -L ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz + curl --fail --output snappy-$(SNAPPY_VER).tar.gz --location ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz endif SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \ @@ -1882,7 +1882,7 @@ endif liblz4.a: -rm -rf lz4-$(LZ4_VER) ifeq (,$(wildcard ./lz4-$(LZ4_VER).tar.gz)) - curl --output lz4-$(LZ4_VER).tar.gz -L ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz + curl --fail --output lz4-$(LZ4_VER).tar.gz --location ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz endif LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \ @@ -1896,7 +1896,7 @@ endif libzstd.a: -rm -rf zstd-$(ZSTD_VER) ifeq (,$(wildcard ./zstd-$(ZSTD_VER).tar.gz)) - curl --output zstd-$(ZSTD_VER).tar.gz -L ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz + curl --fail --output zstd-$(ZSTD_VER).tar.gz --location ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz endif ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \ diff --git a/appveyor.yml b/appveyor.yml index bae8374f7..416b6550d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ install: - md %THIRDPARTY_HOME% - echo "Building Snappy dependency..." - cd %THIRDPARTY_HOME% - - curl -fsSL -o snappy-1.1.7.zip https://github.com/google/snappy/archive/1.1.7.zip + - curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip - unzip snappy-1.1.7.zip - cd snappy-1.1.7 - mkdir build @@ -39,7 +39,7 @@ install: - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64 - echo "Building LZ4 dependency..." - cd %THIRDPARTY_HOME% - - curl -fsSL -o lz4-1.8.3.zip https://github.com/lz4/lz4/archive/v1.8.3.zip + - curl --fail --silent --show-error --output lz4-1.8.3.zip --location https://github.com/lz4/lz4/archive/v1.8.3.zip - unzip lz4-1.8.3.zip - cd lz4-1.8.3\visual\VS2010 - ps: $CMD="$Env:DEV_ENV"; & $CMD lz4.sln /upgrade @@ -47,7 +47,7 @@ install: - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64 - echo "Building ZStd dependency..." - cd %THIRDPARTY_HOME% - - curl -fsSL -o zstd-1.4.0.zip https://github.com/facebook/zstd/archive/v1.4.0.zip + - curl --fail --silent --show-error --output zstd-1.4.0.zip --location https://github.com/facebook/zstd/archive/v1.4.0.zip - unzip zstd-1.4.0.zip - cd zstd-1.4.0\build\VS2010 - ps: $CMD="$Env:DEV_ENV"; & $CMD zstd.sln /upgrade diff --git a/build_tools/regression_build_test.sh b/build_tools/regression_build_test.sh index bcf8907cc..31fb90eb2 100755 --- a/build_tools/regression_build_test.sh +++ b/build_tools/regression_build_test.sh @@ -378,7 +378,7 @@ function send_to_ods { echo >&2 "ERROR: Key $key doesn't have a value." return fi - curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \ + curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \ --connect-timeout 60 } diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator index 8c4c91bfc..948488c49 100755 --- a/build_tools/rocksdb-lego-determinator +++ b/build_tools/rocksdb-lego-determinator @@ -884,7 +884,7 @@ run_regression() # parameters: $1 -- key, $2 -- value function send_size_to_ods { - curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ + curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ --connect-timeout 60 } diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 8be8a7414..750195c37 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -313,17 +313,19 @@ if(NOT EXISTS ${JAVA_TEST_LIBDIR}) file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR}) endif() -if (DEFINED CUSTOM_REPO_URL) - set(SEARCH_REPO_URL ${CUSTOM_REPO_URL}/) - set(CENTRAL_REPO_URL ${CUSTOM_REPO_URL}/) +if (DEFINED CUSTOM_DEPS_URL) + set(DEPS_URL ${CUSTOM_DEPS_URL}/) else () - set(SEARCH_REPO_URL "http://search.maven.org/remotecontent?filepath=") - set(CENTRAL_REPO_URL "https://repo1.maven.org/maven2/") + # This is a URL for artifacts from a "fake" release on pdillinger's fork, + # so as not to put binaries in git (ew). We should move to hosting these + # under the facebook account on github, or something else more reliable + # than maven.org, which has been failing frequently from Travis. + set(DEPS_URL "https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps") endif() if(NOT EXISTS ${JAVA_JUNIT_JAR}) message("Downloading ${JAVA_JUNIT_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}") @@ -332,7 +334,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR}) endif() if(NOT EXISTS ${JAVA_HAMCR_JAR}) message("Downloading ${JAVA_HAMCR_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}") @@ -341,7 +343,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR}) endif() if(NOT EXISTS ${JAVA_MOCKITO_JAR}) message("Downloading ${JAVA_MOCKITO_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}") @@ -350,7 +352,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR}) endif() if(NOT EXISTS ${JAVA_CGLIB_JAR}) message("Downloading ${JAVA_CGLIB_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}") @@ -359,7 +361,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR}) endif() if(NOT EXISTS ${JAVA_ASSERTJ_JAR}) message("Downloading ${JAVA_ASSERTJ_JAR}") - file(DOWNLOAD ${CENTRAL_REPO_URL}org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}") diff --git a/java/Makefile b/java/Makefile index 96d41a4a7..378d21b04 100644 --- a/java/Makefile +++ b/java/Makefile @@ -213,8 +213,11 @@ ifneq ($(DEBUG_LEVEL),0) JAVAC_ARGS = -Xlint:deprecation -Xlint:unchecked endif -SEARCH_REPO_URL?=http://search.maven.org/remotecontent?filepath= -CENTRAL_REPO_URL?=https://repo1.maven.org/maven2/ +# This is a URL for artifacts from a "fake" release on pdillinger's fork, +# so as not to put binaries in git (ew). We should move to hosting these +# under the facebook account on github, or something else more reliable +# than maven.org, which has been failing frequently from Travis. +DEPS_URL?=https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps clean: clean-not-downloaded clean-downloaded @@ -283,11 +286,11 @@ optimistic_transaction_sample: java resolve_test_deps: test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)" - test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) $(SEARCH_REPO_URL)junit/junit/4.12/junit-4.12.jar - test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) $(SEARCH_REPO_URL)org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar - test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" $(SEARCH_REPO_URL)org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar - test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" $(SEARCH_REPO_URL)cglib/cglib/2.2.2/cglib-2.2.2.jar - test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" $(CENTRAL_REPO_URL)org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar + test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_JUNIT_JAR) --location $(DEPS_URL)/junit-4.12.jar + test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_HAMCR_JAR) --location $(DEPS_URL)/hamcrest-core-1.3.jar + test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_MOCKITO_JAR)" --location $(DEPS_URL)/mockito-all-1.10.19.jar + test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_CGLIB_JAR)" --location $(DEPS_URL)/cglib-2.2.2.jar + test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_ASSERTJ_JAR)" --location $(DEPS_URL)/assertj-core-1.7.1.jar java_test: java resolve_test_deps $(AM_V_GEN)mkdir -p $(TEST_CLASSES)