From 27b9859391b65730e27db43c0b1de86569fa75ab Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Fri, 18 Feb 2022 19:28:35 +0100 Subject: [PATCH] Download CI dependencies only inside docker --- Dockerfile | 7 +- .../docker/build-natives.sh | 86 ++++++++++- scripts/core/compile_tdjni.sh | 2 +- scripts/core/configure_td.sh | 2 +- scripts/core/install-dependencies.sh | 146 ++++++------------ scripts/utils/compile-natives-package.sh | 2 + scripts/utils/compile-tdapi-package.sh | 2 + 7 files changed, 139 insertions(+), 108 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd71c46..cb5db06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster +FROM debian:oldstable ARG REVISION ARG GH_MATRIX_OS @@ -27,6 +27,11 @@ ADD src /usr/src/tdlight-java-natives/src WORKDIR /usr/src/tdlight-java-natives/ +ENV CROSS_BUILD_DEPS_DIR "/usr/src/tdlight-java-natives" + +ENV JAVA_HOME="$(find "/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)/" +ENV JAVA_INCLUDE_PATH="$JAVA_HOME/include" + RUN /bin/bash /usr/src/tdlight-java-natives/scripts/continuous-integration/docker/build-natives.sh #WORKDIR /usr/src/tdlight-java-natives/generated/ diff --git a/scripts/continuous-integration/docker/build-natives.sh b/scripts/continuous-integration/docker/build-natives.sh index 7bcb5e3..8c1f2d5 100644 --- a/scripts/continuous-integration/docker/build-natives.sh +++ b/scripts/continuous-integration/docker/build-natives.sh @@ -1,13 +1,91 @@ -#!/bin/bash -e -export JAVA_HOME -JAVA_HOME="$(find "/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)/" -export JAVA_INCLUDE_PATH="$JAVA_HOME/include" source ./scripts/continuous-integration/github-workflows/setup-variables.sh # Check variables correctness if [ -z "${CPU_ARCH_DPKG}" ]; then echo "Missing parameter: CPU_ARCH_DPKG" exit 1 fi +if [ -z "${CROSS_BUILD_DEPS_DIR}" ]; then + echo "Missing parameter: CROSS_BUILD_DEPS_DIR" + exit 1 +fi + +CROSS_OPENJDK_PATH="" +fix_jdk_path() { + # Setup OpenJDK path + CROSS_OPENJDK_PATH="$(find "$CROSS_BUILD_DEPS_DIR/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)" +} + +check_jdk_existance() { + if [[ ! -d "$CROSS_OPENJDK_PATH" ]]; then + echo "Can't find cross OpenJDK at location $CROSS_OPENJDK_PATH" + exit 1 + fi +} + +fix_jdk_path +if [[ ! -f "$CROSS_BUILD_DEPS_DIR/ok-012" ]]; then + rm -rf "$CROSS_BUILD_DEPS_DIR" || true + mkdir -p "$CROSS_BUILD_DEPS_DIR" + cd "$CROSS_BUILD_DEPS_DIR" + # LibZ-Dev + apt-get download "zlib1g-dev:${CPU_ARCH_DPKG}" + ZLIB1G_DEV_DEB=$(find . -name "zlib1g-dev_*.deb") + dpkg -x "$ZLIB1G_DEV_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$ZLIB1G_DEV_DEB" + # LibZ + apt-get download "zlib1g:${CPU_ARCH_DPKG}" + ZLIB1G_DEB=$(find . -name "zlib1g_*.deb") + dpkg -x "$ZLIB1G_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$ZLIB1G_DEB" + # LibSSL-Dev + apt-get download "libssl-dev:${CPU_ARCH_DPKG}" + LIBSSL_DEV_DEB=$(find . -name "libssl-dev_*.deb") + dpkg -x "$LIBSSL_DEV_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$LIBSSL_DEV_DEB" + # LibSSL + apt-get download "libssl:${CPU_ARCH_DPKG}" + LIBSSL_DEB=$(find . -name "libssl_*.deb") + dpkg -x "$LIBSSL_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$LIBSSL_DEB" + # Java Common + apt-get download "java-common:${CPU_ARCH_DPKG}" + JC_DEB=$(find . -name "java-common_*.deb") + dpkg -x "$JC_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$JC_DEB" + # OpenJDK-JRE-Headless + apt-get download "jdk-11-jre-headless:${CPU_ARCH_DPKG}" + OJDKRH_DEB=$(find . -name "jdk-11-jre-headless_*.deb") + dpkg -x "$OJDKRH_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$OJDKRH_DEB" + # OpenJDK-JRE + apt-get download "jdk-11-jre:${CPU_ARCH_DPKG}" + OJDKR_DEB=$(find . -name "jdk-11-jre_*.deb") + dpkg -x "$OJDKR_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$OJDKR_DEB" + # OpenJDK-JDK + apt-get download "jdk-11-jdk-headless:${CPU_ARCH_DPKG}" + OJDKJ_DEB=$(find . -name "jdk-11-jdk-headless_*.deb") + dpkg -x "$OJDKJ_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$OJDKJ_DEB" + # OpenJDK-GUI + apt-get download "jdk-11-jdk:${CPU_ARCH_DPKG}" + OJDKG_DEB=$(find . -name "jdk-11-jdk_*.deb") + dpkg -x "$OJDKG_DEB" "$CROSS_BUILD_DEPS_DIR" + rm "$OJDKG_DEB" + + # Fix symlinks + find "$CROSS_BUILD_DEPS_DIR" -lname "/*" \ + -exec sh -c "ln -sf \"\`echo \"$CROSS_BUILD_DEPS_DIR\$(readlink \$0)\"\`\" \"\$0\"" {} \; + + # Check if openjdk is found + fix_jdk_path + check_jdk_existance + + touch "$CROSS_BUILD_DEPS_DIR/ok-012" +fi + +check_jdk_existance + source ./scripts/continuous-integration/github-workflows/install-dependencies.sh sudo apt install -y "crossbuild-essential-${CPU_ARCH_DPKG}" source ./scripts/continuous-integration/github-workflows/build-natives.sh diff --git a/scripts/core/compile_tdjni.sh b/scripts/core/compile_tdjni.sh index 9f3b165..6c4b521 100755 --- a/scripts/core/compile_tdjni.sh +++ b/scripts/core/compile_tdjni.sh @@ -91,7 +91,7 @@ echo "Configuring CMake..." cd ./generated/tdjni_build/ CMAKE_EXTRA_ARGUMENTS_TDJNI="" -if [[ ! -z "$CROSS_BUILD_DEPS_DIR" ]]; then +if [[ -n "$CROSS_BUILD_DEPS_DIR" ]]; then # shellcheck disable=SC2089 CMAKE_EXTRA_ARGUMENTS_TDJNI="${CMAKE_EXTRA_ARGUMENTS_TDJNI} -DCMAKE_TOOLCHAIN_FILE=${CROSS_BUILD_DEPS_DIR}/toolchain.cmake" fi diff --git a/scripts/core/configure_td.sh b/scripts/core/configure_td.sh index 080c269..fae12e0 100755 --- a/scripts/core/configure_td.sh +++ b/scripts/core/configure_td.sh @@ -61,7 +61,7 @@ if [[ "$IMPLEMENTATION_NAME" = "tdlight" ]]; then else CMAKE_EXTRA_ARGUMENTS_TD="" fi -if [[ ! -z "$CROSS_BUILD_DEPS_DIR" ]]; then +if [[ -n "$CROSS_BUILD_DEPS_DIR" ]]; then # shellcheck disable=SC2089 CMAKE_EXTRA_ARGUMENTS_TD="${CMAKE_EXTRA_ARGUMENTS_TD} -DCMAKE_TOOLCHAIN_FILE=${CROSS_BUILD_DEPS_DIR}/toolchain.cmake" fi diff --git a/scripts/core/install-dependencies.sh b/scripts/core/install-dependencies.sh index 3f36d30..d27ac06 100755 --- a/scripts/core/install-dependencies.sh +++ b/scripts/core/install-dependencies.sh @@ -7,6 +7,8 @@ # CPU_CORES = "-- -j" or "-m" on Windows # OTHER REQUIRED ENVIRONMENT VARIABLES: # CMAKE_EXTRA_ARGUMENTS = +# OPTIONAL ENVIRONMENT VARIABLES: +# CROSS_BUILD_DEPS_DIR = # Check variables correctness if [ -z "${OPERATING_SYSTEM_NAME}" ]; then @@ -44,110 +46,52 @@ echo "Td implementation: ${IMPLEMENTATION_NAME}" echo "Build type: ${BUILD_TYPE}" echo "CPU cores count: ${CPU_CORES}" echo "CMake extra arguments: '${CMAKE_EXTRA_ARGUMENTS}'" - -CROSS_OPENJDK_PATH="" -fix_jdk_path() { - # Setup OpenJDK path - CROSS_OPENJDK_PATH="$(find "$CROSS_BUILD_DEPS_DIR/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)" -} - -check_jdk_existance() { - if [[ ! -d "$CROSS_OPENJDK_PATH" ]]; then - echo "Can't find cross OpenJDK at location $CROSS_OPENJDK_PATH" - exit 1 - fi -} +echo "Cross build deps dir: '${CROSS_BUILD_DEPS_DIR}'" if [[ "$OPERATING_SYSTEM_NAME" == "linux" ]]; then - fix_jdk_path - if [[ ! -f "$CROSS_BUILD_DEPS_DIR/ok-012" ]]; then - rm -rf "$CROSS_BUILD_DEPS_DIR" || true - mkdir -p "$CROSS_BUILD_DEPS_DIR" - cd "$CROSS_BUILD_DEPS_DIR" - # LibZ-Dev - wget "http://ftp.debian.org/debian/pool/main/z/zlib/zlib1g-dev_1.2.11.dfsg-1_${CPU_ARCH_DPKG}.deb" -O zlib1g-dev.deb - ZLIB1G_DEV_DEB=zlib1g-dev.deb - dpkg -x "$ZLIB1G_DEV_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$ZLIB1G_DEV_DEB" - # LibZ - wget "http://ftp.debian.org/debian/pool/main/z/zlib/zlib1g_1.2.11.dfsg-1_${CPU_ARCH_DPKG}.deb" -O zlib1g.deb - ZLIB1G_DEB=zlib1g.deb - dpkg -x "$ZLIB1G_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$ZLIB1G_DEB" - # LibSSL-Dev - wget "http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl-dev_1.1.1d-0+deb10u7_${CPU_ARCH_DPKG}.deb" -O libssl-dev.deb - LIBSSL_DEV_DEB=libssl-dev.deb - dpkg -x "$LIBSSL_DEV_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$LIBSSL_DEV_DEB" - # LibSSL - wget "http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl1.1_1.1.1d-0+deb10u7_${CPU_ARCH_DPKG}.deb" -O libssl.deb - LIBSSL_DEB=libssl.deb - dpkg -x "$LIBSSL_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$LIBSSL_DEB" - # Java Common - wget "http://ftp.debian.org/debian/pool/main/j/java-common/java-common_0.71_all.deb" -O java-common.deb - JC_DEB=java-common.deb - dpkg -x "$JC_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$JC_DEB" - # OpenJDK-JRE-Headless - wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jre-headless_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jre-headless.deb - OJDKRH_DEB=jdk-11-jre-headless.deb - dpkg -x "$OJDKRH_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$OJDKRH_DEB" - # OpenJDK-JRE - wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jre_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jre.deb - OJDKR_DEB=jdk-11-jre.deb - dpkg -x "$OJDKR_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$OJDKR_DEB" - # OpenJDK-JDK - wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jdk-headless_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jdk-headless.deb - OJDKJ_DEB=jdk-11-jdk-headless.deb - dpkg -x "$OJDKJ_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$OJDKJ_DEB" - # OpenJDK-GUI - wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jdk_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jdk.deb - OJDKG_DEB=jdk-11-jdk.deb - dpkg -x "$OJDKG_DEB" "$CROSS_BUILD_DEPS_DIR" - rm "$OJDKG_DEB" + if [ -z "${CROSS_BUILD_DEPS_DIR}" ]; then - # Fix symlinks - find "$CROSS_BUILD_DEPS_DIR" -lname "/*" \ - -exec sh -c "ln -sf \"\`echo \"$CROSS_BUILD_DEPS_DIR\$(readlink \$0)\"\`\" \"\$0\"" {} \; + CROSS_OPENJDK_PATH="" + fix_jdk_path() { + # Setup OpenJDK path + CROSS_OPENJDK_PATH="$(find "$CROSS_BUILD_DEPS_DIR/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)" + } - # Check if openjdk is found - fix_jdk_path - check_jdk_existance + check_jdk_existance() { + if [[ ! -d "$CROSS_OPENJDK_PATH" ]]; then + echo "Can't find cross OpenJDK at location $CROSS_OPENJDK_PATH" + exit 1 + fi + } - touch "$CROSS_BUILD_DEPS_DIR/ok-012" - fi + fix_jdk_path + check_jdk_existance - check_jdk_existance + # Create toolchain + echo "Creating toolchain file..." + cd "$ROOT_DIR" + { + echo "set(CMAKE_SYSTEM_PROCESSOR ${CPU_ARCH_LINUX})"; + echo "set(CMAKE_C_COMPILER ${CROSS_CC})"; + echo "set(CMAKE_C_COMPILER_TARGET ${CLANG_TRIPLE})"; + echo "set(CMAKE_CXX_COMPILER ${CROSS_CXX})"; + echo "set(CMAKE_CXX_COMPILER_TARGET ${CLANG_TRIPLE})"; + echo "set(CMAKE_ASM_COMPILER ${CROSS_CXX})"; + echo "set(CMAKE_ASM_COMPILER_TARGET ${CLANG_TRIPLE})"; + echo "set(CMAKE_LIBRARY_PATH \"$CROSS_BUILD_DEPS_DIR/\")"; + echo "include_directories(\"${CROSS_BUILD_DEPS_DIR}/usr/include/${CPU_ARCH_LINUX}-linux-${CPU_COMPILATION_TOOL}/\")"; + echo "SET(CMAKE_FIND_ROOT_PATH \"$CROSS_BUILD_DEPS_DIR\" \"$JAVA_HOME\" \"/\" \"$JAVA_HOME/include\")"; + echo "include_directories(\"${CROSS_OPENJDK_PATH}/include\")"; + echo "include_directories(\"${CROSS_OPENJDK_PATH}/include/linux\")"; - # Create toolchain - echo "Creating toolchain file..." - cd "$ROOT_DIR" - { - echo "set(CMAKE_SYSTEM_PROCESSOR ${CPU_ARCH_LINUX})"; - echo "set(CMAKE_C_COMPILER ${CROSS_CC})"; - echo "set(CMAKE_C_COMPILER_TARGET ${CLANG_TRIPLE})"; - echo "set(CMAKE_CXX_COMPILER ${CROSS_CXX})"; - echo "set(CMAKE_CXX_COMPILER_TARGET ${CLANG_TRIPLE})"; - echo "set(CMAKE_ASM_COMPILER ${CROSS_CXX})"; - echo "set(CMAKE_ASM_COMPILER_TARGET ${CLANG_TRIPLE})"; - echo "set(CMAKE_LIBRARY_PATH \"$CROSS_BUILD_DEPS_DIR/\")"; - echo "include_directories(\"${CROSS_BUILD_DEPS_DIR}/usr/include/${CPU_ARCH_LINUX}-linux-${CPU_COMPILATION_TOOL}/\")"; - echo "SET(CMAKE_FIND_ROOT_PATH \"$CROSS_BUILD_DEPS_DIR\" \"$JAVA_HOME\" \"/\" \"$JAVA_HOME/include\")"; - echo "include_directories(\"${CROSS_OPENJDK_PATH}/include\")"; - echo "include_directories(\"${CROSS_OPENJDK_PATH}/include/linux\")"; + echo "SET(JAVA_HOME \"$JAVA_HOME/\")"; + echo "SET(JAVA_INCLUDE_PATH \"$JAVA_HOME/include\")"; + echo "SET(JAVA_AWT_INCLUDE_PATH \"$JAVA_HOME/include\")"; + echo "SET(JAVA_INCLUDE_PATH2 \"$JAVA_HOME/include/linux\")"; - echo "SET(JAVA_HOME \"$JAVA_HOME/\")"; - echo "SET(JAVA_INCLUDE_PATH \"$JAVA_HOME/include\")"; - echo "SET(JAVA_AWT_INCLUDE_PATH \"$JAVA_HOME/include\")"; - echo "SET(JAVA_INCLUDE_PATH2 \"$JAVA_HOME/include/linux\")"; - - echo "SET(JAVA_JVM_LIBRARY \"$CROSS_OPENJDK_PATH/lib/${JAVA_INSTALLATION_TYPE}/libjvm.so\")"; - echo "SET(JAVA_AWT_LIBRARY \"$CROSS_OPENJDK_PATH/lib/libawt.so\")"; - cat < "$CROSS_BUILD_DEPS_DIR/toolchain.cmake" - + fi + } > "$CROSS_BUILD_DEPS_DIR/toolchain.cmake" + fi fi; echo "Done." diff --git a/scripts/utils/compile-natives-package.sh b/scripts/utils/compile-natives-package.sh index 10fcc6b..a5f6919 100755 --- a/scripts/utils/compile-natives-package.sh +++ b/scripts/utils/compile-natives-package.sh @@ -10,6 +10,8 @@ # CPU_CORES = "-- -j" or "-m" on Windows # CMAKE_EXTRA_ARGUMENTS = # REVISION = +# OPTIONAL ENVIRONMENT VARIABLES: +# CROSS_BUILD_DEPS_DIR = cd ../core diff --git a/scripts/utils/compile-tdapi-package.sh b/scripts/utils/compile-tdapi-package.sh index 0583633..83b4d3c 100755 --- a/scripts/utils/compile-tdapi-package.sh +++ b/scripts/utils/compile-tdapi-package.sh @@ -10,6 +10,8 @@ # CPU_CORES = "-- -j" or "-m" on Windows # CMAKE_EXTRA_ARGUMENTS = # REVISION = +# OPTIONAL ENVIRONMENT VARIABLES: +# CROSS_BUILD_DEPS_DIR = cd ../core