Download CI dependencies only inside docker

2022-02-18 19:28:35 +01:00 · 2022-02-18 19:28:35 +01:00 · 27b9859391
parent d85eb6a435
commit 27b9859391
7 changed files with 139 additions and 108 deletions
--- a/7
+++ b/7
@ -1,4 +1,4 @@
-FROM debian:buster
+FROM debian:oldstable

 ARG REVISION
 ARG GH_MATRIX_OS
@ -27,6 +27,11 @@ ADD src /usr/src/tdlight-java-natives/src

 WORKDIR /usr/src/tdlight-java-natives/

+ENV CROSS_BUILD_DEPS_DIR "/usr/src/tdlight-java-natives"
+
+ENV JAVA_HOME="$(find "/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)/"
+ENV JAVA_INCLUDE_PATH="$JAVA_HOME/include"
+
 RUN /bin/bash /usr/src/tdlight-java-natives/scripts/continuous-integration/docker/build-natives.sh

 #WORKDIR /usr/src/tdlight-java-natives/generated/
--- a/scripts/continuous-integration/docker/build-natives.sh
+++ b/scripts/continuous-integration/docker/build-natives.sh
@ -1,13 +1,91 @@
-#!/bin/bash -e
-export JAVA_HOME
-JAVA_HOME="$(find "/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)/"
-export JAVA_INCLUDE_PATH="$JAVA_HOME/include"
 source ./scripts/continuous-integration/github-workflows/setup-variables.sh
 # Check variables correctness
 if [ -z "${CPU_ARCH_DPKG}" ]; then
 	echo "Missing parameter: CPU_ARCH_DPKG"
 	exit 1
 fi
+if [ -z "${CROSS_BUILD_DEPS_DIR}" ]; then
+	echo "Missing parameter: CROSS_BUILD_DEPS_DIR"
+	exit 1
+fi
+
+CROSS_OPENJDK_PATH=""
+fix_jdk_path() {
+  # Setup OpenJDK path
+  CROSS_OPENJDK_PATH="$(find "$CROSS_BUILD_DEPS_DIR/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)"
+}
+
+check_jdk_existance() {
+  if [[ ! -d "$CROSS_OPENJDK_PATH" ]]; then
+    echo "Can't find cross OpenJDK at location $CROSS_OPENJDK_PATH"
+    exit 1
+  fi
+}
+
+fix_jdk_path
+if [[ ! -f "$CROSS_BUILD_DEPS_DIR/ok-012" ]]; then
+	rm -rf "$CROSS_BUILD_DEPS_DIR" || true
+	mkdir -p "$CROSS_BUILD_DEPS_DIR"
+	cd "$CROSS_BUILD_DEPS_DIR"
+	# LibZ-Dev
+	apt-get download "zlib1g-dev:${CPU_ARCH_DPKG}"
+	ZLIB1G_DEV_DEB=$(find . -name "zlib1g-dev_*.deb")
+	dpkg -x "$ZLIB1G_DEV_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$ZLIB1G_DEV_DEB"
+	# LibZ
+	apt-get download "zlib1g:${CPU_ARCH_DPKG}"
+	ZLIB1G_DEB=$(find . -name "zlib1g_*.deb")
+	dpkg -x "$ZLIB1G_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$ZLIB1G_DEB"
+	# LibSSL-Dev
+	apt-get download "libssl-dev:${CPU_ARCH_DPKG}"
+	LIBSSL_DEV_DEB=$(find . -name "libssl-dev_*.deb")
+	dpkg -x "$LIBSSL_DEV_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$LIBSSL_DEV_DEB"
+	# LibSSL
+	apt-get download "libssl:${CPU_ARCH_DPKG}"
+	LIBSSL_DEB=$(find . -name "libssl_*.deb")
+	dpkg -x "$LIBSSL_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$LIBSSL_DEB"
+	# Java Common
+	apt-get download "java-common:${CPU_ARCH_DPKG}"
+	JC_DEB=$(find . -name "java-common_*.deb")
+	dpkg -x "$JC_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$JC_DEB"
+	# OpenJDK-JRE-Headless
+	apt-get download "jdk-11-jre-headless:${CPU_ARCH_DPKG}"
+	OJDKRH_DEB=$(find . -name "jdk-11-jre-headless_*.deb")
+	dpkg -x "$OJDKRH_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$OJDKRH_DEB"
+	# OpenJDK-JRE
+	apt-get download "jdk-11-jre:${CPU_ARCH_DPKG}"
+	OJDKR_DEB=$(find . -name "jdk-11-jre_*.deb")
+	dpkg -x "$OJDKR_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$OJDKR_DEB"
+	# OpenJDK-JDK
+	apt-get download "jdk-11-jdk-headless:${CPU_ARCH_DPKG}"
+	OJDKJ_DEB=$(find . -name "jdk-11-jdk-headless_*.deb")
+	dpkg -x "$OJDKJ_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$OJDKJ_DEB"
+	# OpenJDK-GUI
+	apt-get download "jdk-11-jdk:${CPU_ARCH_DPKG}"
+	OJDKG_DEB=$(find . -name "jdk-11-jdk_*.deb")
+	dpkg -x "$OJDKG_DEB" "$CROSS_BUILD_DEPS_DIR"
+	rm "$OJDKG_DEB"
+
+	# Fix symlinks
+	find "$CROSS_BUILD_DEPS_DIR" -lname "/*" \
+			-exec  sh -c "ln -sf \"\`echo \"$CROSS_BUILD_DEPS_DIR\$(readlink \$0)\"\`\" \"\$0\"" {} \;
+
+	# Check if openjdk is found
+	fix_jdk_path
+	check_jdk_existance
+
+	touch "$CROSS_BUILD_DEPS_DIR/ok-012"
+fi
+
+check_jdk_existance
+
 source ./scripts/continuous-integration/github-workflows/install-dependencies.sh
 sudo apt install -y "crossbuild-essential-${CPU_ARCH_DPKG}"
 source ./scripts/continuous-integration/github-workflows/build-natives.sh
--- a/scripts/core/compile_tdjni.sh
+++ b/scripts/core/compile_tdjni.sh
@ -91,7 +91,7 @@ echo "Configuring CMake..."
 cd ./generated/tdjni_build/

 CMAKE_EXTRA_ARGUMENTS_TDJNI=""
-if [[ ! -z "$CROSS_BUILD_DEPS_DIR" ]]; then
+if [[ -n "$CROSS_BUILD_DEPS_DIR" ]]; then
 # shellcheck disable=SC2089
 CMAKE_EXTRA_ARGUMENTS_TDJNI="${CMAKE_EXTRA_ARGUMENTS_TDJNI} -DCMAKE_TOOLCHAIN_FILE=${CROSS_BUILD_DEPS_DIR}/toolchain.cmake"
 fi
--- a/scripts/core/configure_td.sh
+++ b/scripts/core/configure_td.sh
@ -61,7 +61,7 @@ if [[ "$IMPLEMENTATION_NAME" = "tdlight" ]]; then
 else
  CMAKE_EXTRA_ARGUMENTS_TD=""
 fi
-if [[ ! -z "$CROSS_BUILD_DEPS_DIR" ]]; then
+if [[ -n "$CROSS_BUILD_DEPS_DIR" ]]; then
 # shellcheck disable=SC2089
 CMAKE_EXTRA_ARGUMENTS_TD="${CMAKE_EXTRA_ARGUMENTS_TD} -DCMAKE_TOOLCHAIN_FILE=${CROSS_BUILD_DEPS_DIR}/toolchain.cmake"
 fi
--- a/scripts/core/install-dependencies.sh
+++ b/scripts/core/install-dependencies.sh
@ -7,6 +7,8 @@
 #   CPU_CORES = "-- -j<cores>" or "-m" on Windows
 # OTHER REQUIRED ENVIRONMENT VARIABLES:
 #   CMAKE_EXTRA_ARGUMENTS = <args>
+# OPTIONAL ENVIRONMENT VARIABLES:
+#   CROSS_BUILD_DEPS_DIR = <args>

 # Check variables correctness
 if [ -z "${OPERATING_SYSTEM_NAME}" ]; then
@ -44,110 +46,52 @@ echo "Td implementation: ${IMPLEMENTATION_NAME}"
 echo "Build type: ${BUILD_TYPE}"
 echo "CPU cores count: ${CPU_CORES}"
 echo "CMake extra arguments: '${CMAKE_EXTRA_ARGUMENTS}'"
-
-CROSS_OPENJDK_PATH=""
-fix_jdk_path() {
-  # Setup OpenJDK path
-  CROSS_OPENJDK_PATH="$(find "$CROSS_BUILD_DEPS_DIR/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)"
-}
-
-check_jdk_existance() {
-  if [[ ! -d "$CROSS_OPENJDK_PATH" ]]; then
-    echo "Can't find cross OpenJDK at location $CROSS_OPENJDK_PATH"
-    exit 1
-  fi
-}
+echo "Cross build deps dir: '${CROSS_BUILD_DEPS_DIR}'"

 if [[ "$OPERATING_SYSTEM_NAME" == "linux" ]]; then
-  fix_jdk_path
-  if [[ ! -f "$CROSS_BUILD_DEPS_DIR/ok-012" ]]; then
-    rm -rf "$CROSS_BUILD_DEPS_DIR" || true
-    mkdir -p "$CROSS_BUILD_DEPS_DIR"
-    cd "$CROSS_BUILD_DEPS_DIR"
-    # LibZ-Dev
-    wget "http://ftp.debian.org/debian/pool/main/z/zlib/zlib1g-dev_1.2.11.dfsg-1_${CPU_ARCH_DPKG}.deb" -O zlib1g-dev.deb
-    ZLIB1G_DEV_DEB=zlib1g-dev.deb
-    dpkg -x "$ZLIB1G_DEV_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$ZLIB1G_DEV_DEB"
-    # LibZ
-    wget "http://ftp.debian.org/debian/pool/main/z/zlib/zlib1g_1.2.11.dfsg-1_${CPU_ARCH_DPKG}.deb" -O zlib1g.deb
-    ZLIB1G_DEB=zlib1g.deb
-    dpkg -x "$ZLIB1G_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$ZLIB1G_DEB"
-    # LibSSL-Dev
-    wget "http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl-dev_1.1.1d-0+deb10u7_${CPU_ARCH_DPKG}.deb" -O libssl-dev.deb
-    LIBSSL_DEV_DEB=libssl-dev.deb
-    dpkg -x "$LIBSSL_DEV_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$LIBSSL_DEV_DEB"
-    # LibSSL
-    wget "http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl1.1_1.1.1d-0+deb10u7_${CPU_ARCH_DPKG}.deb" -O libssl.deb
-    LIBSSL_DEB=libssl.deb
-    dpkg -x "$LIBSSL_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$LIBSSL_DEB"
-    # Java Common
-    wget "http://ftp.debian.org/debian/pool/main/j/java-common/java-common_0.71_all.deb" -O java-common.deb
-    JC_DEB=java-common.deb
-    dpkg -x "$JC_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$JC_DEB"
-    # OpenJDK-JRE-Headless
-    wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jre-headless_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jre-headless.deb
-    OJDKRH_DEB=jdk-11-jre-headless.deb
-    dpkg -x "$OJDKRH_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$OJDKRH_DEB"
-    # OpenJDK-JRE
-    wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jre_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jre.deb
-    OJDKR_DEB=jdk-11-jre.deb
-    dpkg -x "$OJDKR_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$OJDKR_DEB"
-    # OpenJDK-JDK
-    wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jdk-headless_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jdk-headless.deb
-    OJDKJ_DEB=jdk-11-jdk-headless.deb
-    dpkg -x "$OJDKJ_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$OJDKJ_DEB"
-    # OpenJDK-GUI
-    wget "http://security.debian.org/debian-security/pool/updates/main/o/openjdk-11/openjdk-11-jdk_11.0.13+8-1~deb10u1_${CPU_ARCH_DPKG}.deb" -O jdk-11-jdk.deb
-    OJDKG_DEB=jdk-11-jdk.deb
-    dpkg -x "$OJDKG_DEB" "$CROSS_BUILD_DEPS_DIR"
-    rm "$OJDKG_DEB"
+	if [ -z "${CROSS_BUILD_DEPS_DIR}" ]; then

-    # Fix symlinks
-    find "$CROSS_BUILD_DEPS_DIR" -lname "/*" \
-        -exec  sh -c "ln -sf \"\`echo \"$CROSS_BUILD_DEPS_DIR\$(readlink \$0)\"\`\" \"\$0\"" {} \;
+		CROSS_OPENJDK_PATH=""
+		fix_jdk_path() {
+			# Setup OpenJDK path
+			CROSS_OPENJDK_PATH="$(find "$CROSS_BUILD_DEPS_DIR/usr/lib/jvm/" -maxdepth 1 -type d -iname "java*jdk*" | head -n 1)"
+		}

-    # Check if openjdk is found
-    fix_jdk_path
-    check_jdk_existance
+		check_jdk_existance() {
+			if [[ ! -d "$CROSS_OPENJDK_PATH" ]]; then
+				echo "Can't find cross OpenJDK at location $CROSS_OPENJDK_PATH"
+				exit 1
+			fi
+		}

-    touch "$CROSS_BUILD_DEPS_DIR/ok-012"
-  fi
+		fix_jdk_path
+		check_jdk_existance

-  check_jdk_existance
+		# Create toolchain
+		echo "Creating toolchain file..."
+		cd "$ROOT_DIR"
+		{
+			echo "set(CMAKE_SYSTEM_PROCESSOR ${CPU_ARCH_LINUX})";
+			echo "set(CMAKE_C_COMPILER ${CROSS_CC})";
+			echo "set(CMAKE_C_COMPILER_TARGET ${CLANG_TRIPLE})";
+			echo "set(CMAKE_CXX_COMPILER ${CROSS_CXX})";
+			echo "set(CMAKE_CXX_COMPILER_TARGET ${CLANG_TRIPLE})";
+			echo "set(CMAKE_ASM_COMPILER ${CROSS_CXX})";
+			echo "set(CMAKE_ASM_COMPILER_TARGET ${CLANG_TRIPLE})";
+			echo "set(CMAKE_LIBRARY_PATH \"$CROSS_BUILD_DEPS_DIR/\")";
+			echo "include_directories(\"${CROSS_BUILD_DEPS_DIR}/usr/include/${CPU_ARCH_LINUX}-linux-${CPU_COMPILATION_TOOL}/\")";
+			echo "SET(CMAKE_FIND_ROOT_PATH \"$CROSS_BUILD_DEPS_DIR\" \"$JAVA_HOME\" \"/\" \"$JAVA_HOME/include\")";
+			echo "include_directories(\"${CROSS_OPENJDK_PATH}/include\")";
+			echo "include_directories(\"${CROSS_OPENJDK_PATH}/include/linux\")";

-  # Create toolchain
-  echo "Creating toolchain file..."
-  cd "$ROOT_DIR"
-  {
-    echo "set(CMAKE_SYSTEM_PROCESSOR ${CPU_ARCH_LINUX})";
-    echo "set(CMAKE_C_COMPILER ${CROSS_CC})";
-    echo "set(CMAKE_C_COMPILER_TARGET ${CLANG_TRIPLE})";
-    echo "set(CMAKE_CXX_COMPILER ${CROSS_CXX})";
-    echo "set(CMAKE_CXX_COMPILER_TARGET ${CLANG_TRIPLE})";
-    echo "set(CMAKE_ASM_COMPILER ${CROSS_CXX})";
-    echo "set(CMAKE_ASM_COMPILER_TARGET ${CLANG_TRIPLE})";
-    echo "set(CMAKE_LIBRARY_PATH \"$CROSS_BUILD_DEPS_DIR/\")";
-    echo "include_directories(\"${CROSS_BUILD_DEPS_DIR}/usr/include/${CPU_ARCH_LINUX}-linux-${CPU_COMPILATION_TOOL}/\")";
-    echo "SET(CMAKE_FIND_ROOT_PATH \"$CROSS_BUILD_DEPS_DIR\" \"$JAVA_HOME\" \"/\" \"$JAVA_HOME/include\")";
-    echo "include_directories(\"${CROSS_OPENJDK_PATH}/include\")";
-    echo "include_directories(\"${CROSS_OPENJDK_PATH}/include/linux\")";
+			echo "SET(JAVA_HOME \"$JAVA_HOME/\")";
+			echo "SET(JAVA_INCLUDE_PATH \"$JAVA_HOME/include\")";
+			echo "SET(JAVA_AWT_INCLUDE_PATH \"$JAVA_HOME/include\")";
+			echo "SET(JAVA_INCLUDE_PATH2 \"$JAVA_HOME/include/linux\")";

-    echo "SET(JAVA_HOME \"$JAVA_HOME/\")";
-    echo "SET(JAVA_INCLUDE_PATH \"$JAVA_HOME/include\")";
-    echo "SET(JAVA_AWT_INCLUDE_PATH \"$JAVA_HOME/include\")";
-    echo "SET(JAVA_INCLUDE_PATH2 \"$JAVA_HOME/include/linux\")";
-
-    echo "SET(JAVA_JVM_LIBRARY \"$CROSS_OPENJDK_PATH/lib/${JAVA_INSTALLATION_TYPE}/libjvm.so\")";
-    echo "SET(JAVA_AWT_LIBRARY \"$CROSS_OPENJDK_PATH/lib/libawt.so\")";
-    cat <<EOF
+			echo "SET(JAVA_JVM_LIBRARY \"$CROSS_OPENJDK_PATH/lib/${JAVA_INSTALLATION_TYPE}/libjvm.so\")";
+			echo "SET(JAVA_AWT_LIBRARY \"$CROSS_OPENJDK_PATH/lib/libawt.so\")";
+    	cat <<EOF
 SET(CMAKE_SYSTEM_NAME Linux)

 # Use our definitions for compiler tools
@ -157,17 +101,17 @@ SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
 EOF
-   if [[ ${CPU_ARCHITECTURE_NAME} == "aarch64" || ${CPU_ARCHITECTURE_NAME} == "armhf" ]]; then
-     cat <<EOF
+			 if [[ ${CPU_ARCHITECTURE_NAME} == "aarch64" || ${CPU_ARCHITECTURE_NAME} == "armhf" ]]; then
+				 cat <<EOF
 set(CMAKE_THREAD_LIBS_INIT "-lpthread")
 set(CMAKE_HAVE_THREADS_LIBRARY 1)
 set(CMAKE_USE_WIN32_THREADS_INIT 0)
 set(CMAKE_USE_PTHREADS_INIT 1)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 EOF
-    fi
-  } > "$CROSS_BUILD_DEPS_DIR/toolchain.cmake"
-
+			fi
+		} > "$CROSS_BUILD_DEPS_DIR/toolchain.cmake"
+	fi
 fi;

 echo "Done."
--- a/scripts/utils/compile-natives-package.sh
+++ b/scripts/utils/compile-natives-package.sh
@ -10,6 +10,8 @@
 #   CPU_CORES = "-- -j<cores>" or "-m" on Windows
 #   CMAKE_EXTRA_ARGUMENTS = <args>
 #   REVISION = <revision>
+# OPTIONAL ENVIRONMENT VARIABLES:
+#   CROSS_BUILD_DEPS_DIR = <args>

 cd ../core

--- a/scripts/utils/compile-tdapi-package.sh
+++ b/scripts/utils/compile-tdapi-package.sh
@ -10,6 +10,8 @@
 #   CPU_CORES = "-- -j<cores>" or "-m" on Windows
 #   CMAKE_EXTRA_ARGUMENTS = <args>
 #   REVISION = <revision>
+# OPTIONAL ENVIRONMENT VARIABLES:
+#   CROSS_BUILD_DEPS_DIR = <args>

 cd ../core