From ee0523478dff5f6c9e61e25f45b6e6afa6e4c295 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Wed, 7 Oct 2020 15:20:59 +0200 Subject: [PATCH] Add tdapi generator --- .gitmodules | 6 +++ Jenkinsfile | 29 ++++++++++++++ dependencies/tdlib-serializer | 1 + dependencies/tdlight | 1 + jenkins/docker/dockerfile | 62 ++++++++++++++++++++++++++++++ jenkins/scripts/generate_tdapi.sh | 27 +++++++++++++ jenkins/scripts/setup_variables.sh | 59 ++++++++++++++++++++++++++++ jenkins/scripts/setup_workspace.sh | 13 +++++++ 8 files changed, 198 insertions(+) create mode 100644 .gitmodules create mode 160000 dependencies/tdlib-serializer create mode 160000 dependencies/tdlight create mode 100644 jenkins/docker/dockerfile create mode 100644 jenkins/scripts/generate_tdapi.sh create mode 100755 jenkins/scripts/setup_variables.sh create mode 100755 jenkins/scripts/setup_workspace.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..505926e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "dependencies/tdlight"] + path = dependencies/tdlight + url = https://git.ignuranza.net/tdlight-team/tdlight.git +[submodule "dependencies/tdlib-serializer"] + path = dependencies/tdlib-serializer + url = https://git.ignuranza.net/windoz/tdlib-serializer.git diff --git a/Jenkinsfile b/Jenkinsfile index aaeec08..45d2488 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,6 +17,34 @@ pipeline { defaultValue: false) } stages { + stage("Setup workspace") { + agent none + steps { + sh "mkdir -p \"/var/jenkins_cache/.m2\"" + sh "chown 1000:1000 -R \"/var/jenkins_cache/.m2\"" + sh "mkdir -p \"/var/jenkins_cache/.ccache\"" + sh "chown 1000:1000 -R \"/var/jenkins_cache/.ccache\"" + sh "mkdir -p \"${workspace}/tdlight-java/src/main\"" + //sh "chown 1000:1000 -R \"${workspace}\"" + //sh "chmod 771 -R \"${workspace}\"" + } + } + + stage("Generate TdApi.java") { + agent { + dockerfile { + dir 'jenkins/docker' + filename 'dockerfile' + additionalBuildArgs '--build-arg version=1.0.0 --build-arg UID=1000 --build-arg GID=1000 --build-arg UNAME=jenkins' + args "-v \"${workspace}/src/main:/home/jenkins/output:rw\" -v \"/var/jenkins_cache/.m2:/home/jenkins/.m2:rw\" -v \"/var/jenkins_cache/.ccache:/home/jenkins/.ccache:rw\" -v \"${workspace}:/home/jenkins/work:rw\"" + reuseNode true + } + } + steps { + sh "./jenkins/scripts/generate_tdapi.sh" + } + } + stage("Build & Deploy SNAPSHOT") { agent { docker { @@ -44,6 +72,7 @@ pipeline { steps { sh "git config user.email \"jenkins@mchv.eu\"" sh "git config user.name \"Jenkins\"" + sh "git add --all; git commit -m \"Add generated files\"" sh "mvn -s $MVN_SET -DpushChanges=false -DlocalCheckout=true -DpreparationGoals=initialize release:prepare release:perform -B" } } diff --git a/dependencies/tdlib-serializer b/dependencies/tdlib-serializer new file mode 160000 index 0000000..02e9f27 --- /dev/null +++ b/dependencies/tdlib-serializer @@ -0,0 +1 @@ +Subproject commit 02e9f2796f092637930280e2d075d81bfca0f645 diff --git a/dependencies/tdlight b/dependencies/tdlight new file mode 160000 index 0000000..459e08c --- /dev/null +++ b/dependencies/tdlight @@ -0,0 +1 @@ +Subproject commit 459e08c1b46d67e80777355e011e615f703184c8 diff --git a/jenkins/docker/dockerfile b/jenkins/docker/dockerfile new file mode 100644 index 0000000..184771b --- /dev/null +++ b/jenkins/docker/dockerfile @@ -0,0 +1,62 @@ +FROM debian:stable-slim + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get upgrade -y + +RUN apt-get install wget -y + +RUN wget https://cdn.azul.com/zulu/bin/zulu15.27.17-ca-jdk15.0.0-linux_amd64.deb -O /tmp/install.deb 2>/dev/null + +RUN mkdir -p /usr/share/man/man1 + +RUN apt-get install /tmp/install.deb -y + +RUN rm /tmp/install.deb + +RUN apt-get install maven -y + +RUN apt-get install gcc gperf cmake ccache -y + +RUN apt-get install libssl-dev liblz4-dev libgflags-dev zlib1g-dev -y + +RUN apt-get install build-essential -y + +RUN apt-get install python3 -y + +RUN apt-get install php-cli -y + +RUN apt-get install libreadline-dev git -y + +#RUN ./build.sh + +#ENTRYPOINT [ "/bin/bash" ] + +ARG UNAME=debian +ARG UID=1000 +ARG GID=1000 +RUN groupadd -g $GID -o $UNAME +RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME +USER $UNAME + +RUN mkdir /home/$UNAME/work + +RUN mkdir /home/$UNAME/output + +WORKDIR /home/$UNAME/work + +ENV CCACHE_DIR=/home/$UNAME/.ccache + +ENV JAVA_HOME=/usr/lib/jvm/zulu-15-amd64 + +ENV JAVA_INCLUDE_PATH=/usr/lib/jvm/zulu-15-amd64/include + +ENV TRAVIS_CPU_ARCH=amd64 + +ENV TRAVIS_OS_NAME=linux + +ENV TRAVIS_BUILD_DIR=/home/$UNAME/work + +ENV TRAVIS_OUTPUT_DIR=/home/$UNAME/output + +CMD ["bash"] diff --git a/jenkins/scripts/generate_tdapi.sh b/jenkins/scripts/generate_tdapi.sh new file mode 100644 index 0000000..f79c84f --- /dev/null +++ b/jenkins/scripts/generate_tdapi.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e +# ====== Setup workspace +source ./jenkins/scripts/setup_workspace.sh + +# ====== Prepare tools +cd $TD_CROSS_BIN_DIR +nice -n 5 chrt -b 0 cmake \ + -DTD_ENABLE_JNI=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=${TD_BIN_DIR} \ + -DCMAKE_INSTALL_BINDIR:PATH=${TD_BIN_DIR}/bin \ + $TD_SRC_DIR +nice -n 5 chrt -b 0 cmake --build . --target prepare_cross_compiling --parallel ${TRAVIS_CPU_CORES} +nice -n 5 chrt -b 0 cmake --build . --target td_generate_java_api --parallel ${TRAVIS_CPU_CORES} + +# ====== Build TdApi +cd $TDNATIVES_CPP_BUILD_DIR +nice -n 5 chrt -b 0 cmake -DCMAKE_BUILD_TYPE=Release -DTD_SRC_DIR=${TD_SRC_DIR} -DTD_GENERATED_BINARIES_DIR=${TD_BIN_DIR}/td/generate \ + -DTDNATIVES_BIN_DIR=${TDNATIVES_BIN_DIR} -DTDNATIVES_DOCS_BIN_DIR=${TDNATIVES_DOCS_BIN_DIR} -DJAVA_SRC_DIR=${JAVA_SRC_DIR} -DTDNATIVES_CPP_SRC_DIR:PATH=$TDNATIVES_CPP_SRC_DIR $TDNATIVES_CPP_SRC_DIR +nice -n 5 chrt -b 0 cmake --build $TDNATIVES_CPP_BUILD_DIR --target td_generate_java_api --config Release -- -j${TRAVIS_CPU_CORES} + +# ====== Patch generated java code +echo "Compilation done. Patching TdApi.java" +python3 $TDLIB_SERIALIZER_DIR $JAVA_SRC_DIR/it/tdlight/tdlib/TdApi.java $JAVA_SRC_DIR/it/tdlight/tdlib/new_TdApi.java $TDLIB_SERIALIZER_DIR/headers.txt +rm $JAVA_SRC_DIR/it/tdlight/tdlib/TdApi.java +unexpand --tabs=2 $JAVA_SRC_DIR/it/tdlight/tdlib/new_TdApi.java > $JAVA_SRC_DIR/it/tdlight/tdlib/TdApi.java +rm $JAVA_SRC_DIR/it/tdlight/tdlib/new_TdApi.java diff --git a/jenkins/scripts/setup_variables.sh b/jenkins/scripts/setup_variables.sh new file mode 100755 index 0000000..4e6e390 --- /dev/null +++ b/jenkins/scripts/setup_variables.sh @@ -0,0 +1,59 @@ +#!/bin/bash -e + +# ====== Variables +export TD_SRC_DIR=$TRAVIS_BUILD_DIR/dependencies/tdlight +export TD_BIN_DIR=$TRAVIS_BUILD_DIR/dependencies/tdlight/bin-td-cross +export TDNATIVES_BIN_DIR=$TRAVIS_BUILD_DIR/bin-tdlight +export TDNATIVES_CPP_SRC_DIR=$TRAVIS_BUILD_DIR/tdapi +export TDNATIVES_DOCS_BIN_DIR=$TRAVIS_BUILD_DIR/bin-docs +export TD_BUILD_DIR=$TRAVIS_BUILD_DIR/build-td +export TD_CROSS_BIN_DIR=$TD_SRC_DIR/bin-td-cross +export TDNATIVES_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/tdapi-build +export JAVA_SRC_DIR=$TRAVIS_BUILD_DIR/src/main/java +export TDLIB_SERIALIZER_DIR=$TRAVIS_BUILD_DIR/dependencies/tdlib-serializer +export MAVEN_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/javax.crypto=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED" +TRAVIS_CPU_ARCH_STANDARD="$(tr '[:upper:]' '[:lower:]'<<<"${TRAVIS_CPU_ARCH}")" +export TRAVIS_CPU_ARCH_STANDARD; +export TRAVIS_CPU_CORES="2" +TRAVIS_OS_NAME_STANDARD="$(tr '[:upper:]' '[:lower:]'<<<"${TRAVIS_OS_NAME}")" +export TRAVIS_OS_NAME_STANDARD +export TRAVIS_OS_NAME_SHORT=$TRAVIS_OS_NAME_STANDARD +export SRC_TDJNI_LIBNAME="libtdjni.so" +export DEST_TDJNI_LIBNAME="tdjni.so" + +# ====== OS Variables +TRAVIS_CPU_ARCH_JAVA="$(tr '[:upper:]' '[:lower:]'<<<"${TRAVIS_CPU_ARCH}")" +export TRAVIS_CPU_ARCH_JAVA +export TDLIB_RESOURCES_OUTPUT_DIR="$TRAVIS_OUTPUT_DIR/resources/libs/$TRAVIS_OS_NAME_SHORT/$TRAVIS_CPU_ARCH_STANDARD" + +# ====== Print variables +echo "TD_SRC_DIR=${TD_SRC_DIR}" +echo "TD_BIN_DIR=${TD_BIN_DIR}" +echo "TDNATIVES_BIN_DIR=${TDNATIVES_BIN_DIR}" +echo "TDNATIVES_CPP_SRC_DIR=${TDNATIVES_CPP_SRC_DIR}" +echo "TDNATIVES_DOCS_BIN_DIR=${TDNATIVES_DOCS_BIN_DIR}" +echo "TD_BUILD_DIR=${TD_BUILD_DIR}" +echo "TD_CROSS_BIN_DIR=${TD_CROSS_BIN_DIR}" +echo "TDNATIVES_CPP_BUILD_DIR=${TDNATIVES_CPP_BUILD_DIR}" +echo "JAVA_SRC_DIR=${JAVA_SRC_DIR}" +echo "TDLIB_SERIALIZER_DIR=${TDLIB_SERIALIZER_DIR}" +echo "PATH=${PATH}" +echo "JAVA_HOME=${JAVA_HOME}" +echo "JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}" +echo "JAVA_INCLUDE_PATH2=${JAVA_INCLUDE_PATH2}" +echo "JAVA_AWT_INCLUDE_PATH=${JAVA_AWT_INCLUDE_PATH}" +echo "VCPKG_DIR=${VCPKG_DIR}" +echo "MAVEN_OPTS=${MAVEN_OPTS}" +echo "TRAVIS_CPU_ARCH=${TRAVIS_CPU_ARCH}" +echo "TRAVIS_CPU_ARCH_JAVA=${TRAVIS_CPU_ARCH_JAVA}" +echo "TRAVIS_CPU_ARCH_STANDARD=${TRAVIS_CPU_ARCH_STANDARD}" +echo "TRAVIS_OS_NAME=${TRAVIS_OS_NAME}" +echo "TRAVIS_OS_NAME_STANDARD=${TRAVIS_OS_NAME_STANDARD}" +echo "TRAVIS_OS_NAME_SHORT=${TRAVIS_OS_NAME_SHORT}" +echo "SRC_TDJNI_LIBNAME=${SRC_TDJNI_LIBNAME}" +echo "DEST_TDJNI_LIBNAME=${DEST_TDJNI_LIBNAME}" +echo "TDLIB_RESOURCES_OUTPUT_DIR=${TDLIB_RESOURCES_OUTPUT_DIR}" +echo "CURRENT DIRECTORY* ($(pwd)):" +ls -alch "$(pwd)" +echo "WORK DIRECTORY CONTENT* (${TRAVIS_BUILD_DIR}):" +ls -alch "$TRAVIS_BUILD_DIR" diff --git a/jenkins/scripts/setup_workspace.sh b/jenkins/scripts/setup_workspace.sh new file mode 100755 index 0000000..53b3cb9 --- /dev/null +++ b/jenkins/scripts/setup_workspace.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e +# ====== Setup environment variables +source ./jenkins/scripts/setup_variables.sh + +# ====== Environment setup +[ -d $TD_BUILD_DIR ] || mkdir -p $TD_BUILD_DIR +[ -d $TD_BIN_DIR ] || mkdir -p $TD_BIN_DIR +[ -d $TD_NATIVE_BIN_DIR ] || mkdir -p $TD_NATIVE_BIN_DIR +[ -d $TD_CROSS_BIN_DIR ] || mkdir -p $TD_CROSS_BIN_DIR +[ -d $TDNATIVES_CPP_BUILD_DIR ] || mkdir -p $TDNATIVES_CPP_BUILD_DIR + +echo "CCACHE statistics:" +ccache -s