From 5a97d4410adc1101affc9c8ef2496930cd6b440e Mon Sep 17 00:00:00 2001 From: Norman Maurer Date: Thu, 11 Feb 2021 20:13:56 +0100 Subject: [PATCH] Release workflow --- .github/scripts/merge_local_staging.sh | 20 +++ .github/scripts/release_checkout_tag.sh | 13 ++ .github/scripts/release_rollback.sh | 14 ++ .github/workflows/ci-pr.yml | 2 +- .github/workflows/ci-release.yml | 210 ++++++++++++++++++++++++ .github/workflows/codeql-analysis.yml | 2 +- docker/docker-compose.centos-6.18.yaml | 3 + docker/docker-compose.centos-7.yaml | 13 ++ docker/docker-compose.yaml | 13 ++ 9 files changed, 288 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/merge_local_staging.sh create mode 100755 .github/scripts/release_checkout_tag.sh create mode 100755 .github/scripts/release_rollback.sh create mode 100644 .github/workflows/ci-release.yml diff --git a/.github/scripts/merge_local_staging.sh b/.github/scripts/merge_local_staging.sh new file mode 100755 index 0000000000..a008da0aee --- /dev/null +++ b/.github/scripts/merge_local_staging.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e +if [ "$#" -lt 2 ]; then + echo "Expected target directory and at least one local staging directory" + exit 1 +fi +TARGET=$1 + +for ((i=2; i<=$#; i++)) +do + DIR="${!i}" + SUB_DIR=$(ls -d "${DIR}"/* | awk -F / '{print $NF}') + + if [ ! -d "${TARGET}/${SUB_DIR}" ] + then + mkdir -p "${TARGET}/${SUB_DIR}" + fi + cat "${DIR}"/"${SUB_DIR}"/.index >> "${TARGET}/${SUB_DIR}"/.index + cp -r "${DIR}"/"${SUB_DIR}"/* "${TARGET}/${SUB_DIR}"/ +done diff --git a/.github/scripts/release_checkout_tag.sh b/.github/scripts/release_checkout_tag.sh new file mode 100755 index 0000000000..b87e4b8857 --- /dev/null +++ b/.github/scripts/release_checkout_tag.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +if [ "$#" -ne 1 ]; then + echo "Expected release.properties file" + exit 1 +fi + +TAG=$(grep scm.tag= "$1" | cut -d'=' -f2) + +echo "Checkout tag $TAG" +git checkout "$TAG" +exit 0 diff --git a/.github/scripts/release_rollback.sh b/.github/scripts/release_rollback.sh new file mode 100755 index 0000000000..c0b1deb643 --- /dev/null +++ b/.github/scripts/release_rollback.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +if [ "$#" -ne 3 ]; then + echo "Expected release.properties file, repository name and branch" + exit 1 +fi + +TAG=$(grep scm.tag= "$1" | cut -d'=' -f2) +git remote set-url origin git@github.com:"$2".git +git fetch +git checkout "$3" +mvn -B --file pom.xml release:rollback +git push origin :"$TAG" diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index a9c51b4ced..247c678572 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -2,7 +2,7 @@ name: Build PR on: pull_request: - branches: [ "4.1"] + branches: [ "4.1_copy"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml new file mode 100644 index 0000000000..cbc8664cd6 --- /dev/null +++ b/.github/workflows/ci-release.yml @@ -0,0 +1,210 @@ +name: Release + +on: + + pull_request: + branches: [ "4.1"] + + # Releases can only be triggered via the action tab + workflow_dispatch: + +jobs: + prepare-release: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + ref: main + + - name: Set up JDK 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + + - name: Setup git configuration + run: | + git config --global user.email "netty-project-bot@users.noreply.github.com" + git config --global user.name "Netty Project Bot" + + - name: Install SSH key + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_PRIVATE_KEY_PEM }} + known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} + + # Cache .m2/repository + - uses: actions/cache@v2 + env: + cache-name: release-cache-m2-repository + with: + path: ~/.m2/repository + key: ${{ runner.os }}-pr-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-pr-${{ env.cache-name }}- + ${{ runner.os }}-pr- + + - name: Prepare release with Maven + run: | + mvn -DpreparationGoals=clean release:prepare -B --file pom.xml -DskipTests=true + mvn clean + + - name: Checkout tag + run: ./.github/scripts/release_checkout_tag.sh release.properties + + - name: Upload workspace + uses: actions/upload-artifact@v2 + with: + name: prepare-release-workspace + path: ${{ github.workspace }}/** + + stage-release-linux: + runs-on: ubuntu-latest + needs: prepare-release + strategy: + matrix: + include: + - setup: linux-x86_64-java8 + docker-compose-build: "-f docker/docker-compose.yaml -f docker/docker-compose.centos-6.18.yaml build" + docker-compose-run: "-f docker/docker-compose.yaml -f docker/docker-compose.centos-6.18.yaml run stage-release" + - setup: linux-aarch64 + docker-compose-build: "-f docker/docker-compose.centos-7.yaml build" + docker-compose-run: "-f docker/docker-compose.centos-7.yaml run cross-compile-aarch64-stage-release" + + name: stage-release-${{ matrix.setup }} + + steps: + - name: Download release-workspace + uses: actions/download-artifact@v2 + with: + name: prepare-release-workspace + path: ./prepare-release-workspace/ + + - name: Set up JDK 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + + - name: Setup git configuration + run: | + git config --global user.email "netty-project-bot@users.noreply.github.com" + git config --global user.name "Netty Project Bot" + + - name: Install SSH key + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_PRIVATE_KEY_PEM }} + known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} + + # Enable caching of Docker layers + - uses: satackey/action-docker-layer-caching@v0.0.11 + env: + docker-cache-name: staging-${{ matrix.setup }}-cache-docker + continue-on-error: true + with: + key: ${{ runner.os }}-staging-${{ env.docker-cache-name }}-{hash} + restore-keys: | + ${{ runner.os }}-staging-${{ env.docker-cache-name }}- + + - uses: s4u/maven-settings-action@v2.2.0 + with: + servers: | + [{ + "id": "sonatype-nexus-staging", + "username": "${{ secrets.SONATYPE_USERNAME }}", + "password": "${{ secrets.SONATYPE_PASSWORD }}" + }] + + - name: Create local staging directory + run: mkdir -p ~/local-staging + + - name: Build docker image + working-directory: ./prepare-release-workspace/ + run: docker-compose ${{ matrix.docker-compose-build }} + + - name: Stage release to local staging directory + working-directory: ./prepare-release-workspace/ + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_KEYNAME: ${{ secrets.GPG_KEYNAME }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: docker-compose ${{ matrix.docker-compose-run }} + + - name: Upload local staging directory + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.setup }}-local-staging + path: ~/local-staging + if-no-files-found: error + + - name: Rollback release on failure + working-directory: ./prepare-release-workspace/ + if: ${{ failure() }} + # Rollback the release in case of an failure + run: bash ./.github/scripts/release_rollback.sh release.properties netty/netty-tcnative main + + deploy-staged-release: + runs-on: ubuntu-18.04 + # Wait until we have staged everything + needs: stage-release-linux + steps: + - name: Download release-workspace + uses: actions/download-artifact@v2 + with: + name: prepare-release-workspace + path: ./prepare-release-workspace/ + + - name: Set up JDK 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + + - name: Setup git configuration + run: | + git config --global user.email "netty-project-bot@users.noreply.github.com" + git config --global user.name "Netty Project Bot" + + - name: Install SSH key + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_PRIVATE_KEY_PEM }} + known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} + + # Hardcode the staging artifacts that need to be downloaded. + # These must match the matrix setups. There is currently no way to pull this out of the config. + - name: Download linux-aarch64 staging directory + uses: actions/download-artifact@v2 + with: + name: linux-aarch64-local-staging + path: ~/linux-aarch64-local-staging + + - name: Download linux-x86_64-java8 staging directory + uses: actions/download-artifact@v2 + with: + name: linux-x86_64-java8-local-staging + path: ~/linux-x86_64-java8-local-staging + + # This step takes care of merging all the previous staged repositories in a way that will allow us to deploy + # all together with one maven command. + - name: Merge staging repositories + working-directory: ./prepare-release-workspace/ + run: bash ./.github/scripts/merge_local_staging.sh /home/runner/local-staging/staging ~/linux-aarch64-local-staging/staging ~/linux-x86_64-java8-local-staging/staging + + - uses: s4u/maven-settings-action@v2.2.0 + with: + servers: | + [{ + "id": "sonatype-nexus-staging", + "username": "${{ secrets.SONATYPE_USERNAME }}", + "password": "${{ secrets.SONATYPE_PASSWORD }}" + }] + + - name: Deploy local staged artifacts + working-directory: ./prepare-release-workspace/ + # If we don't want to close the repository we can add -DskipStagingRepositoryClose=true + run: mvn -B --file pom.xml org.sonatype.plugins:nexus-staging-maven-plugin:deploy-staged -DnexusUrl=https://oss.sonatype.org -DserverId=sonatype-nexus-staging -DaltStagingDirectory=/home/runner/local-staging -DskipStagingRepositoryClose=true + + - name: Rollback release on failure + working-directory: ./prepare-release-workspace/ + if: ${{ failure() }} + # Rollback the release in case of an failure + run: bash ./.github/scripts/release_rollback.sh release.properties netty/netty-tcnative main diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3d3eddc83e..abc7714fc4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -10,7 +10,7 @@ on: branches: ["4.1", master] pull_request: # The branches below must be a subset of the branches above - branches: ["4.1", master] + branches: ["4.1_copy", master] schedule: - cron: '0 13 * * 3' diff --git a/docker/docker-compose.centos-6.18.yaml b/docker/docker-compose.centos-6.18.yaml index e522f6348b..5e290fbebe 100644 --- a/docker/docker-compose.centos-6.18.yaml +++ b/docker/docker-compose.centos-6.18.yaml @@ -23,6 +23,9 @@ services: stage-snapshot: image: netty:centos-6-1.8 + stage-release: + image: netty:centos-6-1.8 + deploy: image: netty:centos-6-1.8 diff --git a/docker/docker-compose.centos-7.yaml b/docker/docker-compose.centos-7.yaml index c0b55bd6d2..e730a19212 100644 --- a/docker/docker-compose.centos-7.yaml +++ b/docker/docker-compose.centos-7.yaml @@ -36,6 +36,19 @@ services: - ..:/code command: /bin/bash -cl "./mvnw -Plinux-aarch64 -pl transport-native-unix-common,transport-native-epoll -am clean package org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true" + cross-compile-aarch64-stage-release: + <<: *cross-compile-aarch64-common + environment: + - GPG_KEYNAME + - GPG_PASSPHRASE + - GPG_PRIVATE_KEY + volumes: + - ~/.ssh:/root/.ssh + - ~/.m2:/root/.m2 + - ~/local-staging:/root/local-staging + - ..:/code + command: /bin/bash -cl "cat <(echo -e \"${GPG_PRIVATE_KEY}\") | gpg --batch --import && ./mvnw -B -Plinux-aarch64 -pl transport-native-unix-common,transport-native-epoll -am clean javadoc:jar package gpg:sign org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DnexusUrl=https://oss.sonatype.org -DserverId=sonatype-nexus-staging -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true -Dgpg.passphrase=${GPG_PASSPHRASE} -Dgpg.keyname=${GPG_KEYNAME}" + cross-compile-aarch64-shell: <<: *cross-compile-aarch64-common entrypoint: /bin/bash diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index e516a26bc4..1dfe765495 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -40,6 +40,19 @@ services: - ..:/code command: /bin/bash -cl "./mvnw clean package org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true" + stage-release: + <<: *common + environment: + - GPG_KEYNAME + - GPG_PASSPHRASE + - GPG_PRIVATE_KEY + volumes: + - ~/.ssh:/root/.ssh + - ~/.m2:/root/.m2 + - ~/local-staging:/root/local-staging + - ..:/code + command: /bin/bash -cl "cat <(echo -e \"${GPG_PRIVATE_KEY}\") | gpg --batch --import && ./mvnw clean javadoc:jar package gpg:sign org.sonatype.plugins:nexus-staging-maven-plugin:deploy -DnexusUrl=https://oss.sonatype.org -DserverId=sonatype-nexus-staging -DaltStagingDirectory=/root/local-staging -DskipRemoteStaging=true -DskipTests=true -Dgpg.passphrase=${GPG_PASSPHRASE} -Dgpg.keyname=${GPG_KEYNAME}" + build-boringssl-static: <<: *common command: /bin/bash -cl "./mvnw -P boringssl clean install -Dio.netty.testsuite.badHost=netty.io -Dxml.skip=true"