ci(ct): detect necessary rebuilds and calculate revision number #11
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Container Images Scheduled Maintenance | |
| on: | |
| # TODO: remove "push" in final PR | |
| push: | |
| branches: | |
| - '10478-version-base-img' | |
| paths: | |
| - 'modules/container-base/**' | |
| - 'modules/dataverse-parent/pom.xml' | |
| - '.github/workflows/container_maintenance.yml' | |
| # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC | |
| env: | |
| PLATFORMS: linux/amd64,linux/arm64 | |
| NUM_PAST_RELEASES: 3 | |
| # TODO: change to "develop" in final PR | |
| DEVELOP_BRANCH: 10478-version-base-img | |
| jobs: | |
| discover: | |
| name: Discover Release Matrix | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| # TODO: re-enable for final PR | |
| # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled | |
| #if: ${{ github.repository_owner == 'IQSS' }} | |
| outputs: | |
| branches: ${{ steps.matrix.outputs.branches }} | |
| current_release: ${{ steps.matrix.outputs.current_release }} | |
| steps: | |
| - name: Build branch matrix options | |
| id: matrix | |
| run: | | |
| echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ | |
| jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "${{ env.DEVELOP_BRANCH }}" ]')" | tr -d "\n" | tr -s " " | \ | |
| tee -a "$GITHUB_OUTPUT" | |
| echo "current_release=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '.[0].tag_name' )" | tee -a "$GITHUB_OUTPUT" | |
| build: | |
| name: Build image | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| needs: discover | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| branch: ${{ fromJson(needs.discover.outputs.branches) }} | |
| # TODO: re-enable for final PR | |
| # Only run in upstream repo - avoid unnecessary runs in forks | |
| #if: ${{ github.repository_owner == 'IQSS' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ matrix.branch }} | |
| - name: Determine Java version from Parent POM | |
| run: | | |
| echo "JAVA_VERSION=$(grep '<target.java.version>' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} | |
| - name: Set up JDK ${{ env.JAVA_VERSION }} | |
| id: setup-java | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ env.JAVA_VERSION }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| cache-dependency-path: | | |
| modules/container-base/pom.xml | |
| - name: Download common cache on branch cache miss | |
| if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: dataverse-maven-cache | |
| path: ~/.m2/repository | |
| # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and | |
| # on events in context of upstream because secrets. PRs run in context of forks by default! | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up QEMU for multi-arch builds | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: ${{ env.PLATFORMS }} | |
| # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) | |
| # and try to apply them if present | |
| - name: Get and apply backported patches | |
| # There might be no patches - ignore errors | |
| continue-on-error: true | |
| run: | | |
| mkdir -p "${GITHUB_WORKSPACE}/patches" | |
| curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ | |
| tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 | |
| find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i | |
| # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates | |
| - name: Check for recent Temurin image updates | |
| id: temurin-check | |
| run: | | |
| JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" | |
| JAVA_IMAGE_NS="library" | |
| JAVA_IMAGE_REPO="$( echo "$JAVA_IMAGE" | cut -f1 -d: )" | |
| JAVA_IMAGE_TAG="$( echo "$JAVA_IMAGE" | cut -f2 -d: )" | |
| JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" | |
| if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then | |
| echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" | |
| exit 1 | |
| fi | |
| BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | |
| BASE_IMAGE_NS="$( echo "$BASE_IMAGE" | cut -f1 -d/ )" | |
| BASE_IMAGE_REPO="$( echo "$BASE_IMAGE" | cut -f1 -d: | cut -f2 -d/ )" | |
| BASE_IMAGE_TAG="$( echo "$BASE_IMAGE" | cut -f2 -d: )" | |
| BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" | |
| if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then | |
| echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" | |
| echo "newer_java_image=true" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "Java image $JAVA_IMAGE ($JAVA_IMAGE_LAST_UPDATE) is older than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" | |
| echo "newer_java_image=false" >> "${GITHUB_OUTPUT}" | |
| fi | |
| # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates | |
| - name: Check for package updates in base image | |
| id: package-check | |
| if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} | |
| run: | | |
| BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | |
| PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" | |
| if [[ ! $( docker run -it --rm -u 0 "${BASE_IMAGE}" sh -c "apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then | |
| echo "Base image $BASE_IMAGE needs package updates" | |
| echo "newer_packages=true" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "Base image $BASE_IMAGE has no package updates" | |
| echo "newer_packages=false" >> "${GITHUB_OUTPUT}" | |
| fi | |
| - name: Calculate revision number for immutable tag | |
| run: | | |
| BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | |
| BASE_IMAGE_NS_REPO="$( echo "$BASE_IMAGE" | cut -d: -f1 )" | |
| BASE_IMAGE_TAG=""$( echo "$BASE_IMAGE" | cut -d: -f2 )"" | |
| function get_all_tags() { | |
| ref="$1" | |
| case "$ref" in | |
| */*) :;; # namespace/repository syntax, leave as is | |
| *) ref="library/$ref";; # bare repository name (docker official image); must convert to namespace/repository syntax | |
| esac | |
| token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ref}:pull" | jq -r '.token' ) | |
| i=0 | |
| while [ $? == 0 ]; do | |
| i=$((i+1)) | |
| curl -sS -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/${ref}/tags/?page=$i&page_size=100" | jq -r '."results"[]["name"]' 2>/dev/null | |
| done | |
| } | |
| CURRENT=$( get_all_tags "${BASE_IMAGE_NS_REPO}" | grep "${BASE_IMAGE_TAG}-r" | sed -e "s#${BASE_IMAGE_TAG}-r##" | sort -h | tail -n1 ) | |
| # If there is a current number, increment it - otherwise this is the initial version, set to 0 | |
| if [[ "$CURRENT" ]]; then | |
| echo "REVISION_OPTION=-Dbase.image.revision=$((CURRENT+1))" | tee -a "${GITHUB_ENV}" | |
| else | |
| echo "REVISION_OPTION=-Dbase.image.revision=0" | tee -a "${GITHUB_ENV}" | |
| fi | |
| - name: Configure update of "latest" tag for development branch | |
| if: ${{ matrix.branch == env.DEVELOP_BRANCH }} | |
| run: | | |
| echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" | |
| - name: Deploy multi-arch base container image to Docker Hub | |
| if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' }} | |
| id: build | |
| run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} | |
| # - if: always() | |
| # name: Save status (workaround for matrix outputs) | |
| # run: | | |
| # # steps.build.outcome is the status BEFORE continue-on-error | |
| # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" | |
| #push-app-img: | |
| # name: "Rebase & Publish App Image" | |
| # permissions: | |
| # contents: read | |
| # packages: write | |
| # pull-requests: write | |
| # secrets: inherit | |
| # needs: | |
| # - discover | |
| # - build | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # branch: ${{ fromJson(needs.discover.outputs.branches) }} | |
| # uses: ./.github/workflows/container_app_push.yml | |
| # with: | |
| # branch: ${{ matrix.branch }} | |
| # TODO: job to update the docker hub description with supported tags and all | |
| # - name: Push description to DockerHub | |
| # uses: peter-evans/dockerhub-description@v3 | |
| # with: | |
| # username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| # password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # repository: gdcc/base | |
| # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" | |
| # readme-filepath: ./modules/container-base/README.md |