Container Images Scheduled Maintenance #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Container Images Scheduled Maintenance | |
| on: | |
| # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) | |
| workflow_dispatch: | |
| inputs: | |
| force_build: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: "Build and deploy even if no newer Java images or package updates are found." | |
| schedule: | |
| - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC | |
| env: | |
| PLATFORMS: linux/amd64,linux/arm64 | |
| NUM_PAST_RELEASES: 3 | |
| # TODO: change to "develop" in final PR | |
| DEVELOP_BRANCH: 10478-version-base-img | |
| jobs: | |
| discover: | |
| name: Discover Release Matrix | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| # TODO: re-enable for final PR | |
| # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled | |
| #if: ${{ github.repository_owner == 'IQSS' }} | |
| outputs: | |
| branches: ${{ steps.matrix.outputs.branches }} | |
| current_release: ${{ steps.matrix.outputs.current_release }} | |
| steps: | |
| - name: Build branch matrix options | |
| id: matrix | |
| run: | | |
| echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ | |
| jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "${{ env.DEVELOP_BRANCH }}" ]')" | tr -d "\n" | tr -s " " | \ | |
| tee -a "$GITHUB_OUTPUT" | |
| echo "current_release=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '.[0].tag_name' )" | tee -a "$GITHUB_OUTPUT" | |
| build: | |
| name: Build image | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| needs: discover | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| branch: ${{ fromJson(needs.discover.outputs.branches) }} | |
| # TODO: re-enable for final PR | |
| # Only run in upstream repo - avoid unnecessary runs in forks | |
| #if: ${{ github.repository_owner == 'IQSS' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ matrix.branch }} | |
| - name: Determine Java version from Parent POM | |
| run: | | |
| echo "JAVA_VERSION=$(grep '<target.java.version>' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} | |
| - name: Set up JDK ${{ env.JAVA_VERSION }} | |
| id: setup-java | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ env.JAVA_VERSION }} | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| cache-dependency-path: | | |
| modules/container-base/pom.xml | |
| - name: Download common cache on branch cache miss | |
| if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: dataverse-maven-cache | |
| path: ~/.m2/repository | |
| # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and | |
| # on events in context of upstream because secrets. PRs run in context of forks by default! | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up QEMU for multi-arch builds | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: ${{ env.PLATFORMS }} | |
| # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) | |
| # and try to apply them if present | |
| - name: Get and apply backported patches | |
| # There might be no patches - ignore errors | |
| continue-on-error: true | |
| run: | | |
| mkdir -p "${GITHUB_WORKSPACE}/patches" | |
| curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ | |
| tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 | |
| find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i | |
| # Determine the base image name we are going to use from here on | |
| - name: Determine base image name | |
| run: | | |
| if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then | |
| NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) | |
| else | |
| NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag='${base.image.tag.release}' -q -DforceStdout ) | |
| fi | |
| echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" | |
| # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates | |
| - name: Check for recent Temurin image updates | |
| id: temurin-check | |
| run: | | |
| JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" | |
| JAVA_IMAGE_NS="library" | |
| JAVA_IMAGE_REPO="${JAVA_IMAGE%:*}" | |
| JAVA_IMAGE_TAG="${JAVA_IMAGE#*:}" | |
| JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" | |
| if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then | |
| echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" | |
| exit 1 | |
| fi | |
| BASE_IMAGE_NS="${BASE_IMAGE%/*}" | |
| BASE_IMAGE_REPO="$( echo "${BASE_IMAGE%:*}" | cut -f2 -d/ )" | |
| BASE_IMAGE_TAG="${BASE_IMAGE#*:}" | |
| BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" | |
| if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then | |
| echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" | |
| echo "newer_java_image=true" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "Java image $JAVA_IMAGE ($JAVA_IMAGE_LAST_UPDATE) is older than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" | |
| echo "newer_java_image=false" >> "${GITHUB_OUTPUT}" | |
| fi | |
| # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates | |
| - name: Check for package updates in base image | |
| id: package-check | |
| if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} | |
| run: | | |
| PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" | |
| if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then | |
| echo "Base image $BASE_IMAGE needs package updates" | |
| echo "newer_packages=true" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "Base image $BASE_IMAGE has no package updates" | |
| echo "newer_packages=false" >> "${GITHUB_OUTPUT}" | |
| fi | |
| - name: Calculate revision number for immutable tag (on release branches only) | |
| if: ${{ matrix.branch != env.DEVELOP_BRANCH }} | |
| run: | | |
| # Get the revision logic script (not present on older releases) | |
| curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ | |
| tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 | |
| # Now get the new revision number | |
| REVISION_TAG="'\${base.image.tag}-$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )'" | |
| echo "DOCKER_TAGS=-Dbase.image.tag='\${base.image.tag.release}' -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=${REVISION_TAG}" | tee -a "${GITHUB_ENV}" | |
| - name: Configure update of "latest" tag for development branch | |
| if: ${{ matrix.branch == env.DEVELOP_BRANCH }} | |
| run: | | |
| echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" | |
| - name: Deploy multi-arch base container image to Docker Hub | |
| if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} | |
| id: build | |
| run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} | |
| # - if: always() | |
| # name: Save status (workaround for matrix outputs) | |
| # run: | | |
| # # steps.build.outcome is the status BEFORE continue-on-error | |
| # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" | |
| #push-app-img: | |
| # name: "Rebase & Publish App Image" | |
| # permissions: | |
| # contents: read | |
| # packages: write | |
| # pull-requests: write | |
| # secrets: inherit | |
| # needs: | |
| # - discover | |
| # - build | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # branch: ${{ fromJson(needs.discover.outputs.branches) }} | |
| # uses: ./.github/workflows/container_app_push.yml | |
| # with: | |
| # branch: ${{ matrix.branch }} | |
| # TODO: job to update the docker hub description with supported tags and all | |
| # - name: Push description to DockerHub | |
| # uses: peter-evans/dockerhub-description@v3 | |
| # with: | |
| # username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| # password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # repository: gdcc/base | |
| # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" | |
| # readme-filepath: ./modules/container-base/README.md |