Updates from aiidalab stack

.docker/docker-bake.hcl

@@ -13,7 +13,7 @@ variable "ORGANIZATION" {
 }
 
 variable "REGISTRY" {
-  default = "docker.io/"
+  default = "ghcr.io/"
 }
 
 variable "PLATFORMS" {

.github/workflows/docker-build.yml

@@ -1,8 +1,6 @@
-name: Build image and then upload the image, tags and manifests to GitHub artifacts
+name: Build Docker images and upload them to ghcr.io
 
 env:
-  OWNER: ${{ github.repository_owner }}
-  # Full logs for CI build
   BUILDKIT_PROGRESS: plain
 
 on:
@@ -12,24 +10,36 @@ on:
         description: GitHub Actions Runner image
         required: true
         type: string
+      platforms:
+        description: Target platforms for the build (linux/amd64 and/or linux/arm64)
+        required: true
+        type: string
+    outputs:
+      images:
+        description: Images identified by digests
+        value: ${{ jobs.build.outputs.images }}
 
 jobs:
   build:
+    name: ${{ inputs.platforms }}
     runs-on: ${{ inputs.runsOn }}
+    timeout-minutes: 60
     defaults:
       run:
-        shell: bash
+            # Make sure we fail if any command in a piped command sequence fails
+        shell: bash -e -o pipefail {0}
         working-directory: .docker
 
+    outputs:
+      images: ${{ steps.bake_metadata.outputs.images }}
+
     steps:
-    - name: Checkout Repo
-      uses: actions/checkout@v4
 
-    - name: Build aiida-core images for amd64
-      # The order of the buildx bake files is important, as the second one will overwrite the first one
-      run: docker buildx bake -f docker-bake.hcl -f build.json --set *.platform=linux/$amd64 --load
+    - name: Checkout Repo ⚡️
+      uses: actions/checkout@v4
 
     - name: Set up QEMU
+      if: ${{ inputs.platforms != 'linux/amd64' }}
       uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
@@ -38,50 +48,30 @@ jobs:
     - name: Login to GitHub Container Registry 🔑
       uses: docker/login-action@v3
       with:
-        registry: ${{ env.REGISTRY }}
+        registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    # TODO: Separate amd64 build just just for testing now
-    - name: Build amd64 images 🏗️
-      id: build-amd64
+    - name: Build and upload to ghcr.io 📤
+      id: build
       uses: docker/bake-action@v4
       with:
         push: true
-        set: |
-          *.platform=linux/amd64
-        files: |
-          docker-bake.hcl
-          build.json
-
-    - name: Set output variables
-      run: .github/workflows/extract-docker-image-names.sh >> "${GITHUB_OUTPUT}"
-      env:
-        BAKE_METADATA: ${{ steps.build-amd64.outputs.metadata }}
-
-    # Here we build arm64 images (with help of QEMU virtualization)
-    # and upload both amd64 and arm64 images to ghcr.io
-    - name: Build ARM64 and upload to ghcr.io 🍎📤
-      id: build-upload
-      if: false
-      uses: docker/bake-action@v4
-      with:
-          # TODO: Actually push, add tags first though!
-        load: true
-        push: false
-          # Using provenance to disable default attestation so it will build only desired images:
-          # https://github.com/orgs/community/discussions/45969
+                  # Using provenance to disable default attestation so it will build only desired images:
+                  # https://github.com/orgs/community/discussions/45969
         provenance: false
-          # NOTE: linux/amd64 images are taken from previous step
         set: |
-          *.platform=linux/amd64,linux/arm64
+          *.platform=${{ inputs.platforms }}
+          *.output=type=registry,push-by-digest=true,name-canonical=true
+          *.cache-to=type=gha,scope=${{ github.workflow }},mode=max
+          *.cache-from=type=gha,scope=${{ github.workflow }}
         files: |
           docker-bake.hcl
           build.json
 
     - name: Set output variables
-      if: false
       id: bake_metadata
-      run: .github/workflows/extract-docker-image-names.sh >> "${GITHUB_OUTPUT}"
+      run: |
+        .github/workflows/extract-docker-image-names.sh | tee -a "${GITHUB_OUTPUT}"
       env:
-        BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }}
+        BAKE_METADATA: ${{ steps.build.outputs.metadata }}

.github/workflows/docker-publish.yml

@@ -1,7 +1,8 @@
-name: Publish images to DockerHub
+name: Publish images to Docker container registries
 
 env:
-  OWNER: ${{ github.repository_owner }}
+    # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables
+  DOCKER_METADATA_PR_HEAD_SHA: true
 
 on:
   workflow_call:
@@ -14,25 +15,23 @@ on:
         description: Images built in build step
         required: true
         type: string
+      registry:
+        description: Docker container registry
+        required: true
+        type: string
 
 jobs:
-  tag-push:
-    runs-on: ubuntu-latest
+
+  release:
+    runs-on: ${{ inputs.runsOn }}
     timeout-minutes: 30
     strategy:
       fail-fast: true
       matrix:
-        image: [aiida-core-base, aiida-core-with-services, aiida-core-dev]
-    defaults:
-      run:
-        shell: bash
-        working-directory: .docker
-    permissions:
-      packages: read
+        target: [aiida-core-base, aiida-core-with-services, aiida-core-dev]
 
     steps:
-    - name: Checkout Repo ⚡️
-      uses: actions/checkout@v4
+    - uses: actions/checkout@v4
 
     - name: Login to GitHub Container Registry 🔑
       uses: docker/login-action@v3
@@ -43,6 +42,7 @@ jobs:
 
     - name: Login to DockerHub 🔑
       uses: docker/login-action@v3
+      if: inputs.registry == 'docker.io'
       with:
         registry: docker.io
         username: ${{ secrets.DOCKER_USERNAME }}
@@ -52,33 +52,31 @@ jobs:
       id: build_vars
       run: |
         vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries')
-        echo "vars=$vars"
-        echo "vars=$vars" >> "${GITHUB_OUTPUT}"
+        echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}"
 
-    - name: Docker meta 📝
+    - name: Docker meta
       id: meta
       uses: docker/metadata-action@v5
-      env: ${{ fromJson(steps.build_vars.outputs.vars) }}
+      env: ${{ fromJSON(steps.build_vars.outputs.vars) }}
       with:
-        images: |
-          name=docker.io/${{ env.OWNER }}/${{ matrix.image }}
+                  # e.g. ghcr.io/aiidalab/full-stack
+        images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ matrix.target }}
         tags: |
+          type=ref,event=pr
           type=edge,enable={{is_default_branch}}
-          type=match,pattern=v(\d+\.\d+.\d+),group=1
-          type=raw,value={{tag}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
-          type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
-          type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+          type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ github.ref_type == 'tag'  && startsWith(github.ref_name, 'v') }}
+          type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}
+          type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}
+          type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1
 
-    - name: Determine src image tag
+    - name: Determine source image
       id: images
       run: |
         src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]')
-        echo "src=$src"
-        echo "src=$src" >> "${GITHUB_OUTPUT}"
+        echo "src=$src" | tee -a "${GITHUB_OUTPUT}"
 
-    - name: Push image to docker.io
+    - name: Push image
       uses: akhilerm/[email protected]
-      if: false # TODO: Enable this!
       with:
         src: ${{ steps.images.outputs.src }}
         dst: ${{ steps.meta.outputs.tags }}

.github/workflows/docker-test.yml

@@ -11,15 +11,16 @@ on:
         description: Images built in build step
         required: true
         type: string
+      target:
+        description: Target image for testing
+        required: true
+        type: string
 
 jobs:
 
   test:
     runs-on: ${{ inputs.runsOn }}
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        target: [aiida-core-base, aiida-core-with-services, aiida-core-dev]
+    timeout-minutes: 20
     defaults:
       run:
         shell: bash
@@ -38,7 +39,7 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Set Up Python 🐍
-      if: ${{ inputs.runsOn != 'ARM64' }}
+      if: ${{ startsWith(inputs.runsOn, 'ubuntu') }}
       uses: actions/setup-python@v5
       with:
         python-version: '3.11'
@@ -50,5 +51,6 @@ jobs:
         pip freeze
 
     - name: Run tests
+              # TODO: This probably needs tweaking
       run: pytest -s
       env: ${{ fromJSON(inputs.images) }}

.github/workflows/docker.yml

@@ -3,6 +3,8 @@ name: Build, test and push Docker Images
 on:
   pull_request:
     paths-ignore:
+    - '**.md'
+    - '**.txt'
     - docs/**
     - tests/**
   push:
@@ -22,24 +24,56 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
   build:
     uses: ./.github/workflows/docker-build.yml
     with:
       runsOn: ubuntu-22.04
+      platforms: linux/amd64,linux/arm64
 
-  test:
+  test-amd64:
     needs: build
     uses: ./.github/workflows/docker-test.yml
     strategy:
       matrix:
-        runsOn: [ARM64, ubuntu-22.04]
+        target: [aiida-core-base, aiida-core-with-services, aiida-core-dev]
+    with:
+      runsOn: ubuntu-22.04
+      images: ${{ needs.build.outputs.images }}
+
+  # IMPORTANT: To save arm64 runners resources,
+  # we run the test only when pushing to main.
+  # We also only test the aiida-core-dev image
+  test-arm64:
+    needs: build
+    if: >-
+      github.repository == 'aiidateam/aiida-core'
+      && (github.ref_type == 'tag' || github.ref_name == 'main')
+    uses: ./.github/workflows/docker-test.yml
     with:
-      runsOn: ${{ matrix.runsOn }}
-      images: ${{ needs.build.images }}
+      runsOn: buildjet-4vcpu-ubuntu-2204-arm
+      images: ${{ needs.build.outputs.images }}
+      target: aiida-core-dev
 
-  publish:
-    if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
-    needs: [build, test]
+  publish-ghcr:
+    needs: [build, test-amd64]
     uses: ./.github/workflows/docker-publish.yml
     secrets: inherit
+    with:
+      runsOn: ubuntu-22.04
+      images: ${{ needs.build.outputs.images }}
+      registry: ghcr.io
+
+  publish-dockerhub:
+    if: >-
+      github.repository == 'aiidateam/aiida-core'
+      && (github.ref_type == 'tag' || github.ref_name == 'main')
+    needs: [build, test-amd64, test-arm64, publish-ghcr]
+    uses: ./.github/workflows/docker-publish.yml
+    secrets: inherit
+    with:
+      images: ${{ needs.build.outputs.images }}
+      registry: docker.io