diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl index 41fa93b6f5..0785e27190 100644 --- a/.docker/docker-bake.hcl +++ b/.docker/docker-bake.hcl @@ -13,7 +13,7 @@ variable "ORGANIZATION" { } variable "REGISTRY" { - default = "docker.io/" + default = "ghcr.io/" } variable "PLATFORMS" { diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d42b1018ea..0784c696f3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,8 +1,6 @@ -name: Build image and then upload the image, tags and manifests to GitHub artifacts +name: Build Docker images and upload them to ghcr.io env: - OWNER: ${{ github.repository_owner }} - # Full logs for CI build BUILDKIT_PROGRESS: plain on: @@ -12,24 +10,36 @@ on: description: GitHub Actions Runner image required: true type: string + platforms: + description: Target platforms for the build (linux/amd64 and/or linux/arm64) + required: true + type: string + outputs: + images: + description: Images identified by digests + value: ${{ jobs.build.outputs.images }} jobs: build: + name: ${{ inputs.platforms }} runs-on: ${{ inputs.runsOn }} + timeout-minutes: 60 defaults: run: - shell: bash + # Make sure we fail if any command in a piped command sequence fails + shell: bash -e -o pipefail {0} working-directory: .docker + outputs: + images: ${{ steps.bake_metadata.outputs.images }} + steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Build aiida-core images for amd64 - # The order of the buildx bake files is important, as the second one will overwrite the first one - run: docker buildx bake -f docker-bake.hcl -f build.json --set *.platform=linux/$amd64 --load + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 - name: Set up QEMU + if: ${{ inputs.platforms != 'linux/amd64' }} uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -38,50 +48,30 @@ jobs: - name: Login to GitHub Container Registry 🔑 uses: docker/login-action@v3 with: - registry: ${{ env.REGISTRY }} + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # TODO: Separate amd64 build just just for testing now - - name: Build amd64 images 🏗️ - id: build-amd64 + - name: Build and upload to ghcr.io 📤 + id: build uses: docker/bake-action@v4 with: push: true - set: | - *.platform=linux/amd64 - files: | - docker-bake.hcl - build.json - - - name: Set output variables - run: .github/workflows/extract-docker-image-names.sh >> "${GITHUB_OUTPUT}" - env: - BAKE_METADATA: ${{ steps.build-amd64.outputs.metadata }} - - # Here we build arm64 images (with help of QEMU virtualization) - # and upload both amd64 and arm64 images to ghcr.io - - name: Build ARM64 and upload to ghcr.io 🍎📤 - id: build-upload - if: false - uses: docker/bake-action@v4 - with: - # TODO: Actually push, add tags first though! - load: true - push: false - # Using provenance to disable default attestation so it will build only desired images: - # https://github.com/orgs/community/discussions/45969 + # Using provenance to disable default attestation so it will build only desired images: + # https://github.com/orgs/community/discussions/45969 provenance: false - # NOTE: linux/amd64 images are taken from previous step set: | - *.platform=linux/amd64,linux/arm64 + *.platform=${{ inputs.platforms }} + *.output=type=registry,push-by-digest=true,name-canonical=true + *.cache-to=type=gha,scope=${{ github.workflow }},mode=max + *.cache-from=type=gha,scope=${{ github.workflow }} files: | docker-bake.hcl build.json - name: Set output variables - if: false id: bake_metadata - run: .github/workflows/extract-docker-image-names.sh >> "${GITHUB_OUTPUT}" + run: | + .github/workflows/extract-docker-image-names.sh | tee -a "${GITHUB_OUTPUT}" env: - BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }} + BAKE_METADATA: ${{ steps.build.outputs.metadata }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6adc5370eb..259df9b03e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,7 +1,8 @@ -name: Publish images to DockerHub +name: Publish images to Docker container registries env: - OWNER: ${{ github.repository_owner }} + # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables + DOCKER_METADATA_PR_HEAD_SHA: true on: workflow_call: @@ -14,25 +15,23 @@ on: description: Images built in build step required: true type: string + registry: + description: Docker container registry + required: true + type: string jobs: - tag-push: - runs-on: ubuntu-latest + + release: + runs-on: ${{ inputs.runsOn }} timeout-minutes: 30 strategy: fail-fast: true matrix: - image: [aiida-core-base, aiida-core-with-services, aiida-core-dev] - defaults: - run: - shell: bash - working-directory: .docker - permissions: - packages: read + target: [aiida-core-base, aiida-core-with-services, aiida-core-dev] steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - name: Login to GitHub Container Registry 🔑 uses: docker/login-action@v3 @@ -43,6 +42,7 @@ jobs: - name: Login to DockerHub 🔑 uses: docker/login-action@v3 + if: inputs.registry == 'docker.io' with: registry: docker.io username: ${{ secrets.DOCKER_USERNAME }} @@ -52,33 +52,31 @@ jobs: id: build_vars run: | vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') - echo "vars=$vars" - echo "vars=$vars" >> "${GITHUB_OUTPUT}" + echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" - - name: Docker meta 📝 + - name: Docker meta id: meta uses: docker/metadata-action@v5 - env: ${{ fromJson(steps.build_vars.outputs.vars) }} + env: ${{ fromJSON(steps.build_vars.outputs.vars) }} with: - images: | - name=docker.io/${{ env.OWNER }}/${{ matrix.image }} + # e.g. ghcr.io/aiidalab/full-stack + images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ matrix.target }} tags: | + type=ref,event=pr type=edge,enable={{is_default_branch}} - type=match,pattern=v(\d+\.\d+.\d+),group=1 - type=raw,value={{tag}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1 - - name: Determine src image tag + - name: Determine source image id: images run: | src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]') - echo "src=$src" - echo "src=$src" >> "${GITHUB_OUTPUT}" + echo "src=$src" | tee -a "${GITHUB_OUTPUT}" - - name: Push image to docker.io + - name: Push image uses: akhilerm/tag-push-action@v2.2.0 - if: false # TODO: Enable this! with: src: ${{ steps.images.outputs.src }} dst: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index b06bd41ddd..2cc40f6580 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -11,15 +11,16 @@ on: description: Images built in build step required: true type: string + target: + description: Target image for testing + required: true + type: string jobs: test: runs-on: ${{ inputs.runsOn }} - timeout-minutes: 30 - strategy: - matrix: - target: [aiida-core-base, aiida-core-with-services, aiida-core-dev] + timeout-minutes: 20 defaults: run: shell: bash @@ -38,7 +39,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set Up Python 🐍 - if: ${{ inputs.runsOn != 'ARM64' }} + if: ${{ startsWith(inputs.runsOn, 'ubuntu') }} uses: actions/setup-python@v5 with: python-version: '3.11' @@ -50,5 +51,6 @@ jobs: pip freeze - name: Run tests + # TODO: This probably needs tweaking run: pytest -s env: ${{ fromJSON(inputs.images) }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c5dbb92441..c0269e773c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,6 +3,8 @@ name: Build, test and push Docker Images on: pull_request: paths-ignore: + - '**.md' + - '**.txt' - docs/** - tests/** push: @@ -22,24 +24,56 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: build: uses: ./.github/workflows/docker-build.yml with: runsOn: ubuntu-22.04 + platforms: linux/amd64,linux/arm64 - test: + test-amd64: needs: build uses: ./.github/workflows/docker-test.yml strategy: matrix: - runsOn: [ARM64, ubuntu-22.04] + target: [aiida-core-base, aiida-core-with-services, aiida-core-dev] + with: + runsOn: ubuntu-22.04 + images: ${{ needs.build.outputs.images }} + + # IMPORTANT: To save arm64 runners resources, + # we run the test only when pushing to main. + # We also only test the aiida-core-dev image + test-arm64: + needs: build + if: >- + github.repository == 'aiidateam/aiida-core' + && (github.ref_type == 'tag' || github.ref_name == 'main') + uses: ./.github/workflows/docker-test.yml with: - runsOn: ${{ matrix.runsOn }} - images: ${{ needs.build.images }} + runsOn: buildjet-4vcpu-ubuntu-2204-arm + images: ${{ needs.build.outputs.images }} + target: aiida-core-dev - publish: - if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - needs: [build, test] + publish-ghcr: + needs: [build, test-amd64] uses: ./.github/workflows/docker-publish.yml secrets: inherit + with: + runsOn: ubuntu-22.04 + images: ${{ needs.build.outputs.images }} + registry: ghcr.io + + publish-dockerhub: + if: >- + github.repository == 'aiidateam/aiida-core' + && (github.ref_type == 'tag' || github.ref_name == 'main') + needs: [build, test-amd64, test-arm64, publish-ghcr] + uses: ./.github/workflows/docker-publish.yml + secrets: inherit + with: + images: ${{ needs.build.outputs.images }} + registry: docker.io diff --git a/.github/workflows/extract-docker-image-names.sh b/.github/workflows/extract-docker-image-names.sh index c40c602904..fd642555a1 100755 --- a/.github/workflows/extract-docker-image-names.sh +++ b/.github/workflows/extract-docker-image-names.sh @@ -2,47 +2,53 @@ set -euo pipefail -# Extract image names together with their digests -# to uniquely identify newly built images in subsequent steps. - -# TODO: Make these examples specific to aiida-core +# Extract image names together with their sha256 digests +# from the docker/bake-action metadata output. +# These together uniquely identify newly built images. # +# TODO: Make these examples specific to aiida-core + # The input to this script is a JSON string passed via BAKE_METADATA env variable # Here's example input (trimmed to relevant bits): -# BAKE_META: { +# BAKE_METADATA: { # "base": { -# "buildx.build.ref": "builder-9dc30f03-42f5-4fd5-8c9a-0d54be5ad996/builder-9dc30f03-42f5-4fd5-8c9a-0d54be5ad9960/jex1w6zvslbbomtkedn4no62l", -# "containerimage.config.digest": "sha256:b76dc61672dd0efbd586d56393d3a57f6309654e6903d738168892bc09017e8b", # "containerimage.descriptor": { # "mediaType": "application/vnd.docker.distribution.manifest.v2+json", # "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", # "size": 6170, -# "platform": { -# "architecture": "amd64", -# "os": "linux" -# } # }, # "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", -# "image.name": "ghcr.io/aiidalab/base:pr-439,ghcr.io/aiidalab/base:sha-a0cd2be" +# "image.name": "ghcr.io/aiidalab/base" # }, # "base-with-services": { +# "image.name": "ghcr.io/aiidalab/base-with-services" # "containerimage.digest": "sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26", # "...": "" # } # "full-stack": { -# "containerimage.digest": "sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", -# "...": "" +# "image.name": "ghcr.io/aiidalab/full-stack" +# "containerimage.digest": "sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", +# "...": "" +# } +# "lab": { +# "image.name": "ghcr.io/aiidalab/lab" +# "containerimage.digest": "sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97", +# "...": "" # } # } # # Example output (real output is on one line): # # images={ -# "BASE_IMAGE":"ghcr.io/aiidalab/base:pr-439@sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", -# "BASE_WITH_SERVICES_IMAGE":"ghcr.io/aiidalab/base-with-services:pr-439@sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26", -# "FULL_STACK_IMAGE":"ghcr.io/aiidalab/full-stack:pr-439@sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", -# "LAB_IMAGE":"ghcr.io/aiidalab/lab:pr-439@sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97" +# "BASE_IMAGE": "ghcr.io/aiidalab/base@sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "BASE_WITH_SERVICES_IMAGE": "ghcr.io/aiidalab/base-with-services@sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26", +# "FULL_STACK_IMAGE": "ghcr.io/aiidalab/full-stack@sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", +# "LAB_IMAGE": "ghcr.io/aiidalab/lab@sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97" # } +# +# This json output is later turned to environment variables using fromJson() GHA builtin +# (e.g. BASE_IMAGE=ghcr.io/aiidalab/base@sha256:8e57a52b...) +# and these are in turn read in the docker-compose..yml files for tests. if [[ -z ${BAKE_METADATA-} ]];then echo "ERROR: Environment variable BAKE_METADATA is not set!"