diff --git a/.docker/aiida-core-base/Dockerfile b/.docker/aiida-core-base/Dockerfile index 47f517d2fe..7de1ddb5bd 100644 --- a/.docker/aiida-core-base/Dockerfile +++ b/.docker/aiida-core-base/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # Inspired by jupyter's docker-stacks-fundation image: -# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/Dockerfile +# https://github.com/jupyter/docker-stacks/tree/main/images/docker-stacks-foundation/Dockerfile ARG BASE=ubuntu:22.04 @@ -87,7 +87,7 @@ RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashr # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc -# Create SYSTEM_USER with name jovyan user with UID=1000 and in the 'users' group +# Create $SYSTEM_USER user with UID=1000 and 'users' group # and make sure these dirs are writable by the `users` group. RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ @@ -112,7 +112,7 @@ ARG MAMBA_VERSION # Similar projects using Micromamba: # - Micromamba-Docker: # - repo2docker: -# Install Python, Mamba and jupyter_core +# Install Python, Mamba # Cleanup temporary files and remove Micromamba # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the diff --git a/.docker/docker-bake.hcl b/.docker/docker-bake.hcl index 533995af8d..0785e27190 100644 --- a/.docker/docker-bake.hcl +++ b/.docker/docker-bake.hcl @@ -13,7 +13,7 @@ variable "ORGANIZATION" { } variable "REGISTRY" { - default = "docker.io/" + default = "ghcr.io/" } variable "PLATFORMS" { @@ -27,7 +27,7 @@ variable "TARGETS" { function "tags" { params = [image] result = [ - "${REGISTRY}${ORGANIZATION}/${image}:newly-baked" + "${REGISTRY}${ORGANIZATION}/${image}" ] } diff --git a/.docker/docker-compose.aiida-core-base.yml b/.docker/docker-compose.aiida-core-base.yml index 4524938065..478d752db4 100644 --- a/.docker/docker-compose.aiida-core-base.yml +++ b/.docker/docker-compose.aiida-core-base.yml @@ -29,7 +29,7 @@ services: retries: 10 aiida: - image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/aiida-core-base}:${TAG:-latest} + image: ${REGISTRY:-}${AIIDA_CORE_BASE_IMAGE:-aiidateam/aiida-core-base}${TAG:-} environment: RMQHOST: messaging TZ: Europe/Zurich diff --git a/.docker/docker-compose.aiida-core-dev.yml b/.docker/docker-compose.aiida-core-dev.yml index c07fb3a5cf..b59f6484a5 100644 --- a/.docker/docker-compose.aiida-core-dev.yml +++ b/.docker/docker-compose.aiida-core-dev.yml @@ -3,7 +3,7 @@ version: '3.4' services: aiida: - image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/aiida-core-dev}:${TAG:-latest} + image: ${REGISTRY:-}${AIIDA_CORE_DEV_IMAGE:-aiidateam/aiida-core-dev}${TAG:-} environment: TZ: Europe/Zurich SETUP_DEFAULT_AIIDA_PROFILE: 'true' diff --git a/.docker/docker-compose.aiida-core-with-services.yml b/.docker/docker-compose.aiida-core-with-services.yml index 445259b428..73cf83e7e3 100644 --- a/.docker/docker-compose.aiida-core-with-services.yml +++ b/.docker/docker-compose.aiida-core-with-services.yml @@ -3,7 +3,7 @@ version: '3.4' services: aiida: - image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/aiida-core-with-services}:${TAG:-latest} + image: ${REGISTRY:-}${AIIDA_CORE_WITH_SERVICES_IMAGE:-aiidateam/aiida-core-with-services}${TAG:-} environment: TZ: Europe/Zurich SETUP_DEFAULT_AIIDA_PROFILE: 'true' diff --git a/.docker/pytest.ini b/.docker/pytest.ini index d1e7877377..e6c356c3eb 100644 --- a/.docker/pytest.ini +++ b/.docker/pytest.ini @@ -1,5 +1,5 @@ [pytest] minversion = 7.0 -addopts = -ra -q +addopts = -ra -q --strict-markers testpaths = tests diff --git a/.docker/requirements.txt b/.docker/requirements.txt index 3ba15482bb..6c84d9fa30 100644 --- a/.docker/requirements.txt +++ b/.docker/requirements.txt @@ -1,8 +1,4 @@ -docker -pre-commit -pytest -requests -tabulate -pytest-docker -docker-compose -pyyaml<=5.3.1 +docker~=7.0.0 +pytest~=8.2.0 +requests~=2.32.0 +pytest-docker~=3.1.0 diff --git a/.docker/tests/conftest.py b/.docker/tests/conftest.py index 9eb79ddede..968e941b29 100644 --- a/.docker/tests/conftest.py +++ b/.docker/tests/conftest.py @@ -3,21 +3,33 @@ import pytest +TARGETS = ('aiida-core-base', 'aiida-core-with-services', 'aiida-core-dev') -@pytest.fixture( - scope='session', - params=[ - 'aiida-core-base', - 'aiida-core-with-services', - 'aiida-core-dev', - ], -) -def variant(request): - return request.param + +def target_checker(value): + msg = f"Invalid image target '{value}', must be one of: {TARGETS}" + if value not in TARGETS: + raise pytest.UsageError(msg) + return value + + +def pytest_addoption(parser): + parser.addoption( + '--variant', + action='store', + required=True, + help='target (image name) of the docker-compose file to use.', + type=target_checker, + ) + + +@pytest.fixture(scope='session') +def variant(pytestconfig): + return pytestconfig.getoption('variant') @pytest.fixture(scope='session') -def docker_compose_file(pytestconfig, variant): +def docker_compose_file(variant): return f'docker-compose.{variant}.yml' diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml deleted file mode 100644 index 8ad3a4a125..0000000000 --- a/.github/actions/create-dev-env/action.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Build environment -description: Create build environment - -inputs: - architecture: - description: architecture to be run on - required: true - type: string - -runs: - using: composite - steps: - # actions/setup-python doesn't support Linux arm64 runners - # See: https://github.com/actions/setup-python/issues/108 - # python3 is manually preinstalled in the arm64 VM self-hosted runner - - name: Set Up Python 🐍 - uses: actions/setup-python@v5 - with: - python-version: 3.x - if: ${{ inputs.architecture == 'amd64' }} - - - name: Install Dev Dependencies 📦 - run: | - pip install --upgrade pip - pip install --upgrade -r .docker/requirements.txt - shell: bash diff --git a/.github/actions/install-aiida-core/action.yml b/.github/actions/install-aiida-core/action.yml index aacbe774a4..700e565034 100644 --- a/.github/actions/install-aiida-core/action.yml +++ b/.github/actions/install-aiida-core/action.yml @@ -28,7 +28,7 @@ runs: - name: Install uv installer run: curl --proto '=https' --tlsv1.2 -LsSf https://${{ env.UV_URL }} | sh env: - UV_VERSION: 0.1.35 + UV_VERSION: 0.1.44 UV_URL: github.com/astral-sh/uv/releases/download/$UV_VERSION/uv-installer.sh shell: bash diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml deleted file mode 100644 index 6598424fdd..0000000000 --- a/.github/actions/load-image/action.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Load Docker image -description: Download image tar and load it to docker - -inputs: - image: - description: Image name - required: true - type: string - architecture: - description: Image architecture - required: true - type: string - -runs: - using: composite - steps: - - name: Download built image 📥 - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.image }}-${{ inputs.architecture }} - path: /tmp/ - - name: Load downloaded image to docker 📥 - run: | - docker load --input /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar - docker image ls --all - shell: bash - - name: Delete the file 🗑️ - run: rm -f /tmp/${{ inputs.image }}-${{ inputs.architecture }}.tar - shell: bash - if: always() diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml deleted file mode 100644 index 3195206afa..0000000000 --- a/.github/workflows/docker-build-test-upload.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Build image and then upload the image, tags and manifests to GitHub artifacts - -env: - OWNER: ${{ github.repository_owner }} - -on: - workflow_call: - inputs: - architecture: - description: Image architecture, e.g. amd64, arm64 - required: true - type: string - runsOn: - description: GitHub Actions Runner image - required: true - type: string - -jobs: - build-test-upload: - runs-on: ${{ inputs.runsOn }} - defaults: - run: - shell: bash - working-directory: .docker - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Create dev environment 📦 - uses: ./.github/actions/create-dev-env - with: - architecture: ${{ inputs.architecture }} - - # Self-hosted runners share a state (whole VM) between runs - # Also, they might have running or stopped containers, - # which are not cleaned up by `docker system prun` - - name: Reset docker state and cleanup artifacts 🗑️ - if: ${{ inputs.platform != 'x86_64' }} - run: | - docker kill $(docker ps --quiet) || true - docker rm $(docker ps --all --quiet) || true - docker system prune --all --force - rm -rf /tmp/aiida-core/ - shell: bash - - - name: Build aiida-core images - # The order of the buildx bake files is important, as the second one will overwrite the first one - run: docker buildx bake -f docker-bake.hcl -f build.json --set *.platform=linux/${{ inputs.architecture }} --load - env: - # Full logs for CI build - BUILDKIT_PROGRESS: plain - - - name: Run tests ✅ - run: TAG=newly-baked python -m pytest -s tests - - - name: Save images as tar for later use 💾 - run: | - mkdir -p /tmp/aiida-core - docker save ${{ env.OWNER }}/aiida-core-base -o /tmp/aiida-core/aiida-core-base-${{ inputs.architecture }}.tar - docker save ${{ env.OWNER }}/aiida-core-with-services -o /tmp/aiida-core/aiida-core-with-services-${{ inputs.architecture }}.tar - docker save ${{ env.OWNER }}/aiida-core-dev -o /tmp/aiida-core/aiida-core-dev-${{ inputs.architecture }}.tar - - - name: Upload aiida-core-base image as artifact 💾 - uses: actions/upload-artifact@v4 - with: - name: aiida-core-base-${{ inputs.architecture }} - path: /tmp/aiida-core/aiida-core-base-${{ inputs.architecture }}.tar - retention-days: 3 - - - name: Upload aiida-core-with-services image as artifact 💾 - uses: actions/upload-artifact@v4 - with: - name: aiida-core-with-services-${{ inputs.architecture }} - path: /tmp/aiida-core/aiida-core-with-services-${{ inputs.architecture }}.tar - retention-days: 3 - - - name: Upload aiida-core-dev image as artifact 💾 - uses: actions/upload-artifact@v4 - with: - name: aiida-core-dev-${{ inputs.architecture }} - path: /tmp/aiida-core/aiida-core-dev-${{ inputs.architecture }}.tar - retention-days: 3 diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000000..fcc3da6e63 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,77 @@ +name: Build Docker images and upload them to ghcr.io + +env: + BUILDKIT_PROGRESS: plain + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + platforms: + description: Target platforms for the build (linux/amd64 and/or linux/arm64) + required: true + type: string + outputs: + images: + description: Images identified by digests + value: ${{ jobs.build.outputs.images }} + +jobs: + build: + name: ${{ inputs.platforms }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 60 + defaults: + run: + # Make sure we fail if any command in a piped command sequence fails + shell: bash -e -o pipefail {0} + + outputs: + images: ${{ steps.bake_metadata.outputs.images }} + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Set up QEMU + if: ${{ inputs.platforms != 'linux/amd64' }} + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and upload to ghcr.io 📤 + id: build + uses: docker/bake-action@v4 + with: + push: true + workdir: .docker/ + # Using provenance to disable default attestation so it will build only desired images: + # https://github.com/orgs/community/discussions/45969 + provenance: false + set: | + *.platform=${{ inputs.platforms }} + *.output=type=registry,push-by-digest=true,name-canonical=true + *.cache-to=type=gha,scope=${{ github.workflow }},mode=max + *.cache-from=type=gha,scope=${{ github.workflow }} + files: | + docker-bake.hcl + build.json + + - name: Set output variables + id: bake_metadata + run: | + .github/workflows/extract-docker-image-names.sh | tee -a "${GITHUB_OUTPUT}" + env: + BAKE_METADATA: ${{ steps.build.outputs.metadata }} diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml deleted file mode 100644 index e82661e376..0000000000 --- a/.github/workflows/docker-merge-tags.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Download images tags from GitHub artifacts and create multi-platform manifests - -on: - workflow_call: - inputs: - registry: - description: Docker registry, e.g. ghcr.io, docker.io - required: true - type: string - secrets: - REGISTRY_USERNAME: - required: true - REGISTRY_TOKEN: - required: true - - -jobs: - merge-tags: - runs-on: ubuntu-latest - strategy: - matrix: - image: [aiida-core-base, aiida-core-with-services, aiida-core-dev] - permissions: - packages: write - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - name: Create dev environment 📦 - uses: ./.github/actions/create-dev-env - with: - architecture: amd64 - - - name: Download amd64 tags file 📥 - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.registry }}-${{ matrix.image }}-amd64-tags - path: /tmp/aiida-core - - name: Download arm64 tags file 📥 - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.registry }}-${{ matrix.image }}-arm64-tags - path: /tmp/aiida-core - - - name: Login to Container Registry 🔑 - uses: docker/login-action@v2 - with: - registry: ${{ inputs.registry }} - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Merge tags for the images of different arch 🔀 - run: | - for arch_tag in $(cat /tmp/aiida-core/${{ matrix.image }}-amd64-tags.txt); do - tag=$(echo $arch_tag | sed "s/:amd64-/:/") - docker manifest create $tag --amend $arch_tag - docker manifest push $tag - done - - for arch_tag in $(cat /tmp/aiida-core/${{ matrix.image }}-arm64-tags.txt); do - tag=$(echo $arch_tag | sed "s/:arm64-/:/") - docker manifest create $tag --amend $arch_tag - docker manifest push $tag - done - shell: bash diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000000..ca383c6e57 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,86 @@ +name: Publish images to Docker container registries + +env: + # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables + DOCKER_METADATA_PR_HEAD_SHA: true + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + registry: + description: Docker container registry + required: true + type: string + +jobs: + + release: + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 30 + strategy: + fail-fast: true + matrix: + target: [aiida-core-base, aiida-core-with-services, aiida-core-dev] + defaults: + run: + # Make sure we fail if any command in a piped command sequence fails + shell: bash -e -o pipefail {0} + working-directory: .docker + + steps: + - uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub 🔑 + uses: docker/login-action@v3 + if: inputs.registry == 'docker.io' + with: + registry: docker.io + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Read build variables + id: build_vars + run: | + vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') + echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + env: ${{ fromJSON(steps.build_vars.outputs.vars) }} + with: + images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ matrix.target }} + tags: | + type=ref,event=pr + type=edge,enable={{is_default_branch}} + type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1 + + - name: Determine source image + id: images + run: | + src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]') + echo "src=$src" | tee -a "${GITHUB_OUTPUT}" + + - name: Push image + uses: akhilerm/tag-push-action@v2.2.0 + with: + src: ${{ steps.images.outputs.src }} + dst: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml deleted file mode 100644 index 912738dcd9..0000000000 --- a/.github/workflows/docker-push.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Download Docker image and its tags from GitHub artifacts, apply them and push the image to container registry - -env: - OWNER: ${{ github.repository_owner }} - -on: - workflow_call: - inputs: - architecture: - description: Image architecture - required: true - type: string - registry: - description: Docker registry - required: true - type: string - secrets: - REGISTRY_USERNAME: - required: true - REGISTRY_TOKEN: - required: true - -jobs: - tag-push: - runs-on: ubuntu-latest - strategy: - matrix: - image: [aiida-core-base, aiida-core-with-services, aiida-core-dev] - defaults: - run: - shell: bash - working-directory: .docker - permissions: - packages: write - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - name: Create dev environment 📦 - uses: ./.github/actions/create-dev-env - with: - architecture: ${{ inputs.architecture }} - - name: Load image to Docker 📥 - uses: ./.github/actions/load-image - with: - image: ${{ matrix.image }} - architecture: ${{ inputs.architecture }} - - - name: Read build variables - id: build_vars - run: | - vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') - echo "vars=$vars" >> "${GITHUB_OUTPUT}" - - - name: Docker meta 📝 - id: meta - uses: docker/metadata-action@v4 - env: ${{ fromJson(steps.build_vars.outputs.vars) }} - with: - images: | - name=${{ inputs.registry }}/${{ env.OWNER }}/${{ matrix.image }} - tags: | - type=edge,enable={{is_default_branch}} - type=sha,enable=${{ github.ref_type != 'tag' }} - type=ref,event=pr - type=match,pattern=v(\d+\.\d+.\d+),group=1 - type=raw,value={{tag}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - - - name: Login to Container Registry 🔑 - uses: docker/login-action@v2 - with: - registry: ${{ inputs.registry }} - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Set tags for image and push 🏷️📤💾 - run: | - declare -a arr=(${{ steps.meta.outputs.tags }}) - for tag in "${arr[@]}"; do - arch_tag=$(echo ${tag} | sed "s/:/:${{ inputs.architecture }}-/") - docker tag ${{ env.OWNER }}/${{ matrix.image }}:newly-baked ${arch_tag} - docker push ${arch_tag} - - # write tag to file - mkdir -p /tmp/aiida-core - echo ${arch_tag} >> /tmp/aiida-core/${{ matrix.image }}-${{ inputs.architecture }}-tags.txt - done - - - name: Upload tags file 📤 - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.registry }}-${{ matrix.image }}-${{ inputs.architecture }}-tags - path: /tmp/aiida-core/${{ matrix.image }}-${{ inputs.architecture }}-tags.txt - retention-days: 3 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml new file mode 100644 index 0000000000..652c459102 --- /dev/null +++ b/.github/workflows/docker-test.yml @@ -0,0 +1,52 @@ +name: Test newly built images + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + target: + description: Target image for testing + required: true + type: string + +jobs: + + test: + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 20 + defaults: + run: + working-directory: .docker + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Up Python 🐍 + if: ${{ startsWith(inputs.runsOn, 'ubuntu') }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install dependencies 📦 + run: pip install -r requirements.txt + + - name: Run tests + run: pytest -s --variant ${{ inputs.target }} tests/ + env: ${{ fromJSON(inputs.images) }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 30a41ed5b0..5bb0a6147a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,8 +1,10 @@ -name: Build, test and push Docker Images +name: Docker Images on: pull_request: paths-ignore: + - '**.md' + - '**.txt' - docs/** - tests/** push: @@ -12,6 +14,8 @@ on: tags: - v* paths-ignore: + - '**.md' + - '**.txt' - docs/** - tests/** workflow_dispatch: @@ -22,80 +26,67 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: - amd64-build: - uses: ./.github/workflows/docker-build-test-upload.yml + # We build only amd64 first to catch failures faster. + build-amd64: + if: ${{ ! github.event.pull_request.head.repo.fork }} + uses: ./.github/workflows/docker-build.yml with: - architecture: amd64 - runsOn: ubuntu-latest + runsOn: ubuntu-22.04 + platforms: linux/amd64 - arm64-build: - uses: ./.github/workflows/docker-build-test-upload.yml + test-amd64: + needs: build-amd64 + uses: ./.github/workflows/docker-test.yml + strategy: + matrix: + target: [aiida-core-base, aiida-core-with-services, aiida-core-dev] with: - architecture: arm64 - runsOn: ARM64 - if: ${{ !github.event.pull_request.head.repo.fork }} + runsOn: ubuntu-22.04 + images: ${{ needs.build-amd64.outputs.images }} + target: ${{ matrix.target }} - amd64-push-ghcr: - uses: ./.github/workflows/docker-push.yml + build: + needs: test-amd64 + uses: ./.github/workflows/docker-build.yml with: - architecture: amd64 - registry: ghcr.io - secrets: - REGISTRY_USERNAME: ${{ github.actor }} - REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [amd64-build] - if: ${{ !github.event.pull_request.head.repo.fork }} - - arm64-push-ghcr: - uses: ./.github/workflows/docker-push.yml - with: - architecture: arm64 - registry: ghcr.io - secrets: - REGISTRY_USERNAME: ${{ github.actor }} - REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [arm64-build] - if: ${{ !github.event.pull_request.head.repo.fork }} + runsOn: ubuntu-22.04 + platforms: linux/amd64,linux/arm64 - merge-tags-ghcr: - uses: ./.github/workflows/docker-merge-tags.yml + publish-ghcr: + needs: [build, test-amd64] + uses: ./.github/workflows/docker-publish.yml + secrets: inherit with: + runsOn: ubuntu-22.04 + images: ${{ needs.build.outputs.images }} registry: ghcr.io - secrets: - REGISTRY_USERNAME: ${{ github.actor }} - REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [amd64-push-ghcr, arm64-push-ghcr] - if: ${{ !github.event.pull_request.head.repo.fork }} - amd64-push-dockerhub: - if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - uses: ./.github/workflows/docker-push.yml + # IMPORTANT: To save arm64 runners resources, + # we run the test only when pushing to main. + # We also only test the aiida-core-dev image. + test-arm64: + needs: build + if: >- + github.repository == 'aiidateam/aiida-core' + && (github.ref_type == 'tag' || github.ref_name == 'main') + uses: ./.github/workflows/docker-test.yml with: - architecture: amd64 - registry: docker.io - secrets: - REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.DOCKER_TOKEN }} - needs: [amd64-build] - - arm64-push-dockerhub: - if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - uses: ./.github/workflows/docker-push.yml - with: - architecture: arm64 - registry: docker.io - secrets: - REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.DOCKER_TOKEN }} - needs: [arm64-build] + runsOn: buildjet-4vcpu-ubuntu-2204-arm + images: ${{ needs.build.outputs.images }} + target: aiida-core-dev - merge-tags-dockerhub: - if: github.repository == 'aiidateam/aiida-core' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - uses: ./.github/workflows/docker-merge-tags.yml + publish-dockerhub: + if: >- + github.repository == 'aiidateam/aiida-core' + && (github.ref_type == 'tag' || github.ref_name == 'main') + needs: [build, test-amd64, test-arm64, publish-ghcr] + uses: ./.github/workflows/docker-publish.yml + secrets: inherit with: + runsOn: ubuntu-22.04 + images: ${{ needs.build.outputs.images }} registry: docker.io - secrets: - REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.DOCKER_TOKEN }} - needs: [amd64-push-dockerhub, arm64-push-dockerhub] diff --git a/.github/workflows/extract-docker-image-names.sh b/.github/workflows/extract-docker-image-names.sh new file mode 100755 index 0000000000..8609f7c385 --- /dev/null +++ b/.github/workflows/extract-docker-image-names.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail + +# Extract image names together with their sha256 digests +# from the docker/bake-action metadata output. +# These together uniquely identify newly built images. +# +# The input to this script is a JSON string passed via BAKE_METADATA env variable +# Here's example input (trimmed to relevant bits): +# BAKE_METADATA: { +# "base": { +# "containerimage.descriptor": { +# "mediaType": "application/vnd.docker.distribution.manifest.v2+json", +# "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "size": 6170, +# }, +# "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "image.name": "ghcr.io/aiidalab/base" +# }, +# "aiida-core-base": { +# "image.name": "ghcr.io/aiidateam/aiida-core-base" +# "containerimage.digest": "sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26", +# "...": "" +# } +# "aiida-core-with-services": { +# "image.name": "ghcr.io/aiidateam/aiida-core-with-services" +# "containerimage.digest": "sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", +# "...": "" +# } +# "aiida-core-dev": { +# "image.name": "ghcr.io/aiidateam/aiida-core-with-services" +# "containerimage.digest": "sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97", +# "...": "" +# } +# } +# +# Example output (real output is on one line): +# +# images={ +# "AIIDA_CORE_BASE_IMAGE": "ghcr.io/aiidateam/aiida-core-base@sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "AIIDA_CORE_WITH_SERVICES_IMAGE": "ghcr.io/aiidateam/aiida-core-with-services@sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26", +# "AIIDA_CORE_DEV_IMAGE": "ghcr.io/aiidateam/aiida-core-dev@sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", +# } +# +# This json output is later turned to environment variables using fromJson() GHA builtin +# (e.g. AIIDA_CORE_BASE_IMAGE=ghcr.io/aiidateam/aiida-core-base@sha256:8e57a52b...) +# and these are in turn read in the docker-compose..yml files for tests. + +if [[ -z ${BAKE_METADATA-} ]];then + echo "ERROR: Environment variable BAKE_METADATA is not set!" + exit 1 +fi + +images=$(echo "${BAKE_METADATA}" | jq -c '. as $base |[to_entries[] |{"key": (.key|ascii_upcase|sub("-"; "_"; "g") + "_IMAGE"), "value": [(.value."image.name"|split(",")[0]),.value."containerimage.digest"]|join("@")}] |from_entries') +echo "images=$images"