From 7b25a1441dc6465b89953746b0a85e4ecb6a7981 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 13:45:10 +0200 Subject: [PATCH 01/43] add action for releasing new docker image --- .github/workflows/publish_docker_image.yml | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/publish_docker_image.yml diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml new file mode 100644 index 00000000..364587e5 --- /dev/null +++ b/.github/workflows/publish_docker_image.yml @@ -0,0 +1,59 @@ +# +name: Create and publish a Docker image + +# Configures this workflow to run every time a change is pushed to the branch called `release`. +on: + push: + branches: ["529_add_docker_testing_action_gcroci2"] + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true From ded33a70e6433c1954c7ca31d267e9ab19632b48 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 13:46:11 +0200 Subject: [PATCH 02/43] add todo --- .github/workflows/publish_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 364587e5..3a7bb7f4 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -4,7 +4,7 @@ name: Create and publish a Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: push: - branches: ["529_add_docker_testing_action_gcroci2"] + branches: ["529_add_docker_testing_action_gcroci2"] #TODO: Change this to be dependant on the release action # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From d8165c1c8fd9f4552f45cebf9752276d1706e8d8 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 14:10:55 +0200 Subject: [PATCH 03/43] add logs --- .github/workflows/publish_docker_image.yml | 10 ++++++++++ Dockerfile | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 3a7bb7f4..2917630c 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -46,9 +46,19 @@ jobs: uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . + file: ./Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # Output verbose logs + build-args: BUILDKIT_INLINE_CACHE=1 + outputs: type=docker,dest=/tmp/myimage.tar + + - name: Output Docker build logs + run: | + echo "Docker build logs" + docker load < /tmp/myimage.tar + docker history ${{ steps.meta.outputs.tags }} # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." - name: Generate artifact attestation diff --git a/Dockerfile b/Dockerfile index 98d7eb80..5909ac8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,11 @@ RUN \ echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc -ADD ./env/deeprank2.yml /home/deeprank2 +RUN ls -la /home + +ADD ./env/deeprank2.yml /home/deeprank2/ + +RUN ls -la /home/deeprank2 RUN \ ## Create the environment and install the dependencies From 385a74887200accaae65605f85c77e69912c5eb0 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 14:12:21 +0200 Subject: [PATCH 04/43] remove unsupported commands --- .github/workflows/publish_docker_image.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 2917630c..fc783d50 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -50,15 +50,6 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # Output verbose logs - build-args: BUILDKIT_INLINE_CACHE=1 - outputs: type=docker,dest=/tmp/myimage.tar - - - name: Output Docker build logs - run: | - echo "Docker build logs" - docker load < /tmp/myimage.tar - docker history ${{ steps.meta.outputs.tags }} # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." - name: Generate artifact attestation From ed3b17d55821424b295e1280b2dae946a09c48c8 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 14:44:23 +0200 Subject: [PATCH 05/43] specify platform for building the image in the action --- .github/workflows/publish_docker_image.yml | 1 + Dockerfile | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index fc783d50..06231866 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -46,6 +46,7 @@ jobs: uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . + platforms: linux/amd64 file: ./Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/Dockerfile b/Dockerfile index 5909ac8d..b26ada57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,12 +22,8 @@ RUN \ echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc -RUN ls -la /home - ADD ./env/deeprank2.yml /home/deeprank2/ -RUN ls -la /home/deeprank2 - RUN \ ## Create the environment and install the dependencies mamba env create -f /home/deeprank2/deeprank2.yml && \ From 56d4f77935b91ec23bfe87635a68976b0fe779ec Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 15:00:21 +0200 Subject: [PATCH 06/43] add tags and multiarch build --- .github/workflows/publish_docker_image.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 06231866..c835cdca 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -25,7 +25,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: @@ -38,12 +44,20 @@ jobs: uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - name: Build and push Docker image id: push - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64 From c464425f2459ee1f82ac2a81fe18db8b2142dc9f Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 15:57:24 +0200 Subject: [PATCH 07/43] add debug runs --- .github/workflows/publish_docker_image.yml | 61 +++++++--------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index c835cdca..70b341cd 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -1,75 +1,52 @@ -# -name: Create and publish a Docker image +name: Create and publish a Docker image with manifest -# Configures this workflow to run every time a change is pushed to the branch called `release`. on: push: - branches: ["529_add_docker_testing_action_gcroci2"] #TODO: Change this to be dependant on the release action + branches: ["529_add_docker_testing_action_gcroci2"] -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: build-and-push-image: runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read packages: write - attestations: write - id-token: write - # steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image - id: push uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64 file: ./Dockerfile push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64 + platforms: linux/amd64 - # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true + - name: Create and push manifest + run: | + docker manifest create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64 + docker manifest annotate ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64 --os linux --arch amd64 + docker manifest push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + + - name: Check pushed image and manifest + run: | + docker manifest inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest From 5b93d05ea2af76d4e5d3ec989b45a7d055c7827e Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 16:01:24 +0200 Subject: [PATCH 08/43] restore the inital file --- .github/workflows/publish_docker_image.yml | 57 ++++++++++++---------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 70b341cd..0cc2bf66 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -1,52 +1,59 @@ -name: Create and publish a Docker image with manifest +# +name: Create and publish a Docker image +# Configures this workflow to run every time a change is pushed to the branch called `release`. on: push: - branches: ["529_add_docker_testing_action_gcroci2"] + branches: ["release"] +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: build-and-push-image: runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read packages: write + attestations: write + id-token: write + # steps: - name: Checkout repository uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - name: Build and push Docker image - uses: docker/build-push-action@v5 + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . - file: ./Dockerfile push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64 - platforms: linux/amd64 - - - name: Create and push manifest - run: | - docker manifest create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64 - docker manifest annotate ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64 --os linux --arch amd64 - docker manifest push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} - - name: Check pushed image and manifest - run: | - docker manifest inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true From d33106f6d3519604417a0781f215865a7cc0d12a Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 16:52:26 +0200 Subject: [PATCH 09/43] point to the current branch --- .github/workflows/publish_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 0cc2bf66..364587e5 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -4,7 +4,7 @@ name: Create and publish a Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: push: - branches: ["release"] + branches: ["529_add_docker_testing_action_gcroci2"] # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From 03014a873bfc17dc21c886868cf160cf6cc0d554 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 17:24:27 +0200 Subject: [PATCH 10/43] try rsd workflow --- .github/workflows/_ghcr.yml | 70 ++++++++++++++ .github/workflows/publish_docker_image.yml | 103 ++++++++++++--------- 2 files changed, 128 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/_ghcr.yml diff --git a/.github/workflows/_ghcr.yml b/.github/workflows/_ghcr.yml new file mode 100644 index 00000000..abdc910f --- /dev/null +++ b/.github/workflows/_ghcr.yml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +# SPDX-FileCopyrightText: 2022 dv4all +# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +name: reusable ghcr.io module + +on: + workflow_call: + inputs: + ghcr_user: + required: true + description: User for logging to ghcr.io (use github.actor) + type: string + base_image_name: + required: true + description: Base image name incl. ghcr.io + type: string + image_tag: + required: true + description: Image tag (version) + type: string + dockerfile: + required: true + description: Location and name of docker file + type: string + docker_context: + required: true + description: Docker context for the build command + type: string + secrets: + token: + required: true + outputs: + image_created: + description: Full image name after upload to ghcr.io + value: ${{jobs.build_and_push.outputs.image_build}} + image_uploaded: + description: Confirmation that image is uploaded to ghcr.io + value: ${{jobs.build_and_push.outputs.image_pushed}} + +jobs: + build_and_push: + name: build and push image + runs-on: ubuntu-22.04 + outputs: + image_build: ${{steps.build_image.outputs.image_build}} + image_pushed: ${{steps.build_image.outputs.image_pushed}} + steps: + - name: checkout + # https://github.com/actions/checkout + uses: actions/checkout@v4 + - name: build + id: build_image + run: | + IMAGE_TAG_VERSION=${{inputs.base_image_name}}:${{inputs.image_tag}} + IMAGE_TAG_LASTEST=${{inputs.base_image_name}}:latest + echo image_tag_version $IMAGE_TAG_VERSION + docker build -t $IMAGE_TAG_VERSION -t $IMAGE_TAG_LASTEST -f ${{inputs.dockerfile}} ${{inputs.docker_context}} + echo "{image_build}={$IMAGE_TAG_VERSIONvalue}" >> $GITHUB_OUTPUT + - name: push to ghcr.io + id: push_image + run: | + echo login + echo "${{secrets.token}}" | docker login https://ghcr.io -u ${{inputs.ghcr_user}} --password-stdin + echo push auth image with all tags + docker push ${{inputs.base_image_name}} --all-tags + echo "{image_build}={$IMAGE_TAG_VERSIONvalue}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 364587e5..daa9694b 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -1,4 +1,13 @@ +# SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +# SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +# SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +# SPDX-FileCopyrightText: 2022 dv4all +# SPDX-FileCopyrightText: 2023 - 2024 Dusan Mijatovic (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center # +# SPDX-License-Identifier: Apache-2.0 + name: Create and publish a Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. @@ -6,54 +15,58 @@ on: push: branches: ["529_add_docker_testing_action_gcroci2"] -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - attestations: write - id-token: write - # + release_tag: + name: conventional changelog + runs-on: ubuntu-22.04 + outputs: + changelog: ${{steps.changelog.outputs.clean_changelog}} + tag: ${{steps.changelog.outputs.tag}} + skipped: ${{steps.changelog.outputs.skipped}} steps: - - name: Checkout repository + - name: checkout all history + # https://github.com/actions/checkout uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image - id: push - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + # checkout whole history + fetch-depth: 0 - # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 + - name: calculate new version and create changelog content + id: changelog + # https://github.com/TriPSs/conventional-changelog-action + uses: TriPSs/conventional-changelog-action@v5 with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true + # you can also create separate token to trace action + github-token: "${{secrets.GITHUB_TOKEN}}" + # do not create changelog file, the content is used at next step for relase body + output-file: false + # do not create additional commit, just tag current commit with the version + skip-commit: true + # do not pull - we already checked out the selection we want to use for versioning in previous step + skip-git-pull: true + # skip tag push - it will not push but it will tag + git-push: false + + log_release_tag: + needs: release_tag + name: log version output + runs-on: ubuntu-22.04 + steps: + - name: info + run: | + echo skipped=${{needs.release_tag.outputs.skipped}} + echo tag=${{needs.release_tag.outputs.tag}} + auth: + # it needs to be checked on string value + if: needs.release_tag.outputs.skipped == 'false' + needs: release_tag + name: docker_image_deeprank2 + uses: ./.github/workflows/_ghcr.yml + with: + ghcr_user: ${{github.actor}} + base_image_name: ghcr.io/deeprank/deeprank2 + image_tag: ${{needs.release_tag.outputs.tag}} + dockerfile: ./Dockerfile + docker_context: . + secrets: + token: ${{secrets.GITHUB_TOKEN}} From 4547855a921c2b37ca579cf33a170ef8a962a35e Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 17:25:52 +0200 Subject: [PATCH 11/43] remove skip --- .github/workflows/publish_docker_image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index daa9694b..f91640a5 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -56,9 +56,9 @@ jobs: run: | echo skipped=${{needs.release_tag.outputs.skipped}} echo tag=${{needs.release_tag.outputs.tag}} - auth: - # it needs to be checked on string value - if: needs.release_tag.outputs.skipped == 'false' + docker_image_deeprank2: + # # it needs to be checked on string value + # if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: docker_image_deeprank2 uses: ./.github/workflows/_ghcr.yml From f01cfc379e892cdf36addd2e8905fac4cdf9418c Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 17:27:49 +0200 Subject: [PATCH 12/43] fix base_image_name --- .github/workflows/publish_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index f91640a5..c638bda0 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -64,7 +64,7 @@ jobs: uses: ./.github/workflows/_ghcr.yml with: ghcr_user: ${{github.actor}} - base_image_name: ghcr.io/deeprank/deeprank2 + base_image_name: ghcr.io/DeepRank/deeprank2 image_tag: ${{needs.release_tag.outputs.tag}} dockerfile: ./Dockerfile docker_context: . From 92e4be6e38af89af9d0df40cedbe10fb2b6c1aab Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 17:29:23 +0200 Subject: [PATCH 13/43] use latest --- .github/workflows/publish_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index c638bda0..5b65fa4b 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -65,7 +65,7 @@ jobs: with: ghcr_user: ${{github.actor}} base_image_name: ghcr.io/DeepRank/deeprank2 - image_tag: ${{needs.release_tag.outputs.tag}} + image_tag: latest dockerfile: ./Dockerfile docker_context: . secrets: From ff9d1a9e5af259025265228f4150d1fa4ffac21e Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Wed, 4 Sep 2024 17:30:24 +0200 Subject: [PATCH 14/43] lowercase --- .github/workflows/publish_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 5b65fa4b..553b40b7 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -64,7 +64,7 @@ jobs: uses: ./.github/workflows/_ghcr.yml with: ghcr_user: ${{github.actor}} - base_image_name: ghcr.io/DeepRank/deeprank2 + base_image_name: ghcr.io/deeprank/deeprank2 image_tag: latest dockerfile: ./Dockerfile docker_context: . From 394c13242afac2add0eb9c6658287a542e234716 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:03:12 +0200 Subject: [PATCH 15/43] move ghcr to actions/ folder --- .../_ghcr.yml => actions/install-python-and-package/ghcr.yml} | 0 .github/workflows/publish_docker_image.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/{workflows/_ghcr.yml => actions/install-python-and-package/ghcr.yml} (100%) diff --git a/.github/workflows/_ghcr.yml b/.github/actions/install-python-and-package/ghcr.yml similarity index 100% rename from .github/workflows/_ghcr.yml rename to .github/actions/install-python-and-package/ghcr.yml diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 553b40b7..7abce280 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -61,7 +61,7 @@ jobs: # if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: docker_image_deeprank2 - uses: ./.github/workflows/_ghcr.yml + uses: ./.github/actions/ghcr.yml with: ghcr_user: ${{github.actor}} base_image_name: ghcr.io/deeprank/deeprank2 From 0051d34b221204017aa85890ea19c25863762802 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:06:21 +0200 Subject: [PATCH 16/43] fix folders structure --- .github/actions/{install-python-and-package => }/ghcr.yml | 0 .../action.yml => install-python-and-package.yml} | 0 .github/workflows/build-latest-release.yml | 2 +- .github/workflows/build-repo-frozen-env.yml | 2 +- .github/workflows/build-repo.yml | 2 +- .github/workflows/coveralls.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/release.yml | 2 +- 8 files changed, 6 insertions(+), 6 deletions(-) rename .github/actions/{install-python-and-package => }/ghcr.yml (100%) rename .github/actions/{install-python-and-package/action.yml => install-python-and-package.yml} (100%) diff --git a/.github/actions/install-python-and-package/ghcr.yml b/.github/actions/ghcr.yml similarity index 100% rename from .github/actions/install-python-and-package/ghcr.yml rename to .github/actions/ghcr.yml diff --git a/.github/actions/install-python-and-package/action.yml b/.github/actions/install-python-and-package.yml similarity index 100% rename from .github/actions/install-python-and-package/action.yml rename to .github/actions/install-python-and-package.yml diff --git a/.github/workflows/build-latest-release.yml b/.github/workflows/build-latest-release.yml index 9daa9936..bc1c8d40 100644 --- a/.github/workflows/build-latest-release.yml +++ b/.github/workflows/build-latest-release.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} pkg-installation-type: "latest" diff --git a/.github/workflows/build-repo-frozen-env.yml b/.github/workflows/build-repo-frozen-env.yml index 47f29dce..d3c71dbb 100644 --- a/.github/workflows/build-repo-frozen-env.yml +++ b/.github/workflows/build-repo-frozen-env.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/build-repo.yml b/.github/workflows/build-repo.yml index 7af4cc7a..20654ae5 100644 --- a/.github/workflows/build-repo.yml +++ b/.github/workflows/build-repo.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index eb4feff2..437f76ac 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index b1e59ea9..1546c06f 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -45,7 +45,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, notebooks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f85061b..fae4c93b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: publishing From 0bb5a1bf7bccbe511e831c896d1247bc6fc6fca5 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:11:22 +0200 Subject: [PATCH 17/43] move actions to workflows --- .github/{actions/ghcr.yml => workflows/ _ghcr.yml} | 0 .../_install-python-and-package.yml} | 0 .github/workflows/build-latest-release.yml | 2 +- .github/workflows/build-repo-frozen-env.yml | 2 +- .github/workflows/build-repo.yml | 2 +- .github/workflows/coveralls.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/publish_docker_image.yml | 2 +- .github/workflows/release.yml | 2 +- 9 files changed, 7 insertions(+), 7 deletions(-) rename .github/{actions/ghcr.yml => workflows/ _ghcr.yml} (100%) rename .github/{actions/install-python-and-package.yml => workflows/_install-python-and-package.yml} (100%) diff --git a/.github/actions/ghcr.yml b/.github/workflows/ _ghcr.yml similarity index 100% rename from .github/actions/ghcr.yml rename to .github/workflows/ _ghcr.yml diff --git a/.github/actions/install-python-and-package.yml b/.github/workflows/_install-python-and-package.yml similarity index 100% rename from .github/actions/install-python-and-package.yml rename to .github/workflows/_install-python-and-package.yml diff --git a/.github/workflows/build-latest-release.yml b/.github/workflows/build-latest-release.yml index bc1c8d40..ecd8e4af 100644 --- a/.github/workflows/build-latest-release.yml +++ b/.github/workflows/build-latest-release.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/workflows/_install-python-and-package.yml with: python-version: ${{ matrix.python-version }} pkg-installation-type: "latest" diff --git a/.github/workflows/build-repo-frozen-env.yml b/.github/workflows/build-repo-frozen-env.yml index d3c71dbb..dc1bc3c3 100644 --- a/.github/workflows/build-repo-frozen-env.yml +++ b/.github/workflows/build-repo-frozen-env.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/workflows/_install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/build-repo.yml b/.github/workflows/build-repo.yml index 20654ae5..2b4e2015 100644 --- a/.github/workflows/build-repo.yml +++ b/.github/workflows/build-repo.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/workflows/_install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index 437f76ac..2fd4a1a0 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/workflows/_install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 1546c06f..75e1ea39 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -45,7 +45,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/workflows/_install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, notebooks diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 7abce280..553b40b7 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -61,7 +61,7 @@ jobs: # if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: docker_image_deeprank2 - uses: ./.github/actions/ghcr.yml + uses: ./.github/workflows/_ghcr.yml with: ghcr_user: ${{github.actor}} base_image_name: ghcr.io/deeprank/deeprank2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fae4c93b..ca949fa1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/workflows/_install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: publishing From 56b5e878da133dad423f6181963bb29c2b7076b6 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:15:13 +0200 Subject: [PATCH 18/43] fix action error --- .../actions/install-python-and-package.yml | 84 +++++++++++++++++++ .../workflows/_install-python-and-package.yml | 81 ------------------ .github/workflows/build-latest-release.yml | 2 +- .github/workflows/build-repo-frozen-env.yml | 2 +- .github/workflows/build-repo.yml | 2 +- .github/workflows/coveralls.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/release.yml | 2 +- 8 files changed, 90 insertions(+), 87 deletions(-) create mode 100644 .github/actions/install-python-and-package.yml delete mode 100644 .github/workflows/_install-python-and-package.yml diff --git a/.github/actions/install-python-and-package.yml b/.github/actions/install-python-and-package.yml new file mode 100644 index 00000000..b95bdc7f --- /dev/null +++ b/.github/actions/install-python-and-package.yml @@ -0,0 +1,84 @@ +name: "Install Python and DeepRank2" + +description: "Installs Python, updates pip and installs DeepRank2 together with its dependencies." + +on: + workflow_call: + + inputs: + python-version: + required: false + description: "The Python version to use. Specify major and minor version, e.g. '3.10'." + default: "3.10" + + extras-require: + required: false + description: "The extras dependencies packages to be installed, for instance 'dev' or 'dev,publishing,notebooks'." + default: "test" + + pkg-installation-type: + required: false + description: "The package installation type to install, the latest released version on PyPI ('latest') or the GitHub repository one ('repository')." + default: "repository" + + runs: + using: "composite" + + steps: + - name: Cancel Previous Runs and Set up Python + uses: styfle/cancel-workflow-action@0.4.0 + with: + access_token: ${{ github.token }} + + - uses: actions/checkout@v3 + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniforge-variant: Mambaforge + channels: conda-forge + python-version: ${{ inputs.python-version }} + activate-environment: deeprank2 + environment-file: env/deeprank2.yml + use-mamba: true + if: ${{ inputs.pkg-installation-type != 'frozen' }} + + - name: Setup miniconda with frozen dependencies + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniforge-variant: Mambaforge + channels: conda-forge + python-version: ${{ inputs.python-version }} + activate-environment: deeprank2 + environment-file: env/deeprank2_frozen.yml + use-mamba: true + if: ${{ inputs.pkg-installation-type == 'frozen' }} + + - run: | + conda --version + conda env list + shell: bash -l {0} + + - name: Python info + shell: bash -l {0} + run: | + which python3 + python3 --version + + - name: Install the GitHub repository version of the package + shell: bash -l {0} + if: ${{ inputs.pkg-installation-type == 'repository' || inputs.pkg-installation-type == 'frozen' }} + run: | + conda activate deeprank2 + pip install .'[${{ inputs.extras-require }}]' + + - name: Install the latest released PyPI version of the package + shell: bash -l {0} + if: ${{ inputs.pkg-installation-type == 'latest' }} + run: | + conda activate deeprank2 + pip install pytest + rm -r deeprank2 + pip install deeprank2 diff --git a/.github/workflows/_install-python-and-package.yml b/.github/workflows/_install-python-and-package.yml deleted file mode 100644 index 539cfe42..00000000 --- a/.github/workflows/_install-python-and-package.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: "Install Python and DeepRank2" - -description: "Installs Python, updates pip and installs DeepRank2 together with its dependencies." - -inputs: - python-version: - required: false - description: "The Python version to use. Specify major and minor version, e.g. '3.10'." - default: "3.10" - - extras-require: - required: false - description: "The extras dependencies packages to be installed, for instance 'dev' or 'dev,publishing,notebooks'." - default: "test" - - pkg-installation-type: - required: false - description: "The package installation type to install, the latest released version on PyPI ('latest') or the GitHub repository one ('repository')." - default: "repository" - -runs: - using: "composite" - - steps: - - name: Cancel Previous Runs and Set up Python - uses: styfle/cancel-workflow-action@0.4.0 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 - - - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniforge-variant: Mambaforge - channels: conda-forge - python-version: ${{ inputs.python-version }} - activate-environment: deeprank2 - environment-file: env/deeprank2.yml - use-mamba: true - if: ${{ inputs.pkg-installation-type != 'frozen' }} - - - name: Setup miniconda with frozen dependencies - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniforge-variant: Mambaforge - channels: conda-forge - python-version: ${{ inputs.python-version }} - activate-environment: deeprank2 - environment-file: env/deeprank2_frozen.yml - use-mamba: true - if: ${{ inputs.pkg-installation-type == 'frozen' }} - - - run: | - conda --version - conda env list - shell: bash -l {0} - - - name: Python info - shell: bash -l {0} - run: | - which python3 - python3 --version - - - name: Install the GitHub repository version of the package - shell: bash -l {0} - if: ${{ inputs.pkg-installation-type == 'repository' || inputs.pkg-installation-type == 'frozen' }} - run: | - conda activate deeprank2 - pip install .'[${{ inputs.extras-require }}]' - - - name: Install the latest released PyPI version of the package - shell: bash -l {0} - if: ${{ inputs.pkg-installation-type == 'latest' }} - run: | - conda activate deeprank2 - pip install pytest - rm -r deeprank2 - pip install deeprank2 diff --git a/.github/workflows/build-latest-release.yml b/.github/workflows/build-latest-release.yml index ecd8e4af..bc1c8d40 100644 --- a/.github/workflows/build-latest-release.yml +++ b/.github/workflows/build-latest-release.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/workflows/_install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} pkg-installation-type: "latest" diff --git a/.github/workflows/build-repo-frozen-env.yml b/.github/workflows/build-repo-frozen-env.yml index dc1bc3c3..d3c71dbb 100644 --- a/.github/workflows/build-repo-frozen-env.yml +++ b/.github/workflows/build-repo-frozen-env.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/workflows/_install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/build-repo.yml b/.github/workflows/build-repo.yml index 2b4e2015..20654ae5 100644 --- a/.github/workflows/build-repo.yml +++ b/.github/workflows/build-repo.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/workflows/_install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index 2fd4a1a0..437f76ac 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/workflows/_install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 75e1ea39..1546c06f 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -45,7 +45,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/workflows/_install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: test, notebooks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ca949fa1..fae4c93b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/workflows/_install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package.yml with: python-version: ${{ matrix.python-version }} extras-require: publishing From 77c4f0194643eda02314d50e574b8af5cb1ea775 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:16:51 +0200 Subject: [PATCH 19/43] restore old folder --- .../action.yml} | 0 .github/workflows/build-latest-release.yml | 2 +- .github/workflows/build-repo-frozen-env.yml | 2 +- .github/workflows/build-repo.yml | 2 +- .github/workflows/coveralls.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/release.yml | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename .github/actions/{install-python-and-package.yml => install-python-and-package/action.yml} (100%) diff --git a/.github/actions/install-python-and-package.yml b/.github/actions/install-python-and-package/action.yml similarity index 100% rename from .github/actions/install-python-and-package.yml rename to .github/actions/install-python-and-package/action.yml diff --git a/.github/workflows/build-latest-release.yml b/.github/workflows/build-latest-release.yml index bc1c8d40..417174d9 100644 --- a/.github/workflows/build-latest-release.yml +++ b/.github/workflows/build-latest-release.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package/action.yml with: python-version: ${{ matrix.python-version }} pkg-installation-type: "latest" diff --git a/.github/workflows/build-repo-frozen-env.yml b/.github/workflows/build-repo-frozen-env.yml index d3c71dbb..7fb126c8 100644 --- a/.github/workflows/build-repo-frozen-env.yml +++ b/.github/workflows/build-repo-frozen-env.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package/action.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/build-repo.yml b/.github/workflows/build-repo.yml index 20654ae5..b5395395 100644 --- a/.github/workflows/build-repo.yml +++ b/.github/workflows/build-repo.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package/action.yml with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index 437f76ac..e3a0f120 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package/action.yml with: python-version: ${{ matrix.python-version }} extras-require: test diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 1546c06f..fce2b216 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -45,7 +45,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package/action.yml with: python-version: ${{ matrix.python-version }} extras-require: test, notebooks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fae4c93b..af65141d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package.yml + - uses: ./.github/actions/install-python-and-package/action.yml with: python-version: ${{ matrix.python-version }} extras-require: publishing From 24eea9047c120cc277a36ccc09d0bb0c66b3d77f Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:17:44 +0200 Subject: [PATCH 20/43] reindent yml content --- .../install-python-and-package/action.yml | 135 +++++++++--------- 1 file changed, 66 insertions(+), 69 deletions(-) diff --git a/.github/actions/install-python-and-package/action.yml b/.github/actions/install-python-and-package/action.yml index b95bdc7f..539cfe42 100644 --- a/.github/actions/install-python-and-package/action.yml +++ b/.github/actions/install-python-and-package/action.yml @@ -2,83 +2,80 @@ name: "Install Python and DeepRank2" description: "Installs Python, updates pip and installs DeepRank2 together with its dependencies." -on: - workflow_call: +inputs: + python-version: + required: false + description: "The Python version to use. Specify major and minor version, e.g. '3.10'." + default: "3.10" - inputs: - python-version: - required: false - description: "The Python version to use. Specify major and minor version, e.g. '3.10'." - default: "3.10" + extras-require: + required: false + description: "The extras dependencies packages to be installed, for instance 'dev' or 'dev,publishing,notebooks'." + default: "test" - extras-require: - required: false - description: "The extras dependencies packages to be installed, for instance 'dev' or 'dev,publishing,notebooks'." - default: "test" + pkg-installation-type: + required: false + description: "The package installation type to install, the latest released version on PyPI ('latest') or the GitHub repository one ('repository')." + default: "repository" - pkg-installation-type: - required: false - description: "The package installation type to install, the latest released version on PyPI ('latest') or the GitHub repository one ('repository')." - default: "repository" +runs: + using: "composite" - runs: - using: "composite" + steps: + - name: Cancel Previous Runs and Set up Python + uses: styfle/cancel-workflow-action@0.4.0 + with: + access_token: ${{ github.token }} - steps: - - name: Cancel Previous Runs and Set up Python - uses: styfle/cancel-workflow-action@0.4.0 - with: - access_token: ${{ github.token }} + - uses: actions/checkout@v3 - - uses: actions/checkout@v3 + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniforge-variant: Mambaforge + channels: conda-forge + python-version: ${{ inputs.python-version }} + activate-environment: deeprank2 + environment-file: env/deeprank2.yml + use-mamba: true + if: ${{ inputs.pkg-installation-type != 'frozen' }} - - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniforge-variant: Mambaforge - channels: conda-forge - python-version: ${{ inputs.python-version }} - activate-environment: deeprank2 - environment-file: env/deeprank2.yml - use-mamba: true - if: ${{ inputs.pkg-installation-type != 'frozen' }} + - name: Setup miniconda with frozen dependencies + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniforge-variant: Mambaforge + channels: conda-forge + python-version: ${{ inputs.python-version }} + activate-environment: deeprank2 + environment-file: env/deeprank2_frozen.yml + use-mamba: true + if: ${{ inputs.pkg-installation-type == 'frozen' }} - - name: Setup miniconda with frozen dependencies - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniforge-variant: Mambaforge - channels: conda-forge - python-version: ${{ inputs.python-version }} - activate-environment: deeprank2 - environment-file: env/deeprank2_frozen.yml - use-mamba: true - if: ${{ inputs.pkg-installation-type == 'frozen' }} + - run: | + conda --version + conda env list + shell: bash -l {0} - - run: | - conda --version - conda env list - shell: bash -l {0} + - name: Python info + shell: bash -l {0} + run: | + which python3 + python3 --version - - name: Python info - shell: bash -l {0} - run: | - which python3 - python3 --version + - name: Install the GitHub repository version of the package + shell: bash -l {0} + if: ${{ inputs.pkg-installation-type == 'repository' || inputs.pkg-installation-type == 'frozen' }} + run: | + conda activate deeprank2 + pip install .'[${{ inputs.extras-require }}]' - - name: Install the GitHub repository version of the package - shell: bash -l {0} - if: ${{ inputs.pkg-installation-type == 'repository' || inputs.pkg-installation-type == 'frozen' }} - run: | - conda activate deeprank2 - pip install .'[${{ inputs.extras-require }}]' - - - name: Install the latest released PyPI version of the package - shell: bash -l {0} - if: ${{ inputs.pkg-installation-type == 'latest' }} - run: | - conda activate deeprank2 - pip install pytest - rm -r deeprank2 - pip install deeprank2 + - name: Install the latest released PyPI version of the package + shell: bash -l {0} + if: ${{ inputs.pkg-installation-type == 'latest' }} + run: | + conda activate deeprank2 + pip install pytest + rm -r deeprank2 + pip install deeprank2 From 6218944c3e8746f3c9b79b0ea7ba9d10c46c954c Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:19:51 +0200 Subject: [PATCH 21/43] fix path --- .github/workflows/build-latest-release.yml | 2 +- .github/workflows/build-repo-frozen-env.yml | 2 +- .github/workflows/build-repo.yml | 2 +- .github/workflows/coveralls.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/release.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-latest-release.yml b/.github/workflows/build-latest-release.yml index 417174d9..9daa9936 100644 --- a/.github/workflows/build-latest-release.yml +++ b/.github/workflows/build-latest-release.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package/action.yml + - uses: ./.github/actions/install-python-and-package with: python-version: ${{ matrix.python-version }} pkg-installation-type: "latest" diff --git a/.github/workflows/build-repo-frozen-env.yml b/.github/workflows/build-repo-frozen-env.yml index 7fb126c8..47f29dce 100644 --- a/.github/workflows/build-repo-frozen-env.yml +++ b/.github/workflows/build-repo-frozen-env.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package/action.yml + - uses: ./.github/actions/install-python-and-package with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/build-repo.yml b/.github/workflows/build-repo.yml index b5395395..7af4cc7a 100644 --- a/.github/workflows/build-repo.yml +++ b/.github/workflows/build-repo.yml @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package/action.yml + - uses: ./.github/actions/install-python-and-package with: python-version: ${{ matrix.python-version }} extras-require: test, publishing diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index e3a0f120..eb4feff2 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package/action.yml + - uses: ./.github/actions/install-python-and-package with: python-version: ${{ matrix.python-version }} extras-require: test diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index fce2b216..b1e59ea9 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -45,7 +45,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package/action.yml + - uses: ./.github/actions/install-python-and-package with: python-version: ${{ matrix.python-version }} extras-require: test, notebooks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index af65141d..0f85061b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/install-python-and-package/action.yml + - uses: ./.github/actions/install-python-and-package with: python-version: ${{ matrix.python-version }} extras-require: publishing From 45fa50afa84303b87716747a447cca938e9ea674 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:24:31 +0200 Subject: [PATCH 22/43] fix _ghcr link --- .github/workflows/{ _ghcr.yml => _ghcr.yml} | 0 .github/workflows/ghcr.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ _ghcr.yml => _ghcr.yml} (100%) create mode 100644 .github/workflows/ghcr.yml diff --git a/.github/workflows/ _ghcr.yml b/.github/workflows/_ghcr.yml similarity index 100% rename from .github/workflows/ _ghcr.yml rename to .github/workflows/_ghcr.yml diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml new file mode 100644 index 00000000..e69de29b From a647d756f98a761e864b80262b8e9df212e8f3fd Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:35:26 +0200 Subject: [PATCH 23/43] add tag based on software version --- .github/workflows/publish_docker_image.yml | 53 ++++++---------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index 553b40b7..d3962f68 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -16,56 +16,29 @@ on: branches: ["529_add_docker_testing_action_gcroci2"] jobs: - release_tag: - name: conventional changelog - runs-on: ubuntu-22.04 + read_version: + name: Read version from TOML + runs-on: ubuntu-latest outputs: - changelog: ${{steps.changelog.outputs.clean_changelog}} - tag: ${{steps.changelog.outputs.tag}} - skipped: ${{steps.changelog.outputs.skipped}} + version: ${{ steps.get_version.outputs.VERSION }} steps: - - name: checkout all history - # https://github.com/actions/checkout + - name: Checkout code uses: actions/checkout@v4 - with: - # checkout whole history - fetch-depth: 0 - - name: calculate new version and create changelog content - id: changelog - # https://github.com/TriPSs/conventional-changelog-action - uses: TriPSs/conventional-changelog-action@v5 - with: - # you can also create separate token to trace action - github-token: "${{secrets.GITHUB_TOKEN}}" - # do not create changelog file, the content is used at next step for relase body - output-file: false - # do not create additional commit, just tag current commit with the version - skip-commit: true - # do not pull - we already checked out the selection we want to use for versioning in previous step - skip-git-pull: true - # skip tag push - it will not push but it will tag - git-push: false - - log_release_tag: - needs: release_tag - name: log version output - runs-on: ubuntu-22.04 - steps: - - name: info + - name: Read version from TOML + id: get_version run: | - echo skipped=${{needs.release_tag.outputs.skipped}} - echo tag=${{needs.release_tag.outputs.tag}} + VERSION=$(grep '^version =' pyproject.toml | awk -F '"' '{print $2}') + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + docker_image_deeprank2: - # # it needs to be checked on string value - # if: needs.release_tag.outputs.skipped == 'false' - needs: release_tag + needs: read_version name: docker_image_deeprank2 uses: ./.github/workflows/_ghcr.yml with: ghcr_user: ${{github.actor}} - base_image_name: ghcr.io/deeprank/deeprank2 - image_tag: latest + base_image_name: ghcr.io/${{ github.repository }} + image_tag: ${{ needs.read_version.outputs.version }} dockerfile: ./Dockerfile docker_context: . secrets: From a1ec7e59ff1e13a58b8fff6730bae7728b5bbc94 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:37:39 +0200 Subject: [PATCH 24/43] lowercase the organization --- .github/workflows/publish_docker_image.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index d3962f68..24a29898 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -21,6 +21,7 @@ jobs: runs-on: ubuntu-latest outputs: version: ${{ steps.get_version.outputs.VERSION }} + repo_lowercase: ${{ steps.repo_lowercase.outputs.REPO_LOWERCASE }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -31,13 +32,19 @@ jobs: VERSION=$(grep '^version =' pyproject.toml | awk -F '"' '{print $2}') echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + - name: Convert repository name to lowercase + id: repo_lowercase + run: | + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + echo "REPO_LOWERCASE=$REPO_LOWERCASE" >> $GITHUB_OUTPUT + docker_image_deeprank2: needs: read_version name: docker_image_deeprank2 uses: ./.github/workflows/_ghcr.yml with: ghcr_user: ${{github.actor}} - base_image_name: ghcr.io/${{ github.repository }} + base_image_name: ghcr.io/${{ needs.read_version.outputs.repo_lowercase }} image_tag: ${{ needs.read_version.outputs.version }} dockerfile: ./Dockerfile docker_context: . From bcb67b2a2d08da08bccab62dd5b073d4e4d54cd7 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 12:46:19 +0200 Subject: [PATCH 25/43] try to increase patch version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4172c9c1..592edf19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "deeprank2" -version = "3.0.5" +version = "3.0.6" description = "DeepRank2 is an open-source deep learning framework for data mining of protein-protein interfaces or single-residue missense variants." readme = "README.md" requires-python = ">=3.10" From 82ea4bc795e65db6f01900b4eb3582f75675e08c Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 15:07:22 +0200 Subject: [PATCH 26/43] move docker upload to release.yml --- .github/workflows/ghcr.yml | 0 .github/workflows/publish_docker_image.yml | 52 ---------------------- .github/workflows/release.yml | 40 ++++++++++++++++- pyproject.toml | 2 +- 4 files changed, 39 insertions(+), 55 deletions(-) delete mode 100644 .github/workflows/ghcr.yml delete mode 100644 .github/workflows/publish_docker_image.yml diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml deleted file mode 100644 index e69de29b..00000000 diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml deleted file mode 100644 index 24a29898..00000000 --- a/.github/workflows/publish_docker_image.yml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) -# SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) -# SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences -# SPDX-FileCopyrightText: 2022 dv4all -# SPDX-FileCopyrightText: 2023 - 2024 Dusan Mijatovic (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center -# -# SPDX-License-Identifier: Apache-2.0 - -name: Create and publish a Docker image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - push: - branches: ["529_add_docker_testing_action_gcroci2"] - -jobs: - read_version: - name: Read version from TOML - runs-on: ubuntu-latest - outputs: - version: ${{ steps.get_version.outputs.VERSION }} - repo_lowercase: ${{ steps.repo_lowercase.outputs.REPO_LOWERCASE }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Read version from TOML - id: get_version - run: | - VERSION=$(grep '^version =' pyproject.toml | awk -F '"' '{print $2}') - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Convert repository name to lowercase - id: repo_lowercase - run: | - REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - echo "REPO_LOWERCASE=$REPO_LOWERCASE" >> $GITHUB_OUTPUT - - docker_image_deeprank2: - needs: read_version - name: docker_image_deeprank2 - uses: ./.github/workflows/_ghcr.yml - with: - ghcr_user: ${{github.actor}} - base_image_name: ghcr.io/${{ needs.read_version.outputs.repo_lowercase }} - image_tag: ${{ needs.read_version.outputs.version }} - dockerfile: ./Dockerfile - docker_context: . - secrets: - token: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f85061b..b3c1c6e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: path: dist/* upload_test_pypi: - needs: [build] + needs: build runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' steps: @@ -51,7 +51,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ upload_pypi: - needs: [build] + needs: build runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' steps: @@ -63,3 +63,39 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_TOKEN_DEEPRANK2 }} + + read_version: + needs: upload_pypi + name: Read version from TOML + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get_version.outputs.VERSION }} + repo_lowercase: ${{ steps.repo_lowercase.outputs.REPO_LOWERCASE }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Read version from TOML + id: get_version + run: | + VERSION=$(grep '^version =' pyproject.toml | awk -F '"' '{print $2}') + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + + - name: Convert repository name to lowercase + id: repo_lowercase + run: | + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + echo "REPO_LOWERCASE=$REPO_LOWERCASE" >> $GITHUB_OUTPUT + + upload_docker_image: + needs: read_version + name: Upload Docker image to ghcr.io + uses: ./.github/workflows/_ghcr.yml + with: + ghcr_user: ${{github.actor}} + base_image_name: ghcr.io/${{ needs.read_version.outputs.repo_lowercase }} + image_tag: ${{ needs.read_version.outputs.version }} + dockerfile: ./Dockerfile + docker_context: . + secrets: + token: ${{secrets.GITHUB_TOKEN}} diff --git a/pyproject.toml b/pyproject.toml index 592edf19..4172c9c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "deeprank2" -version = "3.0.6" +version = "3.0.5" description = "DeepRank2 is an open-source deep learning framework for data mining of protein-protein interfaces or single-residue missense variants." readme = "README.md" requires-python = ">=3.10" From 389548a7a05ad6676886fa310245b78fd89b1517 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 15:11:14 +0200 Subject: [PATCH 27/43] fix ubuntu to 22.04 --- .github/workflows/build-latest-release.yml | 2 +- .github/workflows/build-repo-frozen-env.yml | 2 +- .github/workflows/build-repo.yml | 2 +- .github/workflows/cffconvert.yml | 2 +- .github/workflows/coveralls.yml | 2 +- .github/workflows/draft-pdf.yml | 2 +- .github/workflows/fair-software.yml | 2 +- .github/workflows/linting.yml | 2 +- .github/workflows/markdown-link-check.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/release.yml | 8 ++++---- .github/workflows/stale_issue_pr.yml | 2 +- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-latest-release.yml b/.github/workflows/build-latest-release.yml index 9daa9936..28d37f6a 100644 --- a/.github/workflows/build-latest-release.yml +++ b/.github/workflows/build-latest-release.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-22.04"] python-version: ["3.10"] # ["3.10", "3.11"] # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell diff --git a/.github/workflows/build-repo-frozen-env.yml b/.github/workflows/build-repo-frozen-env.yml index 47f29dce..47ab259e 100644 --- a/.github/workflows/build-repo-frozen-env.yml +++ b/.github/workflows/build-repo-frozen-env.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-22.04"] python-version: ["3.10"] # ["3.10", "3.11"] # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell diff --git a/.github/workflows/build-repo.yml b/.github/workflows/build-repo.yml index 7af4cc7a..e777c5d9 100644 --- a/.github/workflows/build-repo.yml +++ b/.github/workflows/build-repo.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-22.04"] python-version: ["3.10"] # ["3.10", "3.11"] # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell diff --git a/.github/workflows/cffconvert.yml b/.github/workflows/cffconvert.yml index 6851c52d..e63baf7f 100644 --- a/.github/workflows/cffconvert.yml +++ b/.github/workflows/cffconvert.yml @@ -8,7 +8,7 @@ on: jobs: validate: name: "validate" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check out a copy of the repository uses: actions/checkout@v3 diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index eb4feff2..0193c441 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-22.04"] python-version: ["3.10"] # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 8b5159f7..d044846b 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -5,7 +5,7 @@ on: jobs: paper: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 name: Paper Draft steps: - name: Checkout diff --git a/.github/workflows/fair-software.yml b/.github/workflows/fair-software.yml index f20d3c84..f336c6ac 100644 --- a/.github/workflows/fair-software.yml +++ b/.github/workflows/fair-software.yml @@ -11,7 +11,7 @@ jobs: verify: if: github.event.pull_request.draft == false name: "fair-software" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: fair-software/howfairis-github-action@0.2.1 name: Measure compliance with fair-software.eu recommendations diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 4dd13612..030e4cbc 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-22.04"] python-version: ["3.10"] # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index 016e3812..a988faf8 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -23,7 +23,7 @@ jobs: markdown-link-check: if: github.event.pull_request.draft == false name: Check markdown links - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: gaurav-nelson/github-action-markdown-link-check@v1 diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index b1e59ea9..93433352 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-22.04"] python-version: ["3.10"] # ["3.10", "3.11"] # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b3c1c6e1..dfc13ef1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-22.04"] python-version: ["3.10"] # https://github.com/marketplace/actions/setup-miniconda#use-a-default-shell @@ -37,7 +37,7 @@ jobs: upload_test_pypi: needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 if: github.event_name == 'workflow_dispatch' steps: - uses: actions/download-artifact@v3 @@ -52,7 +52,7 @@ jobs: upload_pypi: needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 if: github.event_name == 'release' && github.event.action == 'published' steps: - uses: actions/download-artifact@v3 @@ -67,7 +67,7 @@ jobs: read_version: needs: upload_pypi name: Read version from TOML - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: version: ${{ steps.get_version.outputs.VERSION }} repo_lowercase: ${{ steps.repo_lowercase.outputs.REPO_LOWERCASE }} diff --git a/.github/workflows/stale_issue_pr.yml b/.github/workflows/stale_issue_pr.yml index d184c9c9..74ef3f5a 100644 --- a/.github/workflows/stale_issue_pr.yml +++ b/.github/workflows/stale_issue_pr.yml @@ -5,7 +5,7 @@ on: jobs: close-issues: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: issues: write pull-requests: write From 3f91ab01a88bdca45f21ec11a93db47423fefc5f Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 15:27:52 +0200 Subject: [PATCH 28/43] add test for docker image --- .github/workflows/build-latest-release.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test-docker-image.yml | 40 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-docker-image.yml diff --git a/.github/workflows/build-latest-release.yml b/.github/workflows/build-latest-release.yml index 28d37f6a..9689e5cd 100644 --- a/.github/workflows/build-latest-release.yml +++ b/.github/workflows/build-latest-release.yml @@ -3,7 +3,7 @@ name: build (latest release) # Only trigger, when the release workflow succeeded on: workflow_run: - workflows: ["Build and upload to PyPI"] + workflows: ["Build and upload to PyPI and ghcr.io"] types: - completed diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dfc13ef1..cce554ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Build and upload to PyPI +name: Build and upload to PyPI and ghcr.io on: workflow_dispatch: diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml new file mode 100644 index 00000000..16a3179b --- /dev/null +++ b/.github/workflows/test-docker-image.yml @@ -0,0 +1,40 @@ +name: Test Latest Docker Image (Dev) + +on: + push: + branches: + - 529_add_docker_testing_action_gcroci2 + +jobs: + test_latest_docker_image: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull latest Docker image + run: | + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + docker pull ghcr.io/$REPO_LOWERCASE:latest + + - name: Run tests in Docker container + run: | + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null + docker cp tests test_container:/app/tests + docker exec test_container pip install pytest + docker exec test_container pytest /app/tests + docker stop test_container + docker rm test_container + + - name: Output test results + if: failure() + run: | + echo "Tests failed. Please check the test output above for more details." From 65b123cfc5e910616ab71a98bd2d065d07d77d13 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 15:32:57 +0200 Subject: [PATCH 29/43] debug docker container running --- .github/workflows/test-docker-image.yml | 28 +++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index 16a3179b..99b27aa5 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -1,9 +1,9 @@ -name: Test Latest Docker Image (Dev) +name: test latest docker image (dev) # TODO: change on: push: branches: - - 529_add_docker_testing_action_gcroci2 + - 529_add_docker_testing_action_gcroci2 # TODO: change jobs: test_latest_docker_image: @@ -28,9 +28,29 @@ jobs: run: | REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null - docker cp tests test_container:/app/tests + # Inspect container filesystem + echo "Container filesystem structure:" + docker exec test_container ls -R / + + # Determine the appropriate directory for tests + if docker exec test_container [ -d "/app" ]; then + TEST_DIR="/app/tests" + elif docker exec test_container [ -d "/usr/src/app" ]; then + TEST_DIR="/usr/src/app/tests" + else + TEST_DIR="/tests" + fi + + echo "Using test directory: $TEST_DIR" + + # Copy tests to the container + docker cp tests test_container:$TEST_DIR + + # Install pytest and run tests docker exec test_container pip install pytest - docker exec test_container pytest /app/tests + docker exec -e PYTHONPATH=$TEST_DIR test_container pytest $TEST_DIR + + # Clean up docker stop test_container docker rm test_container From 22745b61fa390af514be38271857d86635ed281d Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 15:41:58 +0200 Subject: [PATCH 30/43] copy tests folder recursively --- .github/workflows/test-docker-image.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index 99b27aa5..1994abf7 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -28,9 +28,10 @@ jobs: run: | REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null + # Inspect container filesystem - echo "Container filesystem structure:" - docker exec test_container ls -R / + echo "Container filesystem structure before copying tests:" + docker exec test_container ls / # Determine the appropriate directory for tests if docker exec test_container [ -d "/app" ]; then @@ -44,7 +45,10 @@ jobs: echo "Using test directory: $TEST_DIR" # Copy tests to the container - docker cp tests test_container:$TEST_DIR + docker cp -r tests/. test_container:$TEST_DIR + + echo "Container filesystem structure after copying tests:" + docker exec test_container ls $TEST_DIR # Install pytest and run tests docker exec test_container pip install pytest From de8c6266ab0e7aa129a8d4a652442ac13b67ff17 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 15:46:58 +0200 Subject: [PATCH 31/43] print copied tests folder --- .github/workflows/test-docker-image.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index 1994abf7..1b80e43c 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -29,10 +29,6 @@ jobs: REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null - # Inspect container filesystem - echo "Container filesystem structure before copying tests:" - docker exec test_container ls / - # Determine the appropriate directory for tests if docker exec test_container [ -d "/app" ]; then TEST_DIR="/app/tests" @@ -45,14 +41,17 @@ jobs: echo "Using test directory: $TEST_DIR" # Copy tests to the container - docker cp -r tests/. test_container:$TEST_DIR + docker cp tests test_container:$TEST_DIR - echo "Container filesystem structure after copying tests:" - docker exec test_container ls $TEST_DIR + # Verify the tests were copied correctly + echo "Contents of $TEST_DIR:" + docker exec test_container ls -la $TEST_DIR + echo "Contents of $TEST_DIR/data/:" + docker exec test_container ls -la $TEST_DIR/data || echo "data directory not found" # Install pytest and run tests docker exec test_container pip install pytest - docker exec -e PYTHONPATH=$TEST_DIR test_container pytest $TEST_DIR + docker exec -e PYTHONPATH=$TEST_DIR test_container pytest $TEST_DIR -v # Clean up docker stop test_container From f61f7c1942bd43fa0e9d2f19a1f2eebf331dffa7 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 15:55:42 +0200 Subject: [PATCH 32/43] try to copy the whole repo --- .github/workflows/test-docker-image.yml | 40 ++++++++++++++----------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index 1b80e43c..1691db92 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -29,29 +29,35 @@ jobs: REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null - # Determine the appropriate directory for tests - if docker exec test_container [ -d "/app" ]; then - TEST_DIR="/app/tests" - elif docker exec test_container [ -d "/usr/src/app" ]; then - TEST_DIR="/usr/src/app/tests" - else - TEST_DIR="/tests" - fi + WORK_DIR="/" - echo "Using test directory: $TEST_DIR" + echo "Using working directory: $WORK_DIR" - # Copy tests to the container - docker cp tests test_container:$TEST_DIR + # Copy entire project directory to the container + docker cp . test_container:$WORK_DIR # Verify the tests were copied correctly - echo "Contents of $TEST_DIR:" - docker exec test_container ls -la $TEST_DIR - echo "Contents of $TEST_DIR/data/:" - docker exec test_container ls -la $TEST_DIR/data || echo "data directory not found" + echo "Contents of $WORK_DIR/tests:" + docker exec test_container ls -la $WORK_DIR/tests + echo "Contents of $WORK_DIR/tests/data:" + docker exec test_container ls -la $WORK_DIR/tests/data - # Install pytest and run tests + # Print debugging information + echo "Current working directory:" + docker exec -w $WORK_DIR test_container pwd + echo "Full path of test directory:" + docker exec test_container readlink -f $WORK_DIR/tests + + # Print content of a test file + echo "Content of a test file that accesses data (first 20 lines):" + docker exec test_container head -n 20 $WORK_DIR/tests/test_*.py + + # Install pytest docker exec test_container pip install pytest - docker exec -e PYTHONPATH=$TEST_DIR test_container pytest $TEST_DIR -v + + # Run pytest from the working directory + echo "Running pytest from the working directory:" + docker exec -w $WORK_DIR test_container pytest tests -v # Clean up docker stop test_container From 243721aab7772a7a6b31a06e9ea0f5a2b05d4e22 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 16:01:01 +0200 Subject: [PATCH 33/43] try to fix work dire --- .github/workflows/test-docker-image.yml | 32 ++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index 1691db92..edcc4d26 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -30,34 +30,38 @@ jobs: docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null WORK_DIR="/" - echo "Using working directory: $WORK_DIR" # Copy entire project directory to the container docker cp . test_container:$WORK_DIR - # Verify the tests were copied correctly - echo "Contents of $WORK_DIR/tests:" - docker exec test_container ls -la $WORK_DIR/tests - echo "Contents of $WORK_DIR/tests/data:" - docker exec test_container ls -la $WORK_DIR/tests/data + # Verify the directory structure + echo "Contents of root directory:" + docker exec test_container ls -la $WORK_DIR + echo "Contents of tests directory (if it exists):" + docker exec test_container ls -la $WORK_DIR/tests || echo "No tests directory found" + echo "Contents of data directory (if it exists):" + docker exec test_container ls -la $WORK_DIR/data || echo "No data directory found" # Print debugging information echo "Current working directory:" docker exec -w $WORK_DIR test_container pwd - echo "Full path of test directory:" - docker exec test_container readlink -f $WORK_DIR/tests - # Print content of a test file - echo "Content of a test file that accesses data (first 20 lines):" - docker exec test_container head -n 20 $WORK_DIR/tests/test_*.py + # Find and print content of a test file + echo "Content of a test file (first 20 lines):" + docker exec test_container find $WORK_DIR -name "test_*.py" -type f | head -n 1 | xargs -I {} docker exec test_container head -n 20 {} # Install pytest docker exec test_container pip install pytest - # Run pytest from the working directory - echo "Running pytest from the working directory:" - docker exec -w $WORK_DIR test_container pytest tests -v + # Find the directory containing test files and run pytest + TEST_DIR=$(docker exec test_container find $WORK_DIR -name "test_*.py" -type f | head -n 1 | xargs dirname) + if [ -n "$TEST_DIR" ]; then + echo "Running pytest in directory: $TEST_DIR" + docker exec -w $TEST_DIR test_container pytest -v + else + echo "No test files found" + fi # Clean up docker stop test_container From 0660c5a71cf087a89b581b36565182c64758127d Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 16:06:08 +0200 Subject: [PATCH 34/43] try to copy tests only again --- .github/workflows/test-docker-image.yml | 39 +++++++++++-------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index edcc4d26..bbfe6855 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -32,36 +32,31 @@ jobs: WORK_DIR="/" echo "Using working directory: $WORK_DIR" - # Copy entire project directory to the container - docker cp . test_container:$WORK_DIR + REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null - # Verify the directory structure - echo "Contents of root directory:" - docker exec test_container ls -la $WORK_DIR - echo "Contents of tests directory (if it exists):" - docker exec test_container ls -la $WORK_DIR/tests || echo "No tests directory found" - echo "Contents of data directory (if it exists):" - docker exec test_container ls -la $WORK_DIR/data || echo "No data directory found" + TEST_DIR="/tests" + echo "Tests directory: $TEST_DIR" - # Print debugging information - echo "Current working directory:" - docker exec -w $WORK_DIR test_container pwd + # Copy only the tests folder to the container + docker cp tests test_container:$TEST_DIR + + # Verify the directory structure + echo "Contents of tests directory:" + docker exec test_container ls -la $TEST_DIR + echo "Contents of tests/data directory:" + docker exec test_container ls -la $TEST_DIR/data - # Find and print content of a test file + # Print content of a test file echo "Content of a test file (first 20 lines):" - docker exec test_container find $WORK_DIR -name "test_*.py" -type f | head -n 1 | xargs -I {} docker exec test_container head -n 20 {} + docker exec test_container head -n 20 $TEST_DIR/test_*.py | head -n 20 # Install pytest docker exec test_container pip install pytest - # Find the directory containing test files and run pytest - TEST_DIR=$(docker exec test_container find $WORK_DIR -name "test_*.py" -type f | head -n 1 | xargs dirname) - if [ -n "$TEST_DIR" ]; then - echo "Running pytest in directory: $TEST_DIR" - docker exec -w $TEST_DIR test_container pytest -v - else - echo "No test files found" - fi + # Run pytest + echo "Running pytest in the tests directory:" + docker exec -w $TEST_DIR test_container pytest -v # Clean up docker stop test_container From fcd14ebdeab95ef574715c129c12148a90bad195 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 16:08:24 +0200 Subject: [PATCH 35/43] remove duplicated run --- .github/workflows/test-docker-image.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index bbfe6855..eb297282 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -26,11 +26,6 @@ jobs: - name: Run tests in Docker container run: | - REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null - - WORK_DIR="/" - echo "Using working directory: $WORK_DIR" REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null From 233fb7772dd3d731b8f52a06a039ac45637289f9 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 16:13:26 +0200 Subject: [PATCH 36/43] add project dir --- .github/workflows/test-docker-image.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index eb297282..6271bbe7 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -30,28 +30,33 @@ jobs: REPO_LOWERCASE=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') docker run --name test_container -d ghcr.io/$REPO_LOWERCASE:latest tail -f /dev/null - TEST_DIR="/tests" + PROJECT_DIR="/app" + TEST_DIR="$PROJECT_DIR/tests" + echo "Project directory: $PROJECT_DIR" echo "Tests directory: $TEST_DIR" - # Copy only the tests folder to the container + # Create project directory and copy tests folder + docker exec test_container mkdir -p $PROJECT_DIR docker cp tests test_container:$TEST_DIR # Verify the directory structure + echo "Contents of project directory:" + docker exec test_container ls -la $PROJECT_DIR echo "Contents of tests directory:" docker exec test_container ls -la $TEST_DIR - echo "Contents of tests/data directory:" - docker exec test_container ls -la $TEST_DIR/data - # Print content of a test file - echo "Content of a test file (first 20 lines):" - docker exec test_container head -n 20 $TEST_DIR/test_*.py | head -n 20 + # List test files and print content of the first one + echo "List of test files:" + docker exec test_container find $TEST_DIR -name "test_*.py" + echo "Content of the first test file (first 20 lines):" + docker exec test_container bash -c "head -n 20 \$(find $TEST_DIR -name 'test_*.py' | head -n 1)" # Install pytest docker exec test_container pip install pytest - # Run pytest - echo "Running pytest in the tests directory:" - docker exec -w $TEST_DIR test_container pytest -v + # Run pytest from the project directory + echo "Running pytest from the project directory:" + docker exec -w $PROJECT_DIR test_container python -m pytest tests -v # Clean up docker stop test_container From 195530083d5af1971a3edd48b118669d44531e7d Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Thu, 5 Sep 2024 16:19:58 +0200 Subject: [PATCH 37/43] remove debugging comment and trigger wf only when release is completed --- .github/workflows/test-docker-image.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-docker-image.yml b/.github/workflows/test-docker-image.yml index 6271bbe7..35eb6bbe 100644 --- a/.github/workflows/test-docker-image.yml +++ b/.github/workflows/test-docker-image.yml @@ -1,9 +1,11 @@ -name: test latest docker image (dev) # TODO: change +name: test latest docker image +# Only trigger, when the release workflow succeeded on: - push: - branches: - - 529_add_docker_testing_action_gcroci2 # TODO: change + workflow_run: + workflows: ["Build and upload to PyPI and ghcr.io"] + types: + - completed jobs: test_latest_docker_image: @@ -45,12 +47,6 @@ jobs: echo "Contents of tests directory:" docker exec test_container ls -la $TEST_DIR - # List test files and print content of the first one - echo "List of test files:" - docker exec test_container find $TEST_DIR -name "test_*.py" - echo "Content of the first test file (first 20 lines):" - docker exec test_container bash -c "head -n 20 \$(find $TEST_DIR -name 'test_*.py' | head -n 1)" - # Install pytest docker exec test_container pip install pytest From 6bcb480502ef3a1477be64c8e167c45fec184f0c Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Fri, 6 Sep 2024 11:15:55 +0200 Subject: [PATCH 38/43] fix tutorials add --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b26ada57..9cbe9900 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ RUN \ echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc ADD ./env/deeprank2.yml /home/deeprank2/ +ADD ./tutorials /home/deeprank2/tutorials RUN \ ## Create the environment and install the dependencies @@ -35,7 +36,7 @@ RUN \ # Get the data for running the tutorials if [ -d "/home/deeprank2/tutorials/data_raw" ]; then rm -Rf /home/deeprank2/tutorials/data_raw; fi && \ if [ -d "/home/deeprank2/tutorials/data_processed" ]; then rm -Rf /home/deeprank2/tutorials/data_processed; fi && \ - wget https://zenodo.org/records/8349335/files/data_raw.zip && \ + wget https://zenodo.org/records/13709906/files/data_raw.zip && \ unzip data_raw.zip -d data_raw && \ mv data_raw /home/deeprank2/tutorials && \ apt-get clean && \ @@ -45,8 +46,6 @@ RUN \ find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \ conda clean --force-pkgs-dirs --all --yes -ADD ./tutorials /home/deeprank2/tutorials - ENV PATH /opt/conda/envs/deeprank2/bin:$PATH # Define working directory From f931154b780d1b3efc7796b082422129449611a4 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Fri, 6 Sep 2024 11:16:23 +0200 Subject: [PATCH 39/43] update zenodo address --- tutorials/data_generation_ppi.ipynb | 2 +- tutorials/data_generation_srv.ipynb | 2 +- tutorials/training.ipynb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorials/data_generation_ppi.ipynb b/tutorials/data_generation_ppi.ipynb index 2d1d9650..8553d4a7 100644 --- a/tutorials/data_generation_ppi.ipynb +++ b/tutorials/data_generation_ppi.ipynb @@ -29,7 +29,7 @@ "source": [ "### Input Data\n", "\n", - "The example data used in this tutorial are available on Zenodo at [this record address](https://zenodo.org/record/8349335). To download the raw data used in this tutorial, please visit the link and download `data_raw.zip`. Unzip it, and save the `data_raw/` folder in the same directory as this notebook. The name and the location of the folder are optional but recommended, as they are the name and the location we will use to refer to the folder throughout the tutorial.\n", + "The example data used in this tutorial are available on Zenodo at [this record address](https://zenodo.org/record/13709906). To download the raw data used in this tutorial, please visit the link and download `data_raw.zip`. Unzip it, and save the `data_raw/` folder in the same directory as this notebook. The name and the location of the folder are optional but recommended, as they are the name and the location we will use to refer to the folder throughout the tutorial.\n", "\n", "Note that the dataset contains only 100 data points, which is not enough to develop an impactful predictive model, and the scope of its use is indeed only demonstrative and informative for the users.\n" ] diff --git a/tutorials/data_generation_srv.ipynb b/tutorials/data_generation_srv.ipynb index 1a68f31a..a4fb3c16 100644 --- a/tutorials/data_generation_srv.ipynb +++ b/tutorials/data_generation_srv.ipynb @@ -29,7 +29,7 @@ "source": [ "### Input Data\n", "\n", - "The example data used in this tutorial are available on Zenodo at [this record address](https://zenodo.org/record/8349335). To download the raw data used in this tutorial, please visit the link and download `data_raw.zip`. Unzip it, and save the `data_raw/` folder in the same directory as this notebook. The name and the location of the folder are optional but recommended, as they are the name and the location we will use to refer to the folder throughout the tutorial.\n", + "The example data used in this tutorial are available on Zenodo at [this record address](https://zenodo.org/record/13709906). To download the raw data used in this tutorial, please visit the link and download `data_raw.zip`. Unzip it, and save the `data_raw/` folder in the same directory as this notebook. The name and the location of the folder are optional but recommended, as they are the name and the location we will use to refer to the folder throughout the tutorial.\n", "\n", "Note that the dataset contains only 96 data points, which is not enough to develop an impactful predictive model, and the scope of its use is indeed only demonstrative and informative for the users.\n" ] diff --git a/tutorials/training.ipynb b/tutorials/training.ipynb index 499c3f8e..2c662b8d 100644 --- a/tutorials/training.ipynb +++ b/tutorials/training.ipynb @@ -33,7 +33,7 @@ "\n", "If you have previously run `data_generation_ppi.ipynb` or `data_generation_srv.ipynb` notebook, then their output can be directly used as input for this tutorial.\n", "\n", - "Alternatively, preprocessed HDF5 files can be downloaded directly from Zenodo at [this record address](https://zenodo.org/record/8349335). To download the data used in this tutorial, please visit the link and download `data_processed.zip`. Unzip it, and save the `data_processed/` folder in the same directory as this notebook. The name and the location of the folder are optional but recommended, as they are the name and the location we will use to refer to the folder throughout the tutorial.\n", + "Alternatively, preprocessed HDF5 files can be downloaded directly from Zenodo at [this record address](https://zenodo.org/record/13709906). To download the data used in this tutorial, please visit the link and download `data_processed.zip`. Unzip it, and save the `data_processed/` folder in the same directory as this notebook. The name and the location of the folder are optional but recommended, as they are the name and the location we will use to refer to the folder throughout the tutorial.\n", "\n", "Note that the datasets contain only ~100 data points each, which is not enough to develop an impactful predictive model, and the scope of their use is indeed only demonstrative and informative for the users.\n" ] From 3d784caddc9200abe95222b263a11d5e52dac038 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Fri, 6 Sep 2024 11:16:35 +0200 Subject: [PATCH 40/43] update zenodo address --- .github/workflows/notebooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 93433352..9f3f2902 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -54,7 +54,7 @@ jobs: - name: Download the data for the tutorials shell: bash -l {0} run: | - wget https://zenodo.org/records/8349335/files/data_raw.zip + wget https://zenodo.org/records/13709906/files/data_raw.zip unzip data_raw.zip -d data_raw mv data_raw tutorials From b61662b3929aa8dff2fcd22d1823df2b3b8e13f0 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Fri, 6 Sep 2024 11:16:49 +0200 Subject: [PATCH 41/43] update docs --- README.md | 79 +++++++++++++++++++++++++------------ docs/source/docking.md | 4 +- docs/source/features.md | 30 +++++++------- docs/source/getstarted.md | 12 +++--- docs/source/installation.md | 48 ++++++++++++++++------ 5 files changed, 114 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index c9cd5187..8042303a 100644 --- a/README.md +++ b/README.md @@ -41,27 +41,30 @@ Main features: 📣 [Discussions](https://github.com/DeepRank/deeprank2/discussions) -## Table of contents +## Table of Contents - [DeepRank2](#deeprank2) - [Overview](#overview) - - [Table of contents](#table-of-contents) + - [Table of Contents](#table-of-contents) - [Installation](#installation) - [Containerized Installation](#containerized-installation) - - [Local/remote installation](#localremote-installation) - - [YML file installation (recommended)](#yml-file-installation-recommended) - - [Manual installation (customizable)](#manual-installation-customizable) - - [Testing DeepRank2 installation](#testing-deeprank2-installation) + - [Pull and Run the Pre-build Docker Image (Recommended)](#pull-and-run-the-pre-build-docker-image-recommended) + - [Build the Docker Image Manually](#build-the-docker-image-manually) + - [Removing the Docker Image](#removing-the-docker-image) + - [Local/remote Installation](#localremote-installation) + - [YML File Installation (Recommended)](#yml-file-installation-recommended) + - [Manual Installation (Customizable)](#manual-installation-customizable) + - [Testing DeepRank2 Installation](#testing-deeprank2-installation) - [Contributing](#contributing) - [Using DeepRank2](#using-deeprank2) - - [Data generation](#data-generation) + - [Data Generation](#data-generation) - [Datasets](#datasets) - [GraphDataset](#graphdataset) - [GridDataset](#griddataset) - [Training](#training) - - [Run a pre-trained model on new data](#run-a-pre-trained-model-on-new-data) - - [Computational performances](#computational-performances) - - [Package development](#package-development) + - [Run a Pre-trained Model on New Data](#run-a-pre-trained-model-on-new-data) + - [Computational Performances](#computational-performances) + - [Package Development](#package-development) ## Installation @@ -74,33 +77,59 @@ There are two ways to install DeepRank2: ### Containerized Installation -In order to try out the package without worrying about your OS and without the need of installing all the required dependencies, we created a `Dockerfile` that can be used for taking care of everything in a suitable container. +We provide a pre-built Docker image hosted on GitHub Packages, allowing you to use DeepRank2 without worrying about installing dependencies or configuring your system. This is the recommended method for trying out the package quickly. -For this, you first need to install [Docker](https://docs.docker.com/engine/install/) on your system. Then run the following commands. You may need to have sudo permission for some steps, in which case the commands below can be preceded by `sudo`: +#### Pull and Run the Pre-build Docker Image (Recommended) + +- Install [Docker](https://docs.docker.com/engine/install/) on your system, if not already installed. +- Pull the latest Docker image from GitHub Packages by running the following command: + +```bash +docker pull ghcr.io/deeprank/deeprank2:latest +``` + +- Run the container from the pulled image: + +```bash +docker run -p 8888:8888 ghcr.io/deeprank/deeprank2:latest +``` + +- Once the container is running, open your browser and navigate to `http://localhost:8888` to access the DeepRank2 application. + +From here, you can use DeepRank2, including running the tutorial notebooks. More details about the tutorials can be found [here](https://github.com/DeepRank/deeprank2/blob/main/tutorials/TUTORIAL.md). Note that the Docker container downloads only the raw PDB files required for the tutorials. To generate processed HDF5 files, you will need to run the `data_generation_xxx.ipynb` notebooks. Since Docker containers may have limited memory resources, we reduce the number of data points processed in the tutorials. To fully utilize the package, consider [installing it locally](#localremote-installation). + +#### Build the Docker Image Manually + +If you prefer to build the Docker image yourself or run into issues with the pre-built image, you can manually build and run the container as follows: + +- Install [Docker](https://docs.docker.com/engine/install/) on your system, if not already installed. +- Clone the DeepRank2 repository and navigate to its root directory: ```bash -# Clone the DeepRank2 repository and enter its root directory git clone https://github.com/DeepRank/deeprank2 cd deeprank2 +``` -# Build and run the Docker image +- Build and run the Docker image: + +```bash docker build -t deeprank2 . docker run -p 8888:8888 deeprank2 ``` -Next, open a browser and go to `http://localhost:8888` to access the application running inside the Docker container. From there you can use DeepRank2, e.g. to run the tutorial notebooks. +- Once the container is running, open your browser and navigate to `http://localhost:8888` to access the DeepRank2 application. -More details about the tutorials' contents can be found [here](https://github.com/DeepRank/deeprank2/blob/main/tutorials/TUTORIAL.md). Note that in the docker container only the raw PDB files are downloaded, which needed as a starting point for the tutorials. You can obtain the processed HDF5 files by running the `data_generation_xxx.ipynb` notebooks. Because Docker containers are limited in memory resources, we limit the number of data points processed in the tutorials. Please [install the package locally](#localremote-installation) to fully leverage its capabilities. +#### Removing the Docker Image -If after running the tutorials you want to remove the (quite large) Docker image from your machine, you must first [stop the container](https://docs.docker.com/engine/reference/commandline/stop/) and can then [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). More general information about Docker can be found on the [official website docs](https://docs.docker.com/get-started/). +If you no longer need the Docker image (which can be quite large), you can remove it after stopping the container. Follow the [container stop instructions](https://docs.docker.com/engine/reference/commandline/stop/) and [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). For more general information on Docker, refer to the [official Docker documentation](https://docs.docker.com/get-started/). -### Local/remote installation +### Local/remote Installation Local installation is formally only supported on the latest stable release of ubuntu, for which widespread automated testing through continuous integration workflows has been set up. However, it is likely that the package runs smoothly on other operating systems as well. Before installing DeepRank2 please ensure you have [GCC](https://gcc.gnu.org/install/) installed: if running `gcc --version` gives an error, run `sudo apt-get install gcc`. -#### YML file installation (recommended) +#### YML File Installation (Recommended) You can use the provided YML file for creating a [conda environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) via [mamba](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html), containing the latest stable release of DeepRank2 and all its dependencies. This will install the CPU-only version of DeepRank2 on Python 3.10. @@ -117,7 +146,7 @@ pip install deeprank2 We also provide a frozen environment YML file located at `env/deeprank2_frozen.yml` with all dependencies set to fixed versions. The `env/deeprank2_frozen.yml` file provides a frozen environment with all dependencies set to fixed versions. This ensures reproducibility of experiments and results by preventing changes in package versions that could occur due to updates or modifications in the default `env/deeprank2.yml`. Use this frozen environment file for a stable and consistent setup, particularly if you encounter issues with the default environment file. -#### Manual installation (customizable) +#### Manual Installation (Customizable) If you want to use the GPUs, choose a specific python version (note that at the moment we support python 3.10 only), are a MacOS user, or if the YML installation was not successful, you can install the package manually. We advise to do this inside a [conda virtual environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html). @@ -143,7 +172,7 @@ pip install -e .'[test]' The `test` extra is optional, and can be used to install test-related dependencies, useful during development. -#### Testing DeepRank2 installation +#### Testing DeepRank2 Installation If you have cloned the repository, you can check that all components were installed correctly using `pytest`. We especially recommend doing this in case you installed DeepRank2 and its dependencies manually (the latter option above). @@ -160,7 +189,7 @@ If you would like to contribute to the package in any way, please see [our guide The following section serves as a first guide to start using the package, using protein-protein Interface (PPI) queries as example. For an enhanced learning experience, we provide in-depth [tutorial notebooks](https://github.com/DeepRank/deeprank2/tree/main/tutorials) for generating PPI data, generating SVR data, and for the training pipeline. For more details, see the [extended documentation](https://deeprank2.rtfd.io/). -### Data generation +### Data Generation For each protein-protein complex (or protein structure containing a missense variant), a `Query` can be created and added to the `QueryCollection` object, to be processed later on. Two subtypes of `Query` exist: `ProteinProteinInterfaceQuery` and `SingleResidueVariantQuery`. @@ -370,7 +399,7 @@ trainer.test() ``` -#### Run a pre-trained model on new data +#### Run a Pre-trained Model on New Data If you want to analyze new PDB files using a pre-trained model, the first step is to process and save them into HDF5 files [as we have done above](#data-generation). @@ -404,7 +433,7 @@ trainer.test() For more details about how to run a pre-trained model on new data, see the [docs](https://deeprank2.readthedocs.io/en/latest/getstarted.html#run-a-pre-trained-model-on-new-data). -## Computational performances +## Computational Performances We measured the efficiency of data generation in DeepRank2 using the tutorials' [PDB files](https://zenodo.org/record/8187806) (~100 data points per data set), averaging the results run on Apple M1 Pro, using a single CPU. Parameter settings were: atomic resolution, `distance_cutoff` of 5.5 Å, radius (for SRV only) of 10 Å. The [features modules](https://deeprank2.readthedocs.io/en/latest/features.html) used were `components`, `contact`, `exposure`, `irc`, `secondary_structure`, `surfacearea`, for a total of 33 features for PPIs and 26 for SRVs (the latter do not use `irc` features). @@ -414,6 +443,6 @@ Parameter settings were: atomic resolution, `distance_cutoff` of 5.5 Å, radius | PPIs | graph only: **2.99** (std 0.23)
graph+grid: **11.35** (std 1.30) | graph only: **0.54** (std 0.07)
graph+grid: **16.09** (std 0.44) | | SRVs | graph only: **2.20** (std 0.08)
graph+grid: **2.85** (std 0.10) | graph only: **0.05** (std 0.01)
graph+grid: **17.52** (std 0.59) | -## Package development +## Package Development If you're looking for developer documentation, go [here](https://github.com/DeepRank/deeprank2/blob/dev/README.dev.md). diff --git a/docs/source/docking.md b/docs/source/docking.md index eb5f83ae..f7e05091 100644 --- a/docs/source/docking.md +++ b/docs/source/docking.md @@ -1,4 +1,4 @@ -# Docking scores +# Docking Scores The following scores have been developed for evaluating the quality of the protein-protein models produced by computational methods (docking models), and all of them compare the structural similarity between the decoys (computationally generated structures) and the experimentally solved native structures. To calculate these measures, the interface between the two interacting protein molecules is defined as any pair of heavy atoms from the two molecules within 5Å of each other. @@ -11,7 +11,7 @@ The following scores have been developed for evaluating the quality of the prote See https://onlinelibrary.wiley.com/doi/abs/10.1002/prot.10393 for more details about `capri_class`, `lrmsd`, `irmsd`, and `fnat`. See https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0161879 for more details about `dockq`. -## Compute and add docking scores +## Compute and Add Docking Scores The following code snippet shows an example of how to use deeprank2 to compute the docking scores for a given docking model, and how to add one of the scores (e.g., `dockq`) as a target to the already processed data. diff --git a/docs/source/features.md b/docs/source/features.md index 6fa59ea6..3271f6e9 100644 --- a/docs/source/features.md +++ b/docs/source/features.md @@ -2,7 +2,7 @@ Features implemented in the code-base are defined in `deeprank2.feature` subpackage. -## Custom features +## Custom Features Users can add custom features by cloning the repository, creating a new module and placing it in `deeprank2.feature` subpackage. The custom features can then be used by installing the package in editable mode (see [here](https://deeprank2.readthedocs.io/en/latest/installation.html#install-deeprank2) for more details). We strongly recommend submitting a pull request (PR) to merge the new feature into the official repository. @@ -77,15 +77,15 @@ dataset = GraphDataset( The following is a brief description of the features already implemented in the code-base, for each features' module. -## Default node features +## Default Node Features For atomic graphs, when features relate to residues then _all_ atoms of one residue receive the feature value for that residue. -### Core properties of atoms and residues: `deeprank2.features.components` +### Core Properties of Atoms and Residues: `deeprank2.features.components` These features relate to the chemical components (atoms and amino acid residues) of which the graph is composed. Detailed information and descrepancies between sources are described can be found in `deeprank2.domain.aminoacidlist.py`. -#### Atom properties: +#### Atom Properties: These features are only used in atomic graphs. @@ -93,7 +93,7 @@ These features are only used in atomic graphs. - `atom_charge`: Atomic charge in Coulomb (float). Taken from `deeprank2.domain.forcefield.patch.top`. - `pdb_occupancy`: Proportion of structures where the atom was detected at this position (float). In some cases a single atom was detected at different positions, in which case separate structures exist whose occupancies sum to 1. Only the highest occupancy atom is used by deeprank2. -#### Residue properties: +#### Residue Properties: - `res_type`: One-hot encoding of the amino acid residue (size 20). - `polarity`: One-hot encoding of the polarity of the amino acid (options: NONPOLAR, POLAR, NEGATIVE, POSITIVE). Note that sources vary on the polarity for few of the amino acids; see detailed information in `deeprank2.domain.aminoacidlist.py`. @@ -104,14 +104,14 @@ These features are only used in atomic graphs. - `hb_donors`, `hb_acceptors`: The number of hydrogen bond donor/acceptor atoms in the residue (int). Hydrogen bonds are noncovalent intermolecular interactions formed between an hydrogen atom (partially positively charged) bound to a small, highly electronegative atom (O, N, F) with an unshared electron pair. -#### Properties related to variant residues: +#### Properties Related to Variant Residues: These features are only used in SingleResidueVariant queries. - `variant_res`: One-hot encoding of variant amino acid (size 20). - `diff_charge`, `diff_polarity`, `diff_size`, `diff_mass`, `diff_pI`, `diff_hb_donors`, `diff_hb_acceptors`: Subtraction of the wildtype value of indicated feature from the variant value. For example, if the variant has 4 hb_donors and the wildtype has 5, then `diff_hb_donors == -1`. -### Conservation features: `deeprank2.features.conservation` +### Conservation Features: `deeprank2.features.conservation` These features relate to the conservation state of individual residues. @@ -120,36 +120,36 @@ These features relate to the conservation state of individual residues. - `conservation` (only used in SingleResidueVariant queries): Conservation of the wild type amino acid (float). _More details required._ - `diff_conservation` (only used in SingleResidueVariant queries): Subtraction of wildtype conservation from the variant conservation (float). -### Protein context features: +### Protein Context Features: -#### Surface exposure: `deeprank2.features.exposure` +#### Surface Exposure: `deeprank2.features.exposure` These features relate to the exposure of residues to the surface, and are computed using [biopython](https://biopython.org/docs/1.81/api/Bio.PDB.html). Note that these features can only be calculated per residue and not per atom. - `res_depth`: [Residue depth](https://en.wikipedia.org/wiki/Residue_depth) is the average distance (in Å) of the residue to the closest molecule of bulk water (float). See also [`Bio.PDB.ResidueDepth`](https://biopython.org/docs/1.75/api/Bio.PDB.ResidueDepth.html). - `hse`: [Half sphere exposure (HSE)](https://en.wikipedia.org/wiki/Half_sphere_exposure) is a protein solvent exposure measure indicating how buried an amino acid residue is in a protein (3 float values, see [Bio.PDB.HSExposure](https://biopython.org/docs/dev/api/Bio.PDB.HSExposure.html#module-Bio.PDB.HSExposure) for details). -#### Surface accessibility: `deeprank2.features.surfacearea` +#### Surface Accessibility: `deeprank2.features.surfacearea` These features relate to the surface area of the residue, and are computed using [freesasa](https://freesasa.github.io). Note that these features can only be calculated per residue and not per atom. - `sasa`: [Solvent-Accessible Surface Area](https://en.wikipedia.org/wiki/Accessible_surface_area) is the surface area (in Å^2) of a biomolecule that is accessible to the solvent (float). - `bsa`: Buried Surface Area is the surface area (in Å^2) that is buried away from the solvent when two or more proteins or subunits associate to form a complex, i.e. it measures the size of the complex interface (float). -#### Secondary structure: `deeprank2.features.secondary_structure` +#### Secondary Structure: `deeprank2.features.secondary_structure` - `sec_struct`: One-hot encoding of the [DSSP]() assigned secondary structure of the amino acid, using the three major classes (HELIX, STRAND, COIL). Calculated using [DSSP4](https://github.com/PDB-REDO/dssp). -#### Inter-residue contacts (IRCs): `deeprank2.features.irc` +#### Inter-residue Contacts (IRCs): `deeprank2.features.irc` These features are only calculated for ProteinProteinInterface queries. - `irc_total`: The number of residues on the other chain that are within a cutoff distance of 5.5 Å (int). - `irc_nonpolar_nonpolar`, `irc_nonpolar_polar`, `irc_nonpolar_negative`, `irc_nonpolar_positive`, `irc_polar_polar`, `irc_polar_negative`, `irc_polar_positive`, `irc_negative_negative`, `irc_positive_positive`, `irc_negative_positive`: As above, but for specific residue polarity pairings. -## Default edge features +## Default Edge Features -### Contact features: `deeprank2.features.contact` +### Contact Features: `deeprank2.features.contact` These features relate to relationships between individual nodes. For atomic graphs, when features relate to residues then _all_ atoms of one residue receive the feature value for that residue. @@ -166,7 +166,7 @@ These features relate to the structural relationship between nodes. - `same_res`: Boolean indicating whether atoms belong to the same residue (1) or separate residues (0). Only used in atomic graphs. - `covalent`: Boolean indicating whether nodes are covalently bound (1) or not (0). Note that covalency is not directly assessed, but any edge with a maximum distance of 2.1 Å is considered covalent. -#### Nonbond energies: +#### Nonbond Energies: These features measure nonbond energy potentials between nodes, and are calculated using [OPLS forcefield](https://en.wikipedia.org/wiki/OPLS). For residue graphs, the pairwise sum of potentials for all atoms from each residue is used. Note that no distance cutoff is used and the radius of influence is assumed to be infinite, although the potentials tends to 0 at large distance. Also edges are only assigned within a given cutoff radius when graphs are created. diff --git a/docs/source/getstarted.md b/docs/source/getstarted.md index cca87829..9c264df0 100644 --- a/docs/source/getstarted.md +++ b/docs/source/getstarted.md @@ -1,9 +1,9 @@ -# Get started +# Get Started The following section serves as a first guide to start using the package, using protein-protein interface (PPI) queries as example. For an enhanced learning experience, we provide in-depth [tutorial notebooks](https://github.com/DeepRank/deeprank2/tree/main/tutorials) for generating PPI data, generating SRVs data, and for the training pipeline. -## Data generation +## Data Generation For each protein-protein complex (or protein structure containing a missense variant), a `Query` can be created and added to the `QueryCollection` object, to be processed later on. Two subtypes of `Query` exist: `ProteinProteinInterfaceQuery` and `SingleResidueVariantQuery`. @@ -89,7 +89,7 @@ hdf5_paths = queries.process( grid_map_method = MapMethod.GAUSSIAN) ``` -## Data exploration +## Data Exploration As representative example, the following is the HDF5 structure generated by the previous phase for `1ATN_1w.pdb`, so for one single graph, for the graph + grid case: @@ -199,7 +199,7 @@ dataset_test = GraphDataset( ) ``` -#### Transforming features +#### Transforming Features For the `GraphDataset` class it is possible to define a dictionary to indicate which transformations to apply to the features, being the transformations lambda functions and/or standardization. If `True`, standardization is applied after transformation, if the latter is present. Example: @@ -360,7 +360,7 @@ trainer.test() ``` -### Results export and visualization +### Results Export and Visualization The user can specify a DeepRank2 exporter or a custom one in `output_exporters` parameter of the Trainer class, together with the path where to save the results. Exporters are used for storing predictions information collected later on during training and testing. Example: @@ -411,7 +411,7 @@ fig.update_layout( ) ``` -## Run a pre-trained model on new data +## Run a Pre-trained Model on New Data If you want to run a pre-trained model on new PDB files, the first step is to process and save them into HDF5 files. Let's suppose that the model has been trained with `ProteinProteinInterfaceQuery` queries mapped to graphs: diff --git a/docs/source/installation.md b/docs/source/installation.md index b945f260..904bb3ff 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -11,27 +11,53 @@ There are two ways to install DeepRank2: (containerized-installation)= -In order to try out the package without worrying about your OS and without the need of installing all the required dependencies, we created a `Dockerfile` that can be used for taking care of everything in a suitable container. +We provide a pre-built Docker image hosted on GitHub Packages, allowing you to use DeepRank2 without worrying about installing dependencies or configuring your system. This is the recommended method for trying out the package quickly. -For this, you first need to install [Docker](https://docs.docker.com/engine/install/) on your system. Then run the following commands. You may need to have sudo permission for some steps, in which case the commands below can be preceded by `sudo`: +### Pull and Run the Pre-build Docker Image (Recommended) + +- Install [Docker](https://docs.docker.com/engine/install/) on your system, if not already installed. +- Pull the latest Docker image from GitHub Packages by running the following command: + +```bash +docker pull ghcr.io/deeprank/deeprank2:latest +``` + +- Run the container from the pulled image: + +```bash +docker run -p 8888:8888 ghcr.io/deeprank/deeprank2:latest +``` + +- Once the container is running, open your browser and navigate to `http://localhost:8888` to access the DeepRank2 application. + +From here, you can use DeepRank2, including running the tutorial notebooks. More details about the tutorials can be found [here](https://github.com/DeepRank/deeprank2/blob/main/tutorials/TUTORIAL.md). Note that the Docker container downloads only the raw PDB files required for the tutorials. To generate processed HDF5 files, you will need to run the `data_generation_xxx.ipynb` notebooks. Since Docker containers may have limited memory resources, we reduce the number of data points processed in the tutorials. To fully utilize the package, consider [installing it locally](#localremote-installation). + +### Build the Docker Image Manually + +If you prefer to build the Docker image yourself or run into issues with the pre-built image, you can manually build and run the container as follows: + +- Install [Docker](https://docs.docker.com/engine/install/) on your system, if not already installed. +- Clone the DeepRank2 repository and navigate to its root directory: ```bash -# Clone the DeepRank2 repository and enter its root directory git clone https://github.com/DeepRank/deeprank2 cd deeprank2 +``` -# Build and run the Docker image +- Build and run the Docker image: + +```bash docker build -t deeprank2 . docker run -p 8888:8888 deeprank2 ``` -Next, open a browser and go to `http://localhost:8888` to access the application running inside the Docker container. From there you can use DeepRank2, e.g. to run the tutorial notebooks. +- Once the container is running, open your browser and navigate to `http://localhost:8888` to access the DeepRank2 application. -More details about the tutorials' contents can be found [here](https://github.com/DeepRank/deeprank2/blob/main/tutorials/TUTORIAL.md). Note that in the docker container only the raw PDB files are downloaded, which needed as a starting point for the tutorials. You can obtain the processed HDF5 files by running the `data_generation_xxx.ipynb` notebooks. Because Docker containers are limited in memory resources, we limit the number of data points processed in the tutorials. Please [install the package locally](#localremote-installation) to fully leverage its capabilities. +### Removing the Docker Image -If after running the tutorials you want to remove the (quite large) Docker image from your machine, you must first [stop the container](https://docs.docker.com/engine/reference/commandline/stop/) and can then [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). More general information about Docker can be found on the [official website docs](https://docs.docker.com/get-started/). +If you no longer need the Docker image (which can be quite large), you can remove it after stopping the container. Follow the [container stop instructions](https://docs.docker.com/engine/reference/commandline/stop/) and [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). For more general information on Docker, refer to the [official Docker documentation](https://docs.docker.com/get-started/). -## Local/remote installation +## Local/remote Installation (localremote-installation)= @@ -39,7 +65,7 @@ Local installation is formally only supported on the latest stable release of ub Before installing DeepRank2 please ensure you have [GCC](https://gcc.gnu.org/install/) installed: if running `gcc --version` gives an error, run `sudo apt-get install gcc`. -## YML file installation (recommended) +## YML File Installation (Recommended) You can use the provided YML file for creating a [conda environment](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) via [mamba](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html), containing the latest stable release of DeepRank2 and all its dependencies. This will install the CPU-only version of DeepRank2 on Python 3.10. @@ -56,7 +82,7 @@ pip install deeprank2 We also provide a frozen environment YML file located at `env/deeprank2_frozen.yml` with all dependencies set to fixed versions. The `env/deeprank2_frozen.yml` file provides a frozen environment with all dependencies set to fixed versions. This ensures reproducibility of experiments and results by preventing changes in package versions that could occur due to updates or modifications in the default `env/deeprank2.yml`. Use this frozen environment file for a stable and consistent setup, particularly if you encounter issues with the default environment file. -## Manual installation (customizable) +## Manual Installation (Customizable) (manual-installation)= @@ -84,7 +110,7 @@ pip install -e .'[test]' The `test` extra is optional, and can be used to install test-related dependencies, useful during development. -## Testing DeepRank2 installation +## Testing DeepRank2 Installation If you have cloned the repository, you can check that all components were installed correctly using `pytest`. We especially recommend doing this in case you installed DeepRank2 and its dependencies manually (the latter option above). From 55bd679e3de15f5b6b2c83eace07900098a6a043 Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Fri, 6 Sep 2024 11:23:33 +0200 Subject: [PATCH 42/43] update target values file for svr --- tests/perf/srv_perf.py | 2 +- tutorials/data_generation_srv.ipynb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/perf/srv_perf.py b/tests/perf/srv_perf.py index 94be374e..645030cb 100644 --- a/tests/perf/srv_perf.py +++ b/tests/perf/srv_perf.py @@ -88,7 +88,7 @@ def get_pdb_files_and_target_data(data_path: str) -> tuple[list[str], list, list, list, list]: - csv_data = pd.read_csv(os.path.join(data_path, "srv_target_values.csv")) + csv_data = pd.read_csv(os.path.join(data_path, "srv_target_values_curated.csv")) # before running this script change .ent to .pdb pdb_files = glob.glob(os.path.join(data_path, "pdb", "*.pdb")) pdb_files.sort() diff --git a/tutorials/data_generation_srv.ipynb b/tutorials/data_generation_srv.ipynb index a4fb3c16..f8606c7e 100644 --- a/tutorials/data_generation_srv.ipynb +++ b/tutorials/data_generation_srv.ipynb @@ -114,7 +114,7 @@ "metadata": {}, "source": [ "- Raw data are PDB files in `data_raw/srv/pdb/`, which contains atomic coordinates of the protein structure containing the variant.\n", - "- Target data, so in our case pathogenic versus benign labels, are in `data_raw/srv/srv_target_values.csv`.\n", + "- Target data, so in our case pathogenic versus benign labels, are in `data_raw/srv/srv_target_values_curated.csv`.\n", "- The final SRV processed data will be saved in `data_processed/srv/` folder, which in turns contains a folder for residue-level data and another one for atomic-level data. More details about such different levels will come a few cells below.\n" ] }, @@ -133,7 +133,7 @@ "outputs": [], "source": [ "def get_pdb_files_and_target_data(data_path):\n", - " csv_data = pd.read_csv(os.path.join(data_path, \"srv_target_values.csv\"))\n", + " csv_data = pd.read_csv(os.path.join(data_path, \"srv_target_values_curated.csv\"))\n", " pdb_files = glob.glob(os.path.join(data_path, \"pdb\", \"*.ent\"))\n", " pdb_files.sort()\n", " pdb_file_names = [os.path.basename(pdb_file) for pdb_file in pdb_files]\n", From d65507ba71f963dc2ff62ad4584d4e43d65441ed Mon Sep 17 00:00:00 2001 From: gcroci2 Date: Fri, 13 Sep 2024 18:54:19 +0200 Subject: [PATCH 43/43] add suggestions --- .github/workflows/release.yml | 8 ++++---- README.md | 2 +- docs/source/installation.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cce554ce..7639baa2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: user: __token__ password: ${{ secrets.PYPI_TOKEN_DEEPRANK2 }} - read_version: + read_only_version: needs: upload_pypi name: Read version from TOML runs-on: ubuntu-22.04 @@ -88,13 +88,13 @@ jobs: echo "REPO_LOWERCASE=$REPO_LOWERCASE" >> $GITHUB_OUTPUT upload_docker_image: - needs: read_version + needs: read_only_version name: Upload Docker image to ghcr.io uses: ./.github/workflows/_ghcr.yml with: ghcr_user: ${{github.actor}} - base_image_name: ghcr.io/${{ needs.read_version.outputs.repo_lowercase }} - image_tag: ${{ needs.read_version.outputs.version }} + base_image_name: ghcr.io/${{ needs.read_only_version.outputs.repo_lowercase }} + image_tag: ${{ needs.read_only_version.outputs.version }} dockerfile: ./Dockerfile docker_context: . secrets: diff --git a/README.md b/README.md index 8042303a..49e028af 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ docker run -p 8888:8888 deeprank2 #### Removing the Docker Image -If you no longer need the Docker image (which can be quite large), you can remove it after stopping the container. Follow the [container stop instructions](https://docs.docker.com/engine/reference/commandline/stop/) and [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). For more general information on Docker, refer to the [official Docker documentation](https://docs.docker.com/get-started/). +If you no longer need the Docker image (which can be quite large), you can remove it after stopping the container. Follow the [container stop](https://docs.docker.com/engine/reference/commandline/stop/) and [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/) instructions. For more general information on Docker, refer to the [Docker documentation](https://docs.docker.com/get-started/) directly. ### Local/remote Installation diff --git a/docs/source/installation.md b/docs/source/installation.md index 904bb3ff..9a0f22ea 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -55,7 +55,7 @@ docker run -p 8888:8888 deeprank2 ### Removing the Docker Image -If you no longer need the Docker image (which can be quite large), you can remove it after stopping the container. Follow the [container stop instructions](https://docs.docker.com/engine/reference/commandline/stop/) and [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/). For more general information on Docker, refer to the [official Docker documentation](https://docs.docker.com/get-started/). +If you no longer need the Docker image (which can be quite large), you can remove it after stopping the container. Follow the [container stop](https://docs.docker.com/engine/reference/commandline/stop/) and [remove the image](https://docs.docker.com/engine/reference/commandline/image_rm/) instructions. For more general information on Docker, refer to the [Docker documentation](https://docs.docker.com/get-started/) directly. ## Local/remote Installation