Merge branch 'optimize_notebook_validation' of https://github.com/sac… #8749
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| workflow_dispatch: | |
| inputs: | |
| cache_base: | |
| required: false | |
| type: string | |
| description: 'The name of the branch to use as cache base.' | |
| default: main | |
| export_environment: | |
| type: boolean | |
| description: Export the build environment as tar artifact that can be imported with Docker. | |
| # The GitHub application copy-pr-bot copies the source code for every pull request | |
| # into the repository. Approving such upstream pushes effectively marks code as trusted, | |
| # and is necessary to use the self-hosted NVIDIA runners. | |
| push: | |
| branches: | |
| - "pull-request/[0-9]+" | |
| name: CI # do not change name without updating workflow_run triggers | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| metadata: | |
| name: Retrieve PR info | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| pull_request_number: ${{ steps.pr_info.outputs.pr_number }} | |
| pull_request_base: ${{ steps.pr_info.outputs.pr_base }} | |
| cache_base: ${{ steps.pr_info.outputs.pr_base }} | |
| llvm_commit: ${{ steps.repo_info.outputs.llvm_commit }} | |
| pybind11_commit: ${{ steps.repo_info.outputs.pybind11_commit }} | |
| platform_config: ${{ steps.config.outputs.platforms }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - id: config | |
| run: | | |
| platforms="{}" | |
| for platform_id in amd64 arm64; do | |
| if [ "$platform_id" == "amd64" ]; then minimal_base_image=ghcr.io/nvidia/amd64/almalinux:8 | |
| elif [ "$platform_id" == "arm64" ]; then minimal_base_image=ghcr.io/nvidia/arm64v8/almalinux:8 | |
| fi | |
| platform={\"$platform_id\":{\"minimal_base_image\":\"$minimal_base_image\"}} | |
| platforms=`echo $platforms | jq ". |= . + $platform"` | |
| done | |
| echo "platforms=$(echo $platforms)" >> $GITHUB_OUTPUT | |
| - id: pr_info | |
| run: | | |
| pr_number=`echo ${{ github.ref_name }} | grep pull-request/ | (grep -o [0-9]* || true)` | |
| pr_number=${pr_number:-${{ github.event.pull_request.number }}} | |
| if [ -n "$pr_number" ]; then | |
| pr_base=`gh pr view $pr_number -R ${{ github.repository }} --json baseRefName --jq .baseRefName` | |
| echo "pr_number=$pr_number" >> $GITHUB_OUTPUT | |
| echo "pr_base=$pr_base" >> $GITHUB_OUTPUT | |
| fi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - id: repo_info | |
| run: | | |
| echo "llvm_commit=$(git rev-parse @:./tpls/llvm)" >> $GITHUB_OUTPUT | |
| echo "pybind11_commit=$(git rev-parse @:./tpls/pybind11)" >> $GITHUB_OUTPUT | |
| devdeps: | |
| name: Load dependencies | |
| needs: metadata | |
| strategy: | |
| matrix: | |
| platform: [amd64, arm64] | |
| toolchain: [clang16, gcc11, gcc12] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: linux/${{ matrix.platform }} | |
| dockerfile: build/devdeps.Dockerfile | |
| toolchain: ${{ matrix.toolchain }} | |
| registry_cache_from: ${{ inputs.cache_base || needs.metadata.outputs.cache_base }} | |
| checkout_submodules: true | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-${{ matrix.toolchain }} | |
| wheeldeps: | |
| name: Load wheel dependencies | |
| needs: metadata | |
| strategy: | |
| matrix: | |
| platform: [amd64, arm64] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: linux/${{ matrix.platform }} | |
| dockerfile: build/devdeps.manylinux.Dockerfile | |
| toolchain: gcc11 | |
| build_args: | | |
| base_image=ghcr.io/nvidia/pypa/manylinux_2_28${{ (matrix.platform == 'arm64' && '_aarch64') || (matrix.platform == 'amd64' && '_x86_64') || '' }}:latest | |
| distro=rhel8 | |
| llvm_commit=${{ needs.metadata.outputs.llvm_commit }} | |
| pybind11_commit=${{ needs.metadata.outputs.pybind11_commit }} | |
| registry_cache_from: ${{ inputs.cache_base || needs.metadata.outputs.cache_base }} | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-python | |
| source_build: | |
| name: Load source build cache | |
| needs: metadata | |
| strategy: | |
| matrix: | |
| platform: [amd64, arm64] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: linux/${{ matrix.platform }} | |
| dockerfile: build/assets.Dockerfile | |
| build_target: prereqs | |
| toolchain: llvm | |
| build_args: | | |
| base_image=${{ fromJson(needs.metadata.outputs.platform_config)[format('{0}', matrix.platform)].minimal_base_image }} | |
| registry_cache_from: ${{ inputs.cache_base || needs.metadata.outputs.cache_base }} | |
| checkout_submodules: true | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-installer | |
| # This job is needed only when using the cloudposse GitHub action to read | |
| # the output of a matrix job. This is a workaround due to current GitHub | |
| # limitations that may not be needed if the work started here concludes: | |
| # https://github.com/actions/runner/pull/2477 | |
| config: | |
| name: Configure build | |
| runs-on: ubuntu-latest | |
| needs: [devdeps, wheeldeps, source_build] | |
| outputs: | |
| json: "${{ steps.read_json.outputs.result }}" | |
| steps: | |
| - uses: cloudposse/[email protected] | |
| id: read_json | |
| with: | |
| matrix-step-name: dev_environment | |
| build_and_test: | |
| name: Build and test | |
| needs: config | |
| strategy: | |
| matrix: | |
| platform: [amd64, arm64] | |
| toolchain: [clang16, gcc11, gcc12] | |
| mpi: [openmpi, mpich] | |
| exclude: | |
| - toolchain: llvm | |
| mpi: mpich | |
| - toolchain: clang16 | |
| mpi: mpich | |
| fail-fast: false | |
| uses: ./.github/workflows/test_in_devenv.yml | |
| with: | |
| platform: linux/${{ matrix.platform }} | |
| mpi: ${{ matrix.mpi }} | |
| devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} | |
| devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} | |
| devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} | |
| export_environment: ${{ github.event_name == 'workflow_dispatch' && inputs.export_environment }} | |
| docker_image: | |
| name: Create Docker images | |
| needs: config | |
| strategy: | |
| matrix: | |
| platform: [amd64, arm64] | |
| fail-fast: false | |
| uses: ./.github/workflows/docker_images.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: linux/${{ matrix.platform }} | |
| devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-gcc11', matrix.platform)] }} | |
| devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-gcc11', matrix.platform)] }} | |
| devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-gcc11', matrix.platform)] }} | |
| python_wheels: | |
| name: Create Python wheels | |
| needs: config | |
| strategy: | |
| matrix: | |
| platform: [amd64, arm64] | |
| python_version: ['3.8', '3.11'] | |
| fail-fast: false | |
| uses: ./.github/workflows/python_wheels.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platform: linux/${{ matrix.platform }} | |
| python_version: ${{ matrix.python_version }} | |
| devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-python', matrix.platform)] }} | |
| devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-python', matrix.platform)] }} | |
| devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-python', matrix.platform)] }} | |
| binaries: | |
| name: Create CUDA Quantum installer | |
| needs: [metadata, config] | |
| strategy: | |
| matrix: | |
| platform: [amd64, arm64] | |
| fail-fast: false | |
| uses: ./.github/workflows/prebuilt_binaries.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platform: linux/${{ matrix.platform }} | |
| platform_base_image: ${{ fromJson(needs.metadata.outputs.platform_config)[format('{0}', matrix.platform)].minimal_base_image }} | |
| build_cache: ${{ fromJson(needs.config.outputs.json).build_cache[format('{0}-installer', matrix.platform)] }} | |
| toolchain: llvm | |
| clean_up: | |
| name: Prepare cache clean-up | |
| runs-on: ubuntu-latest | |
| needs: [metadata, config, build_and_test, docker_image, wheeldeps, python_wheels] | |
| # We need to clean up even if the workflow is cancelled or fails. | |
| if: always() | |
| steps: | |
| - name: Save cache keys and metadata | |
| id: workflow_inputs | |
| run: | | |
| set -e | |
| key_matrix='${{ needs.config.outputs.json }}' | |
| keys=`echo $key_matrix | jq '.cache_key | to_entries | .[].value' --raw-output` | |
| echo "$keys" >> cache_keys.txt | |
| echo "pr-number: ${{ needs.metadata.outputs.pull_request_number }}" >> metadata.txt | |
| echo "pr-base: ${{ needs.metadata.outputs.pull_request_base }}" >> metadata.txt | |
| - name: Upload cache keys | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cache_keys_ci | |
| path: cache_keys.txt | |
| retention-days: 1 | |
| if-no-files-found: error | |
| - name: Upload metadata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: metadata_ci | |
| path: metadata.txt | |
| retention-days: 1 | |
| if-no-files-found: error |