From 0d7c3ff7d61479d1f2b5ab78975aa42290013211 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 5 Dec 2023 10:47:26 -0800 Subject: [PATCH] fix ci scripts and workflows --- .github/workflows/build.yaml | 87 ++++++++++++++++++++++++++------ .github/workflows/pr.yaml | 97 ++++++++++++++++++++++++++++-------- ci/build_python.sh | 5 +- ci/build_wheel.sh | 4 ++ ci/checks/style.sh | 27 ---------- ci/test_python.sh | 3 +- ci/test_wheel.sh | 3 ++ 7 files changed, 161 insertions(+), 65 deletions(-) delete mode 100755 ci/checks/style.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8118872..d1e8a82 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,15 +25,53 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true +defaults: + run: + shell: bash + +permissions: + actions: read + checks: none + contents: read + deployments: none + discussions: none + id-token: write + issues: none + packages: read + pages: none + pull-requests: read + repository-projects: none + security-events: none + statuses: none + jobs: python-build: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} + runs-on: "linux-amd64-cpu8" + container: + image: rapidsai/ci-conda:latest + env: + RAPIDS_BUILD_TYPE: ${{ inputs.build_type || 'branch' }} + PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }} + steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + - name: Standardize repository information + run: | + echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}" + echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" + echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" + echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + - name: Python build + run: ci/build_python.sh + env: + GH_TOKEN: ${{ github.token }} upload-conda: needs: [python-build] secrets: inherit @@ -44,15 +82,32 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} wheel-build: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - script: ci/build_wheel.sh - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + runs-on: "linux-amd64-cpu8" + container: + image: rapidsai/ci-wheel:latest + env: + RAPIDS_BUILD_TYPE: ${{ inputs.build_type || 'branch' }} + PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }} + steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + - name: Standardize repository information + run: | + echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}" + echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" + echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" + echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + - name: Python build + run: ci/build_wheel.sh + env: + GH_TOKEN: ${{ github.token }} wheel-publish: needs: wheel-build secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 43d5ab0..1632e33 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -32,65 +32,122 @@ jobs: pr-builder: needs: - checks - - conda-python-build-tests - - wheel-build-test + - conda-python-build + - conda-python-tests + - wheel-build + - wheel-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 - conda-python-build-tests: + conda-python-build: needs: checks - runs-on: "linux-amd64-gpu-v100-latest-1" + runs-on: "linux-amd64-cpu8" container: image: rapidsai/ci-conda:latest env: - RAPIDS_BUILD_TYPE: branch + RAPIDS_BUILD_TYPE: pull-request PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }} - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Standardize repository information run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" + echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}" echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}" + echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}" - name: Python build run: ci/build_python.sh env: GH_TOKEN: ${{ github.token }} - - name: Python test + conda-python-tests: + needs: conda-python-build + runs-on: "linux-amd64-gpu-v100-latest-1" + container: + image: rapidsai/ci-conda:latest + env: + RAPIDS_BUILD_TYPE: pull-request + PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }} + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Standardize repository information + run: | + echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}" + echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" + echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}" + echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}" + - name: Python tests run: ci/test_python.sh env: GH_TOKEN: ${{ github.token }} - - wheel-build-test: + wheel-build: needs: checks - runs-on: "linux-amd64-gpu-v100-latest-1" + runs-on: "linux-amd64-cpu8" container: image: rapidsai/ci-wheel:latest env: - RAPIDS_BUILD_TYPE: branch + RAPIDS_BUILD_TYPE: pull-request PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }} - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Standardize repository information run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" + echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}" echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}" + echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}" - name: Python build run: ci/build_wheel.sh env: GH_TOKEN: ${{ github.token }} - - name: Python test + wheel-tests: + needs: wheel-build + runs-on: "linux-amd64-gpu-v100-latest-1" + container: + image: rapidsai/ci-wheel:latest + env: + RAPIDS_BUILD_TYPE: pull-request + PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }} + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Standardize repository information + run: | + echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}" + echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" + echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}" + echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}" + - name: Python tests run: ci/test_wheel.sh env: - GH_TOKEN: ${{ github.token }} \ No newline at end of file + GH_TOKEN: ${{ github.token }} diff --git a/ci/build_python.sh b/ci/build_python.sh index 3a54494..9d9d8ca 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -25,4 +25,7 @@ echo "__version__ = '$version'" > jupyterlab_nvdashboard/_version.py # TODO: Remove `--no-test` flag once importing on a CPU # node works correctly rapids-logger "Building JupyterLab NVDashboard conda package" -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild --no-test conda/recipes/jupyterlab-nvdashboard +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild --no-test conda/recipes/jupyterlab-nvdashboard --output + +rapids-logger "Uploading JupyterLab NVDashboard conda package to S3" +rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 77e798b..5a765f1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -37,3 +37,7 @@ python -m pip install build # Build the Python package python -m build -s -w + +rapids-logger "Uploading JupyterLab NVDashboard wheels to S3" +# Upload Python wheels to S3 +RAPIDS_PY_WHEEL_NAME="${package_name}" rapids-upload-wheels-to-s3 dist diff --git a/ci/checks/style.sh b/ci/checks/style.sh deleted file mode 100755 index 0351c81..0000000 --- a/ci/checks/style.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. - -# Ignore errors and set path -set +e -PATH=/opt/conda/bin:$PATH - -# Activate common conda env -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Run flake8 and get results/return code -FLAKE=`flake8 --exclude=src,style,test` -RETVAL=$? - -# Output results if failure otherwise show pass -if [ "$FLAKE" != "" ]; then - echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" - echo -e "$FLAKE" - echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: flake8 style check\n\n" -fi - -#TODO Fix flake8 issues then re-enable this check -#exit $RETVAL -exit 0 \ No newline at end of file diff --git a/ci/test_python.sh b/ci/test_python.sh index bfeaace..8eedcf9 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -18,7 +18,8 @@ set +u conda activate test set -u -PYTHON_CHANNEL="/tmp/conda-bld-output/noarch" +# rapids-logger "Downloading artifacts from previous jobs" +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-print-env diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e6fbc9b..8497bd5 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -6,6 +6,9 @@ set -eou pipefail # Set the package name package_name="jupyterlab-nvdashboard" +rapids-logger "Downloading artifacts from previous jobs" +RAPIDS_PY_WHEEL_NAME="${package_name}" rapids-download-wheels-from-s3 ./dist + # echo to expand wildcard before adding `[extra]` required for pip python -m pip install $(echo ./dist/jupyterlab_nvdashboard*.whl)[test]