Add workflows to build and test nightly and release (#155)

* add workflow to build and test

* fix indent

* clean up

* add project id

* minor update

* fix a bug

* test release

* add test

* fix test reporting issue

* add upload and use compressed-tensors bucket

* use secret for bucket

* not install

* still uses k8s-util

* clean up

* try GCP_GHA_SA

* fix path

* use NM_PYPI_SA again

* authenticate with GCP_GHA_SA and NM_PYPI_SA

* fix id

* seperate install and test steps

* test more configs

* fix error

* fix a bug

* fix again

* add step-status

* fix failure

* test release

* clean up

* clean up

* update

* test nightly

* test release

* fix a bug

* fix a bug

* change default back to NIGHTLY

* clean up

---------

Co-authored-by: dhuangnm <[email protected]>
Co-authored-by: dhuangnm <[email protected]>

.github/actions/test/action.yml

@@ -0,0 +1,32 @@
+name: test compressed-tensors
+description: 'test compressed-tensors'
+inputs:
+  venv:
+    description: "path of virtualenv"
+    required: true
+outputs:
+  status:
+    description: "final status from test"
+    value: ${{ steps.test.outputs.status }}
+runs:
+  using: composite
+  steps:
+
+    - name: install wheel
+      uses: neuralmagic/nm-actions/actions/[email protected]
+      with:
+          venv: ${{ inputs.venv }}
+          name: compressed
+          extra: "[dev,accelerate]"
+
+    - name: test
+      id: test
+      run: |
+          source ${{ inputs.venv }}/bin/activate
+          rm -rf src
+          SUCCESS=0
+          pytest tests --junitxml=test-results/report.xml || SUCCESS=$?
+          echo "status=${SUCCESS}" >> "$GITHUB_OUTPUT"
+          deactivate
+          exit ${SUCCESS}
+      shell: bash

.github/scripts/step-status

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+# echo "green encased checkmark" if "${1} == 0"
+# echo "red X"                   if "${1} != 0"
+
+STEP_STATUS=${1}
+
+if [ "$STEP_STATUS" -eq 0 ]; then
+    # green check
+    echo -e "\xE2\x9C\x85"
+else
+    # red x
+    echo -e "\xE2\x9D\x8C"
+fi

.github/workflows/build-test.yml

@@ -0,0 +1,68 @@
+name: build-test
+on:
+
+  # makes workflow reusable
+  workflow_call:
+    inputs:
+      wf_category:
+        description: "workflow category: NIGHTLY, RELEASE"
+        type: string
+        default: NIGHTLY
+      push_to_pypi:
+        description: "When set to true, built whl and tar.gz will be pushed to public pypi if all tests pass"
+        type: boolean
+        default: false
+      gitref:
+        description: "git commit hash or tag name"
+        type: string
+        default: main
+
+      # build related parameters
+      build_label:
+        description: "requested runner label for build (specifies instance)"
+        type: string
+        default: ubuntu-20.04
+
+      # test related parameters
+      test_configs:
+        description: "python, label, timeout"
+        type: string
+        required: true
+
+jobs:
+
+    BUILD:
+        uses: ./.github/workflows/build.yml
+        with:
+            wf_category: ${{ inputs.wf_category }}
+            build_label: ${{ inputs.build_label }}
+            gitref: ${{ inputs.gitref }}
+            timeout: 20
+        secrets: inherit
+
+    TEST:
+        needs: [BUILD]
+        strategy:
+            fail-fast: false
+            matrix:
+                test_config: ${{ fromJson(inputs.test_configs) }}
+        uses: ./.github/workflows/test.yml
+        with:
+            gitref: ${{ inputs.gitref }}
+            test_label: ${{ matrix.test_config.label }}
+            python: ${{ matrix.test_config.python }}
+            timeout: ${{ matrix.test_config.timeout }}
+            whl: ${{ needs.BUILD.outputs.whl }}
+            testmo_run_id: ${{ needs.BUILD.outputs.testmo_run_id }}
+        secrets: inherit
+
+    UPLOAD:
+        needs: [BUILD, TEST]
+        uses: ./.github/workflows/upload.yml
+        with:
+            label: k8s-util
+            timeout: 40
+            run_id: ${{ github.run_id }}
+            push_to_pypi: ${{ inputs.push_to_pypi }}
+            testmo_run_id: ${{ needs.BUILD.outputs.testmo_run_id }}
+        secrets: inherit

.github/workflows/build.yml

@@ -0,0 +1,161 @@
+name: build
+on:
+  # makes workflow reusable
+  workflow_call:
+    inputs:
+      wf_category:
+        description: "categories: NIGHTLY, RELEASE"
+        type: string
+        default: NIGHTLY
+      build_label:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      timeout:
+        description: "time limit for run in minutes "
+        type: string
+        default: 20
+      gitref:
+        description: "git commit hash or branch name"
+        type: string
+        default: main
+    outputs:
+      whl:
+        description: 'basename for generated whl'
+        value: ${{ jobs.BUILD.outputs.whl }}
+      testmo_run_id:
+        description: 'testmo run id'
+        value: ${{ jobs.BUILD.outputs.testmo_run_id }}
+
+  # makes workflow manually callable
+  workflow_dispatch:
+    inputs:
+      wf_category:
+        description: "categories: NIGHTLY, RELEASE"
+        type: string
+        default: NIGHTLY
+      build_label:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      timeout:
+        description: "time limit for run in minutes "
+        type: string
+        default: 20
+      gitref:
+        description: "git commit hash or branch name"
+        type: string
+        default: main
+
+jobs:
+
+    BUILD:
+
+        runs-on: ${{ inputs.build_label }}
+        timeout-minutes: ${{ fromJson(inputs.timeout) }}
+        permissions:
+            contents: 'read'
+            id-token: 'write'
+
+        outputs:
+            run_id: ${{ github.run_id }}
+            whl: ${{ steps.build.outputs.whlname }}
+            tarfile: ${{ steps.build.outputs.tarname }}
+            testmo_run_id: ${{ steps.create_testmo_run.outputs.id }}
+
+        steps:
+
+            - name: set python
+              uses: actions/setup-python@v4
+              with:
+                  python-version: '3.10'
+
+            - name: checkout code
+              id: checkout
+              uses: actions/checkout@v4
+              with:
+                  ref: ${{ inputs.gitref }}
+
+            - name: install testmo
+              uses: neuralmagic/nm-actions/actions/[email protected]
+
+            - name: create testmo run
+              id: create_testmo_run
+              uses: neuralmagic/nm-actions/actions/[email protected]
+              if: success()
+              with:
+                testmo_url: https://neuralmagic.testmo.net
+                testmo_token: ${{ secrets.TESTMO_TEST_TOKEN }}
+                source: 'build-test'
+                project_id: 14
+                run_name: compressedtensors-${{ inputs.wf_category }}-${{ inputs.gitref }}-${GITHUB_ACTOR}
+
+            - name: build
+              id: build
+              uses: neuralmagic/nm-actions/actions/[email protected]
+              with:
+                  dev: false
+                  release: ${{ inputs.wf_category == 'RELEASE' }}
+
+            # GCP
+            - name: 'Authenticate to Google Cloud'
+              id: auth
+              uses: google-github-actions/[email protected]
+              with:
+                  project_id: ${{ secrets.GCP_PROJECT }}
+                  workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+                  service_account: ${{ secrets.GCP_GHA_SA }}
+
+            - name: 'Set up Cloud SDK'
+              uses: 'google-github-actions/setup-gcloud@v2'
+              with:
+                  version: '>= 473.0.0'
+
+            - name: copy whl and source distribution
+              run: |
+                  gcloud storage cp dist/${{ steps.build.outputs.whlname }} ${{ secrets.GCP_BUILD_ML_ASSETS2 }}/${{ github.run_id }}/${{ steps.build.outputs.whlname }}
+                  gcloud storage cp dist/${{ steps.build.outputs.tarname }} ${{ secrets.GCP_BUILD_ML_ASSETS2 }}/${{ github.run_id }}/${{ steps.build.outputs.tarname }}
+
+            - name: upload whl
+              uses: actions/upload-artifact@v4
+              if: success() || failure()
+              with:
+                  name: ${{ steps.build.outputs.whlname }}
+                  path: dist/${{ steps.build.outputs.whlname }}
+                  retention-days: 5
+
+            - name: upload tar.gz
+              uses: actions/upload-artifact@v4
+              if: success() || failure()
+              with:
+                  name: ${{ steps.build.outputs.tarname }}
+                  path: dist/${{ steps.build.outputs.tarname }}
+                  retention-days: 5
+
+            - name: summary
+              uses: neuralmagic/nm-actions/actions/[email protected]
+              if: success() || failure()
+              with:
+                  label: ${{ inputs.build_label }}
+                  gitref: ${{ inputs.gitref }}
+                  whl_status: ${{ steps.build.outputs.status }}
+
+            - name: report build status to testmo
+              id: report_build
+              uses: neuralmagic/nm-actions/actions/[email protected]
+              if: (success() || failure()) && ${{ inputs.testmo_run_id != '' }}
+              with:
+                  testmo_url: https://neuralmagic.testmo.net
+                  testmo_token: ${{ secrets.TESTMO_TEST_TOKEN }}
+                  testmo_run_id: ${{ steps.create_testmo_run.outputs.id }}
+                  results: build-results
+                  step_status: ${{ steps.build.outputs.status }}
+
+            - name: run status
+              id: run_status
+              if: success() || failure()
+              env:
+                  WHL_STATUS: ${{ steps.build.outputs.status }}
+              run: |
+                  echo "build status: ${WHL_STATUS}"
+                  if [ -z "${WHL_STATUS}" ] || [ "${WHL_STATUS}" -ne "0" ]; then exit 1; fi