test_e2eshark.yml

# Copyright 2024 Advanced Micro Devices
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: E2ESHARK Test Suite
on:
  workflow_dispatch:
  schedule:
  # Runs at 12:00 PM UTC, which is 5:00 AM PST
   - cron: '0 12 * * *'

jobs:
  turbine:
    runs-on: nodai-amdgpu-mi250-x86-64
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          repository: nod-ai/SHARK-Turbine
          ref: torch_2.1
          path: turbine

  e2eshark:
    needs: [turbine]
    runs-on: nodai-amdgpu-mi250-x86-64
    env:
      E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv
      TURBINE_VENV_DIR: ${{ github.workspace }}/turbine_venv
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          repository: nod-ai/SHARK-TestSuite
          path: test-suite

      - name: "Setup e2eshark python venv"
        run: |
          python3.11 -m venv ${E2E_VENV_DIR}
          source ${E2E_VENV_DIR}/bin/activate
          pip install --upgrade pip
          pip install -r ./e2eshark/requirements.txt
          pip uninstall -y numpy
          pip install numpy
        working-directory: ./test-suite

      - name: Run Onnx Mode
        run: |
          source ${E2E_VENV_DIR}/bin/activate
          pip list
          cd e2eshark
          free -mh
          HF_TOKEN=${{ secrets.HF_TOKEN }} AZ_PRIVATE_CONNECTION=${{ secrets.AZ_PRIVATE_CONNECTION }} python3.11 ./run.py \
            -r ./test-onnx \
            --report \
            --cachedir ~/huggingface_cache \
            --mode onnx \
            -f onnx pytorch \
            -g models \
            --tolerance .001 .001 \
            --cleanup \
            --postprocess \
            --ci \
            -v
        working-directory: ./test-suite

      - name: Setup turbine python venv
        run: |
          python3.11 -m venv ${TURBINE_VENV_DIR}
          source ${TURBINE_VENV_DIR}/bin/activate
          pip install --upgrade pip
          pip install --index-url https://download.pytorch.org/whl/cpu \
            -r ../turbine/core/pytorch-cpu-requirements.txt \
            -r ../turbine/core/torchvision-requirements.txt
          pip install --upgrade -r ../turbine/core/requirements.txt
          pip install -e ../turbine/core[testing]
          pip install -e ../turbine/models
          pip install -r ./e2eshark/ci-requirements.txt
          pip uninstall -y transformers
          pip install transformers -U
        working-directory: ./test-suite

      - name: Run Turbine Mode
        run: |
          source ${TURBINE_VENV_DIR}/bin/activate
          cd e2eshark
          free -mh
          HF_TOKEN=${{ secrets.HF_TOKEN }} python3.11 ./run.py \
          -r ./test-turbine \
          --report \
          --cachedir ~/huggingface_cache \
          --mode turbine \
          -g models \
          --cleanup \
          --postprocess \
          --ci \
          -v
        working-directory: ./test-suite
      
      - uses: actions/upload-artifact@master
        with:
          name: ci_reports
          path: ./test-suite/e2eshark/ci_reports

  upload_artifacts:
    needs: [e2eshark]
    runs-on: nodai-amdgpu-mi250-x86-64
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          repository: nod-ai/e2eshark-reports
          ref: 'main'
          token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }}
          path: e2eshark-reports
      
      - uses: actions/download-artifact@master
        with:
          name: ci_reports
          path: ./e2eshark-reports

      - name: Push artifacts
        run: |
          date=$(date '+%Y-%m-%d')
          git config user.name "GitHub Actions Bot"
          git config user.email "<>"
          git add $date
          git commit -m "add CI reports for e2eshark"
          git push origin main
        working-directory: ./e2eshark-reports