ci_pipeline.yaml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: CI Pipe

on:
  pull_request:
    branches: [ "master" ]
    paths:
      - 'mindnlp/**'
      - 'tests/ut/**'
      - '!mindnlp/dataset/**'
      - '!tests/ut/dataset/**'
      - '!docs/**'
      - '.github/workflows/**'
  push:
    branches: [ "master" ]
    paths:
      - 'mindnlp/**'
      - '!mindnlp/dataset/**'
      - '!docs/**'

permissions:
  contents: read

jobs:
  pylint-check:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.9
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip==24.0
        pip install -r requirements/requirements.txt
    - name: Install MindSpore
      shell: bash
      env:
        OS: ubuntu-latest
        PYTHON: 3.9
      run: |
        python .github/install_mindspore.py
        pip install -r download.txt
    - name: Analysing the mindnlp code with pylint
      run: |
        NUM_CORES=$(nproc)
        pylint --jobs=$NUM_CORES mindnlp --rcfile=.github/pylint.conf

  ut-test:
    needs: pylint-check
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python: [3.9]
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3
    - name: avoid macos error
      if: matrix.os == 'macos-latest'
      run: rm -rf $RUNNER_TOOL_CACHE/Python/3.8.17

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip==24.0
        pip install -r requirements/requirements.txt
    - name: Install MindSpore
      shell: bash
      env:
        OS: ${{ matrix.os }}
        PYTHON: ${{ matrix.python }}
      run: |
        python .github/install_mindspore.py
        pip install -r download.txt
    - name: Test with pytest
      run: |
        pytest -c pytest.ini -m 'not download and not gpu_only' --ignore=tests/ut/transformers tests/ut

  release-test:
    needs: pylint-check
    strategy:
      matrix:
        ms_version: ['2.2.14', '2.3.1']
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.9
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip==24.0
        pip install -r requirements/requirements.txt
    - name: Install MindSpore
      shell: bash
      run: |
        pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${{matrix.ms_version}}/MindSpore/unified/x86_64/mindspore-${{matrix.ms_version}}-cp39-cp39-linux_x86_64.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com
    - name: Test with pytest
      run: |
        pytest -c pytest.ini -m 'not download and not gpu_only' --ignore=tests/ut/transformers tests/ut 
        # pytest -c pytest.ini -m 'not download and not gpu_only' tests/ut

  transformers-model-test:
    needs: pylint-check
    strategy:
      matrix:
        alpha: ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y']
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.9
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip==24.0
        pip install -r requirements/requirements.txt
    - name: Install MindSpore
      shell: bash
      env:
        OS: ubuntu-latest
        PYTHON: 3.9
      run: |
        python .github/install_mindspore.py
        pip install -r download.txt
    - name: Test with pytest
      run: |
        pytest -vs tests/ut/transformers/models/${{ matrix.alpha }}*/test_modeling*

  st-test:
    needs: ut-test
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        python: [3.8]
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip==24.0
        pip install -r requirements/requirements.txt
    - name: Install MindSpore
      shell: bash
      env:
        OS: ${{ matrix.os }}
        PYTHON: ${{ matrix.python }}
      run: |
        python .github/install_mindspore.py
        pip install -r download.txt
    - name: Test ST with pytest
      run: |
        pytest -c pytest.ini tests/st

  kaggle-gpu-test:
    needs: pylint-check
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.repository_owner == 'mindspore-lab'
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python 3.8
      uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - uses: "lvyufeng/action-kaggle-gpu-test@latest"
      with:
        kaggle_username: "${{ secrets.KAGGLE_USERNAME }}"
        kaggle_key: "${{ secrets.KAGGLE_API_KEY }}"
        kernel_name: mindnlp
        kernel_job_file: .github/kaggle_gpu_test.py