Add required jobs for pretrained_model_test.

Signed-off-by: Jay Zhang <[email protected]>
onnx · Jun 25, 2024 · 9b88331 · 9b88331
1 parent c0645f2
commit 9b88331
Show file tree

Hide file tree

Showing 8 changed files with 320 additions and 110 deletions.
diff --git a/.github/actions/keras_application_test/action.yml b/.github/actions/keras_application_test/action.yml
@@ -0,0 +1,64 @@
+name: Unit Test Run
+
+inputs:
+  os:
+    description: 'OS type'
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+  opset_version:
+    description: 'OPSET version'
+  skip_tflite:
+    description: 'Skip TFLite tests'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+        chmod +x ./tests/utils/setup_test_env.sh
+        ./tests/utils/setup_test_env.sh ${{ inputs.tf_version }} ${{ inputs.ort_version }} ${{ inputs.onnx_version }}
+
+    - name: Fix Paths (Windows only)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        $site_dir = python -c "import site; print(site.getsitepackages()[1])"
+        echo "##vso[task.prependpath]$site_dir\numpy\.libs"
+        $base_dir = python -c "import site; print(site.getsitepackages()[0])"
+        echo "##vso[task.prependpath]$base_dir/Library/bin"
+
+    - name: Run unit_test (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+        export TF2ONNX_TEST_BACKEND=onnxruntime
+        export TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        export TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        export TF2ONNX_SKIP_TFJS_TESTS=True
+        export TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls
+
+    - name: Run unit_test (Windows)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        set TF2ONNX_TEST_BACKEND=onnxruntime
+        set TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        set TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        set TF2ONNX_SKIP_TFJS_TESTS=True
+        set TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls
diff --git a/.github/actions/keras_unit_test/action.yml b/.github/actions/keras_unit_test/action.yml
@@ -0,0 +1,64 @@
+name: Unit Test Run
+
+inputs:
+  os:
+    description: 'OS type'
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+  opset_version:
+    description: 'OPSET version'
+  skip_tflite:
+    description: 'Skip TFLite tests'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+        chmod +x ./tests/utils/setup_test_env.sh
+        ./tests/utils/setup_test_env.sh ${{ inputs.tf_version }} ${{ inputs.ort_version }} ${{ inputs.onnx_version }}
+
+    - name: Fix Paths (Windows only)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        $site_dir = python -c "import site; print(site.getsitepackages()[1])"
+        echo "##vso[task.prependpath]$site_dir\numpy\.libs"
+        $base_dir = python -c "import site; print(site.getsitepackages()[0])"
+        echo "##vso[task.prependpath]$base_dir/Library/bin"
+
+    - name: Run unit_test (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+        export TF2ONNX_TEST_BACKEND=onnxruntime
+        export TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        export TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        export TF2ONNX_SKIP_TFJS_TESTS=True
+        export TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls
+
+    - name: Run unit_test (Windows)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        set TF2ONNX_TEST_BACKEND=onnxruntime
+        set TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        set TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        set TF2ONNX_SKIP_TFJS_TESTS=True
+        set TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls
diff --git a/.github/actions/pretrained_model_test/action.yml b/.github/actions/pretrained_model_test/action.yml
@@ -0,0 +1,45 @@
+name: Unit Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+  opset_version:
+    description: 'OPSET version'
+  skip_tflite:
+    description: 'Skip TFLite tests'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+        chmod +x ./tests/utils/setup_test_env.sh
+        ./tests/utils/setup_test_env.sh ${{ inputs.tf_version }} ${{ inputs.ort_version }} ${{ inputs.onnx_version }}
+
+    - name: Fix Paths (Windows only)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        $site_dir = python -c "import site; print(site.getsitepackages()[1])"
+        echo "##vso[task.prependpath]$site_dir\numpy\.libs"
+        $base_dir = python -c "import site; print(site.getsitepackages()[0])"
+        echo "##vso[task.prependpath]$base_dir/Library/bin"
+
+    - name: Run pretrained_model_test
+      run: |
+        # TODO: fix unity model path
+        # python tests/run_pretrained_models.py --backend $CI_ONNX_BACKEND --opset $CI_ONNX_OPSET --config tests/unity.yaml || status=$?
+        python tests/run_pretrained_models.py --backend onnxruntime --opset ${{ inputs.opset_version }} --skip_tf_tests False --skip_tflite_tests ${{ inputs.skip_tflite }} --skip_tfjs_tests True --config tests/run_pretrained_models.yaml || status=$?
+        ls
diff --git a/.github/actions/unit_test/action.yml b/.github/actions/unit_test/action.yml
@@ -1,8 +1,6 @@
 name: Unit Test Run
 
 inputs:
-  os:
-    description: 'OS type'
   tf_version:
     description: 'TensorFlow version'
   python_version:

diff --git a/.github/workflows/pretrained_model_test_ci.yml b/.github/workflows/pretrained_model_test_ci.yml
@@ -0,0 +1,139 @@
+name: Pretrained Model Test (CI)
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+  cancel-in-progress: true
+
+jobs:
+
+  Test_min_py_with_min_tf: # Do not change this name because it is used in Ruleset of this repo.
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Tests (Py38-TF2.9-18)
+        uses: ./.github/actions/pretrained_model_test
+        with:
+          os: 'ubuntu-latest'
+          tf_version: '2.9.0'
+          python_version: '3.8'
+          ort_version: '1.16.3'
+          onnx_version: '1.16.1'
+          opset_version: '18'
+          skip_tflite: 'False'
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Py38-TF2.9-18-ubuntu)
+          path: ./**/test-results-*.xml
+
+  Test_max_py_with_latest_tf: # Do not change this name because it is used in Ruleset of this repo.
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Tests (Py39-TF2.15-18)
+        uses: ./.github/actions/pretrained_model_test
+        with:
+          os: 'ubuntu-latest'
+          tf_version: '2.15.0'
+          python_version: '3.9'
+          ort_version: '1.16.3'
+          onnx_version: '1.16.1'
+          opset_version: '18'
+          skip_tflite: 'False'
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Py39-TF2.15-18-ubuntu)
+          path: ./**/test-results-*.xml
+
+  Extra_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        name:
+          - 'py38-tf2.13'
+          - 'py39-tf2.14'
+        os: ['windows-2022']
+        opset_version: ['18', '15']
+        ort_version: ['1.16.3']
+        onnx_version: ['1.16.1']
+        skip_tflite: ['False']
+        include:
+          - name: 'py38-tf2.13'
+            tf_version: '2.13.0'
+            python_version: '3.8'
+          - name: 'py39-tf2.14'
+            tf_version: '2.14.0'
+            python_version: '3.9'
+          - name: 'py37-tf1.15'
+            tf_version: '1.15.5'
+            python_version: '3.7'
+            os: 'ubuntu-latest'
+            opset_version: '15'
+            ort_version: '1.14.1'
+            onnx_version: '1.14.1'
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Tests (Py${{ matrix.python_version }}_TF${{ matrix.tf_version }}_${{ matrix.os }}_${{ matrix.opset_version }}_${{ matrix.ort_version }}_${{ matrix.skip_tflite }})
+        uses: ./.github/actions/pretrained_model_test
+        with:
+          tf_version: ${{ matrix.tf_version }}
+          python_version: ${{ matrix.python_version }}
+          ort_version: ${{ matrix.ort_version }}
+          onnx_version: ${{ matrix.onnx_version }}
+          opset_version: ${{ matrix.opset_version }}
+          skip_tflite: ${{ matrix.skip_tflite }}
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Py${{ matrix.python_version }}_TF${{ matrix.tf_version }}_${{ matrix.os }}_${{ matrix.opset_version }}_${{ matrix.ort_version }}_${{ matrix.skip_tflite }})
+          path: ./**/test-results-*.xml
+
+  publish-test-results:
+    name: "Publish Tests Results to Github"
+    needs: [Test_min_py_with_min_tf, Test_max_py_with_latest_tf, Extra_tests]
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+    if: always()
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: "**/pytest*.xml"