Merge branch 'main' into patch-1

onnx · Dec 24, 2024 · 609b1c1 · 609b1c1
2 parents 4b034e8 + 4b97bcf
commit 609b1c1
Show file tree

Hide file tree

Showing 108 changed files with 3,528 additions and 2,059 deletions.
diff --git a/.github/actions/keras_application_test/action.yml b/.github/actions/keras_application_test/action.yml
@@ -0,0 +1,75 @@
+name: Keras2onnx Application Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+          python -m pip install --upgrade pip
+          pip install onnxconverter-common
+          pip install onnx==${{ inputs.onnx_version }}
+          pip uninstall -y protobuf
+          pip install "protobuf~=3.20"
+          pip install h5py==3.7.0
+          pip install parameterized
+          pip install timeout-decorator
+          pip install coloredlogs flatbuffers
+          pip install tensorflow==${{ inputs.tf_version }}
+          pip install onnxruntime==${{ inputs.ort_version }}
+          pip install pillow
+          pip install opencv-python
+          pip install tqdm
+          pip install keras-segmentation==0.2.0
+          git clone https://github.com/matterport/Mask_RCNN
+          cd Mask_RCNN
+          pip install -r requirements.txt
+          python setup.py install
+          cd ..
+          pip install matplotlib
+          git clone https://github.com/qqwweee/keras-yolo3
+          pip install keras-resnet
+          pip install git+https://www.github.com/keras-team/keras-contrib.git
+          pip install keras-tcn==2.8.3
+          pip install git+https://github.com/qubvel/efficientnet
+          pip install keras-self-attention
+          pip install pytest pytest-cov pytest-runner
+
+          if [[ ${{ inputs.tf_version }} == 1.* ]]; then 
+              pip install keras==2.3.1
+              pip install transformers==4.2.0
+              pip uninstall -y h5py
+              pip install h5py==2.9.0
+              pip install numpy==1.19.0
+          else 
+              pip install transformers
+              pip install "numpy<2"
+          fi
+
+          pip install -e .
+
+          echo "----- List all of depdencies:"
+          pip freeze --all
+
+    - name: Run keras_application_test (${{ runner.os }})
+      shell: bash
+      run: |
+          python -c "import onnxruntime"
+          pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
+          cd tests/keras2onnx_applications/nightly_build
+          python run_all_v2.py
diff --git a/.github/actions/keras_unit_test/action.yml b/.github/actions/keras_unit_test/action.yml
@@ -0,0 +1,53 @@
+name: Keras2onnx Unit Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+          python -m pip install --upgrade pip
+          pip install onnxconverter-common
+          pip install onnx==${{ inputs.onnx_version }}
+          pip install h5py==3.7.0
+          pip install parameterized
+          pip install timeout-decorator
+          pip install coloredlogs flatbuffers
+          pip install tensorflow==${{ inputs.tf_version }}
+          pip install pytest pytest-cov pytest-runner
+          pip install onnxruntime==${{ inputs.ort_version }}
+          pip uninstall -y protobuf
+          pip install "protobuf~=3.20"
+          if [[ ${{ inputs.tf_version }} == 1.* ]]; then 
+              pip install numpy==1.19.0
+          else 
+              pip install "numpy<2"
+          fi
+
+          pip install -e .
+
+          echo "----- List all of depdencies:"
+          pip freeze --all
+
+    - name: Run keras_unit_test (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+          python -c "import onnxruntime"
+          python -c "import onnxconverter_common"
+          pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
diff --git a/.github/actions/pretrained_model_test/action.yml b/.github/actions/pretrained_model_test/action.yml
@@ -0,0 +1,46 @@
+name: Pretrained Model Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+  opset_version:
+    description: 'OPSET version'
+  skip_tflite:
+    description: 'Skip TFLite tests'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+        chmod +x ./tests/utils/setup_test_env.sh
+        ./tests/utils/setup_test_env.sh ${{ inputs.tf_version }} ${{ inputs.ort_version }} ${{ inputs.onnx_version }}
+
+    - name: Fix Paths (Windows only)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        $site_dir = python -c "import site; print(site.getsitepackages()[1])"
+        echo "##vso[task.prependpath]$site_dir\numpy\.libs"
+        $base_dir = python -c "import site; print(site.getsitepackages()[0])"
+        echo "##vso[task.prependpath]$base_dir/Library/bin"
+
+    - name: Run pretrained_model_test
+      shell: bash
+      run: |
+        # TODO: fix unity model path
+        # python tests/run_pretrained_models.py --backend $CI_ONNX_BACKEND --opset $CI_ONNX_OPSET --config tests/unity.yaml || status=$?
+        python tests/run_pretrained_models.py --backend onnxruntime --opset ${{ inputs.opset_version }} --skip_tf_tests False --skip_tflite_tests ${{ inputs.skip_tflite }} --skip_tfjs_tests True --config tests/run_pretrained_models.yaml || status=$?
+        ls
diff --git a/.github/actions/unit_test/action.yml b/.github/actions/unit_test/action.yml
@@ -0,0 +1,62 @@
+name: Unit Test Run
+
+inputs:
+  tf_version:
+    description: 'TensorFlow version'
+  python_version:
+    description: 'Python version'
+  ort_version:
+    description: 'ONNXRuntime version'
+  onnx_version:
+    description: 'ONNX version'
+  opset_version:
+    description: 'OPSET version'
+  skip_tflite:
+    description: 'Skip TFLite tests'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python (${{ inputs.python_version }})
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python_version }}
+
+    - name: Install dependencies (TF-v${{ inputs.tf_version }})
+      shell: bash
+      run: |
+        chmod +x ./tests/utils/setup_test_env.sh
+        ./tests/utils/setup_test_env.sh ${{ inputs.tf_version }} ${{ inputs.ort_version }} ${{ inputs.onnx_version }}
+
+    - name: Fix Paths (Windows only)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        $site_dir = python -c "import site; print(site.getsitepackages()[1])"
+        echo "##vso[task.prependpath]$site_dir\numpy\.libs"
+        $base_dir = python -c "import site; print(site.getsitepackages()[0])"
+        echo "##vso[task.prependpath]$base_dir/Library/bin"
+
+    - name: Run unit_test (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+        export TF2ONNX_TEST_BACKEND=onnxruntime
+        export TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        export TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        export TF2ONNX_SKIP_TFJS_TESTS=True
+        export TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls
+
+    - name: Run unit_test (Windows)
+      shell: pwsh
+      if: runner.os == 'Windows'
+      run: |
+        set TF2ONNX_TEST_BACKEND=onnxruntime
+        set TF2ONNX_TEST_OPSET=${{ inputs.opset_version }}
+        set TF2ONNX_SKIP_TFLITE_TESTS=${{ inputs.skip_tflite }}
+        set TF2ONNX_SKIP_TFJS_TESTS=True
+        set TF2ONNX_SKIP_TF_TESTS=False
+        python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append --junitxml=junit/test-results.xml
+        ls
diff --git a/.github/workflows/keras_application_test_ci.yml b/.github/workflows/keras_application_test_ci.yml
@@ -0,0 +1,144 @@
+name: Keras2onnx App Test (CI)
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+  cancel-in-progress: true
+
+jobs:
+
+  Test_min_py_with_min_tf: # Do not change this name because it is used in Ruleset of this repo.
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Tests (Py38-TF2.9)
+        uses: ./.github/actions/keras_application_test
+        with:
+          tf_version: '2.9.0'
+          python_version: '3.8'
+          ort_version: '1.16.3'
+          onnx_version: '1.16.1'
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Py38-TF2.9-ubuntu)
+          path: ./**/test-results-*.xml
+
+  Test_max_py_with_latest_tf: # Do not change this name because it is used in Ruleset of this repo.
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Tests (Py310-TF2.15)
+        uses: ./.github/actions/keras_application_test
+        with:
+          tf_version: '2.15.0'
+          python_version: '3.10'
+          ort_version: '1.16.3'
+          onnx_version: '1.16.1'
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Py310-TF2.15-ubuntu)
+          path: ./**/test-results-*.xml
+
+  Test_py37_with_tf1_15: # Do not change this name because it is used in Ruleset of this repo.
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Tests (Py37-TF1.15)
+        uses: ./.github/actions/keras_application_test
+        with:
+          tf_version: '1.15.5'
+          python_version: '3.7'
+          ort_version: '1.14.1'
+          onnx_version: '1.14.1'
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Py37-TF1.15-ubuntu)
+          path: ./**/test-results-*.xml
+
+  Extra_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        name:
+          - 'py38-tf2.13'
+          - 'py39-tf2.15'
+        os: ['ubuntu-latest', 'windows-2022']
+        ort_version: ['1.16.3']
+        onnx_version: ['1.16.1']
+        include:
+          - name: 'py38-tf2.13'
+            tf_version: '2.13.0'
+            python_version: '3.8'
+          - name: 'py39-tf2.15'
+            tf_version: '2.15.0'
+            python_version: '3.9'
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Tests (Py${{ matrix.python_version }}_TF${{ matrix.tf_version }}_${{ matrix.os }}_${{ matrix.ort_version }})
+        uses: ./.github/actions/keras_application_test
+        with:
+          tf_version: ${{ matrix.tf_version }}
+          python_version: ${{ matrix.python_version }}
+          ort_version: ${{ matrix.ort_version }}
+          onnx_version: ${{ matrix.onnx_version }}
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (${{ matrix.python_version }}-${{ matrix.tf_version }}-${{ matrix.os }})
+          path: ./**/test-results*.xml
+
+  publish-test-results:
+    name: "Publish Tests Results to Github"
+    needs: [Test_min_py_with_min_tf, Test_max_py_with_latest_tf, Test_py37_with_tf1_15, Extra_tests]
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+    if: always()
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: "artifacts/**/*.xml"