Remove global counters in tests (onnx#5661)

Remove global counters in tests from reference_evaluator_backend_test
and automatic_upgrade_test to allow parallel test execution.

Run tests in parallel in CI.

---------

Signed-off-by: Justin Chu <[email protected]>
Signed-off-by: Justin Chu <[email protected]>
Co-authored-by: Chun-Wei Chen <[email protected]>

.azure-pipelines/Linux-CI.yml

@@ -81,7 +81,7 @@ jobs:
   - script: |
       source venv/bin/activate
 
-      pytest -sv --cov=onnx --cov-report=xml --cov-append --cov-branch --junit-xml pytest.xml
+      pytest -sv --cov=onnx --cov-report=xml --cov-append --cov-branch --junit-xml pytest.xml -n auto --dist loadscope
       if [ $? -ne 0 ]; then
         echo "pytest failed"
         exit 1

.azure-pipelines/MacOS-CI.yml

@@ -73,7 +73,7 @@ jobs:
     displayName: 'Install dependencies and ONNX'
 
   - script: |
-      pytest
+      pytest -n auto --dist loadscope
       if [ $? -ne 0 ]; then
         echo "pytest failed"
         exit 1
@@ -83,7 +83,7 @@ jobs:
       export ORT_MAX_IR_SUPPORTED_VERSION=8
       export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
       export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=19
-      pytest
+      pytest -n auto --dist loadscope
       if [ $? -ne 0 ]; then
         echo "pytest failed when testing onnx with onnxruntime"
         exit 1

.azure-pipelines/Windows-CI.yml

@@ -72,7 +72,7 @@ jobs:
       )
 
       pip install -e ".[reference]" -v
-      pytest
+      pytest -n auto --dist loadscope
       IF NOT %ERRORLEVEL% EQU 0 (
         @echo "pytest failed"
         EXIT 1
@@ -118,7 +118,7 @@ jobs:
         pip uninstall -y onnx
         pip install .
 
-        pytest
+        pytest -n auto --dist loadscope
         IF NOT %ERRORLEVEL% EQU 0 (
           @echo "pytest failed when testing onnx with libprotobuf=3.20"
           EXIT 1

onnx/test/version_converter/automatic_upgrade_test.py

@@ -14,12 +14,14 @@
 # to the most recent version and runs checker and shape inference on the final upgraded model.
 ####################################################################################
 
-tested_ops = []
-
 
 class TestAutomaticUpgrade(automatic_conversion_test_base.TestAutomaticConversion):
+    @classmethod
+    def setUpClass(cls):
+        cls.tested_ops = []
+
     def _test_op_upgrade(self, op, *args, **kwargs):
-        tested_ops.append(op)
+        self.tested_ops.append(op)
         self._test_op_conversion(op, *args, **kwargs, is_upgrade=True)
 
     def test_Abs(self) -> None:
@@ -1739,9 +1741,10 @@ def test_RegexFullMatch(self) -> None:
         )
 
     def test_ops_tested(self) -> None:
+        # NOTE: This test is order dependent and needs to run last in this class
         all_schemas = onnx.defs.get_all_schemas()
-        all_op_names = [schema.name for schema in all_schemas if schema.domain == ""]
-        excluded_ops = [
+        all_op_names = {schema.name for schema in all_schemas if schema.domain == ""}
+        excluded_ops = {
             # Sequence-based and Optional-based ops disabled because
             # the version converter doesn't play nicely with sequences
             "ConcatFromSequence",
@@ -1757,12 +1760,11 @@ def test_ops_tested(self) -> None:
             "OptionalGetElement",
             "OptionalHasElement",
             "StringSplit",
-        ]
-        all_op_names = [op for op in all_op_names if op not in excluded_ops]
+        }
+        expected_tested_ops = all_op_names - excluded_ops
 
-        untested_ops = set(all_op_names) - set(tested_ops)
-        print(untested_ops)
-        assert len(untested_ops) == 0
+        untested_ops = expected_tested_ops - set(self.tested_ops)
+        self.assertEqual(untested_ops, set())
 
 
 if __name__ == "__main__":

requirements-dev.txt

@@ -6,6 +6,7 @@ parameterized
 protobuf
 pytest
 pytest-cov
+pytest-xdist
 setuptools
 twine
 wheel

requirements-release.txt

@@ -6,6 +6,7 @@ parameterized
 protobuf==4.21.12
 pytest
 pytest-cov
+pytest-xdist
 setuptools
 twine
 wheel