From 59995e53b86796a639bfe7a3dcb97011860aff04 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Wed, 18 Oct 2023 14:45:37 -0700 Subject: [PATCH] Remove global counters in tests (#5661) Remove global counters in tests from reference_evaluator_backend_test and automatic_upgrade_test to allow parallel test execution. Run tests in parallel in CI. --------- Signed-off-by: Justin Chu Signed-off-by: Justin Chu Co-authored-by: Chun-Wei Chen --- .azure-pipelines/Linux-CI.yml | 2 +- .azure-pipelines/MacOS-CI.yml | 4 ++-- .azure-pipelines/Windows-CI.yml | 4 ++-- .../automatic_upgrade_test.py | 22 ++++++++++--------- requirements-dev.txt | 1 + requirements-release.txt | 1 + 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.azure-pipelines/Linux-CI.yml b/.azure-pipelines/Linux-CI.yml index 7a5bc1477e8..aed2b44e214 100644 --- a/.azure-pipelines/Linux-CI.yml +++ b/.azure-pipelines/Linux-CI.yml @@ -81,7 +81,7 @@ jobs: - script: | source venv/bin/activate - pytest -sv --cov=onnx --cov-report=xml --cov-append --cov-branch --junit-xml pytest.xml + pytest -sv --cov=onnx --cov-report=xml --cov-append --cov-branch --junit-xml pytest.xml -n auto --dist loadscope if [ $? -ne 0 ]; then echo "pytest failed" exit 1 diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml index b28ebd6232a..f8ee1eb1838 100644 --- a/.azure-pipelines/MacOS-CI.yml +++ b/.azure-pipelines/MacOS-CI.yml @@ -73,7 +73,7 @@ jobs: displayName: 'Install dependencies and ONNX' - script: | - pytest + pytest -n auto --dist loadscope if [ $? -ne 0 ]; then echo "pytest failed" exit 1 @@ -83,7 +83,7 @@ jobs: export ORT_MAX_IR_SUPPORTED_VERSION=8 export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3 export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=19 - pytest + pytest -n auto --dist loadscope if [ $? -ne 0 ]; then echo "pytest failed when testing onnx with onnxruntime" exit 1 diff --git a/.azure-pipelines/Windows-CI.yml b/.azure-pipelines/Windows-CI.yml index 0b4a955b40a..1a6a27bc38a 100644 --- a/.azure-pipelines/Windows-CI.yml +++ b/.azure-pipelines/Windows-CI.yml @@ -72,7 +72,7 @@ jobs: ) pip install -e ".[reference]" -v - pytest + pytest -n auto --dist loadscope IF NOT %ERRORLEVEL% EQU 0 ( @echo "pytest failed" EXIT 1 @@ -118,7 +118,7 @@ jobs: pip uninstall -y onnx pip install . - pytest + pytest -n auto --dist loadscope IF NOT %ERRORLEVEL% EQU 0 ( @echo "pytest failed when testing onnx with libprotobuf=3.20" EXIT 1 diff --git a/onnx/test/version_converter/automatic_upgrade_test.py b/onnx/test/version_converter/automatic_upgrade_test.py index 763ee663de1..e8c61cdae68 100644 --- a/onnx/test/version_converter/automatic_upgrade_test.py +++ b/onnx/test/version_converter/automatic_upgrade_test.py @@ -14,12 +14,14 @@ # to the most recent version and runs checker and shape inference on the final upgraded model. #################################################################################### -tested_ops = [] - class TestAutomaticUpgrade(automatic_conversion_test_base.TestAutomaticConversion): + @classmethod + def setUpClass(cls): + cls.tested_ops = [] + def _test_op_upgrade(self, op, *args, **kwargs): - tested_ops.append(op) + self.tested_ops.append(op) self._test_op_conversion(op, *args, **kwargs, is_upgrade=True) def test_Abs(self) -> None: @@ -1739,9 +1741,10 @@ def test_RegexFullMatch(self) -> None: ) def test_ops_tested(self) -> None: + # NOTE: This test is order dependent and needs to run last in this class all_schemas = onnx.defs.get_all_schemas() - all_op_names = [schema.name for schema in all_schemas if schema.domain == ""] - excluded_ops = [ + all_op_names = {schema.name for schema in all_schemas if schema.domain == ""} + excluded_ops = { # Sequence-based and Optional-based ops disabled because # the version converter doesn't play nicely with sequences "ConcatFromSequence", @@ -1757,12 +1760,11 @@ def test_ops_tested(self) -> None: "OptionalGetElement", "OptionalHasElement", "StringSplit", - ] - all_op_names = [op for op in all_op_names if op not in excluded_ops] + } + expected_tested_ops = all_op_names - excluded_ops - untested_ops = set(all_op_names) - set(tested_ops) - print(untested_ops) - assert len(untested_ops) == 0 + untested_ops = expected_tested_ops - set(self.tested_ops) + self.assertEqual(untested_ops, set()) if __name__ == "__main__": diff --git a/requirements-dev.txt b/requirements-dev.txt index 0a7aa99b1e2..15daaa3ecea 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,7 @@ parameterized protobuf pytest pytest-cov +pytest-xdist setuptools twine wheel diff --git a/requirements-release.txt b/requirements-release.txt index 4cada27e188..747d481e8fc 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -6,6 +6,7 @@ parameterized protobuf==4.21.12 pytest pytest-cov +pytest-xdist setuptools twine wheel