Merge pull request #55 from darrenburns/parameterised-testing

Parameterised testing

.all-contributorsrc

@@ -30,7 +30,9 @@
       "avatar_url": "https://avatars0.githubusercontent.com/u/1615476?v=4",
       "profile": "https://github.com/khusrokarim",
       "contributions": [
-        "ideas"
+        "ideas",
+        "code",
+        "bug"
       ]
     }
   ],

README.md

@@ -1,7 +1,6 @@
 # Ward
 ![](https://github.com/darrenburns/ward/workflows/Ward%20CI/badge.svg)
-[![PyPI version](https://badge.fury.io/py/ward.svg)](https://badge.fury.io/py/ward) <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-)
-<!-- ALL-CONTRIBUTORS-BADGE:END -->
+[![PyPI version](https://badge.fury.io/py/ward.svg)](https://badge.fury.io/py/ward) <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-)<!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 See the full documentation and feature set [here](https://wardpy.com).
 
@@ -70,7 +69,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
 <table>
   <tr>
     <td align="center"><a href="https://darrenburns.net"><img src="https://avatars0.githubusercontent.com/u/5740731?v=4" width="60px;" alt="Darren Burns"/><br /><sub><b>Darren Burns</b></sub></a><br /><a href="https://github.com/darrenburns/ward/commits?author=darrenburns" title="Code">💻</a> <a href="https://github.com/darrenburns/ward/commits?author=darrenburns" title="Documentation">📖</a> <a href="#ideas-darrenburns" title="Ideas, Planning, & Feedback">🤔</a> <a href="#review-darrenburns" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/darrenburns/ward/issues?q=author%3Adarrenburns" title="Bug reports">🐛</a> <a href="#example-darrenburns" title="Examples">💡</a></td>
-    <td align="center"><a href="https://github.com/khusrokarim"><img src="https://avatars0.githubusercontent.com/u/1615476?v=4" width="60px;" alt="khusrokarim"/><br /><sub><b>khusrokarim</b></sub></a><br /><a href="#ideas-khusrokarim" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://github.com/khusrokarim"><img src="https://avatars0.githubusercontent.com/u/1615476?v=4" width="60px;" alt="khusrokarim"/><br /><sub><b>khusrokarim</b></sub></a><br /><a href="#ideas-khusrokarim" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/darrenburns/ward/commits?author=khusrokarim" title="Code">💻</a> <a href="https://github.com/darrenburns/ward/issues?q=author%3Akhusrokarim" title="Bug reports">🐛</a></td>
   </tr>
 </table>
 

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import setup
 
-version = "0.14.1a0"
+version = "0.15.0a0"
 description = "A modern Python 3 test framework for finding and fixing flaws faster."
 with open("README.md", "r") as fh:
     if platform.system() != "Windows":

tests/test_suite.py

@@ -4,8 +4,7 @@
 from ward.fixtures import Fixture
 from ward.models import Scope, SkipMarker
 from ward.suite import Suite
-from ward.test_result import TestOutcome, TestResult
-from ward.testing import Test, skip, test
+from ward.testing import Test, skip, test, TestOutcome, TestResult
 
 NUMBER_OF_TESTS = 5
 

tests/test_testing.py

@@ -1,8 +1,10 @@
+from unittest import mock
 from unittest.mock import Mock
 
-from ward import expect
+from ward import expect, raises
+from ward.errors import ParameterisationError
 from ward.fixtures import fixture
-from ward.testing import Test, test
+from ward.testing import Test, test, each, ParamMeta
 
 
 def f():
@@ -60,12 +62,12 @@ def _(dependent_test=dependent_test):
 
 @test("Test.has_deps should return True when test uses fixtures")
 def _(dependent_test=dependent_test):
-    expect(dependent_test.has_deps()).equals(True)
+    expect(dependent_test.has_deps).equals(True)
 
 
 @test("Test.has_deps should return False when test doesn't use fixtures")
 def _(anonymous_test=anonymous_test):
-    expect(anonymous_test.has_deps()).equals(False)
+    expect(anonymous_test.has_deps).equals(False)
 
 
 @test("Test.__call__ should delegate to the function it wraps")
@@ -74,3 +76,72 @@ def _():
     t = Test(fn=mock, module_name=mod)
     t(1, 2, key="val")
     expect(mock).called_once_with(1, 2, key="val")
+
+
+@test("Test.is_parameterised should return True for parameterised test")
+def _():
+    def parameterised_test(a=each(1, 2, 3), b="a value"):
+        pass
+
+    t = Test(fn=parameterised_test, module_name=mod)
+
+    expect(t.is_parameterised).equals(True)
+
+
+@test("Test.is_parameterised should return False for standard tests")
+def _():
+    def test():
+        pass
+
+    t = Test(fn=test, module_name=mod)
+
+    expect(t.is_parameterised).equals(False)
+
+
+@test("Test.get_parameterised_instances returns test in list if not parameterised")
+def _():
+    def test():
+        pass
+
+    t = Test(fn=test, module_name=mod)
+
+    expect(t.get_parameterised_instances()).equals([t])
+
+
+@test("Test.get_parameterised_instances returns correct number of test instances")
+def _():
+    def test(a=each(1, 2), b=each(3, 4)):
+        pass
+
+    t = Test(fn=test, module_name=mod)
+    expect(t.get_parameterised_instances()).equals(
+        [
+            Test(
+                id=mock.ANY,
+                fn=t.fn,
+                module_name=t.module_name,
+                param_meta=ParamMeta(0, 2),
+                sout=mock.ANY,
+                serr=mock.ANY,
+            ),
+            Test(
+                id=mock.ANY,
+                fn=t.fn,
+                module_name=t.module_name,
+                param_meta=ParamMeta(1, 2),
+                sout=mock.ANY,
+                serr=mock.ANY,
+            ),
+        ]
+    )
+
+
+@test("Test.get_parameterised_instances raises exception for arg count mismatch")
+def _():
+    def invalid_test(a=each(1, 2), b=each(3, 4, 5)):
+        pass
+
+    t = Test(fn=invalid_test, module_name=mod)
+
+    with raises(ParameterisationError):
+        a = t.get_parameterised_instances()

tests/test_util.py

@@ -1,6 +1,6 @@
 from tests.test_suite import example_test
 from ward import expect, test
-from ward.test_result import TestOutcome, TestResult
+from ward.testing import TestOutcome, TestResult
 from ward.util import ExitCode, get_exit_code
 
 

ward/__init__.py

@@ -1,4 +1,4 @@
 from .expect import expect, raises
 from .fixtures import fixture
-from .testing import skip, xfail, test
+from .testing import each, skip, test, xfail
 from .models import Scope

ward/errors.py

@@ -1,2 +1,6 @@
 class FixtureError(Exception):
     pass
+
+
+class ParameterisationError(Exception):
+    pass

ward/suite.py

@@ -4,10 +4,9 @@
 from typing import Generator, List
 
 from ward.errors import FixtureError
-from ward.fixtures import FixtureCache
+from ward.fixtures import FixtureCache, Fixture
 from ward.models import Scope
-from ward.test_result import TestOutcome, TestResult
-from ward.testing import Test
+from ward.testing import Test, TestOutcome, TestResult
 
 
 @dataclass
@@ -30,67 +29,42 @@ def generate_test_runs(self) -> Generator[TestResult, None, None]:
                 )
                 self.cache.teardown_fixtures(to_teardown)
 
-            marker = test.marker.name if test.marker else None
-            if marker == "SKIP":
-                yield TestResult(test, TestOutcome.SKIP)
-                previous_test_module = test.module_name
-                continue
+            generated_tests = test.get_parameterised_instances()
+            for i, generated_test in enumerate(generated_tests):
+                marker = generated_test.marker.name if generated_test.marker else None
+                if marker == "SKIP":
+                    yield generated_test.get_result(TestOutcome.SKIP)
+                    previous_test_module = generated_test.module_name
+                    continue
 
-            sout, serr = io.StringIO(), io.StringIO()
-            try:
-                with redirect_stdout(sout), redirect_stderr(serr):
-                    resolved_fixtures = test.resolve_fixtures(self.cache)
-            except FixtureError as e:
-                # We can't run teardown code here because we can't know how much
-                # of the fixture has been executed.
-                yield TestResult(
-                    test,
-                    TestOutcome.FAIL,
-                    e,
-                    captured_stdout=sout.getvalue(),
-                    captured_stderr=serr.getvalue(),
-                )
-                sout.close()
-                serr.close()
-                previous_test_module = test.module_name
-                continue
-            try:
-                resolved_vals = {
-                    k: fix.resolved_val for (k, fix) in resolved_fixtures.items()
-                }
+                try:
+                    resolved_vals = generated_test.resolve_args(self.cache, iteration=i)
 
-                # Run the test, while capturing output.
-                with redirect_stdout(sout), redirect_stderr(serr):
-                    test(**resolved_vals)
+                    # Run the test, while capturing output.
+                    generated_test(**resolved_vals)
 
-                # The test has completed without exception and therefore passed
-                if marker == "XFAIL":
-                    yield TestResult(
-                        test,
-                        TestOutcome.XPASS,
-                        captured_stdout=sout.getvalue(),
-                        captured_stderr=serr.getvalue(),
+                    # The test has completed without exception and therefore passed
+                    outcome = (
+                        TestOutcome.XPASS if marker == "XFAIL" else TestOutcome.PASS
                     )
-                else:
-                    yield TestResult(test, TestOutcome.PASS)
-            except Exception as e:
-                # TODO: Differentiate between ExpectationFailed and other Exceptions.
-                if marker == "XFAIL":
-                    yield TestResult(test, TestOutcome.XFAIL, e)
-                else:
-                    yield TestResult(
-                        test,
-                        TestOutcome.FAIL,
-                        e,
-                        captured_stdout=sout.getvalue(),
-                        captured_stderr=serr.getvalue(),
+                    yield generated_test.get_result(outcome)
+
+                except FixtureError as e:
+                    # We can't run teardown code here because we can't know how much
+                    # of the fixture has been executed.
+                    yield generated_test.get_result(TestOutcome.FAIL, e)
+                    previous_test_module = generated_test.module_name
+                    continue
+
+                except Exception as e:
+                    # TODO: Differentiate between ExpectationFailed and other Exceptions.
+                    outcome = (
+                        TestOutcome.XFAIL if marker == "XFAIL" else TestOutcome.FAIL
                     )
-            finally:
-                sout.close()
-                serr.close()
+                    yield generated_test.get_result(outcome, e)
 
-            self._teardown_fixtures_scoped_to_test(test)
-            previous_test_module = test.module_name
+                self._teardown_fixtures_scoped_to_test(generated_test)
+                previous_test_module = generated_test.module_name
 
         # Take care of any additional teardown.
         self.cache.teardown_all()

ward/terminal.py

@@ -10,7 +10,7 @@
 from ward.diff import make_diff
 from ward.expect import ExpectationFailed, Expected
 from ward.suite import Suite
-from ward.test_result import TestOutcome, TestResult
+from ward.testing import TestOutcome, TestResult
 from ward.util import ExitCode, get_exit_code