Merge pull request #85 from darrenburns/randomise

Add --order "random" option

tests/test_suite.py

@@ -355,13 +355,15 @@ def test_3(a=a, b=b, c=c):
 
     test_1.ward_meta.path = "module1"
     test_2.ward_meta.path = "module2"
-    test_3.ward_meta.path = "module2"
+    test_3.ward_meta.path = "module1"
 
     suite = Suite(
         tests=[
+            # Module ordering is intentionally off here, to ensure correct
+            # interaction between module-scope fixtures and random ordering
             Test(fn=test_1, module_name="module1"),
             Test(fn=test_2, module_name="module2"),
-            Test(fn=test_3, module_name="module2"),
+            Test(fn=test_3, module_name="module1"),
         ]
     )
 
@@ -374,15 +376,15 @@ def test_3(a=a, b=b, c=c):
             "resolve c",  # test fixture resolved at start of test1
             "test1",
             "teardown c",  # test fixture teardown at start of test1
-            "teardown b",  # module fixture teardown at end of module1
             "resolve b",  # module fixture resolved at start of module2
             "resolve c",  # test fixture resolved at start of test2
             "test2",
             "teardown c",  # test fixture teardown at start of test2
+            "teardown b",  # module fixture teardown at end of module2
             "resolve c",  # test fixture resolved at start of test3
             "test3",
             "teardown c",  # test fixture teardown at end of test3
-            "teardown b",  # module fixture teardown at end of module2
+            "teardown b",  # module fixture teardown at end of module1
             "teardown a",  # global fixtures are torn down at the very end
         ]
     )

ward/run.py

@@ -1,5 +1,6 @@
 import sys
 from pathlib import Path
+from random import shuffle
 from timeit import default_timer
 
 import click
@@ -27,12 +28,12 @@
     "--path",
     default=".",
     type=click.Path(exists=True),
-    help="Path to tests.",
+    help="Path to test directory.",
     multiple=True,
 )
 @click.option(
     "--search",
-    help="Search test names, descriptions and module names for the search query and only run matching tests.",
+    help="Search test names, bodies, descriptions and module names for the search query and only run matching tests.",
 )
 @click.option(
     "--fail-limit",
@@ -45,19 +46,27 @@
         ["test-per-line", "dots-global", "dots-module"], case_sensitive=False
     ),
 )
+@click.option(
+    "--order",
+    type=click.Choice(
+        ["standard", "random"], case_sensitive=False
+    ),
+    default="standard",
+    help="Specify the order in which tests should run.",
+)
 @click.version_option(version=__version__)
-def run(path, search, fail_limit, test_output_style):
+def run(path, search, fail_limit, test_output_style, order):
     start_run = default_timer()
 
     paths = [Path(p) for p in path]
     mod_infos = get_info_for_modules(paths)
     modules = list(load_modules(mod_infos))
     unfiltered_tests = get_tests_in_modules(modules)
-    tests = search_generally(unfiltered_tests, query=search)
+    tests = list(search_generally(unfiltered_tests, query=search))
     time_to_collect = default_timer() - start_run
 
-    suite = Suite(tests=list(tests))
-    test_results = suite.generate_test_runs()
+    suite = Suite(tests=tests)
+    test_results = suite.generate_test_runs(order=order)
 
     writer = SimpleTestResultWrite(suite=suite, test_output_style=test_output_style)
     results = writer.output_all_test_results(

ward/suite.py

@@ -1,5 +1,6 @@
 from collections import defaultdict
 from dataclasses import dataclass, field
+from random import shuffle
 from typing import Generator, List
 
 from ward import Scope
@@ -24,7 +25,9 @@ def _test_counts_per_module(self):
             counts[path] += 1
         return counts
 
-    def generate_test_runs(self) -> Generator[TestResult, None, None]:
+    def generate_test_runs(self, order="standard") -> Generator[TestResult, None, None]:
+        if order == "random":
+            shuffle(self.tests)
         num_tests_per_module = self._test_counts_per_module()
         for test in self.tests:
             generated_tests = test.get_parameterised_instances()