diff --git a/tests/test_suite.py b/tests/test_suite.py index 7d7c237a..21288c58 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -355,13 +355,15 @@ def test_3(a=a, b=b, c=c): test_1.ward_meta.path = "module1" test_2.ward_meta.path = "module2" - test_3.ward_meta.path = "module2" + test_3.ward_meta.path = "module1" suite = Suite( tests=[ + # Module ordering is intentionally off here, to ensure correct + # interaction between module-scope fixtures and random ordering Test(fn=test_1, module_name="module1"), Test(fn=test_2, module_name="module2"), - Test(fn=test_3, module_name="module2"), + Test(fn=test_3, module_name="module1"), ] ) @@ -374,15 +376,15 @@ def test_3(a=a, b=b, c=c): "resolve c", # test fixture resolved at start of test1 "test1", "teardown c", # test fixture teardown at start of test1 - "teardown b", # module fixture teardown at end of module1 "resolve b", # module fixture resolved at start of module2 "resolve c", # test fixture resolved at start of test2 "test2", "teardown c", # test fixture teardown at start of test2 + "teardown b", # module fixture teardown at end of module2 "resolve c", # test fixture resolved at start of test3 "test3", "teardown c", # test fixture teardown at end of test3 - "teardown b", # module fixture teardown at end of module2 + "teardown b", # module fixture teardown at end of module1 "teardown a", # global fixtures are torn down at the very end ] ) diff --git a/ward/run.py b/ward/run.py index 0d888282..945d4bb5 100644 --- a/ward/run.py +++ b/ward/run.py @@ -1,5 +1,6 @@ import sys from pathlib import Path +from random import shuffle from timeit import default_timer import click @@ -27,12 +28,12 @@ "--path", default=".", type=click.Path(exists=True), - help="Path to tests.", + help="Path to test directory.", multiple=True, ) @click.option( "--search", - help="Search test names, descriptions and module names for the search query and only run matching tests.", + help="Search test names, bodies, descriptions and module names for the search query and only run matching tests.", ) @click.option( "--fail-limit", @@ -45,19 +46,27 @@ ["test-per-line", "dots-global", "dots-module"], case_sensitive=False ), ) +@click.option( + "--order", + type=click.Choice( + ["standard", "random"], case_sensitive=False + ), + default="standard", + help="Specify the order in which tests should run.", +) @click.version_option(version=__version__) -def run(path, search, fail_limit, test_output_style): +def run(path, search, fail_limit, test_output_style, order): start_run = default_timer() paths = [Path(p) for p in path] mod_infos = get_info_for_modules(paths) modules = list(load_modules(mod_infos)) unfiltered_tests = get_tests_in_modules(modules) - tests = search_generally(unfiltered_tests, query=search) + tests = list(search_generally(unfiltered_tests, query=search)) time_to_collect = default_timer() - start_run - suite = Suite(tests=list(tests)) - test_results = suite.generate_test_runs() + suite = Suite(tests=tests) + test_results = suite.generate_test_runs(order=order) writer = SimpleTestResultWrite(suite=suite, test_output_style=test_output_style) results = writer.output_all_test_results( diff --git a/ward/suite.py b/ward/suite.py index d2aac710..e17125be 100644 --- a/ward/suite.py +++ b/ward/suite.py @@ -1,5 +1,6 @@ from collections import defaultdict from dataclasses import dataclass, field +from random import shuffle from typing import Generator, List from ward import Scope @@ -24,7 +25,9 @@ def _test_counts_per_module(self): counts[path] += 1 return counts - def generate_test_runs(self) -> Generator[TestResult, None, None]: + def generate_test_runs(self, order="standard") -> Generator[TestResult, None, None]: + if order == "random": + shuffle(self.tests) num_tests_per_module = self._test_counts_per_module() for test in self.tests: generated_tests = test.get_parameterised_instances()