diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 5abd0dcf11..d9f8294100 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -97,11 +97,14 @@ jobs: tmt lint cd .. + # Enable local provisioning as this is considered dangerous since tmt v1.38.0 + # see https://github.com/teemtee/tmt/pull/3282 - name: Run integration tests id: run-tests run: | cd tests - tmt run -v \ + tmt --feeling-safe \ + run -v \ -eCONTAINER_USED=integration-test-local \ -eWITH_COVERAGE=1 \ -eLOG_LEVEL=DEBUG \ diff --git a/.packit.yml b/.packit.yml index 2f40bb09a1..2cae788ce4 100644 --- a/.packit.yml +++ b/.packit.yml @@ -1,5 +1,4 @@ --- - upstream_project_url: https://github.com/eclipse-bluechi/bluechi issue_repository: https://github.com/eclipse-bluechi/bluechi specfile_path: bluechi.spec @@ -54,11 +53,14 @@ jobs: labels: - standard env: + # Enable local provisioning as this is considered dangerous since tmt v1.38.0 + # see https://github.com/teemtee/tmt/pull/3282 + TMT_FEELING_SAFE: 1 INSTALL_DEPS: "yes" targets: # Run integration tests on Fedora using CS9 containers, because running integrations tests on CS9 using CS9 # containers is very flaky - # + # # This can be set to fedora-rawhide as soon as the following issue gets resolved: # https://github.com/containers/podman/issues/22422 - fedora-latest-stable-x86_64 @@ -68,13 +70,16 @@ jobs: labels: - valgrind env: + # Enable local provisioning as this is considered dangerous since tmt v1.38.0 + # see https://github.com/teemtee/tmt/pull/3282 + TMT_FEELING_SAFE: 1 INSTALL_DEPS: "yes" WITH_VALGRIND: "1" - job: copr_build trigger: commit branch: main - owner: '@centos-automotive-sig' + owner: "@centos-automotive-sig" project: bluechi-snapshot notifications: failure_comment: diff --git a/tests/README.md b/tests/README.md index b5e4c4aae8..019812c89f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -150,6 +150,19 @@ execution result directory, for example: `/var/tmp/tmt/run-001/plans/tier0/report/default-0/report` +## Changing timeouts for integration tests + +In some cases it might be necessary to adjust the default timeouts that are used in different steps of an integration tests execution cycle. The currently available environment variables as well as their default values are: + +```shell +# in seconds +TIMEOUT_TEST_SETUP=20 +TIMEOUT_TEST_RUN=45 +TIMEOUT_COLLECT_TEST_RESULTS=20 +``` + +These can be set either in the `environment` section of the tmt plan or using the `-e` option when running tmt, e.g. `-eTIMEOUT_TEST_SETUP=40`. + ## Developing integration tests ### Code Style diff --git a/tests/bluechi_test/fixtures.py b/tests/bluechi_test/fixtures.py index ecf99664cd..3dc6e469c3 100644 --- a/tests/bluechi_test/fixtures.py +++ b/tests/bluechi_test/fixtures.py @@ -69,6 +69,33 @@ def machines_ssh_password() -> str: return _get_env_value("SSH_PASSWORD", "") +def _safely_parse_int(input: str, default: int) -> int: + if input.isdigit(): + return int(input) + return default + + +@pytest.fixture(scope="session") +def timeout_test_setup() -> int: + """Returns the timeout for setting up the test setup""" + + return _safely_parse_int(_get_env_value("TIMEOUT_TEST_SETUP", ""), 20) + + +@pytest.fixture(scope="session") +def timeout_test_run() -> int: + """Returns the timeout for executing the actual test""" + + return _safely_parse_int(_get_env_value("TIMEOUT_TEST_RUN", ""), 45) + + +@pytest.fixture(scope="session") +def timeout_collecting_test_results() -> int: + """Returns the timeout for collecting all test results""" + + return _safely_parse_int(_get_env_value("TIMEOUT_COLLECT_TEST_RESULTS", ""), 20) + + def _read_topology() -> Dict[str, Any]: """ Returns the parsed YAML for the tmt guest topology: @@ -216,6 +243,9 @@ def bluechi_test( additional_ports: dict, machines_ssh_user: str, machines_ssh_password: str, + timeout_test_setup: int, + timeout_test_run: int, + timeout_collecting_test_results: int, ) -> BluechiTest: if is_multihost_run: @@ -227,6 +257,9 @@ def bluechi_test( tmt_test_data_dir, run_with_valgrind, run_with_coverage, + timeout_test_setup, + timeout_test_run, + timeout_collecting_test_results, ) return BluechiContainerTest( @@ -238,5 +271,8 @@ def bluechi_test( tmt_test_data_dir, run_with_valgrind, run_with_coverage, + timeout_test_setup, + timeout_test_run, + timeout_collecting_test_results, additional_ports, ) diff --git a/tests/bluechi_test/test.py b/tests/bluechi_test/test.py index 2029066400..2a0e8e0ba1 100644 --- a/tests/bluechi_test/test.py +++ b/tests/bluechi_test/test.py @@ -19,7 +19,7 @@ BluechiControllerMachine, BluechiMachine, ) -from bluechi_test.util import TIMEOUT_GATHER, TIMEOUT_SETUP, TIMEOUT_TEST, Timeout +from bluechi_test.util import Timeout from podman import PodmanClient LOGGER = logging.getLogger(__name__) @@ -32,12 +32,18 @@ def __init__( tmt_test_data_dir: str, run_with_valgrind: bool, run_with_coverage: bool, + timeout_test_setup: int, + timeout_test_run: int, + timeout_collecting_test_results: int, ) -> None: self.tmt_test_serial_number = tmt_test_serial_number self.tmt_test_data_dir = tmt_test_data_dir self.run_with_valgrind = run_with_valgrind self.run_with_coverage = run_with_coverage + self.timeout_test_setup = timeout_test_setup + self.timeout_test_run = timeout_test_run + self.timeout_collecting_test_results = timeout_collecting_test_results self.bluechi_controller_config: BluechiControllerConfig = None self.bluechi_node_configs: List[BluechiAgentConfig] = [] @@ -149,15 +155,12 @@ def run( exec: Callable[ [BluechiControllerMachine, Dict[str, BluechiAgentMachine]], None ], - timeout_setup: int = TIMEOUT_SETUP, - timeout_test: int = TIMEOUT_TEST, - timeout_gather: int = TIMEOUT_GATHER, ): LOGGER.info("Test execution started") successful = False try: - with Timeout(timeout_setup, "Timeout setting up BlueChi system"): + with Timeout(self.timeout_test_setup, "Timeout setting up BlueChi system"): successful, container = self.setup() ctrl_container, node_container = container except TimeoutError as ex: @@ -171,7 +174,7 @@ def run( test_result = None try: - with Timeout(timeout_test, "Timeout running test"): + with Timeout(self.timeout_test_run, "Timeout running test"): exec(ctrl_container, node_container) except Exception as ex: test_result = ex @@ -186,7 +189,10 @@ def run( traceback.print_exc() try: - with Timeout(timeout_gather, "Timeout collecting test artifacts"): + with Timeout( + self.timeout_collecting_test_results, + "Timeout collecting test artifacts", + ): self.gather_logs(ctrl_container, node_container) if self.run_with_valgrind: self.check_valgrind_logs() @@ -215,6 +221,9 @@ def __init__( tmt_test_data_dir: str, run_with_valgrind: bool, run_with_coverage: bool, + timeout_test_setup: int, + timeout_test_run: int, + timeout_collecting_test_results: int, additional_ports: Dict, ) -> None: @@ -223,6 +232,9 @@ def __init__( tmt_test_data_dir, run_with_valgrind, run_with_coverage, + timeout_test_setup, + timeout_test_run, + timeout_collecting_test_results, ) self.podman_client = podman_client @@ -324,6 +336,9 @@ def __init__( tmt_test_data_dir: str, run_with_valgrind: bool, run_with_coverage: bool, + timeout_test_setup: int, + timeout_test_run: int, + timeout_collecting_test_results: int, ) -> None: super().__init__( @@ -331,6 +346,9 @@ def __init__( tmt_test_data_dir, run_with_valgrind, run_with_coverage, + timeout_test_setup, + timeout_test_run, + timeout_collecting_test_results, ) self.available_hosts = available_hosts diff --git a/tests/bluechi_test/util.py b/tests/bluechi_test/util.py index 9dc03d0e1e..2ea657e080 100644 --- a/tests/bluechi_test/util.py +++ b/tests/bluechi_test/util.py @@ -70,14 +70,6 @@ def remove_control_chars(value: str) -> str: return _ANSI_SEQUENCE.sub("", value) -# timeout for setting up tests in s -TIMEOUT_SETUP = 20 -# timeout for running tests in s -TIMEOUT_TEST = 45 -# timeout for collecting test results in s -TIMEOUT_GATHER = 20 - - class Timeout: def __init__(self, seconds=1, error_message="Timeout"): self.seconds = seconds