Make integration test timeouts configurable

.github/workflows/integration-tests.yml

@@ -97,11 +97,14 @@ jobs:
           tmt lint
           cd ..
 
+      # Enable local provisioning as this is considered dangerous since tmt v1.38.0
+      # see https://github.com/teemtee/tmt/pull/3282
       - name: Run integration tests
         id: run-tests
         run: |
           cd tests
-          tmt run -v \
+          tmt --feeling-safe \
+            run -v \
             -eCONTAINER_USED=integration-test-local \
             -eWITH_COVERAGE=1 \
             -eLOG_LEVEL=DEBUG \

.packit.yml

@@ -1,5 +1,4 @@
 ---
-
 upstream_project_url: https://github.com/eclipse-bluechi/bluechi
 issue_repository: https://github.com/eclipse-bluechi/bluechi
 specfile_path: bluechi.spec
@@ -54,11 +53,14 @@ jobs:
     labels:
       - standard
     env:
+      # Enable local provisioning as this is considered dangerous since tmt v1.38.0
+      # see https://github.com/teemtee/tmt/pull/3282
+      TMT_FEELING_SAFE: 1
       INSTALL_DEPS: "yes"
     targets:
       # Run integration tests on Fedora using CS9 containers, because running integrations tests on CS9 using CS9
       # containers is very flaky
-      # 
+      #
       # This can be set to fedora-rawhide as soon as the following issue gets resolved:
       # https://github.com/containers/podman/issues/22422
       - fedora-latest-stable-x86_64
@@ -68,13 +70,16 @@ jobs:
     labels:
       - valgrind
     env:
+      # Enable local provisioning as this is considered dangerous since tmt v1.38.0
+      # see https://github.com/teemtee/tmt/pull/3282
+      TMT_FEELING_SAFE: 1
       INSTALL_DEPS: "yes"
       WITH_VALGRIND: "1"
 
   - job: copr_build
     trigger: commit
     branch: main
-    owner: '@centos-automotive-sig'
+    owner: "@centos-automotive-sig"
     project: bluechi-snapshot
     notifications:
       failure_comment:

tests/README.md

@@ -150,6 +150,19 @@ execution result directory, for example:
 
 `/var/tmp/tmt/run-001/plans/tier0/report/default-0/report`
 
+## Changing timeouts for integration tests
+
+In some cases it might be necessary to adjust the default timeouts that are used in different steps of an integration tests execution cycle. The currently available environment variables as well as their default values are:
+
+```shell
+# in seconds
+TIMEOUT_TEST_SETUP=20
+TIMEOUT_TEST_RUN=45
+TIMEOUT_COLLECT_TEST_RESULTS=20
+```
+
+These can be set either in the `environment` section of the tmt plan or using the `-e` option when running tmt, e.g. `-eTIMEOUT_TEST_SETUP=40`.
+
 ## Developing integration tests
 
 ### Code Style

tests/bluechi_test/fixtures.py

@@ -69,6 +69,33 @@ def machines_ssh_password() -> str:
     return _get_env_value("SSH_PASSWORD", "")
 
 
+def _safely_parse_int(input: str, default: int) -> int:
+    if input.isdigit():
+        return int(input)
+    return default
+
+
+@pytest.fixture(scope="session")
+def timeout_test_setup() -> int:
+    """Returns the timeout for setting up the test setup"""
+
+    return _safely_parse_int(_get_env_value("TIMEOUT_TEST_SETUP", ""), 20)
+
+
+@pytest.fixture(scope="session")
+def timeout_test_run() -> int:
+    """Returns the timeout for executing the actual test"""
+
+    return _safely_parse_int(_get_env_value("TIMEOUT_TEST_RUN", ""), 45)
+
+
+@pytest.fixture(scope="session")
+def timeout_collecting_test_results() -> int:
+    """Returns the timeout for collecting all test results"""
+
+    return _safely_parse_int(_get_env_value("TIMEOUT_COLLECT_TEST_RESULTS", ""), 20)
+
+
 def _read_topology() -> Dict[str, Any]:
     """
     Returns the parsed YAML for the tmt guest topology:
@@ -216,6 +243,9 @@ def bluechi_test(
     additional_ports: dict,
     machines_ssh_user: str,
     machines_ssh_password: str,
+    timeout_test_setup: int,
+    timeout_test_run: int,
+    timeout_collecting_test_results: int,
 ) -> BluechiTest:
 
     if is_multihost_run:
@@ -227,6 +257,9 @@ def bluechi_test(
             tmt_test_data_dir,
             run_with_valgrind,
             run_with_coverage,
+            timeout_test_setup,
+            timeout_test_run,
+            timeout_collecting_test_results,
         )
 
     return BluechiContainerTest(
@@ -238,5 +271,8 @@ def bluechi_test(
         tmt_test_data_dir,
         run_with_valgrind,
         run_with_coverage,
+        timeout_test_setup,
+        timeout_test_run,
+        timeout_collecting_test_results,
         additional_ports,
     )

tests/bluechi_test/test.py

@@ -19,7 +19,7 @@
     BluechiControllerMachine,
     BluechiMachine,
 )
-from bluechi_test.util import TIMEOUT_GATHER, TIMEOUT_SETUP, TIMEOUT_TEST, Timeout
+from bluechi_test.util import Timeout
 from podman import PodmanClient
 
 LOGGER = logging.getLogger(__name__)
@@ -32,12 +32,18 @@ def __init__(
         tmt_test_data_dir: str,
         run_with_valgrind: bool,
         run_with_coverage: bool,
+        timeout_test_setup: int,
+        timeout_test_run: int,
+        timeout_collecting_test_results: int,
     ) -> None:
 
         self.tmt_test_serial_number = tmt_test_serial_number
         self.tmt_test_data_dir = tmt_test_data_dir
         self.run_with_valgrind = run_with_valgrind
         self.run_with_coverage = run_with_coverage
+        self.timeout_test_setup = timeout_test_setup
+        self.timeout_test_run = timeout_test_run
+        self.timeout_collecting_test_results = timeout_collecting_test_results
 
         self.bluechi_controller_config: BluechiControllerConfig = None
         self.bluechi_node_configs: List[BluechiAgentConfig] = []
@@ -149,15 +155,12 @@ def run(
         exec: Callable[
             [BluechiControllerMachine, Dict[str, BluechiAgentMachine]], None
         ],
-        timeout_setup: int = TIMEOUT_SETUP,
-        timeout_test: int = TIMEOUT_TEST,
-        timeout_gather: int = TIMEOUT_GATHER,
     ):
         LOGGER.info("Test execution started")
 
         successful = False
         try:
-            with Timeout(timeout_setup, "Timeout setting up BlueChi system"):
+            with Timeout(self.timeout_test_setup, "Timeout setting up BlueChi system"):
                 successful, container = self.setup()
             ctrl_container, node_container = container
         except TimeoutError as ex:
@@ -171,7 +174,7 @@ def run(
 
         test_result = None
         try:
-            with Timeout(timeout_test, "Timeout running test"):
+            with Timeout(self.timeout_test_run, "Timeout running test"):
                 exec(ctrl_container, node_container)
         except Exception as ex:
             test_result = ex
@@ -186,7 +189,10 @@ def run(
             traceback.print_exc()
 
         try:
-            with Timeout(timeout_gather, "Timeout collecting test artifacts"):
+            with Timeout(
+                self.timeout_collecting_test_results,
+                "Timeout collecting test artifacts",
+            ):
                 self.gather_logs(ctrl_container, node_container)
                 if self.run_with_valgrind:
                     self.check_valgrind_logs()
@@ -215,6 +221,9 @@ def __init__(
         tmt_test_data_dir: str,
         run_with_valgrind: bool,
         run_with_coverage: bool,
+        timeout_test_setup: int,
+        timeout_test_run: int,
+        timeout_collecting_test_results: int,
         additional_ports: Dict,
     ) -> None:
 
@@ -223,6 +232,9 @@ def __init__(
             tmt_test_data_dir,
             run_with_valgrind,
             run_with_coverage,
+            timeout_test_setup,
+            timeout_test_run,
+            timeout_collecting_test_results,
         )
 
         self.podman_client = podman_client
@@ -324,13 +336,19 @@ def __init__(
         tmt_test_data_dir: str,
         run_with_valgrind: bool,
         run_with_coverage: bool,
+        timeout_test_setup: int,
+        timeout_test_run: int,
+        timeout_collecting_test_results: int,
     ) -> None:
 
         super().__init__(
             tmt_test_serial_number,
             tmt_test_data_dir,
             run_with_valgrind,
             run_with_coverage,
+            timeout_test_setup,
+            timeout_test_run,
+            timeout_collecting_test_results,
         )
 
         self.available_hosts = available_hosts

tests/bluechi_test/util.py

@@ -70,14 +70,6 @@ def remove_control_chars(value: str) -> str:
     return _ANSI_SEQUENCE.sub("", value)
 
 
-# timeout for setting up tests in s
-TIMEOUT_SETUP = 20
-# timeout for running tests in s
-TIMEOUT_TEST = 45
-# timeout for collecting test results in s
-TIMEOUT_GATHER = 20
-
-
 class Timeout:
     def __init__(self, seconds=1, error_message="Timeout"):
         self.seconds = seconds