From 7e33f373af8bbd0a9fbb88ec1f370f0411bebc44 Mon Sep 17 00:00:00 2001 From: Michael Engel Date: Fri, 1 Mar 2024 13:59:48 +0100 Subject: [PATCH 1/3] added multihost tag for integration tests Relates to: https://github.com/eclipse-bluechi/bluechi/issues/662 Added a tag on the root main.fmf to mark all tests by default to be able to run in a multihost setting. Certain tests that exceed the available number of hosts (>3) are disabled for now. Signed-off-by: Michael Engel --- tests/tests/main.fmf | 1 + tests/tests/tier0/bluechi-agent-logisquiet/main.fmf | 1 + tests/tests/tier0/bluechi-agent-loglevel/main.fmf | 1 + tests/tests/tier0/bluechi-agent-logtarget/main.fmf | 1 + tests/tests/tier0/bluechi-list-units-on-all-nodes/main.fmf | 4 ++++ tests/tests/tier0/monitor-system-status/main.fmf | 1 + tests/tests/tier0/monitor-wildcard-node-reconnect/main.fmf | 1 + .../proxy-service-multiple-services-multiple-nodes/main.fmf | 1 + 8 files changed, 11 insertions(+) diff --git a/tests/tests/main.fmf b/tests/tests/main.fmf index 9f129c1866..65e8ec66bb 100644 --- a/tests/tests/main.fmf +++ b/tests/tests/main.fmf @@ -1,4 +1,5 @@ component: bluechi +tag: [multihost] test: | pytest -svv \ --confcutdir=../../../ \ diff --git a/tests/tests/tier0/bluechi-agent-logisquiet/main.fmf b/tests/tests/tier0/bluechi-agent-logisquiet/main.fmf index a675fdb7cc..6655a38310 100644 --- a/tests/tests/tier0/bluechi-agent-logisquiet/main.fmf +++ b/tests/tests/tier0/bluechi-agent-logisquiet/main.fmf @@ -1,2 +1,3 @@ summary: Test if bluechi agent can handle different types of values for LogIsQuiet in the configuration id: 96aa0e17-5e23-4cc3-bc34-88368b8cc07b +tag: [] diff --git a/tests/tests/tier0/bluechi-agent-loglevel/main.fmf b/tests/tests/tier0/bluechi-agent-loglevel/main.fmf index 6dad85833c..f51c400e65 100644 --- a/tests/tests/tier0/bluechi-agent-loglevel/main.fmf +++ b/tests/tests/tier0/bluechi-agent-loglevel/main.fmf @@ -1,2 +1,3 @@ summary: Test if bluechi agent can handle different types of values for LogLevel in the configuration id: f3dda8c1-ceb7-446c-9d65-2aa8ca798099 +tag: [] diff --git a/tests/tests/tier0/bluechi-agent-logtarget/main.fmf b/tests/tests/tier0/bluechi-agent-logtarget/main.fmf index cd7e8f10b3..69f5018fc7 100644 --- a/tests/tests/tier0/bluechi-agent-logtarget/main.fmf +++ b/tests/tests/tier0/bluechi-agent-logtarget/main.fmf @@ -1,2 +1,3 @@ summary: Test if bluechi agent can handle different types of values for LogTarget in the configuration id: 5cf92c54-e073-475c-970f-b8e090ec4da2 +tag: [] diff --git a/tests/tests/tier0/bluechi-list-units-on-all-nodes/main.fmf b/tests/tests/tier0/bluechi-list-units-on-all-nodes/main.fmf index 9b27f3e5a2..c6f62a57b2 100644 --- a/tests/tests/tier0/bluechi-list-units-on-all-nodes/main.fmf +++ b/tests/tests/tier0/bluechi-list-units-on-all-nodes/main.fmf @@ -1,3 +1,7 @@ summary: Test if bluechi list-nodes returns the same list of units from all nodes which can be gathered by running systemctl on each node id: efbf2397-de69-4f67-a8a8-b2c10bc68c13 + +# disabled for multihost till this is solved: +# https://github.com/eclipse-bluechi/bluechi/issues/767 +tag: [] diff --git a/tests/tests/tier0/monitor-system-status/main.fmf b/tests/tests/tier0/monitor-system-status/main.fmf index d80381a2e2..3c546b1837 100644 --- a/tests/tests/tier0/monitor-system-status/main.fmf +++ b/tests/tests/tier0/monitor-system-status/main.fmf @@ -1,3 +1,4 @@ summary: Test if the system status changed signal is being emitted when nodes disconnect and reconnect id: 96615a39-229f-4083-8c7d-cb0c540ca598 +tag: [] diff --git a/tests/tests/tier0/monitor-wildcard-node-reconnect/main.fmf b/tests/tests/tier0/monitor-wildcard-node-reconnect/main.fmf index 4888a1da59..9be2d4ce91 100644 --- a/tests/tests/tier0/monitor-wildcard-node-reconnect/main.fmf +++ b/tests/tests/tier0/monitor-wildcard-node-reconnect/main.fmf @@ -1,2 +1,3 @@ summary: Test if the proper virtual signals are emitted when a monitor with wildcard subscription is active and a node disconnects and reconnects again id: 071d8a13-aee6-4dc0-9a91-0fb9a2e38a93 +tag: [] diff --git a/tests/tests/tier0/proxy-service-multiple-services-multiple-nodes/main.fmf b/tests/tests/tier0/proxy-service-multiple-services-multiple-nodes/main.fmf index bf9b4ec9b2..b38159b4c8 100644 --- a/tests/tests/tier0/proxy-service-multiple-services-multiple-nodes/main.fmf +++ b/tests/tests/tier0/proxy-service-multiple-services-multiple-nodes/main.fmf @@ -1,2 +1,3 @@ summary: Test proper lifetimes of all services when multiple services on multiple nodes request the same service on the same node id: 4b5a0d32-4b85-404d-9a02-f58531bf1957 +tag: [] From 346f0bca55bb360fea2242dda31b901b58bdaa7e Mon Sep 17 00:00:00 2001 From: Michael Engel Date: Fri, 1 Mar 2024 14:05:12 +0100 Subject: [PATCH 2/3] implemented cleanup for integration tests Relates to: https://github.com/eclipse-bluechi/bluechi/issues/662 In order to run all tests on a set of provisioned hosts, the state of the host needs to be restored after each test run. This includes: - removing all added files - restoring all changed files - stopping all systemd units and reset failed ones (incl. bluechi) By keeping track of these operations, the test class is able to execute respective cleanup tasks on the hosts. Therefore, the cleanup from machine class has been moved to the ssh test class. The container test class still only removes the created containers. Signed-off-by: Michael Engel --- tests/bluechi_test/bluechictl.py | 9 ++- tests/bluechi_test/fixtures.py | 9 ++- tests/bluechi_test/machine.py | 119 +++++++++++++++++++++---------- tests/bluechi_test/systemctl.py | 10 ++- tests/bluechi_test/test.py | 63 +++++++++++----- 5 files changed, 149 insertions(+), 61 deletions(-) diff --git a/tests/bluechi_test/bluechictl.py b/tests/bluechi_test/bluechictl.py index 1225c40098..376b811714 100644 --- a/tests/bluechi_test/bluechictl.py +++ b/tests/bluechi_test/bluechictl.py @@ -3,7 +3,7 @@ import logging from enum import Enum -from typing import Tuple, Iterator, Any, Optional, Union +from typing import Tuple, Iterator, Any, Optional, Union, Dict, List from bluechi_test.client import Client @@ -24,6 +24,8 @@ class BluechiCtl(): def __init__(self, client: Client) -> None: self.client = client + self.tracked_services: Dict[str, List[str]] = dict() + def _run(self, log_txt: str, cmd: str, check_result: bool, expected_result: int) \ -> Tuple[Optional[int], Union[Iterator[bytes], Any, Tuple[bytes, bytes]]]: LOGGER.debug(log_txt) @@ -78,6 +80,11 @@ def get_node_status(self, node_name: str = None, check_result: bool = True, expe def start_unit(self, node_name: str, unit_name: str, check_result: bool = True, expected_result: int = 0) \ -> Tuple[Optional[int], Union[Iterator[bytes], Any, Tuple[bytes, bytes]]]: + # track started units to stop and reset failures on cleanup + if node_name not in self.tracked_services: + self.tracked_services[node_name] = [] + self.tracked_services[node_name].append(unit_name) + return self._run( f"Starting unit '{unit_name}' on node '{node_name}'", f"start {node_name} {unit_name}", diff --git a/tests/bluechi_test/fixtures.py b/tests/bluechi_test/fixtures.py index b0e5901ed3..1fe3e102f6 100644 --- a/tests/bluechi_test/fixtures.py +++ b/tests/bluechi_test/fixtures.py @@ -76,7 +76,7 @@ def _read_topology() -> Dict[str, Any]: if tmt_yaml_file is None or tmt_yaml_file == "": return get_primary_ip() - topology = "" + topology = dict() with open(tmt_yaml_file, "r") as f: topology = yaml.safe_load(f.read()) return topology @@ -94,9 +94,9 @@ def bluechi_ctrl_host_ip() -> str: if "guests" not in topology: return get_primary_ip() - for guest in topology["guests"]: - if "name" in guest and "controller" in guest["role"]: - return guest["hostname"] + for _, values in topology["guests"].items(): + if values["role"] == "controller": + return values["hostname"] return get_primary_ip() @@ -215,7 +215,6 @@ def bluechi_test( return BluechiSSHTest(available_hosts, machines_ssh_user, machines_ssh_password, - bluechi_ctrl_svc_port, tmt_test_serial_number, tmt_test_data_dir, run_with_valgrind, diff --git a/tests/bluechi_test/machine.py b/tests/bluechi_test/machine.py index beb33f70d0..f2f891960b 100644 --- a/tests/bluechi_test/machine.py +++ b/tests/bluechi_test/machine.py @@ -5,9 +5,9 @@ import time import traceback -from typing import Any, Iterator, Optional, Tuple, Union +from typing import Any, Iterator, Optional, Tuple, Union, List -from bluechi_test.client import Client, ContainerClient, SSHClient +from bluechi_test.client import Client from bluechi_test.config import BluechiAgentConfig, BluechiControllerConfig from bluechi_test.systemctl import SystemCtl from bluechi_test.bluechictl import BluechiCtl @@ -18,15 +18,23 @@ class BluechiMachine(): + valgrind_log_directory = '/var/log/valgrind' valgrind_log_path_controller = '/var/log/valgrind/bluechi-controller-valgrind.log' valgrind_log_path_agent = '/var/log/valgrind/bluechi-agent-valgrind.log' + gcda_file_location = '/var/tmp/bluechi-coverage' + + backup_file_suffix = '.backup' + def __init__(self, name: str, client: Client) -> None: self.name = name self.client = client self.systemctl = SystemCtl(client) + self.created_files: List[str] = [] + self.changed_files: List[str] = [] + def create_file(self, target_dir: str, file_name: str, content: str) -> None: target_file = os.path.join(target_dir, file_name) try: @@ -36,6 +44,27 @@ def create_file(self, target_dir: str, file_name: str, content: str) -> None: traceback.print_exc() return + # keep track of create file for later cleanup + self.created_files.append(os.path.join(target_dir, file_name)) + + def _track_changed_file(self, target_dir: str, file_name: str) -> None: + target_file = os.path.join(target_dir, file_name) + try: + # create backup of original file only the first time + if target_file in self.changed_files: + return + + LOGGER.debug(f"Creating backup of file '{target_file}'...") + backup_file = target_file + BluechiMachine.backup_file_suffix + result, output = self.client.exec_run(f"cp {target_file} {backup_file}") + if result != 0: + raise Exception(output) + self.changed_files.append(target_file) + except Exception as ex: + LOGGER.error(f"Failed to create backup of file '{target_file}': {ex}") + traceback.print_exc() + return + def get_file(self, machine_path: str, local_path: str) -> None: self.client.get_file(machine_path, local_path) @@ -77,9 +106,12 @@ def copy_systemd_service(self, service_file_name: str, source_dir: str, target_d LOGGER.debug(f"Copy local systemd service '{source_path}' to container path '{target_dir}'\ with content:\n{content}") - self.client.create_file(target_dir, service_file_name, content) + self.create_file(target_dir, service_file_name, content) self.systemctl.daemon_reload() + # keep track of created service file to potentially stop it in later cleanup + self.systemctl.tracked_services.append(service_file_name) + def copy_container_script(self, script_file_name: str): curr_dir = os.getcwd() source_path = os.path.join(curr_dir, "..", "..", "..", "bluechi_test", "container_scripts", script_file_name) @@ -89,14 +121,18 @@ def copy_container_script(self, script_file_name: str): LOGGER.info(f"Copy container script '{source_path}' to container path '{curr_dir}'\ with content:\n{content}") - self.client.create_file(target_dir, script_file_name, content) + self.create_file(target_dir, script_file_name, content) def restart_with_config_file(self, config_file_location, service): + unit_dir = "/usr/lib/systemd/system" + service_file = f"{service}.service" + + self._track_changed_file(unit_dir, service_file) self.client.exec_run(f"sed -i '/ExecStart=/c\\ExecStart=/usr/libexec/{service} -c " f"{config_file_location}' " - f"/usr/lib/systemd/system/{service}.service") + f"{os.path.join(unit_dir, service_file)}") self.systemctl.daemon_reload() - self.systemctl.restart_unit(f"{service}.service") + self.systemctl.restart_unit(service_file) def wait_for_bluechi_agent(self): should_wait = True @@ -109,13 +145,21 @@ def wait_for_bluechi_controller(self): should_wait = not self.systemctl.service_is_active("bluechi-controller") def enable_valgrind(self) -> None: + unit_dir = "/usr/lib/systemd/system" + controller_service = "bluechi-controller.service" + agent_service = "bluechi-agent.service" + + self._track_changed_file(unit_dir, controller_service) self.client.exec_run(f"sed -i '/ExecStart=/c\\ExecStart=/usr/bin/valgrind -s --leak-check=yes " f"--log-file={BluechiMachine.valgrind_log_path_controller} " - f"/usr/libexec/bluechi-controller' /usr/lib/systemd/system/bluechi-controller.service") + f"/usr/libexec/bluechi-controller' {os.path.join(unit_dir, controller_service)}") + + self._track_changed_file(unit_dir, agent_service) self.client.exec_run(f"sed -i '/ExecStart=/c\\ExecStart=/usr/bin/valgrind -s --leak-check=yes " f"--log-file={BluechiMachine.valgrind_log_path_agent} /usr/libexec/bluechi-agent' " - f"/usr/lib/systemd/system/bluechi-agent.service") - self.client.exec_run("mkdir -p /var/log/valgrind") + f"{os.path.join(unit_dir, agent_service)}") + + self.client.exec_run(f"mkdir -p {BluechiMachine.valgrind_log_directory}") self.systemctl.daemon_reload() def run_python(self, python_script_path: str) -> \ @@ -124,6 +168,9 @@ def run_python(self, python_script_path: str) -> \ target_file_dir = os.path.join("/", "tmp") target_file_name = get_random_name(10) content = read_file(python_script_path) + + # directly call create_file on client to bypass cleanup as + # the script file will be removed after running it self.client.create_file(target_file_dir, target_file_name, content) target_file_path = os.path.join(target_file_dir, target_file_name) @@ -136,31 +183,16 @@ def run_python(self, python_script_path: str) -> \ finally: return result, output - def cleanup(self): - if isinstance(self.client, ContainerClient): - if self.client.container.status == 'running': - kw_params = {'timeout': 0} - self.client.container.stop(**kw_params) - self.client.container.remove() - elif isinstance(self.client, SSHClient): - # TODO: implement proper cleanup (removing all added files etc.) - pass - def gather_valgrind_logs(self, data_dir: str) -> None: - bluechi_controller_valgrind_filename = f"bluechi-controller-valgrind-{self.name}.log" - bluechi_agent_valgrind_filename = f"bluechi-agent-valgrind-{self.name}.log" - bluechi_controller_valgrind_log_target_path = f"/tmp/{bluechi_controller_valgrind_filename}" - bluechi_agent_valgrind_log_target_path = f"/tmp/{bluechi_agent_valgrind_filename}" - - # Collect valgrind logs by copying log files to the data directory - result, _ = self.client.exec_run( - f'cp -f {BluechiMachine.valgrind_log_path_controller} {bluechi_controller_valgrind_log_target_path}') - if result == 0: - self.client.get_file(bluechi_controller_valgrind_log_target_path, data_dir) - result, _ = self.client.exec_run( - f'cp -f {BluechiMachine.valgrind_log_path_agent} {bluechi_agent_valgrind_log_target_path}') - if result == 0: - self.client.get_file(bluechi_agent_valgrind_log_target_path, data_dir) + try: + self.client.get_file(BluechiMachine.valgrind_log_path_controller, data_dir) + except Exception as ex: + LOGGER.debug(f"Failed to get valgrind logs for controller: {ex}") + + try: + self.client.get_file(BluechiMachine.valgrind_log_path_agent, data_dir) + except Exception as ex: + LOGGER.debug(f"Failed to get valgrind logs for agent: {ex}") def gather_journal_logs(self, data_dir: str) -> None: log_file = f"/tmp/journal-{self.name}.log" @@ -168,11 +200,13 @@ def gather_journal_logs(self, data_dir: str) -> None: self.client.exec_run( f'bash -c "journalctl --no-pager > {log_file}"', tty=True) + # track created logfile for later cleanup + self.created_files.append(log_file) + self.client.get_file(log_file, data_dir) def gather_coverage(self, data_coverage_dir: str) -> None: - gcda_file_location = "/var/tmp/bluechi-coverage" - coverage_file = f"{gcda_file_location}/coverage-{self.name}.info" + coverage_file = f"{BluechiMachine.gcda_file_location}/coverage-{self.name}.info" LOGGER.info(f"Generating info file '{coverage_file}' started") result, output = self.client.exec_run( @@ -183,6 +217,17 @@ def gather_coverage(self, data_coverage_dir: str) -> None: self.client.get_file(f"{coverage_file}", data_coverage_dir) + def cleanup_valgrind_logs(self): + self.client.exec_run(f"rm -f {BluechiMachine.valgrind_log_path_controller}") + self.client.exec_run(f"rm -f {BluechiMachine.valgrind_log_path_agent}") + + def cleanup_journal_logs(self): + self.client.exec_run("journalctl --flush --rotate") + self.client.exec_run("journalctl --vacuum-time=1s") + + def cleanup_coverage(self): + self.client.exec_run(f"rm -rf {BluechiMachine.gcda_file_location}/*") + class BluechiAgentMachine(BluechiMachine): @@ -192,7 +237,7 @@ def __init__(self, name: str, client: Client, agent_config: BluechiAgentConfig) self.config = agent_config # add confd file to container - self.client.create_file(self.config.get_confd_dir(), self.config.file_name, self.config.serialize()) + self.create_file(self.config.get_confd_dir(), self.config.file_name, self.config.serialize()) class BluechiControllerMachine(BluechiMachine): @@ -205,4 +250,4 @@ def __init__(self, name: str, client: Client, ctrl_config: BluechiControllerConf self.config = ctrl_config # add confd file to container - self.client.create_file(self.config.get_confd_dir(), self.config.file_name, self.config.serialize()) + self.create_file(self.config.get_confd_dir(), self.config.file_name, self.config.serialize()) diff --git a/tests/bluechi_test/systemctl.py b/tests/bluechi_test/systemctl.py index 4a749f7a74..d374eb753b 100644 --- a/tests/bluechi_test/systemctl.py +++ b/tests/bluechi_test/systemctl.py @@ -2,7 +2,7 @@ import logging -from typing import Any, Iterator, Optional, Set, Tuple, Union +from typing import Any, Iterator, Optional, Set, Tuple, Union, List from bluechi_test.client import Client @@ -16,6 +16,8 @@ class SystemCtl(): def __init__(self, client: Client) -> None: self.client = client + self.tracked_services: List[str] = [] + def _do_operation_on_unit(self, unit_name: str, operation: str, check_result: bool, expected_result: int) \ -> Tuple[Optional[int], Union[Iterator[bytes], Any, Tuple[bytes, bytes]]]: @@ -36,6 +38,9 @@ def _do_operation(self, operation: str, check_result: bool, expected_result: int def start_unit(self, unit_name: str, check_result: bool = True, expected_result: int = 0) \ -> Tuple[Optional[int], Union[Iterator[bytes], Any, Tuple[bytes, bytes]]]: + # track started units to stop and reset failures on cleanup + self.tracked_services.append(unit_name) + return self._do_operation_on_unit(unit_name, "start", check_result, expected_result) def stop_unit(self, @@ -63,6 +68,9 @@ def stop_unit(self, def restart_unit(self, unit_name: str, check_result: bool = True, expected_result: int = 0) \ -> Tuple[Optional[int], Union[Iterator[bytes], Any, Tuple[bytes, bytes]]]: + # track started units to stop and reset failures on cleanup + self.tracked_services.append(unit_name) + return self._do_operation_on_unit(unit_name, "restart", check_result, expected_result) def reset_failed_for_unit(self, unit_name: str, check_result: bool = True, expected_result: int = 0) \ diff --git a/tests/bluechi_test/test.py b/tests/bluechi_test/test.py index 76ea117276..fb98707b05 100644 --- a/tests/bluechi_test/test.py +++ b/tests/bluechi_test/test.py @@ -13,7 +13,7 @@ from bluechi_test.client import ContainerClient, SSHClient from bluechi_test.command import Command from bluechi_test.config import BluechiControllerConfig, BluechiAgentConfig -from bluechi_test.machine import BluechiAgentMachine, BluechiControllerMachine +from bluechi_test.machine import BluechiMachine, BluechiAgentMachine, BluechiControllerMachine LOGGER = logging.getLogger(__name__) @@ -244,11 +244,18 @@ def setup(self) -> Tuple[bool, Tuple[BluechiControllerMachine, Dict[str, Bluechi def teardown(self, ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]): LOGGER.debug("Stopping and removing all container...") - if ctrl is not None: - ctrl.cleanup() + if ctrl is not None and isinstance(ctrl.client, ContainerClient): + if ctrl.client.container.status == 'running': + kw_params = {'timeout': 0} + ctrl.client.container.stop(**kw_params) + ctrl.client.container.remove() for _, node in nodes.items(): - node.cleanup() + if node is not None and isinstance(node.client, ContainerClient): + if node.client.container.status == 'running': + kw_params = {'timeout': 0} + node.client.container.stop(**kw_params) + node.client.container.remove() class BluechiSSHTest(BluechiTest): @@ -257,7 +264,6 @@ def __init__(self, available_hosts: Dict[str, List[Tuple[str, str]]], ssh_user: str, ssh_password: str, - bluechi_ctrl_port: str, tmt_test_serial_number: str, tmt_test_data_dir: str, run_with_valgrind: bool, @@ -271,7 +277,6 @@ def __init__(self, self.available_hosts = available_hosts self.ssh_user = ssh_user self.ssh_password = ssh_password - self.bluechi_ctrl_port = bluechi_ctrl_port def add_bluechi_agent_config(self, cfg: BluechiAgentConfig): if len(self.bluechi_node_configs) >= len(self.available_hosts["agent"]): @@ -314,6 +319,7 @@ def setup(self) -> Tuple[bool, Tuple[BluechiControllerMachine, Dict[str, Bluechi name = self.assemble_agent_machine_name(cfg) guest, host = self.available_hosts["agent"][i] + i = i + 1 LOGGER.debug(f"Setting up agent machine on guest '{guest}' with host '{host}'") agent_machine = BluechiAgentMachine( @@ -342,17 +348,40 @@ def setup(self) -> Tuple[bool, Tuple[BluechiControllerMachine, Dict[str, Bluechi return (success, (ctrl_machine, agent_machines)) def teardown(self, ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]): - LOGGER.debug("Cleaning up all machines...") - - # TODO: - # Proper cleanup on remote machines. Restore - # - added - # - changed - # - deleted - # files to previous state, e.g. when calling create_file or enable_valgrind. - if ctrl is not None: - ctrl.cleanup() + self._cleanup(ctrl) for _, node in nodes.items(): - node.cleanup() + bluechictl_started_services: List[str] = [] + if node.config.node_name in ctrl.bluechictl.tracked_services: + bluechictl_started_services = ctrl.bluechictl.tracked_services[node.config.node_name] + self._cleanup(node, bluechictl_started_services) + + def _cleanup(self, machine: BluechiMachine, other_tracked_service: List[str] = []): + + LOGGER.info("Stopping and resetting tracked units...") + tracked_services = machine.systemctl.tracked_services + other_tracked_service + for service in tracked_services: + machine.systemctl.stop_unit(service, check_result=False) + machine.systemctl.reset_failed_for_unit(service, check_result=False) + + LOGGER.info("Removing created files...") + cmd_rm_created_files = "rm " + " ".join(machine.created_files) + machine.client.exec_run(cmd_rm_created_files) + + LOGGER.info("Restoring changed files...") + for changed_file in machine.changed_files: + backup_file = changed_file + BluechiMachine.backup_file_suffix + machine.client.exec_run(f"mv {backup_file} {changed_file}") + + # ensure changed systemd services are reloaded + machine.systemctl.daemon_reload() + + # ensure that both, agent and controller can be started again + # if they failed to quickly previously + machine.systemctl.reset_failed_for_unit("bluechi-agent", check_result=False) + machine.systemctl.reset_failed_for_unit("bluechi-controller", check_result=False) + + machine.cleanup_valgrind_logs() + machine.cleanup_journal_logs() + machine.cleanup_coverage() From 7075439222cfd8df1932498765ed5a357cfd0c65 Mon Sep 17 00:00:00 2001 From: Michael Engel Date: Fri, 1 Mar 2024 14:06:26 +0100 Subject: [PATCH 3/3] fixes in integration test logic Signed-off-by: Michael Engel --- .../test_bluechi_agent_cmd_line_c_invalid_config.py | 6 +++--- ...st_bluechi_agent_connect_via_controller_address.py | 6 +++--- .../test_agent_started_and_connected.py | 4 +--- .../test_bluechi_change_port_with_c_cmd_option.py | 11 +++++------ .../test_bluechi_controller_set_loglevel_invalid.py | 5 +++-- .../test_version_option_provided.py | 2 +- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/tests/tests/tier0/bluechi-agent-cmd-line-c-invalid-config/test_bluechi_agent_cmd_line_c_invalid_config.py b/tests/tests/tier0/bluechi-agent-cmd-line-c-invalid-config/test_bluechi_agent_cmd_line_c_invalid_config.py index b3a2ffd4c4..24330643b6 100644 --- a/tests/tests/tier0/bluechi-agent-cmd-line-c-invalid-config/test_bluechi_agent_cmd_line_c_invalid_config.py +++ b/tests/tests/tier0/bluechi-agent-cmd-line-c-invalid-config/test_bluechi_agent_cmd_line_c_invalid_config.py @@ -18,12 +18,12 @@ def exec(ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]): node_foo = nodes[NODE_FOO] - config_file_location = "/var/tmp" + config_file_location = "/tmp" bluechi_agent_str = "bluechi-agent" - invalid_conf_str = "config-files/invalid.conf" + invalid_conf_str = "invalid.conf" # Copying relevant config files into node container - content = read_file(invalid_conf_str) + content = read_file(os.path.join("config-files", invalid_conf_str)) node_foo.create_file(config_file_location, invalid_conf_str, content) LOGGER.debug("Setting invalid.conf as conf file for foo and checking if failed") diff --git a/tests/tests/tier0/bluechi-agent-connect-via-controller-address/test_bluechi_agent_connect_via_controller_address.py b/tests/tests/tier0/bluechi-agent-connect-via-controller-address/test_bluechi_agent_connect_via_controller_address.py index 2d64f2ba37..7eea943be5 100644 --- a/tests/tests/tier0/bluechi-agent-connect-via-controller-address/test_bluechi_agent_connect_via_controller_address.py +++ b/tests/tests/tier0/bluechi-agent-connect-via-controller-address/test_bluechi_agent_connect_via_controller_address.py @@ -13,9 +13,9 @@ def exec(ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]): node_foo = nodes[NODE_FOO] - config_file_location = "/var/tmp" + config_file_location = "/tmp" bluechi_agent_str = "bluechi-agent" - invalid_conf_str = "config/Invalid-ControllerAddress.conf" + invalid_conf_str = "Invalid-ControllerAddress.conf" invalid_controller_address = "Invalid-ControllerAddress.conf" service = "org.eclipse.bluechi" object = "/org/eclipse/bluechi" @@ -25,7 +25,7 @@ def exec(ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]): assert node_foo.wait_for_unit_state_to_be(bluechi_agent_str, "active", delay=2) assert 's "up"' == ctrl.exec_run(f"busctl get-property {service} {object} {interface} {property}")[1] - content = read_file(invalid_conf_str) + content = read_file(os.path.join("config", invalid_conf_str)) node_foo.create_file(config_file_location, invalid_conf_str, content) node_foo.restart_with_config_file(os.path.join(config_file_location, invalid_controller_address), bluechi_agent_str) diff --git a/tests/tests/tier0/bluechi-agent-started-and-connected/test_agent_started_and_connected.py b/tests/tests/tier0/bluechi-agent-started-and-connected/test_agent_started_and_connected.py index beeb1b0d2a..1ba2b7e96a 100644 --- a/tests/tests/tier0/bluechi-agent-started-and-connected/test_agent_started_and_connected.py +++ b/tests/tests/tier0/bluechi-agent-started-and-connected/test_agent_started_and_connected.py @@ -9,9 +9,7 @@ def foo_startup_verify(ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]): - result, output = ctrl.bluechictl.get_unit_status("node-foo", "bluechi-agent.service") - assert result == 0 - assert str(output).split('\n')[2].split('|')[2].strip() == 'active' + nodes["node-foo"].wait_for_bluechi_agent() # TODO: Add code to test that agent on node foo is successfully connected to bluechi controller diff --git a/tests/tests/tier0/bluechi-change-port-with-c-cmd-option/test_bluechi_change_port_with_c_cmd_option.py b/tests/tests/tier0/bluechi-change-port-with-c-cmd-option/test_bluechi_change_port_with_c_cmd_option.py index 5237daf104..c2ec5cf5e5 100644 --- a/tests/tests/tier0/bluechi-change-port-with-c-cmd-option/test_bluechi_change_port_with_c_cmd_option.py +++ b/tests/tests/tier0/bluechi-change-port-with-c-cmd-option/test_bluechi_change_port_with_c_cmd_option.py @@ -16,14 +16,13 @@ def exec(ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]): config_file_location = "/var/tmp" bluechi_agent_str = "bluechi-agent" bluechi_controller_str = "bluechi-controller" - file_location_ctrl = os.path.join("config-files", "ctrl_port_8421.conf") - file_location_agent = os.path.join("config-files", "agent_port_8421.conf") # Copying relevant config files into the nodes container - content = read_file(file_location_agent) - node_foo.create_file(config_file_location, file_location_agent, content) - content = read_file(file_location_ctrl) - ctrl.create_file(config_file_location, file_location_ctrl, content) + content = read_file(os.path.join("config-files", "agent_port_8421.conf")) + node_foo.create_file(config_file_location, "agent_port_8421.conf", content) + + content = read_file(os.path.join("config-files", "ctrl_port_8421.conf")) + ctrl.create_file(config_file_location, "ctrl_port_8421.conf", content) ctrl.restart_with_config_file( os.path.join(config_file_location, "ctrl_port_8421.conf"), bluechi_controller_str) diff --git a/tests/tests/tier0/bluechi-controller-set-loglevel-invalid/test_bluechi_controller_set_loglevel_invalid.py b/tests/tests/tier0/bluechi-controller-set-loglevel-invalid/test_bluechi_controller_set_loglevel_invalid.py index 965bd57a82..69f8644e13 100644 --- a/tests/tests/tier0/bluechi-controller-set-loglevel-invalid/test_bluechi_controller_set_loglevel_invalid.py +++ b/tests/tests/tier0/bluechi-controller-set-loglevel-invalid/test_bluechi_controller_set_loglevel_invalid.py @@ -7,8 +7,9 @@ def exec(ctrl: BluechiControllerMachine, _: Dict[str, BluechiAgentMachine]): - _, output = ctrl.bluechictl.set_log_level_on_controller("INF", check_result=False) - assert "Disconnect" not in output + result, _ = ctrl.bluechictl.set_log_level_on_controller("INF", check_result=False) + # call to set invalid loglevel should fail + assert result != 0 def test_bluechi_controller_set_loglevel_invalid( diff --git a/tests/tests/tier0/bluechi-version-option-provided/test_version_option_provided.py b/tests/tests/tier0/bluechi-version-option-provided/test_version_option_provided.py index ee765c93cb..0eb1af48ec 100644 --- a/tests/tests/tier0/bluechi-version-option-provided/test_version_option_provided.py +++ b/tests/tests/tier0/bluechi-version-option-provided/test_version_option_provided.py @@ -12,7 +12,7 @@ def check_help_option(ctrl: BluechiControllerMachine, _: Dict[str, BluechiAgentM assert rpm_re == 0 # There is no way to strip dist part from the release using rpm query flags, so we need to replace it manually - bc_ver_rel = rpm_out.replace(".el9", "") + bc_ver_rel = ".".join(rpm_out.split(".")[:-1]) executables = [ '/usr/libexec/bluechi-controller', '/usr/libexec/bluechi-agent'