From ca8883993644a1dbde98263e03a968aebee8a250 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 22 Feb 2024 12:44:04 -0500 Subject: [PATCH 1/2] Podman: make current utility class explicit async There's no clear indication that the current Podman utility class is an async implementation. With some use cases that can benefit from a non-async behavior, let's make this current implementation explicitly an async one (with the renaming), so that a sync implementation can have the standard name. Signed-off-by: Cleber Rosa --- avocado/plugins/runners/podman_image.py | 4 ++-- avocado/plugins/spawners/podman.py | 4 ++-- avocado/utils/podman.py | 2 +- selftests/functional/plugin/podman_image.py | 4 ++-- selftests/functional/utils/podman.py | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/avocado/plugins/runners/podman_image.py b/avocado/plugins/runners/podman_image.py index 0b9a5dd5c7..06299d6b21 100644 --- a/avocado/plugins/runners/podman_image.py +++ b/avocado/plugins/runners/podman_image.py @@ -7,7 +7,7 @@ from avocado.core.nrunner.app import BaseRunnerApp from avocado.core.nrunner.runner import RUNNER_RUN_STATUS_INTERVAL, BaseRunner from avocado.core.utils import messages -from avocado.utils.podman import Podman, PodmanException +from avocado.utils.podman import AsyncPodman, PodmanException class PodmanImageRunner(BaseRunner): @@ -34,7 +34,7 @@ def _run_podman_pull(self, uri, queue): # information for debugging in case of errors. logging.getLogger("avocado.utils.podman").addHandler(logging.NullHandler()) try: - podman = Podman() + podman = AsyncPodman() loop = asyncio.get_event_loop() loop.run_until_complete(podman.execute("pull", uri)) queue.put({"result": "pass"}) diff --git a/avocado/plugins/spawners/podman.py b/avocado/plugins/spawners/podman.py index 386860389b..00a33d0288 100644 --- a/avocado/plugins/spawners/podman.py +++ b/avocado/plugins/spawners/podman.py @@ -16,7 +16,7 @@ from avocado.core.version import VERSION from avocado.utils import distro from avocado.utils.asset import Asset -from avocado.utils.podman import Podman, PodmanException +from avocado.utils.podman import AsyncPodman, PodmanException LOG = logging.getLogger(__name__) @@ -163,7 +163,7 @@ def podman(self): if self._podman is None: podman_bin = self.config.get("spawner.podman.bin") try: - self._podman = Podman(podman_bin) + self._podman = AsyncPodman(podman_bin) except PodmanException as ex: LOG.error(ex) return self._podman diff --git a/avocado/utils/podman.py b/avocado/utils/podman.py index a557559701..8595dcfd1c 100644 --- a/avocado/utils/podman.py +++ b/avocado/utils/podman.py @@ -32,7 +32,7 @@ class PodmanException(Exception): pass -class Podman: +class AsyncPodman: def __init__(self, podman_bin=None): path = which(podman_bin or "podman") if not path: diff --git a/selftests/functional/plugin/podman_image.py b/selftests/functional/plugin/podman_image.py index d358f254bf..2fbe95592a 100644 --- a/selftests/functional/plugin/podman_image.py +++ b/selftests/functional/plugin/podman_image.py @@ -1,5 +1,5 @@ from avocado import Test -from avocado.utils.podman import Podman +from avocado.utils.podman import AsyncPodman class PodmanImageTest(Test): @@ -8,7 +8,7 @@ async def test(self): :avocado: dependency={"type": "package", "name": "podman", "action": "check"} :avocado: dependency={"type": "podman-image", "uri": "registry.fedoraproject.org/fedora:38"} """ - podman = Podman() + podman = AsyncPodman() _, stdout, _ = await podman.execute( "images", "--filter", diff --git a/selftests/functional/utils/podman.py b/selftests/functional/utils/podman.py index 8d7586f050..23f775a618 100644 --- a/selftests/functional/utils/podman.py +++ b/selftests/functional/utils/podman.py @@ -1,15 +1,15 @@ from avocado import Test -from avocado.utils.podman import Podman +from avocado.utils.podman import AsyncPodman -class PodmanTest(Test): +class AsyncPodmanTest(Test): async def test_python_version(self): """ :avocado: dependency={"type": "package", "name": "podman", "action": "check"} :avocado: dependency={"type": "podman-image", "uri": "fedora:38"} :avocado: tags=slow """ - podman = Podman() + podman = AsyncPodman() result = await podman.get_python_version("fedora:38") self.assertEqual(result, (3, 11, "/usr/bin/python3")) @@ -19,7 +19,7 @@ async def test_container_info(self): :avocado: dependency={"type": "podman-image", "uri": "fedora:38"} :avocado: tags=slow """ - podman = Podman() + podman = AsyncPodman() _, stdout, _ = await podman.execute("create", "fedora:38", "/bin/bash") container_id = stdout.decode().strip() result = await podman.get_container_info(container_id) From c201be93b16d13de4dda6e85171069f450d9b88a Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 22 Feb 2024 12:44:04 -0500 Subject: [PATCH 2/2] Podman: add synchronous version of utilities While the Podman spawner benefits from the async APIs because of its requirement to also make async APIs available, most common uses of the utilities APIs are not in an async context. This introduces a Podman class that is not async. I've researched some prior art to avoid duplicating code, but it seems there's no magic and bulletproof way. Reference: https://github.com/elastic/elastic-transport-python/blob/main/elastic_transport/_transport.py Reference: https://github.com/elastic/elastic-transport-python/blob/main/elastic_transport/_async_transport.py Signed-off-by: Cleber Rosa --- avocado/utils/podman.py | 158 ++++++++++++++++++++++++--- selftests/check.py | 2 +- selftests/functional/utils/podman.py | 31 +++++- 3 files changed, 172 insertions(+), 19 deletions(-) diff --git a/avocado/utils/podman.py b/avocado/utils/podman.py index 8595dcfd1c..4407b55f6d 100644 --- a/avocado/utils/podman.py +++ b/avocado/utils/podman.py @@ -22,7 +22,9 @@ import json import logging -from asyncio import create_subprocess_exec, subprocess +import subprocess +from asyncio import create_subprocess_exec +from asyncio import subprocess as asyncio_subprocess from shutil import which LOG = logging.getLogger(__name__) @@ -32,7 +34,20 @@ class PodmanException(Exception): pass -class AsyncPodman: +class _Podman: + + PYTHON_VERSION_COMMAND = json.dumps( + [ + "/usr/bin/env", + "python3", + "-c", + ( + "import sys; print(sys.version_info.major, " + "sys.version_info.minor, sys.executable)" + ), + ] + ) + def __init__(self, podman_bin=None): path = which(podman_bin or "podman") if not path: @@ -41,6 +56,125 @@ def __init__(self, podman_bin=None): self.podman_bin = path + +class Podman(_Podman): + def execute(self, *args): + """Execute a command and return the returncode, stdout and stderr. + + :param args: Variable length argument list to be used as argument + during execution. + :rtype: tuple with returncode, stdout and stderr. + """ + try: + LOG.debug("Executing %s", args) + + cmd = [self.podman_bin, *args] + proc = subprocess.Popen( + cmd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = proc.communicate() + LOG.debug("Return code: %s", proc.returncode) + LOG.debug("Stdout: %s", stdout.decode("utf-8", "replace")) + LOG.debug("Stderr: %s", stderr.decode("utf-8", "replace")) + except (FileNotFoundError, PermissionError) as ex: + # Since this method is also used by other methods, let's + # log here as well. + msg = "Could not execute the command." + LOG.error("%s: %s", msg, str(ex)) + raise PodmanException(msg) from ex + + if proc.returncode != 0: + command_args = " ".join(args) + msg = f'Failure from command "{self.podman_bin} {command_args}": returned code "{proc.returncode}" stderr: "{stderr}"' + LOG.error(msg) + raise PodmanException(msg) + + return proc.returncode, stdout, stderr + + def copy_to_container(self, container_id, src, dst): + """Copy artifacts from src to container:dst. + + This method allows copying the contents of src to the dst. Files will + be copied from the local machine to the container. The "src" argument + can be a file or a directory. + + :param str container_id: string with the container identification. + :param str src: what file or directory you are trying to copy. + :param str dst: the destination inside the container. + :rtype: tuple with returncode, stdout and stderr. + """ + try: + return self.execute("cp", src, f"{container_id}:{dst}") + except PodmanException as ex: + error = f"Failed copying data to container {container_id}" + LOG.error(error) + raise PodmanException(error) from ex + + def get_python_version(self, image): + """Return the current Python version installed in an image. + + :param str image: Image name. i.e: 'fedora:33'. + :rtype: tuple with both: major, minor numbers and executable path. + """ + try: + _, stdout, _ = self.execute( + "run", "--rm", f"--entrypoint={self.PYTHON_VERSION_COMMAND}", image + ) + except PodmanException as ex: + raise PodmanException("Failed getting Python version.") from ex + + if stdout: + output = stdout.decode().strip().split() + return int(output[0]), int(output[1]), output[2] + + def get_container_info(self, container_id): + """Return all information about specific container. + + :param container_id: identifier of container + :type container_id: str + :rtype: dict + """ + try: + _, stdout, _ = self.execute( + "ps", "--all", "--format=json", "--filter", f"id={container_id}" + ) + except PodmanException as ex: + raise PodmanException( + f"Failed getting information about container:" f" {container_id}." + ) from ex + containers = json.loads(stdout.decode()) + for container in containers: + if container["Id"] == container_id: + return container + return {} + + def start(self, container_id): + """Starts a container and return the returncode, stdout and stderr. + + :param str container_id: Container identification string to start. + :rtype: tuple with returncode, stdout and stderr. + """ + try: + return self.execute("start", container_id) + except PodmanException as ex: + raise PodmanException("Failed to start the container.") from ex + + def stop(self, container_id): + """Stops a container and return the returncode, stdout and stderr. + + :param str container_id: Container identification string to stop. + :rtype: tuple with returncode, stdout and stderr. + """ + try: + return self.execute("stop", "-t=0", container_id) + except PodmanException as ex: + raise PodmanException("Failed to stop the container.") from ex + + +class AsyncPodman(_Podman): async def execute(self, *args): """Execute a command and return the returncode, stdout and stderr. @@ -52,7 +186,10 @@ async def execute(self, *args): LOG.debug("Executing %s", args) proc = await create_subprocess_exec( - self.podman_bin, *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE + self.podman_bin, + *args, + stdout=asyncio_subprocess.PIPE, + stderr=asyncio_subprocess.PIPE, ) stdout, stderr = await proc.communicate() LOG.debug("Return code: %s", proc.returncode) @@ -98,22 +235,9 @@ async def get_python_version(self, image): :param str image: Image name. i.e: 'fedora:33'. :rtype: tuple with both: major, minor numbers and executable path. """ - - entrypoint = json.dumps( - [ - "/usr/bin/env", - "python3", - "-c", - ( - "import sys; print(sys.version_info.major, " - "sys.version_info.minor, sys.executable)" - ), - ] - ) - try: _, stdout, _ = await self.execute( - "run", "--rm", f"--entrypoint={entrypoint}", image + "run", "--rm", f"--entrypoint={self.PYTHON_VERSION_COMMAND}", image ) except PodmanException as ex: raise PodmanException("Failed getting Python version.") from ex diff --git a/selftests/check.py b/selftests/check.py index a51025844d..d6d6a35eeb 100755 --- a/selftests/check.py +++ b/selftests/check.py @@ -29,7 +29,7 @@ "nrunner-requirement": 16, "unit": 667, "jobs": 11, - "functional-parallel": 300, + "functional-parallel": 302, "functional-serial": 4, "optional-plugins": 0, "optional-plugins-golang": 2, diff --git a/selftests/functional/utils/podman.py b/selftests/functional/utils/podman.py index 23f775a618..a90ed0d39d 100644 --- a/selftests/functional/utils/podman.py +++ b/selftests/functional/utils/podman.py @@ -1,5 +1,34 @@ from avocado import Test -from avocado.utils.podman import AsyncPodman +from avocado.utils.podman import AsyncPodman, Podman + + +class PodmanTest(Test): + def test_python_version(self): + """ + :avocado: dependency={"type": "package", "name": "podman", "action": "check"} + :avocado: dependency={"type": "podman-image", "uri": "fedora:38"} + :avocado: tags=slow + """ + podman = Podman() + result = podman.get_python_version("fedora:38") + self.assertEqual(result, (3, 11, "/usr/bin/python3")) + + def test_container_info(self): + """ + :avocado: dependency={"type": "package", "name": "podman", "action": "check"} + :avocado: dependency={"type": "podman-image", "uri": "fedora:38"} + :avocado: tags=slow + """ + podman = Podman() + _, stdout, _ = podman.execute("create", "fedora:38", "/bin/bash") + container_id = stdout.decode().strip() + result = podman.get_container_info(container_id) + self.assertEqual(result["Id"], container_id) + + podman.execute("rm", container_id) + + result = podman.get_container_info(container_id) + self.assertEqual(result, {}) class AsyncPodmanTest(Test):