From cc6e1bb71fa6db705e421dd7f9c1ed4f421148d0 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Wed, 31 Jan 2024 11:36:33 -0700 Subject: [PATCH] Use pytest fixtures to run tests on local env and agent env --- python/tests/conftest.py | 66 ++++++++++++++++--- python/tests/test_busybox.py | 23 ++++--- ...ocalenvironment.py => test_environment.py} | 27 ++++---- python/tests/test_executor.py | 15 +++-- python/tests/test_inject.py | 17 +++-- python/tests/test_qemu_executor.py | 8 +-- python/tests/test_target_local.py | 11 +--- python/tests/test_target_serialization.py | 24 +++---- 8 files changed, 119 insertions(+), 72 deletions(-) rename python/tests/{test_localenvironment.py => test_environment.py} (71%) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 6c04d3e..45fef33 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -3,9 +3,18 @@ import platform import sys from pathlib import Path +from typing import TYPE_CHECKING, Generator import pytest +from binharness.bootstrap.docker import bootstrap_env_from_image +from binharness.bootstrap.subprocess import SubprocessAgent +from binharness.localenvironment import LocalEnvironment + +if TYPE_CHECKING: + from binharness.agentenvironment import AgentEnvironment + from binharness.types.environment import Environment + try: import docker except ImportError: @@ -24,9 +33,7 @@ def pytest_runtest_setup(item: pytest.Item) -> None: # Skip docker tests if docker is not installed if "docker" in list(item.iter_markers()): - try: - import docker - except ImportError: + if docker is None: pytest.skip("docker is not installed") try: docker.from_env() @@ -34,7 +41,7 @@ def pytest_runtest_setup(item: pytest.Item) -> None: pytest.skip("docker is not running") -@pytest.fixture() +@pytest.fixture(scope="session") def agent_binary_host() -> str: expected_path = ( Path(__file__).parent.parent.parent / "target" / "debug" / "bh_agent_server" @@ -44,7 +51,7 @@ def agent_binary_host() -> str: return str(expected_path) -@pytest.fixture() +@pytest.fixture(scope="session") def agent_binary_linux_host_arch() -> str: arch = platform.machine() # Fixup arch for arm64 @@ -63,11 +70,52 @@ def agent_binary_linux_host_arch() -> str: return str(expected_path) -@pytest.fixture() +@pytest.fixture(scope="session") +def local_env() -> LocalEnvironment: + return LocalEnvironment() + + +@pytest.fixture(scope="session") +@pytest.mark.linux() +def linux_local_env() -> LocalEnvironment: + return LocalEnvironment() + + +@pytest.fixture(scope="session") +def agent_env(agent_binary_host: str) -> Generator[AgentEnvironment, None, None]: + agent = SubprocessAgent(Path(agent_binary_host)) + yield agent.get_environment(0) + agent.stop() + + +@pytest.fixture(scope="session") +@pytest.mark.docker() def docker_client() -> docker.DockerClient: - if docker is None: - pytest.skip("docker is not installed") try: - return docker.from_env() + client = docker.from_env() except docker.errors.DockerException: pytest.skip("docker is not running") + yield client + client.close() + + +@pytest.fixture(scope="session") +def docker_env( + docker_client: docker.APIClient, agent_binary_linux_host_arch: str +) -> Generator[AgentEnvironment, None, None]: + agent = bootstrap_env_from_image( + agent_binary_linux_host_arch, "ubuntu:22.04", docker_client=docker_client + ) + yield agent.get_environment(0) + agent.container.stop() + agent.container.remove() + + +@pytest.fixture(params=["local_env", "agent_env"], scope="session") +def env(request) -> Environment: # noqa: ANN001 + return request.getfixturevalue(request.param) # type: ignore[no-any-return] + + +@pytest.fixture(params=["linux_local_env", "docker_env"], scope="session") +def linux_env(request) -> Environment: # noqa: ANN001 + return request.getfixturevalue(request.param) # type: ignore[no-any-return] diff --git a/python/tests/test_busybox.py b/python/tests/test_busybox.py index 58de298..0cc38e3 100644 --- a/python/tests/test_busybox.py +++ b/python/tests/test_busybox.py @@ -1,14 +1,17 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest from binharness.common.busybox import BusyboxInjection -from binharness.localenvironment import LocalEnvironment + +if TYPE_CHECKING: + from binharness import Environment @pytest.mark.linux() -def test_busybox_injection() -> None: - env = LocalEnvironment() +def test_busybox_injection(env: Environment) -> None: busybox_injection = BusyboxInjection() busybox_injection.install(env) assert busybox_injection.run("true").wait() == 0 @@ -26,19 +29,18 @@ def test_busybox_injection() -> None: @pytest.mark.linux() -def test_busbox_injection_mktemp() -> None: - env = LocalEnvironment() +def test_busbox_injection_mktemp(env: Environment) -> None: busybox_injection = BusyboxInjection() busybox_injection.install(env) assert busybox_injection.mktemp().is_file() assert busybox_injection.mktemp(directory=True).is_dir() +# TODO: Make this work on agents, probably an issue in wait() @pytest.mark.linux() -def test_nc_interaction() -> None: - env = LocalEnvironment() +def test_nc_interaction(local_env: Environment) -> None: busybox = BusyboxInjection() - busybox.install(env) + busybox.install(local_env) server_proc = busybox.nc("localhost", 10001, listen=True) client_proc = busybox.nc("localhost", 10001) @@ -54,3 +56,8 @@ def test_nc_interaction() -> None: assert server_proc.stdout.readline() == b"hello\n" assert client_proc.stdout is not None assert client_proc.stdout.readline() == b"hello back\n" + + client_proc.stdin.close() + client_proc.wait() + server_proc.stdin.close() + server_proc.wait() diff --git a/python/tests/test_localenvironment.py b/python/tests/test_environment.py similarity index 71% rename from python/tests/test_localenvironment.py rename to python/tests/test_environment.py index fc968a0..052554e 100644 --- a/python/tests/test_localenvironment.py +++ b/python/tests/test_environment.py @@ -2,23 +2,24 @@ import pathlib import tempfile +from typing import TYPE_CHECKING import pytest from binharness.common.busybox import BusyboxInjection -from binharness.localenvironment import LocalEnvironment +if TYPE_CHECKING: + from binharness import Environment -def test_run_command() -> None: - env = LocalEnvironment() + +def test_run_command(env: Environment) -> None: proc = env.run_command(["echo", "hello"]) stdout, _ = proc.communicate() assert proc.returncode == 0 assert stdout == b"hello\n" -def test_inject_files() -> None: - env = LocalEnvironment() +def test_inject_files(env: Environment) -> None: env_temp = env.get_tempdir() with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = pathlib.Path(tmp_dir) @@ -32,14 +33,14 @@ def test_inject_files() -> None: assert local_file.read_text() == "hello" -def test_get_tempdir() -> None: - env = LocalEnvironment() - assert env.get_tempdir() == pathlib.Path(tempfile.gettempdir()) +# TODO: Need to think about how to handle this test with remote environments +def test_get_tempdir(local_env: Environment) -> None: + assert local_env.get_tempdir() == pathlib.Path(tempfile.gettempdir()) +# TODO: Need to think about how to test these with non-linux environments @pytest.mark.linux() -def test_stdout() -> None: - env = LocalEnvironment() +def test_stdout(env: Environment) -> None: busybox = BusyboxInjection() busybox.install(env) proc = busybox.shell("echo hello") @@ -48,8 +49,7 @@ def test_stdout() -> None: @pytest.mark.linux() -def test_stderr() -> None: - env = LocalEnvironment() +def test_stderr(env: Environment) -> None: busybox = BusyboxInjection() busybox.install(env) proc = busybox.shell("echo hello 1>&2") @@ -58,8 +58,7 @@ def test_stderr() -> None: @pytest.mark.linux() -def test_process_poll() -> None: - env = LocalEnvironment() +def test_process_poll(env: Environment) -> None: busybox = BusyboxInjection() busybox.install(env) proc = busybox.run("head") diff --git a/python/tests/test_executor.py b/python/tests/test_executor.py index 96c97e7..476c01d 100644 --- a/python/tests/test_executor.py +++ b/python/tests/test_executor.py @@ -4,24 +4,25 @@ import pytest -from binharness.common.busybox import BusyboxShellExecutor -from binharness.localenvironment import LocalEnvironment -from binharness.types.executor import ( +from binharness import ( + Environment, ExecutorEnvironmentMismatchError, + InjectionNotInstalledError, + LocalEnvironment, + Target, ) -from binharness.types.injection import InjectionNotInstalledError -from binharness.types.target import Target +from binharness.common.busybox import BusyboxShellExecutor @pytest.mark.linux() -def test_busybox_injection_without_install() -> None: - env = LocalEnvironment() +def test_busybox_injection_without_install(env: Environment) -> None: target = Target(env, Path("/usr/bin/true")) busybox_shell = BusyboxShellExecutor() with pytest.raises(InjectionNotInstalledError): assert busybox_shell.run_target(target) +# TODO: Maybe LocalEnvironment objects should be interchangable? @pytest.mark.linux() def test_busybox_injection_different_environment() -> None: env1 = LocalEnvironment() diff --git a/python/tests/test_inject.py b/python/tests/test_inject.py index e2f21d1..1039ee0 100644 --- a/python/tests/test_inject.py +++ b/python/tests/test_inject.py @@ -2,10 +2,10 @@ import stat from pathlib import Path +from typing import TYPE_CHECKING import pytest -from binharness.localenvironment import LocalEnvironment from binharness.types.injection import ( ExecutableInjection, Injection, @@ -13,10 +13,12 @@ InjectionNotInstalledError, ) +if TYPE_CHECKING: + from binharness.types.environment import Environment + @pytest.mark.linux() -def test_inject_true() -> None: - env = LocalEnvironment() +def test_inject_true(env: Environment) -> None: true_injection = Injection(Path("/usr/bin/true")) true_injection.install(env) assert true_injection.env_path is not None @@ -24,8 +26,7 @@ def test_inject_true() -> None: @pytest.mark.linux() -def test_inject_true_executable() -> None: - env = LocalEnvironment() +def test_inject_true_executable(env: Environment) -> None: true_injection = ExecutableInjection(Path("/usr/bin/true")) true_injection.install(env) assert true_injection.env_path is not None @@ -36,8 +37,7 @@ def test_inject_true_executable() -> None: @pytest.mark.linux() -def test_inject_true_executable_twice() -> None: - env = LocalEnvironment() +def test_inject_true_executable_twice(env: Environment) -> None: true_injection = ExecutableInjection(Path("/usr/bin/true")) true_injection.install(env) with pytest.raises(InjectionAlreadyInstalledError): @@ -45,8 +45,7 @@ def test_inject_true_executable_twice() -> None: @pytest.mark.linux() -def test_inject_two_true_executables() -> None: - env = LocalEnvironment() +def test_inject_two_true_executables(env: Environment) -> None: true_injection_1 = ExecutableInjection(Path("/usr/bin/true")) true_injection_1.install(env) true_injection_2 = ExecutableInjection(Path("/usr/bin/true")) diff --git a/python/tests/test_qemu_executor.py b/python/tests/test_qemu_executor.py index 323883a..6f45f3b 100644 --- a/python/tests/test_qemu_executor.py +++ b/python/tests/test_qemu_executor.py @@ -4,13 +4,12 @@ import pytest -from binharness import LocalEnvironment, Target +from binharness import Environment, Target from binharness.common.qemu import QemuExecutor @pytest.mark.linux() -def test_run_true() -> None: - env = LocalEnvironment() +def test_run_true(env: Environment) -> None: target = Target(env, Path("/bin/true")) qemu = QemuExecutor() qemu.install(env) @@ -18,8 +17,7 @@ def test_run_true() -> None: @pytest.mark.linux() -def test_run_strace() -> None: - env = LocalEnvironment() +def test_run_strace(env: Environment) -> None: target = Target(env, Path("/bin/true")) qemu = QemuExecutor() qemu.install(env) diff --git a/python/tests/test_target_local.py b/python/tests/test_target_local.py index e550ade..5ae2979 100644 --- a/python/tests/test_target_local.py +++ b/python/tests/test_target_local.py @@ -4,15 +4,11 @@ import pytest -from binharness import LocalEnvironment +from binharness import Environment, NullExecutor, Target from binharness.common.busybox import BusyboxShellExecutor -from binharness.types.executor import NullExecutor -from binharness.types.target import Target -@pytest.mark.linux() -def test_run_target() -> None: - env = LocalEnvironment() +def test_run_target(env: Environment) -> None: target = Target(env, Path("true")) executor = NullExecutor() proc = executor.run_target(target) @@ -20,8 +16,7 @@ def test_run_target() -> None: @pytest.mark.linux() -def test_run_target_busybox() -> None: - env = LocalEnvironment() +def test_run_target_busybox(env: Environment) -> None: target = Target(env, Path("true")) executor = BusyboxShellExecutor() executor.install(env) diff --git a/python/tests/test_target_serialization.py b/python/tests/test_target_serialization.py index adc57bf..e4deb3a 100644 --- a/python/tests/test_target_serialization.py +++ b/python/tests/test_target_serialization.py @@ -6,22 +6,24 @@ import pytest -from binharness.localenvironment import LocalEnvironment -from binharness.serialize import TargetImportError, export_target, import_target -from binharness.types.executor import NullExecutor -from binharness.types.target import Target +from binharness import ( + Environment, + NullExecutor, + Target, + TargetImportError, + export_target, + import_target, +) -def test_local_target_export() -> None: - env = LocalEnvironment() +def test_local_target_export(env: Environment) -> None: target = Target(env, Path("/usr/bin/true")) with tempfile.TemporaryDirectory() as raw_tmpdir: tmpdir = Path(raw_tmpdir) export_target(target, tmpdir / "test_export.zip") -def test_local_target_import() -> None: - env = LocalEnvironment() +def test_local_target_import(env: Environment) -> None: target = Target(env, Path("/usr/bin/true")) with tempfile.TemporaryDirectory() as raw_tmpdir: tmpdir = Path(raw_tmpdir) @@ -35,8 +37,7 @@ def test_local_target_import() -> None: assert executor.run_target(new_target).wait() == 0 -def test_local_target_import_without_metadata() -> None: - env = LocalEnvironment() +def test_local_target_import_without_metadata(env: Environment) -> None: target = Target(env, Path("/usr/bin/true")) with tempfile.TemporaryDirectory() as raw_tmpdir: tmpdir = Path(raw_tmpdir) @@ -54,8 +55,7 @@ def test_local_target_import_without_metadata() -> None: import_target(env, tmpdir / "test_export_modified.zip") -def test_local_target_import_invalid_metadata_archive() -> None: - env = LocalEnvironment() +def test_local_target_import_invalid_metadata_archive(env: Environment) -> None: target = Target(env, Path("/usr/bin/true")) with tempfile.TemporaryDirectory() as raw_tmpdir: tmpdir = Path(raw_tmpdir)