diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 4890e635..28c43209 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -44,6 +44,12 @@ To run the end-to-end tests, follow these steps: ## Additional Notes - The WireMock tool is used to mock Git server endpoints for testing. - Podman is used for container and pod management and to build the container image for the mock API server. +- If the images are not already built, the `make test-e2e` command will build them automatically and remove them at the end of the test. If not, you can build them manually with the following command from the root of the project directory: + + ```bash + podman build -t localhost/mock-server:latest -f tests/e2e/Dockerfile tests/e2e + podman build -t localhost/trestlebot:latest -f Dockerfile . + ``` ## Future Improvements - Provide an option to use pre-built trestle-bot container images from a registry instead of building them locally. diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 373c0c04..3113a6ac 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -24,7 +24,10 @@ import pytest from git.repo import Repo +from trestle.common.model_utils import ModelUtils from trestle.core.commands.init import InitCmd +from trestle.core.models.file_content_type import FileContentType +from trestle.oscal.component import ComponentDefinition from trestle.oscal.profile import Profile from tests.conftest import YieldFixture @@ -34,7 +37,7 @@ setup_for_profile, setup_rules_view, ) -from trestlebot.const import RULES_VIEW_DIR +from trestlebot.const import ERROR_EXIT_CODE, RULES_VIEW_DIR, SUCCESS_EXIT_CODE logger = logging.getLogger(__name__) @@ -48,6 +51,16 @@ test_prof = "simplified_nist_profile" test_filter_prof = "simplified_filter_profile" +test_comp_name = "test-comp" + + +def image_exists(image_name: str) -> bool: + """Check if the image already exists.""" + try: + subprocess.check_output(["podman", "image", "inspect", image_name]) + return True + except subprocess.CalledProcessError: + return False def build_transform_command(data_path: str, command_args: Dict[str, str]) -> List[str]: @@ -91,32 +104,37 @@ def build_create_cd_command(data_path: str, command_args: Dict[str, str]) -> Lis @pytest.fixture(scope="module") def podman_setup() -> YieldFixture[int]: """Build the trestlebot container image and run the mock server in a pod.""" - # Build the container image - subprocess.run( - [ - "podman", - "build", - "-f", - container_file, - "-t", - image_name, - ], - check=True, - ) - # Build mock server container image - subprocess.run( - [ - "podman", - "build", - "-f", - f"{e2e_context}/{container_file}", - "-t", - mock_server_image_name, - e2e_context, - ], - check=True, - ) + cleanup_trestlebot_image = False + cleanup_mock_server_image = False + + if not image_exists(image_name): + subprocess.run( + [ + "podman", + "build", + "-f", + container_file, + "-t", + image_name, + ], + check=True, + ) + cleanup_trestlebot_image = True + + if not image_exists(mock_server_image_name): + subprocess.run( + [ + "podman", + "build", + "-f", + f"{e2e_context}/{container_file}", + "-t", + mock_server_image_name, + e2e_context, + ], + check=True, + ) # Create a pod response = subprocess.run( @@ -130,8 +148,10 @@ def podman_setup() -> YieldFixture[int]: ["podman", "play", "kube", "--down", f"{e2e_context}/play-kube.yml"], check=True, ) - subprocess.run(["podman", "rmi", image_name], check=True) - subprocess.run(["podman", "rmi", mock_server_image_name], check=True) + if cleanup_trestlebot_image: + subprocess.run(["podman", "rmi", image_name], check=True) + if cleanup_mock_server_image: + subprocess.run(["podman", "rmi", mock_server_image_name], check=True) except subprocess.CalledProcessError as e: raise RuntimeError(f"Failed to clean up podman resources: {e}") @@ -148,7 +168,18 @@ def podman_setup() -> YieldFixture[int]: "committer-name": "test", "committer-email": "test@email.com", }, - 0, + SUCCESS_EXIT_CODE, + ), + ( + "success/happy path with model skipping", + { + "branch": "test", + "rules-view-path": RULES_VIEW_DIR, + "committer-name": "test", + "committer-email": "test", + "skip-items": test_comp_name, + }, + SUCCESS_EXIT_CODE, ), ( "failure/missing args", @@ -190,20 +221,31 @@ def test_rules_transform_e2e( init._run(args) # Setup the rules directory - setup_rules_view(tmp_repo_path, "test-comp") + setup_rules_view(tmp_repo_path, test_comp_name) remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - # Build the command to be run in the shell command = build_transform_command(tmp_repo_str, command_args) - - # Run the command - run_response = subprocess.run(command, cwd=tmp_repo_path) - - # Get subprocess response + run_response = subprocess.run(command, capture_output=True) assert run_response.returncode == response + # Check that the component definition was created + if response == SUCCESS_EXIT_CODE: + if "skip-items" in command_args: + assert "input: test-comp.csv" not in run_response.stdout.decode("utf-8") + else: + comp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class( + tmp_repo_path, test_comp_name, ComponentDefinition, FileContentType.JSON + ) + assert comp_path.exists() + assert "input: test-comp.csv" in run_response.stdout.decode("utf-8") + branch = command_args["branch"] + assert ( + f"Changes pushed to {branch} successfully." + in run_response.stdout.decode("utf-8") + ) + @pytest.mark.slow @pytest.mark.parametrize( @@ -221,7 +263,7 @@ def test_rules_transform_e2e( "committer-name": "test", "committer-email": "test@email.com", }, - 0, + SUCCESS_EXIT_CODE, ), ( "success/happy path with filtering", @@ -236,7 +278,7 @@ def test_rules_transform_e2e( "committer-email": "test@email.com", "filter-by-profile": test_filter_prof, }, - 0, + SUCCESS_EXIT_CODE, ), ( "failure/missing args", @@ -263,7 +305,22 @@ def test_rules_transform_e2e( "committer-name": "test", "committer-email": "test@email.com", }, - 1, + ERROR_EXIT_CODE, + ), + ( + "failure/missing filter profile", + { + "profile-name": test_prof, + "component-title": "test-comp", + "compdef-name": "test-compdef", + "component-description": "test", + "markdown-path": "markdown", + "branch": "test", + "committer-name": "test", + "committer-email": "test", + "filter-by-profile": "fake", + }, + ERROR_EXIT_CODE, ), ], ) @@ -303,11 +360,23 @@ def test_create_cd_e2e( remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - # Build the command to be run in the shell command = build_create_cd_command(tmp_repo_str, command_args) - - # Run the command - run_response = subprocess.run(command, cwd=tmp_repo_path) - - # Get subprocess response + run_response = subprocess.run(command, cwd=tmp_repo_path, capture_output=True) assert run_response.returncode == response + + # Check that all expected files were created + if response == SUCCESS_EXIT_CODE: + comp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class( + tmp_repo_path, + command_args["compdef-name"], + ComponentDefinition, + FileContentType.JSON, + ) + assert comp_path.exists() + assert (tmp_repo_path / command_args["markdown-path"]).exists() + assert ( + tmp_repo_path + / RULES_VIEW_DIR + / command_args["compdef-name"] + / command_args["component-title"] + ).exists()