test: updates test_e2e.py with additional assertions

Adds assertions to check the command outcome
Adds a function to check if the image exists
Updates e2e README.md

Signed-off-by: Jennifer Power <[email protected]>

tests/e2e/README.md

@@ -44,6 +44,12 @@ To run the end-to-end tests, follow these steps:
 ## Additional Notes
 - The WireMock tool is used to mock Git server endpoints for testing.
 - Podman is used for container and pod management and to build the container image for the mock API server.
+- If the images are not already built, the `make test-e2e` command will build them automatically and remove them at the end of the test. If not, you can build them manually with the following command from the root of the project directory:
+
+  ```bash
+  podman build -t localhost/mock-server:latest -f tests/e2e/Dockerfile tests/e2e
+  podman build -t localhost/trestlebot:latest -f Dockerfile .
+  ```
 
 ## Future Improvements
 - Provide an option to use pre-built trestle-bot container images from a registry instead of building them locally.

tests/e2e/test_e2e.py

@@ -24,7 +24,10 @@
 
 import pytest
 from git.repo import Repo
+from trestle.common.model_utils import ModelUtils
 from trestle.core.commands.init import InitCmd
+from trestle.core.models.file_content_type import FileContentType
+from trestle.oscal.component import ComponentDefinition
 from trestle.oscal.profile import Profile
 
 from tests.conftest import YieldFixture
@@ -34,7 +37,7 @@
     setup_for_profile,
     setup_rules_view,
 )
-from trestlebot.const import RULES_VIEW_DIR
+from trestlebot.const import ERROR_EXIT_CODE, RULES_VIEW_DIR, SUCCESS_EXIT_CODE
 
 
 logger = logging.getLogger(__name__)
@@ -48,6 +51,16 @@
 
 test_prof = "simplified_nist_profile"
 test_filter_prof = "simplified_filter_profile"
+test_comp_name = "test-comp"
+
+
+def image_exists(image_name: str) -> bool:
+    """Check if the image already exists."""
+    try:
+        subprocess.check_output(["podman", "image", "inspect", image_name])
+        return True
+    except subprocess.CalledProcessError:
+        return False
 
 
 def build_transform_command(data_path: str, command_args: Dict[str, str]) -> List[str]:
@@ -91,32 +104,37 @@ def build_create_cd_command(data_path: str, command_args: Dict[str, str]) -> Lis
 @pytest.fixture(scope="module")
 def podman_setup() -> YieldFixture[int]:
     """Build the trestlebot container image and run the mock server in a pod."""
-    # Build the container image
-    subprocess.run(
-        [
-            "podman",
-            "build",
-            "-f",
-            container_file,
-            "-t",
-            image_name,
-        ],
-        check=True,
-    )
 
-    # Build mock server container image
-    subprocess.run(
-        [
-            "podman",
-            "build",
-            "-f",
-            f"{e2e_context}/{container_file}",
-            "-t",
-            mock_server_image_name,
-            e2e_context,
-        ],
-        check=True,
-    )
+    cleanup_trestlebot_image = False
+    cleanup_mock_server_image = False
+
+    if not image_exists(image_name):
+        subprocess.run(
+            [
+                "podman",
+                "build",
+                "-f",
+                container_file,
+                "-t",
+                image_name,
+            ],
+            check=True,
+        )
+        cleanup_trestlebot_image = True
+
+    if not image_exists(mock_server_image_name):
+        subprocess.run(
+            [
+                "podman",
+                "build",
+                "-f",
+                f"{e2e_context}/{container_file}",
+                "-t",
+                mock_server_image_name,
+                e2e_context,
+            ],
+            check=True,
+        )
 
     # Create a pod
     response = subprocess.run(
@@ -130,8 +148,10 @@ def podman_setup() -> YieldFixture[int]:
             ["podman", "play", "kube", "--down", f"{e2e_context}/play-kube.yml"],
             check=True,
         )
-        subprocess.run(["podman", "rmi", image_name], check=True)
-        subprocess.run(["podman", "rmi", mock_server_image_name], check=True)
+        if cleanup_trestlebot_image:
+            subprocess.run(["podman", "rmi", image_name], check=True)
+        if cleanup_mock_server_image:
+            subprocess.run(["podman", "rmi", mock_server_image_name], check=True)
     except subprocess.CalledProcessError as e:
         raise RuntimeError(f"Failed to clean up podman resources: {e}")
 
@@ -148,7 +168,18 @@ def podman_setup() -> YieldFixture[int]:
                 "committer-name": "test",
                 "committer-email": "[email protected]",
             },
-            0,
+            SUCCESS_EXIT_CODE,
+        ),
+        (
+            "success/happy path with model skipping",
+            {
+                "branch": "test",
+                "rules-view-path": RULES_VIEW_DIR,
+                "committer-name": "test",
+                "committer-email": "test",
+                "skip-items": test_comp_name,
+            },
+            SUCCESS_EXIT_CODE,
         ),
         (
             "failure/missing args",
@@ -190,20 +221,31 @@ def test_rules_transform_e2e(
     init._run(args)
 
     # Setup the rules directory
-    setup_rules_view(tmp_repo_path, "test-comp")
+    setup_rules_view(tmp_repo_path, test_comp_name)
 
     remote_url = "http://localhost:8080/test.git"
     repo.create_remote("origin", url=remote_url)
 
-    # Build the command to be run in the shell
     command = build_transform_command(tmp_repo_str, command_args)
-
-    # Run the command
-    run_response = subprocess.run(command, cwd=tmp_repo_path)
-
-    # Get subprocess response
+    run_response = subprocess.run(command, capture_output=True)
     assert run_response.returncode == response
 
+    # Check that the component definition was created
+    if response == SUCCESS_EXIT_CODE:
+        if "skip-items" in command_args:
+            assert "input: test-comp.csv" not in run_response.stdout.decode("utf-8")
+        else:
+            comp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class(
+                tmp_repo_path, test_comp_name, ComponentDefinition, FileContentType.JSON
+            )
+            assert comp_path.exists()
+            assert "input: test-comp.csv" in run_response.stdout.decode("utf-8")
+        branch = command_args["branch"]
+        assert (
+            f"Changes pushed to {branch} successfully."
+            in run_response.stdout.decode("utf-8")
+        )
+
 
 @pytest.mark.slow
 @pytest.mark.parametrize(
@@ -221,7 +263,7 @@ def test_rules_transform_e2e(
                 "committer-name": "test",
                 "committer-email": "[email protected]",
             },
-            0,
+            SUCCESS_EXIT_CODE,
         ),
         (
             "success/happy path with filtering",
@@ -236,7 +278,7 @@ def test_rules_transform_e2e(
                 "committer-email": "[email protected]",
                 "filter-by-profile": test_filter_prof,
             },
-            0,
+            SUCCESS_EXIT_CODE,
         ),
         (
             "failure/missing args",
@@ -263,7 +305,22 @@ def test_rules_transform_e2e(
                 "committer-name": "test",
                 "committer-email": "[email protected]",
             },
-            1,
+            ERROR_EXIT_CODE,
+        ),
+        (
+            "failure/missing filter profile",
+            {
+                "profile-name": test_prof,
+                "component-title": "test-comp",
+                "compdef-name": "test-compdef",
+                "component-description": "test",
+                "markdown-path": "markdown",
+                "branch": "test",
+                "committer-name": "test",
+                "committer-email": "test",
+                "filter-by-profile": "fake",
+            },
+            ERROR_EXIT_CODE,
         ),
     ],
 )
@@ -303,11 +360,23 @@ def test_create_cd_e2e(
     remote_url = "http://localhost:8080/test.git"
     repo.create_remote("origin", url=remote_url)
 
-    # Build the command to be run in the shell
     command = build_create_cd_command(tmp_repo_str, command_args)
-
-    # Run the command
-    run_response = subprocess.run(command, cwd=tmp_repo_path)
-
-    # Get subprocess response
+    run_response = subprocess.run(command, cwd=tmp_repo_path, capture_output=True)
     assert run_response.returncode == response
+
+    # Check that all expected files were created
+    if response == SUCCESS_EXIT_CODE:
+        comp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class(
+            tmp_repo_path,
+            command_args["compdef-name"],
+            ComponentDefinition,
+            FileContentType.JSON,
+        )
+        assert comp_path.exists()
+        assert (tmp_repo_path / command_args["markdown-path"]).exists()
+        assert (
+            tmp_repo_path
+            / RULES_VIEW_DIR
+            / command_args["compdef-name"]
+            / command_args["component-title"]
+        ).exists()