diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 14ff575f..28c43209 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -44,4 +44,14 @@ To run the end-to-end tests, follow these steps: ## Additional Notes - The WireMock tool is used to mock Git server endpoints for testing. - Podman is used for container and pod management and to build the container image for the mock API server. -- In the future, we plan to provide an option to use pre-built trestle-bot container images from a registry instead of building them locally. \ No newline at end of file +- If the images are not already built, the `make test-e2e` command will build them automatically and remove them at the end of the test. If not, you can build them manually with the following command from the root of the project directory: + + ```bash + podman build -t localhost/mock-server:latest -f tests/e2e/Dockerfile tests/e2e + podman build -t localhost/trestlebot:latest -f Dockerfile . + ``` + +## Future Improvements +- Provide an option to use pre-built trestle-bot container images from a registry instead of building them locally. +- Create endpoints that mock GitHub and GitLab API calls for pull request creation. +- Add more end-to-end tests to cover more use cases. \ No newline at end of file diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index d99fb4a0..384a966a 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -17,51 +17,59 @@ """E2E tests for the rules transform command.""" import argparse +import logging import pathlib import subprocess from typing import Dict, List, Tuple import pytest from git.repo import Repo +from trestle.common.model_utils import ModelUtils from trestle.core.commands.init import InitCmd +from trestle.core.models.file_content_type import FileContentType +from trestle.oscal.component import ComponentDefinition +from trestle.oscal.profile import Profile from tests.conftest import YieldFixture -from tests.testutils import args_dict_to_list, setup_rules_view -from trestlebot.const import RULES_VIEW_DIR +from tests.testutils import ( + args_dict_to_list, + load_from_json, + setup_for_profile, + setup_rules_view, +) +from trestlebot.const import ( + ERROR_EXIT_CODE, + INVALID_ARGS_EXIT_CODE, + RULES_VIEW_DIR, + SUCCESS_EXIT_CODE, +) +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + image_name = "localhost/trestlebot:latest" mock_server_image_name = "localhost/mock-server:latest" pod_name = "trestlebot-e2e-pod" e2e_context = "tests/e2e" container_file = "Dockerfile" +test_prof = "simplified_nist_profile" +test_filter_prof = "simplified_filter_profile" +test_comp_name = "test-comp" + + +def image_exists(image_name: str) -> bool: + """Check if the image already exists.""" + try: + subprocess.check_output(["podman", "image", "inspect", image_name]) + return True + except subprocess.CalledProcessError: + return False + -# Define test cases and expected outcomes -test_cases: List[Tuple[Dict[str, str], int]] = [ - ( - { - "branch": "test", - "rules-view-path": RULES_VIEW_DIR, - "committer-name": "test", - "committer-email": "test@email.com", - "file-patterns": ".", - }, - 0, - ), - ( - { - "branch": "test", - "rules-view-path": RULES_VIEW_DIR, - "file-patterns": ".", - }, - 2, - ), -] - - -def build_image_command(data_path: str, command_args: Dict[str, str]) -> List[str]: - """Build a command to be run in the shell.""" +def build_transform_command(data_path: str, command_args: Dict[str, str]) -> List[str]: + """Build a command to be run in the shell for rules transform.""" return [ "podman", "run", @@ -79,35 +87,78 @@ def build_image_command(data_path: str, command_args: Dict[str, str]) -> List[st ] +def build_create_cd_command(data_path: str, command_args: Dict[str, str]) -> List[str]: + """Build a command to be run in the shell for create cd.""" + return [ + "podman", + "run", + "--pod", + pod_name, + "--entrypoint", + "trestlebot-create-cd", + "--rm", + "-v", + f"{data_path}:/trestle", + "-w", + "/trestle", + image_name, + *args_dict_to_list(command_args), + ] + + +def build_trestlebot_image() -> bool: + """ + Build the trestlebot image. + + Returns: + Returns true if the image was built, false if it already exists. + """ + if not image_exists(image_name): + subprocess.run( + [ + "podman", + "build", + "-f", + container_file, + "-t", + image_name, + ], + check=True, + ) + return True + return False + + +def build_mock_server_image() -> bool: + """ + Build the mock server image. + + Returns: + Returns true if the image was built, false if it already exists. + """ + if not image_exists(mock_server_image_name): + subprocess.run( + [ + "podman", + "build", + "-f", + f"{e2e_context}/{container_file}", + "-t", + mock_server_image_name, + e2e_context, + ], + check=True, + ) + return True + return False + + @pytest.fixture(scope="module") def podman_setup() -> YieldFixture[int]: """Build the trestlebot container image and run the mock server in a pod.""" - # Build the container image - subprocess.run( - [ - "podman", - "build", - "-f", - container_file, - "-t", - image_name, - ], - check=True, - ) - # Build mock server container image - subprocess.run( - [ - "podman", - "build", - "-f", - f"{e2e_context}/{container_file}", - "-t", - mock_server_image_name, - e2e_context, - ], - check=True, - ) + cleanup_trestlebot_image = build_trestlebot_image() + cleanup_mock_server_image = build_mock_server_image() # Create a pod response = subprocess.run( @@ -121,18 +172,53 @@ def podman_setup() -> YieldFixture[int]: ["podman", "play", "kube", "--down", f"{e2e_context}/play-kube.yml"], check=True, ) - subprocess.run(["podman", "rmi", image_name], check=True) - subprocess.run(["podman", "rmi", mock_server_image_name], check=True) + if cleanup_trestlebot_image: + subprocess.run(["podman", "rmi", image_name], check=True) + if cleanup_mock_server_image: + subprocess.run(["podman", "rmi", mock_server_image_name], check=True) except subprocess.CalledProcessError as e: raise RuntimeError(f"Failed to clean up podman resources: {e}") -# Run each test case @pytest.mark.slow -@pytest.mark.parametrize("command_args, response", test_cases) +@pytest.mark.parametrize( + "test_name, command_args, response", + [ + ( + "success/happy path", + { + "branch": "test", + "rules-view-path": RULES_VIEW_DIR, + "committer-name": "test", + "committer-email": "test@email.com", + }, + SUCCESS_EXIT_CODE, + ), + ( + "success/happy path with model skipping", + { + "branch": "test", + "rules-view-path": RULES_VIEW_DIR, + "committer-name": "test", + "committer-email": "test", + "skip-items": test_comp_name, + }, + SUCCESS_EXIT_CODE, + ), + ( + "failure/missing args", + { + "branch": "test", + "rules-view-path": RULES_VIEW_DIR, + }, + INVALID_ARGS_EXIT_CODE, + ), + ], +) def test_rules_transform_e2e( tmp_repo: Tuple[str, Repo], podman_setup: int, + test_name: str, command_args: Dict[str, str], response: int, ) -> None: @@ -141,6 +227,8 @@ def test_rules_transform_e2e( # and the mock server is running assert podman_setup == 0 + logger.info(f"Running test: {test_name}") + tmp_repo_str, repo = tmp_repo tmp_repo_path = pathlib.Path(tmp_repo_str) @@ -157,16 +245,162 @@ def test_rules_transform_e2e( init._run(args) # Setup the rules directory - setup_rules_view(tmp_repo_path, "test-comp") + setup_rules_view(tmp_repo_path, test_comp_name) remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - # Build the command to be run in the shell - command = build_image_command(tmp_repo_str, command_args) + command = build_transform_command(tmp_repo_str, command_args) + run_response = subprocess.run(command, capture_output=True) + assert run_response.returncode == response + + # Check that the component definition was created + if response == SUCCESS_EXIT_CODE: + if "skip-items" in command_args: + assert "input: test-comp.csv" not in run_response.stdout.decode("utf-8") + else: + comp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class( + tmp_repo_path, test_comp_name, ComponentDefinition, FileContentType.JSON + ) + assert comp_path.exists() + assert "input: test-comp.csv" in run_response.stdout.decode("utf-8") + branch = command_args["branch"] + assert ( + f"Changes pushed to {branch} successfully." + in run_response.stdout.decode("utf-8") + ) + + +@pytest.mark.slow +@pytest.mark.parametrize( + "test_name, command_args, response", + [ + ( + "success/happy path", + { + "profile-name": test_prof, + "component-title": "test-comp", + "compdef-name": "test-compdef", + "component-description": "test", + "markdown-path": "markdown", + "branch": "test", + "committer-name": "test", + "committer-email": "test@email.com", + }, + SUCCESS_EXIT_CODE, + ), + ( + "success/happy path with filtering", + { + "profile-name": test_prof, + "component-title": "test-comp", + "compdef-name": "test-compdef", + "component-description": "test", + "markdown-path": "markdown", + "branch": "test", + "committer-name": "test", + "committer-email": "test@email.com", + "filter-by-profile": test_filter_prof, + }, + SUCCESS_EXIT_CODE, + ), + ( + "failure/missing args", + { + "component-title": "test-comp", + "compdef-name": "test-compdef", + "component-description": "test", + "markdown-path": "markdown", + "branch": "test", + "committer-name": "test", + "committer-email": "test@email.com", + }, + INVALID_ARGS_EXIT_CODE, + ), + ( + "failure/missing profile", + { + "profile-name": "fake", + "component-title": "test-comp", + "compdef-name": "test-compdef", + "component-description": "test", + "markdown-path": "markdown", + "branch": "test", + "committer-name": "test", + "committer-email": "test@email.com", + }, + ERROR_EXIT_CODE, + ), + ( + "failure/missing filter profile", + { + "profile-name": test_prof, + "component-title": "test-comp", + "compdef-name": "test-compdef", + "component-description": "test", + "markdown-path": "markdown", + "branch": "test", + "committer-name": "test", + "committer-email": "test", + "filter-by-profile": "fake", + }, + ERROR_EXIT_CODE, + ), + ], +) +def test_create_cd_e2e( + tmp_repo: Tuple[str, Repo], + podman_setup: int, + test_name: str, + command_args: Dict[str, str], + response: int, +) -> None: + """Test the trestlebot rules transform command.""" + # Check that the container image was built successfully + # and the mock server is running + assert podman_setup == 0 + + logger.info(f"Running test: {test_name}") + + tmp_repo_str, repo = tmp_repo - # Run the command - run_response = subprocess.run(command, cwd=tmp_repo_path) + tmp_repo_path = pathlib.Path(tmp_repo_str) + + # Create a trestle workspace in the temporary git repository + args = argparse.Namespace( + verbose=0, + trestle_root=tmp_repo_path, + full=True, + local=False, + govdocs=False, + ) + init = InitCmd() + init._run(args) - # Get subprocess response + # Load profiles into the environment + _ = setup_for_profile(tmp_repo_path, test_prof, "") + load_from_json(tmp_repo_path, test_filter_prof, test_filter_prof, Profile) + + remote_url = "http://localhost:8080/test.git" + repo.create_remote("origin", url=remote_url) + + command = build_create_cd_command(tmp_repo_str, command_args) + run_response = subprocess.run(command, cwd=tmp_repo_path, capture_output=True) assert run_response.returncode == response + + # Check that all expected files were created + if response == SUCCESS_EXIT_CODE: + comp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class( + tmp_repo_path, + command_args["compdef-name"], + ComponentDefinition, + FileContentType.JSON, + ) + assert comp_path.exists() + assert (tmp_repo_path / command_args["markdown-path"]).exists() + assert ( + tmp_repo_path + / RULES_VIEW_DIR + / command_args["compdef-name"] + / command_args["component-title"] + ).exists() diff --git a/trestlebot/const.py b/trestlebot/const.py index 32f8dbaf..7219d566 100644 --- a/trestlebot/const.py +++ b/trestlebot/const.py @@ -22,6 +22,7 @@ # Common exit codes SUCCESS_EXIT_CODE = 0 ERROR_EXIT_CODE = 1 +INVALID_ARGS_EXIT_CODE = 2 # SSP Index Fields