feat: adds image tests to CI

.github/workflows/e2e.yml

@@ -0,0 +1,36 @@
+name: E2E
+
+on:
+  workflow_call:
+    inputs:
+      image:
+        description: "Name of the trestlebot image you want to test."
+        type: string
+        required: true
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v3
+
+      - name: Set up poetry and install
+        uses: ./.github/actions/setup-poetry
+        with:
+          python-version: "3.9"
+
+      # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+      - name: Pull the image
+        run: |
+            podman pull "${IMAGE}"
+            echo "TRESTLEBOT_IMAGE=${IMAGE}" >> "$GITHUB_ENV"
+        env:
+          IMAGE: ${{ inputs.image }}
+
+      - name: Run tests
+        run: make test-e2e

.github/workflows/publish.yml

@@ -9,16 +9,26 @@ on:
         description: "Name of the tag for the published image"
         type: string
         required: true
+      skip_tests:
+        description: "Skip end to end tests when publishing an image."
+        type: boolean
+        required: false
+        default: false
 env:
     IMAGE_NAME: trestle-bot
     IMAGE_REGISTRY: quay.io
 
 jobs:
+
   publish-image:
     runs-on: 'ubuntu-latest'
     permissions:
       contents: read
-      packages: write
+    outputs:
+      skip_tests: ${{ steps.check_event.outputs.event_type == 'release'
+                    || (steps.check_event.outputs.event_type == 'workflow_dispatch'
+                      && github.event.inputs.skip_tests == 'true') }}
+      image: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}@${{ steps.build-image.outputs.digest }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -42,21 +52,45 @@ jobs:
 
       # Using intermediary variable to process event based input
       - name: Set TAG environment variable for Release
-        if: steps.check_event.outputs.event_type == 'release'
+        if: ${{ steps.check_event.outputs.event_type == 'release' }}
         run: echo "TAG=$RELEASE_VERSION" >> "$GITHUB_ENV"
         env:
           RELEASE_VERSION: ${{ github.event.release.tag_name }}
 
       - name: Set TAG environment variable for Workflow Dispatch
-        if: steps.check_event.outputs.event_type == 'workflow_dispatch'
+        if: ${{ steps.check_event.outputs.event_type == 'workflow_dispatch' }}
         run: echo "TAG=$INPUT_VERSION" >> "$GITHUB_ENV"
         env:
           INPUT_VERSION: ${{ github.event.inputs.tag }}
 
+      - name: Build and export to Docker
+        uses: docker/build-push-action@v5
+        with:
+          load: true
+          tags: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }}
+
+      - name: Pre-push Image Scan
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }}
+          exit-code: 1
+          scanners: secret
+          severity: HIGH,CRITICAL,MEDIUM
+
       - name: Build and Push
         uses: docker/build-push-action@v5
+        id: build-image
         with:
           push: true
-          tags: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }}
           cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=max
+          tags: ${{ env.IMAGE_REGISTRY }}/${{ vars.QUAY_ORG }}/${{ env.IMAGE_NAME }}:${{ env.TAG }}
+
+  test:
+    permissions:
+        contents: read
+    needs: publish-image
+    if: ${{ needs.publish-image.outputs.skip_tests != 'true' }}
+    uses: ./.github/workflows/e2e.yml
+    with:
+      image: ${{ needs.publish-image.outputs.image }}

tests/conftest.py

@@ -30,11 +30,11 @@
 from trestle.core.commands.init import InitCmd
 
 from tests.testutils import (
+    CONTAINER_FILE_NAME,
     E2E_BUILD_CONTEXT,
     MOCK_SERVER_IMAGE_NAME,
     TRESTLEBOT_TEST_IMAGE_NAME,
-    build_mock_server_image,
-    build_trestlebot_image,
+    build_test_image,
     clean,
 )
 from trestlebot import const
@@ -199,17 +199,29 @@ def test_rule() -> TrestleRule:
 
 
 @pytest.fixture(scope="package")
-def podman_setup() -> YieldFixture[int]:
-    """Build the trestlebot container image and run the mock server in a pod."""
-
-    cleanup_trestlebot_image = build_trestlebot_image()
-    cleanup_mock_server_image = build_mock_server_image()
+def podman_setup() -> YieldFixture[Tuple[int, str]]:
+    """
+    Build the trestlebot container image and run the mock server in a pod.
+
+    Yields:
+        Tuple[int, str]: The return code from the podman play command and the trestlebot image name.
+    """
+
+    # Get the image information from the environment, if present
+    trestlebot_image = os.environ.get("TRESTLEBOT_IMAGE", TRESTLEBOT_TEST_IMAGE_NAME)
+
+    cleanup_trestlebot_image = build_test_image(trestlebot_image)
+    cleanup_mock_server_image = build_test_image(
+        MOCK_SERVER_IMAGE_NAME,
+        f"{E2E_BUILD_CONTEXT}/{CONTAINER_FILE_NAME}",
+        E2E_BUILD_CONTEXT,
+    )
 
     # Create a pod
     response = subprocess.run(
         ["podman", "play", "kube", f"{E2E_BUILD_CONTEXT}/play-kube.yml"], check=True
     )
-    yield response.returncode
+    yield response.returncode, trestlebot_image
 
     # Clean up the container image, pod and mock server
     try:
@@ -218,7 +230,7 @@ def podman_setup() -> YieldFixture[int]:
             check=True,
         )
         if cleanup_trestlebot_image:
-            subprocess.run(["podman", "rmi", TRESTLEBOT_TEST_IMAGE_NAME], check=True)
+            subprocess.run(["podman", "rmi", trestlebot_image], check=True)
         if cleanup_mock_server_image:
             subprocess.run(["podman", "rmi", MOCK_SERVER_IMAGE_NAME], check=True)
     except subprocess.CalledProcessError as e:

tests/e2e/README.md

@@ -49,11 +49,14 @@ To run the end-to-end tests, follow these steps:
   ```bash
   podman build -t localhost/mock-server:latest -f tests/e2e/Dockerfile tests/e2e
   podman build -t localhost/trestlebot:latest -f Dockerfile .
+
+  # Use a prebuilt image from quay.io
+  podman pull quay.io/continuouscompliance/trestle-bot:latest
+  export TRESTLEBOT_IMAGE=quay.io/continuouscompliance/trestle-bot:latest
   ```
 
 - When created tests that push to a branch, ensure the name is "test". This is because the mock API server is configured to only allow pushes to a branch named "test".
 
 ## Future Improvements
-- Provide an option to use pre-built trestle-bot container images from a registry instead of building them locally.
 - Create endpoints that mock GitHub and GitLab API calls for pull request creation.
 - Add more end-to-end tests to cover more use cases.

tests/e2e/test_e2e_compdef.py

@@ -89,15 +89,16 @@
 )
 def test_rules_transform_e2e(
     tmp_repo: Tuple[str, Repo],
-    podman_setup: int,
+    podman_setup: Tuple[int, str],
     test_name: str,
     command_args: Dict[str, str],
     response: int,
 ) -> None:
     """Test the trestlebot rules transform command."""
     # Check that the container image was built successfully
     # and the mock server is running
-    assert podman_setup == 0
+    exit_code, image_name = podman_setup
+    assert exit_code == 0
 
     logger.info(f"Running test: {test_name}")
 
@@ -122,7 +123,9 @@ def test_rules_transform_e2e(
     remote_url = "http://localhost:8080/test.git"
     repo.create_remote("origin", url=remote_url)
 
-    command = build_test_command(tmp_repo_str, "rules-transform", command_args)
+    command = build_test_command(
+        tmp_repo_str, "rules-transform", command_args, image_name
+    )
     run_response = subprocess.run(command, capture_output=True)
     assert run_response.returncode == response
 
@@ -224,15 +227,16 @@ def test_rules_transform_e2e(
 )
 def test_create_cd_e2e(
     tmp_repo: Tuple[str, Repo],
-    podman_setup: int,
+    podman_setup: Tuple[int, str],
     test_name: str,
     command_args: Dict[str, str],
     response: int,
 ) -> None:
     """Test the trestlebot rules transform command."""
     # Check that the container image was built successfully
     # and the mock server is running
-    assert podman_setup == 0
+    exit_code, image_name = podman_setup
+    assert exit_code == 0
 
     logger.info(f"Running test: {test_name}")
 
@@ -258,7 +262,7 @@ def test_create_cd_e2e(
     remote_url = "http://localhost:8080/test.git"
     repo.create_remote("origin", url=remote_url)
 
-    command = build_test_command(tmp_repo_str, "create-cd", command_args)
+    command = build_test_command(tmp_repo_str, "create-cd", command_args, image_name)
     run_response = subprocess.run(command, cwd=tmp_repo_path, capture_output=True)
     assert run_response.returncode == response
 

tests/e2e/test_e2e_ssp.py

@@ -94,7 +94,7 @@
 )
 def test_ssp_editing_e2e(
     tmp_repo: Tuple[str, Repo],
-    podman_setup: int,
+    podman_setup: Tuple[int, str],
     test_name: str,
     command_args: Dict[str, str],
     response: int,
@@ -103,7 +103,8 @@ def test_ssp_editing_e2e(
     """Test the trestlebot autosync command with SSPs."""
     # Check that the container image was built successfully
     # and the mock server is running
-    assert podman_setup == 0
+    exit_code, image_name = podman_setup
+    assert exit_code == 0
 
     logger.info(f"Running test: {test_name}")
 
@@ -151,7 +152,7 @@ def test_ssp_editing_e2e(
     remote_url = "http://localhost:8080/test.git"
     repo.create_remote("origin", url=remote_url)
 
-    command = build_test_command(tmp_repo_str, "autosync", command_args)
+    command = build_test_command(tmp_repo_str, "autosync", command_args, image_name)
     run_response = subprocess.run(command, capture_output=True)
     assert run_response.returncode == response
 

tests/testutils.py

@@ -279,9 +279,6 @@ def replace_string_in_file(file_path: str, old_string: str, new_string: str) ->
         file.write(updated_content)
 
 
-# E2E test utils
-
-
 def _image_exists(image_name: str) -> bool:
     """Check if the image already exists."""
     try:
@@ -291,46 +288,27 @@ def _image_exists(image_name: str) -> bool:
         return False
 
 
-def build_trestlebot_image() -> bool:
-    """
-    Build the trestlebot image.
-
-    Returns:
-        Returns true if the image was built, false if it already exists.
-    """
-    if not _image_exists(TRESTLEBOT_TEST_IMAGE_NAME):
-        subprocess.run(
-            [
-                "podman",
-                "build",
-                "-f",
-                CONTAINER_FILE_NAME,
-                "-t",
-                TRESTLEBOT_TEST_IMAGE_NAME,
-            ],
-            check=True,
-        )
-        return True
-    return False
-
-
-def build_mock_server_image() -> bool:
+def build_test_image(
+    image_name: str,
+    container_file: str = CONTAINER_FILE_NAME,
+    build_context: str = ".",
+) -> bool:
     """
-    Build the mock server image.
+    Build an image for testing image.
 
     Returns:
         Returns true if the image was built, false if it already exists.
     """
-    if not _image_exists(MOCK_SERVER_IMAGE_NAME):
+    if not _image_exists(image_name):
         subprocess.run(
             [
                 "podman",
                 "build",
                 "-f",
-                f"{E2E_BUILD_CONTEXT}/{CONTAINER_FILE_NAME}",
+                container_file,
                 "-t",
-                MOCK_SERVER_IMAGE_NAME,
-                E2E_BUILD_CONTEXT,
+                image_name,
+                build_context,
             ],
             check=True,
         )
@@ -339,7 +317,10 @@ def build_mock_server_image() -> bool:
 
 
 def build_test_command(
-    data_path: str, command_name: str, command_args: Dict[str, str]
+    data_path: str,
+    command_name: str,
+    command_args: Dict[str, str],
+    image_name: str = TRESTLEBOT_TEST_IMAGE_NAME,
 ) -> List[str]:
     """Build a command to be run in the shell for trestlebot"""
     return [
@@ -354,6 +335,6 @@ def build_test_command(
         f"{data_path}:/trestle",
         "-w",
         "/trestle",
-        TRESTLEBOT_TEST_IMAGE_NAME,
+        image_name,
         *args_dict_to_list(command_args),
     ]