[COS] Add COS substrate fixture (#40)

Add an optional validation suite for COS integration using the an external K8s as the substrate for the stack.

.github/workflows/integration_test.yaml

@@ -1,10 +1,28 @@
+
 name: Integration tests
 
 on:
   pull_request:
 
 jobs:
+  extra-args:
+    runs-on: ubuntu-latest
+    outputs:
+      args: ${{ steps.flags.outputs.args }}
+    steps:
+      - name: Determine extra args
+        id: flags
+        env:
+          TITLE: ${{ github.event.pull_request.title }}
+        run: |
+          EXTRA_ARGS="--crash-dump=on-failure"
+          if [[ "$TITLE" == *"[COS]"* ]]; then
+            EXTRA_ARGS="$EXTRA_ARGS --cos"
+          fi
+          echo "args=$EXTRA_ARGS" >> "$GITHUB_OUTPUT"
+
   build-all-charms:
+    needs: [extra-args]
     strategy:
       matrix:
         path:
@@ -13,14 +31,15 @@ jobs:
     uses: ./.github/workflows/build-charm.yaml
     with:
       working-directory:  ${{ matrix.path }}
+
   integration-tests:
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    needs: [build-all-charms]
+    needs: [build-all-charms, extra-args]
     secrets: inherit
     with:
       provider: lxd
       juju-channel: 3.3/stable
-      extra-arguments: --crash-dump=on-failure
+      extra-arguments: ${{needs.extra-args.outputs.args}}
       load-test-enabled: false
       zap-enabled: false
       trivy-fs-enabled: true

CONTRIBUTING.md

@@ -111,6 +111,10 @@ Running the integration tests with extra arguments can be accomplished with
 tox run -e integration-tests -- --positional --arguments
 ```
 
+#### COS Integration
+
+The COS integration tests are optional as these are slow/heavy tests. Currently, this suite only runs on LXD. If you are modifying something related to the COS integration, you can validate your changes through integration testing using the flag `--cos`. Also, when submitting a Pull Request with changes related to COS, you must include the `[COS]` tag in your Pull Request description. This will instruct GitHub Actions to execute the respective validation tests against your changes.
+
 #### Useful arguments
 `--keep-models`: Doesn't delete the model once the integration tests are finished
 `--model`: Rerun the test with a given model name -- if it already exist, the integration tests will use it

tests/integration/conftest.py

@@ -4,7 +4,9 @@
 """Fixtures for charm tests."""
 import asyncio
 import contextlib
+import json
 import logging
+import shlex
 from dataclasses import dataclass, field
 from itertools import chain
 from pathlib import Path
@@ -13,8 +15,15 @@
 import pytest
 import pytest_asyncio
 import yaml
+from juju.model import Model
+from juju.tag import untag
+from kubernetes import config as k8s_config
+from kubernetes.client import Configuration
 from pytest_operator.plugin import OpsTest
 
+from .cos_substrate import LXDSubstrate
+from .helpers import get_address
+
 log = logging.getLogger(__name__)
 
 
@@ -28,6 +37,19 @@ def pytest_addoption(parser: pytest.Parser):
         parser: Pytest parser.
     """
     parser.addoption("--charm-file", dest="charm_files", action="append", default=[])
+    parser.addoption("--cos", action="store_true", default=False, help="Run COS integration tests")
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "cos: mark COS integration tests")
+
+
+def pytest_collection_modifyitems(config, items):
+    if not config.getoption("--cos"):
+        skip_cos = pytest.mark.skip(reason="need --cos option to run")
+        for item in items:
+            if item.get_closest_marker("cos"):
+                item.add_marker(skip_cos)
 
 
 @dataclass
@@ -191,3 +213,161 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest):
         bundle.switch(charm.app_name, path)
     async with deploy_model(request, ops_test, model, bundle) as the_model:
         yield the_model
+
+
+@pytest_asyncio.fixture(scope="module")
+async def grafana_agent(ops_test: OpsTest, kubernetes_cluster: Model):
+    """Deploy Grafana Agent."""
+    await kubernetes_cluster.deploy("grafana-agent", channel="stable")
+    await kubernetes_cluster.integrate("grafana-agent:cos-agent", "k8s:cos-agent")
+    await kubernetes_cluster.integrate("grafana-agent:cos-agent", "k8s-worker:cos-agent")
+    await kubernetes_cluster.integrate("k8s:cos-worker-tokens", "k8s-worker:cos-tokens")
+
+    yield
+
+    await kubernetes_cluster.remove_application("grafana-agent")
+
+
+@pytest_asyncio.fixture(scope="module")
+async def cos_model(ops_test: OpsTest, kubernetes_cluster: Model, grafana_agent):
+    """Create a COS substrate and a K8s model."""
+    container_name = "cos-substrate"
+    network_name = "cos-network"
+    manager = LXDSubstrate(container_name, network_name)
+
+    config = manager.create_substrate()
+    kubeconfig_path = ops_test.tmp_path / "kubeconfig"
+    kubeconfig_path.write_text(config)
+    config = type.__call__(Configuration)
+    k8s_config.load_config(client_configuration=config, config_file=str(kubeconfig_path))
+
+    k8s_cloud = await ops_test.add_k8s(kubeconfig=config, skip_storage=False)
+    k8s_model = await ops_test.track_model(
+        "cos", cloud_name=k8s_cloud, keep=ops_test.ModelKeep.NEVER
+    )
+    yield k8s_model
+
+    await ops_test.forget_model("cos", timeout=10 * 60, allow_failure=True)
+
+    manager.teardown_substrate()
+
+
+@pytest_asyncio.fixture(scope="module")
+async def cos_lite_installed(ops_test: OpsTest, cos_model: Model):
+    """Install COS Lite bundle."""
+    log.info("Deploying COS bundle ...")
+    cos_charms = ["alertmanager", "catalogue", "grafana", "loki", "prometheus", "traefik"]
+    overlays = [
+        ops_test.Bundle("cos-lite", "edge"),
+        Path("tests/integration/data/cos-offers-overlay.yaml"),
+    ]
+
+    bundle, *overlays = await ops_test.async_render_bundles(*overlays)
+    cmd = f"juju deploy -m {cos_model.name} {bundle} --trust " + " ".join(
+        f"--overlay={f}" for f in overlays
+    )
+    rc, stdout, stderr = await ops_test.run(*shlex.split(cmd))
+    assert rc == 0, f"COS Lite failed to deploy: {(stderr or stdout).strip()}"
+
+    await cos_model.block_until(
+        lambda: all(app in cos_model.applications for app in cos_charms),
+        timeout=60,
+    )
+    await cos_model.wait_for_idle(status="active", timeout=20 * 60, raise_on_error=False)
+
+    yield
+    log.info("Removing COS Lite charms...")
+    with ops_test.model_context("cos"):
+        for charm in cos_charms:
+            log.info(f"Removing {charm}...")
+            cmd = f"remove-application {charm} --destroy-storage --force --no-prompt"
+            rc, stdout, stderr = await ops_test.juju(*shlex.split(cmd))
+            log.info(f"{(stdout or stderr)})")
+            assert rc == 0
+            await cos_model.block_until(
+                lambda: charm not in cos_model.applications, timeout=60 * 10
+            )
+
+
+@pytest_asyncio.fixture(scope="module")
+async def traefik_address(ops_test: OpsTest, cos_model: Model, cos_lite_installed):
+    """Fixture to get Traefik address."""
+    with ops_test.model_context("cos"):
+        address = await get_address(ops_test=ops_test, app_name="traefik")
+    yield address
+
+
+@pytest_asyncio.fixture(scope="module")
+async def expected_dashboard_titles():
+    """Fixture to get expected Grafana dashboard titles."""
+    grafana_dir = Path("charms/worker/k8s/src/grafana_dashboards")
+    grafana_files = [p for p in grafana_dir.iterdir() if p.is_file() and p.name.endswith(".json")]
+    titles = []
+    for path in grafana_files:
+        dashboard = json.loads(path.read_text())
+        titles.append(dashboard["title"])
+    return set(titles)
+
+
+@pytest_asyncio.fixture(scope="module")
+async def related_grafana(ops_test: OpsTest, cos_model: Model, cos_lite_installed):
+    """Fixture to integrate with Grafana."""
+    model_owner = untag("user-", cos_model.info.owner_tag)
+    cos_model_name = cos_model.name
+
+    with ops_test.model_context("main") as model:
+        log.info("Integrating with Grafana")
+        await ops_test.model.integrate(
+            "grafana-agent",
+            f"{model_owner}/{cos_model_name}.grafana-dashboards",
+        )
+        with ops_test.model_context("cos") as k8s_model:
+            await k8s_model.wait_for_idle(status="active")
+        await ops_test.model.wait_for_idle(status="active")
+
+    yield
+
+    with ops_test.model_context("main") as model:
+        log.info("Removing Grafana SAAS ...")
+        await ops_test.model.remove_saas("grafana-dashboards")
+    with ops_test.model_context("cos") as model:
+        log.info("Removing Grafana Offer...")
+        await model.remove_offer(f"{model.name}.grafana-dashboards", force=True)
+
+
+@pytest_asyncio.fixture(scope="module")
+async def grafana_password(ops_test, cos_model: Model, related_grafana):
+    """Fixture to get Grafana password."""
+    with ops_test.model_context("cos"):
+        action = (
+            await ops_test.model.applications["grafana"].units[0].run_action("get-admin-password")
+        )
+        action = await action.wait()
+        yield action.results["admin-password"]
+
+
+@pytest_asyncio.fixture(scope="module")
+async def related_prometheus(ops_test: OpsTest, cos_model, cos_lite_installed):
+    """Fixture to integrate with Prometheus."""
+    model_owner = untag("user-", cos_model.info.owner_tag)
+    cos_model_name = cos_model.name
+
+    with ops_test.model_context("main") as model:
+        log.info("Integrating with Prometheus")
+        relation = await ops_test.model.integrate(
+            "grafana-agent",
+            f"{model_owner}/{cos_model_name}.prometheus-receive-remote-write",
+        )
+        await ops_test.model.wait_for_idle(status="active")
+        with ops_test.model_context("cos") as model:
+            await model.wait_for_idle(status="active")
+
+    yield
+
+    with ops_test.model_context("main") as model:
+        log.info("Removing Prometheus Remote Write SAAS ...")
+        await ops_test.model.remove_saas("prometheus-receive-remote-write")
+
+    with ops_test.model_context("cos") as model:
+        log.info("Removing Prometheus Offer...")
+        await model.remove_offer(f"{model.name}.prometheus-receive-remote-write", force=True)