enable ceph-csi integration (#119)

* add ceph k8s-info integration
* add ceph integration test

.github/workflows/integration_test.yaml

@@ -36,23 +36,24 @@ jobs:
   integration-tests:
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
     needs: [charmcraft-channel, extra-args]
+    secrets: inherit
     strategy:
       matrix:
         arch:
-        - {id: amd64-k8s,  suite: k8s,  builder-label: ubuntu-22.04, tester-arch: x64}
-        - {id: amd64-etcd, suite: etcd, builder-label: ubuntu-22.04, tester-arch: x64}
-    secrets: inherit
+        - {id: amd64, builder-label: ubuntu-22.04, tester-arch: x64}
+        suite: ["k8s", "etcd", "ceph"]
     with:
-      identifier: ${{ matrix.arch.id }}
+      identifier: ${{ matrix.arch.id }}-${{ matrix.suite }}
+      builder-runner-label: ${{ matrix.arch.builder-label }}
       charmcraft-channel: ${{ needs.charmcraft-channel.outputs.channel }}
-      extra-arguments: ${{needs.extra-args.outputs.args}} -k test_${{ matrix.arch.suite }}
+      extra-arguments: ${{needs.extra-args.outputs.args}} -k test_${{ matrix.suite }}
       juju-channel: 3/stable
       load-test-enabled: false
       provider: lxd
       self-hosted-runner: true
       self-hosted-runner-arch: ${{ matrix.arch.tester-arch }}
       test-timeout: 120
-      test-tox-env: integration-${{ matrix.arch.suite }}
+      test-tox-env: integration-${{ matrix.suite }}
       trivy-fs-enabled: false
       trivy-image-config: "trivy.yaml"
       tmate-debug: true

charms/worker/k8s/charmcraft.yaml

@@ -176,6 +176,8 @@ provides:
     interface: cos-k8s-tokens
   containerd:
     interface: containerd
+  ceph-k8s-info:
+    interface: kubernetes-info
 
 requires:
   etcd:

charms/worker/k8s/src/charm.py

@@ -153,6 +153,18 @@ def __init__(self, *args):
             self.etcd = EtcdReactiveRequires(self)
             self.framework.observe(self.on.get_kubeconfig_action, self._get_external_kubeconfig)
 
+    def _k8s_info(self, event: ops.EventBase):
+        """Send cluster information on the kubernetes-info relation.
+
+        Provide applications with cluster characteristics. This should only run on the lead
+        k8s control plane unit.
+
+        Args:
+            event: ops.RelationChangedEvent - event triggered by the relation changed hook
+        """
+        if isinstance(event, ops.RelationChangedEvent) and event.relation.name == "ceph-k8s-info":
+            event.relation.data[self.app]["kubelet-root-dir"] = "/var/lib/kubelet"
+
     @status.on_error(
         ops.WaitingStatus("Installing COS requirements"),
         subprocess.CalledProcessError,
@@ -679,6 +691,7 @@ def _reconcile(self, event: ops.EventBase):
         self._apply_snap_requirements()
         self._check_k8sd_ready()
         if self.lead_control_plane:
+            self._k8s_info(event)
             self._bootstrap_k8s_snap()
             self._enable_functionalities()
             self._update_annotations()

tests/integration/conftest.py

@@ -184,7 +184,7 @@ def switch(self, name: str, path: Path):
     def drop_constraints(self):
         """Remove constraints on applications. Useful for testing on lxd."""
         for app in self.applications.values():
-            app["constraints"] = None
+            app["constraints"] = ""
 
     def add_constraints(self, constraints: Dict[str, str]):
         """Add constraints to applications.
@@ -193,7 +193,10 @@ def add_constraints(self, constraints: Dict[str, str]):
             constraints:  Mapping of constraints to add to applications.
         """
         for app in self.applications.values():
-            val: str = app["constraints"]
+            if app.get("num_units", 0) < 1:
+                log.info("Skipping constraints for subordinate charm: %s", app["charm"])
+                continue
+            val: str = app.get("constraints", "")
             existing = dict(kv.split("=", 1) for kv in val.split())
             existing.update(constraints)
             app["constraints"] = " ".join(f"{k}={v}" for k, v in existing.items())

tests/integration/data/test-bundle-ceph.yaml

@@ -0,0 +1,44 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: integration-test-ceph
+description: |-
+  Used to deploy or refresh within an integration test model
+series: jammy
+applications:
+  k8s:
+    charm: k8s
+    channel: latest/edge
+    constraints: cores=2 mem=8G root-disk=16G
+    num_units: 1
+  k8s-worker:
+    charm: k8s-worker
+    channel: latest/edge
+    constraints: cores=2 mem=8G root-disk=16G
+    num_units: 1
+  ceph-csi:
+    charm: ceph-csi
+    channel: latest/stable
+    options:
+      provisioner-replicas: 1
+  ceph-mon:
+    charm: ceph-mon
+    channel: quincy/stable
+    constraints: cores=2 mem=4G root-disk=16G
+    num_units: 1
+    options:
+      monitor-count: 1
+      expected-osd-count: 1
+  ceph-osd:
+    charm: ceph-osd
+    channel: quincy/stable
+    constraints: cores=2 mem=4G root-disk=16G
+    num_units: 1
+    storage:
+      osd-devices: 1G,2
+      osd-journals: 1G,1
+relations:
+  - [k8s, k8s-worker:cluster]
+  - [ceph-csi, k8s:ceph-k8s-info]
+  - [ceph-csi, ceph-mon:client]
+  - [ceph-mon, ceph-osd:mon]

tests/integration/test_ceph.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# pylint: disable=duplicate-code
+"""Integration tests."""
+
+import pytest
+from juju import model, unit
+
+# This pytest mark configures the test environment to use the Canonical Kubernetes
+# bundle with ceph, for all the test within this module.
+pytestmark = [
+    pytest.mark.bundle_file("test-bundle-ceph.yaml"),
+    pytest.mark.ignore_blocked,
+]
+
+
+@pytest.mark.abort_on_fail
+async def test_ceph_sc(kubernetes_cluster: model.Model):
+    """Test that a ceph storage class is available."""
+    k8s: unit.Unit = kubernetes_cluster.applications["k8s"].units[0]
+    event = await k8s.run("k8s kubectl get sc -o=jsonpath='{.items[*].provisioner}'")
+    result = await event.wait()
+    stdout = result.results["stdout"]
+    assert "rbd.csi.ceph.com" in stdout, f"No ceph provisioner found in: {stdout}"

tox.ini

@@ -84,7 +84,7 @@ deps =
 commands =
     bandit -c {toxinidir}/pyproject.toml -r {[vars]all_path}
 
-[testenv:{integration,integration-k8s,integration-etcd}]
+[testenv:{integration,integration-k8s,integration-etcd,integration-ceph}]
 description = Run integration tests
 deps = -r test_requirements.txt
 commands =