From a275fc64a9338048bf588306eeb72d777c55f7f2 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 26 Feb 2024 10:15:35 +0100 Subject: [PATCH 01/34] coredns integration implementation --- charms/worker/k8s/charmcraft.yaml | 3 ++ .../k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 9 +++++ charms/worker/k8s/requirements.txt | 1 + charms/worker/k8s/src/charm.py | 39 +++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/charms/worker/k8s/charmcraft.yaml b/charms/worker/k8s/charmcraft.yaml index 2d83bfa8..ed60cf25 100644 --- a/charms/worker/k8s/charmcraft.yaml +++ b/charms/worker/k8s/charmcraft.yaml @@ -116,3 +116,6 @@ requires: interface: etcd external-cloud-provider: interface: external_cloud_provider + dns-provider: + interface: kube-dns + limit: 1 diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index d928ceae..c47267fc 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -714,6 +714,15 @@ def update_cluster_config(self, config: UpdateClusterConfigRequest): endpoint = "/1.0/k8sd/cluster/config" body = config.dict(exclude_none=True, by_alias=True) self._send_request(endpoint, "PUT", EmptyResponse, body) + + def configure_dns(self, dns_domain: str, dns_ip: str): + """Configure the DNS for the k8s cluster. + Args: + dns_domain (str): The domain name for the DNS. + dns_ip (str): The IP address for the DNS. + """ + # TODO: Implement the DNS configuration. + pass def get_cluster_status(self) -> GetClusterStatusResponse: """Retrieve cluster status. diff --git a/charms/worker/k8s/requirements.txt b/charms/worker/k8s/requirements.txt index eb134dd4..bdd4b588 100644 --- a/charms/worker/k8s/requirements.txt +++ b/charms/worker/k8s/requirements.txt @@ -2,6 +2,7 @@ charm-lib-contextual-status @ git+https://github.com/charmed-kubernetes/charm-li charm-lib-interface-external-cloud-provider @ git+https://github.com/charmed-kubernetes/charm-lib-interface-external-cloud-provider@e1c5fc69e98100a7d43c0ad5a7969bba1ecbcd40 charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@9b212854e768f13c26cc907bed51444e97e51b50#subdirectory=ops charm-lib-reconciler @ git+https://github.com/charmed-kubernetes/charm-lib-reconciler@f818cc30d1a22be43ffdfecf7fbd9c3fd2967502 +charm-lib-interface-kube-dns @ git+https://github.com/charmed-kubernetes/charm-lib-interface-kube-dns@3b94af979dbb3ecd926faa30b37dc366e90385e5 cosl==0.0.8 ops==2.12.0 pydantic==1.10.15 diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index f37f3cb7..3bad0f5d 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -50,6 +50,7 @@ UserFacingDatastoreConfig, ) from charms.kubernetes_libs.v0.etcd import EtcdReactiveRequires +from charms.interface_kube_dns import KubeDnsRequires from charms.node_base import LabelMaker from charms.reconciler import Reconciler from cos_integration import COSIntegration @@ -105,6 +106,7 @@ def __init__(self, *args): self.reconciler = Reconciler(self, self._reconcile) self.distributor = TokenDistributor(self, self.get_node_name(), self.api_manager) self.collector = TokenCollector(self, self.get_node_name()) + self.kube_dns = KubeDnsRequires(self, endpoint="dns-provider") self.labeler = LabelMaker( self, kubeconfig_path=self._internal_kubeconfig, kubectl=KUBECTL_PATH ) @@ -418,6 +420,43 @@ def _ensure_cluster_config(self): self.api_manager.update_cluster_config(update_request) + def _configure_components(self): + """Enable necessary components for the Kubernetes cluster.""" + if self.dns_charm_integrated(): + status.add(ops.MaintenanceStatus("Disabling DNS")) + self.api_manager.configure_component("dns", False) + self.configure_dns() + else: + status.add(ops.MaintenanceStatus("Enabling DNS")) + self.api_manager.configure_component("dns", True) + + def dns_charm_integrated(self) -> bool: + dns_relation = self.model.get_relation("dns-provider") + if not dns_relation: + return False + return True + + def configure_dns(self): + if isinstance(self.unit.status, ops.BlockedStatus): + return + + if not self._state.joined: + return + + dns_relation = self.model.get_relation("dns-provider") + if not dns_relation: + return + + dns_ip = self.kube_dns.address + dns_domain = self.kube_dns.domain + port = self.kube_dns.port or 53 + + if not dns_ip or not dns_domain: + return + + self.unit.status = ops.MaintenanceStatus("configuring DNS") + self.api_manager.configure_dns(dns_ip, dns_domain) + def _get_scrape_jobs(self): """Retrieve the Prometheus Scrape Jobs. From d49a76b5ca098c9ef19643f699aaf0bb321813d2 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 26 Feb 2024 11:45:50 +0100 Subject: [PATCH 02/34] format --- charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 2 +- charms/worker/k8s/src/charm.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index c47267fc..baf5b9a3 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -714,7 +714,7 @@ def update_cluster_config(self, config: UpdateClusterConfigRequest): endpoint = "/1.0/k8sd/cluster/config" body = config.dict(exclude_none=True, by_alias=True) self._send_request(endpoint, "PUT", EmptyResponse, body) - + def configure_dns(self, dns_domain: str, dns_ip: str): """Configure the DNS for the k8s cluster. Args: diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index 3bad0f5d..9c43a299 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -34,6 +34,7 @@ from charms.contextual_status import WaitingStatus, on_error from charms.grafana_agent.v0.cos_agent import COSAgentProvider from charms.interface_external_cloud_provider import ExternalCloudProvider +from charms.interface_kube_dns import KubeDnsRequires from charms.k8s.v0.k8sd_api_manager import ( BootstrapConfig, ControlPlaneNodeJoinConfig, From bba0036240a2ec10a50e8a60949e94d3c0ae9a01 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 26 Feb 2024 12:56:30 +0100 Subject: [PATCH 03/34] lint --- .../k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 2 +- charms/worker/k8s/src/charm.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index baf5b9a3..61386b90 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -717,12 +717,12 @@ def update_cluster_config(self, config: UpdateClusterConfigRequest): def configure_dns(self, dns_domain: str, dns_ip: str): """Configure the DNS for the k8s cluster. + Args: dns_domain (str): The domain name for the DNS. dns_ip (str): The IP address for the DNS. """ # TODO: Implement the DNS configuration. - pass def get_cluster_status(self) -> GetClusterStatusResponse: """Retrieve cluster status. diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index 9c43a299..88339726 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -423,40 +423,43 @@ def _ensure_cluster_config(self): def _configure_components(self): """Enable necessary components for the Kubernetes cluster.""" - if self.dns_charm_integrated(): + if self._dns_charm_integrated(): status.add(ops.MaintenanceStatus("Disabling DNS")) self.api_manager.configure_component("dns", False) - self.configure_dns() + self._configure_dns() else: status.add(ops.MaintenanceStatus("Enabling DNS")) self.api_manager.configure_component("dns", True) def dns_charm_integrated(self) -> bool: + """Check if the DNS charm is integrated. + + Returns: + bool: True if the DNS charm is integrated, False otherwise. + """ dns_relation = self.model.get_relation("dns-provider") if not dns_relation: return False return True def configure_dns(self): + """Configure DNS with dns config from the dns-provider relation.""" if isinstance(self.unit.status, ops.BlockedStatus): return - if not self._state.joined: - return - dns_relation = self.model.get_relation("dns-provider") if not dns_relation: return dns_ip = self.kube_dns.address dns_domain = self.kube_dns.domain - port = self.kube_dns.port or 53 + # port = self.kube_dns.port or 53 if not dns_ip or not dns_domain: return self.unit.status = ops.MaintenanceStatus("configuring DNS") - self.api_manager.configure_dns(dns_ip, dns_domain) + self.api_manager.configure_dns(dns_domain, dns_ip) def _get_scrape_jobs(self): """Retrieve the Prometheus Scrape Jobs. From d7f9dda677c7e3734ff4274e6c2692d38ab2874a Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 26 Feb 2024 17:11:41 +0100 Subject: [PATCH 04/34] fixtures draft --- tests/integration/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b393833b..a4cfd688 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -11,7 +11,8 @@ from itertools import chain from pathlib import Path from typing import List, Mapping, Optional - +import juju.utils +import shlex import pytest import pytest_asyncio import yaml From 580bea14bdc79665f7bb5ac3a3da3fc4c60b7b89 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 27 Feb 2024 10:59:19 +0100 Subject: [PATCH 05/34] add tests and integration with coredns --- tests/integration/conftest.py | 66 +++++++++++++++++++++++++++++++++++ tests/integration/test_k8s.py | 31 ++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a4cfd688..e9c41da5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -262,6 +262,15 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): bundle.switch(charm.app_name, path) async with deploy_model(request, ops_test, model, bundle, raise_on_blocked) as the_model: yield the_model + + # Creating Kubernetes cloud + await configure_k8s_cloud(ops_test) + + # Creating Coredns model + coredns_model_result = await coredns_model(ops_test) + + # Deploying Coredns charm + await manage_coredns_lifecycle(ops_test, coredns_model_result) @pytest_asyncio.fixture(name="_grafana_agent", scope="module") @@ -277,6 +286,63 @@ async def grafana_agent(kubernetes_cluster: Model): await kubernetes_cluster.remove_application("grafana-agent") +@pytest.fixture(scope="module") +async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model): + """ + This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. + It waits for the deployment to complete and ensures that the Coredns application is active. + """ + log.info(f"Deploying Coredns ") + + #TODO: check what k8s_alias is and what it should be + k8s_alias = coredns_model + with ops_test.model_context(k8s_alias) as model: + await asyncio.gather( + model.deploy(entity_url="coredns", trust=True, channel="edge", ), + ) + + await model.block_until( + lambda: all("coredns" in model.applications), + timeout=60, + ) + await model.wait_for_idle(status="active", timeout=5 * 60) + + coredns_app = model.applications["coredns"] + + # Consume and relate Coredns + await consume_core_dns(ops_test, cluster_model="your_cluster_model_name", k8s_model="k8s-model") + + await ops_test.model.wait_for_idle(status="active", timeout=5 * 60) + + log.info(f"Removing coredns ...") + cmd = "remove-application coredns --destroy-storage --force" + rc, stdout, stderr = await ops_test.juju(*shlex.split(cmd)) + log.info(f"{(stdout or stderr)})") + assert rc == 0 + await m.block_until(lambda: "coredns" not in m.applications, timeout=60 * 10) + + +async def consume_core_dns(ops_test: OpsTest, cluster_model: str, k8s_model: str = "k8s-model"): + """ + This function consumes and relates Coredns in the specified Kubernetes (k8s) model with a cluster model. + """ + log.info("Consuming Coredns...") + # TODO: find out if ops test has something that does this? + + # Juju consume command + consume_cmd = f"juju consume -m {cluster_model} {k8s_model}.coredns" + rc, stdout, stderr = await ops_test.juju(*shlex.split(consume_cmd)) + log.info(f"{(stdout or stderr)}") + assert rc == 0, "Failed to consume Coredns in the cluster model." + # TODO: or wait for the relation to be established? + await ops_test.model.wait_for_idle(status="active", timeout=5 * 60) + + # Juju relate command + relate_cmd = f"juju relate -m {cluster_model} coredns k8s" + rc, stdout, stderr = await ops_test.juju(*shlex.split(relate_cmd)) + log.info(f"{(stdout or stderr)}") + assert rc == 0, "Failed to relate Coredns in the cluster model with k8s" + @pytest_asyncio.fixture(scope="module") async def cos_model( ops_test: OpsTest, kubernetes_cluster, _grafana_agent # pylint: disable=W0613 diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 10c79972..96ec4135 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -175,3 +175,34 @@ async def test_prometheus(traefik_address: str, cos_model: model.Model): ] for query in queries: await prometheus.check_metrics(query) +async def test_coredns_integration(kubernetes_cluster): + """Test the coredns integration.""" + k8s = kubernetes_cluster.applications["k8s"] + + dns_relation = k8s.model.get_relation("dns-provider") + + #TODO make sure this works + # Check if the DNS relation is set, and the domain is set to cluster.local + assert dns_relation, "No DNS relation found" + assert dns_relation.data["domain"] == "cluster.local", "Domain not set to cluster.local" + + +async def test_dns(kubernetes_cluster): + """ + This function performs a DNS test on the specified Kubernetes (k8s) unit in the cluster model. + The test is performed by running a pod in the k8s unit and + checking if it can resolve the domain name (See: https://charmhub.io/microk8s/docs/how-to-advanced-dns). + """ + log.info("Running DNS test...") + k8s = kubernetes_cluster.applications["k8s"] + k8s_unit = k8s.units[0] + # Do we need to switch models? + exec_cmd = f"juju exec --unit {k8s_unit} -- k8s kubectl run --rm -it --image alpine --restart=Never test-dns -- nslookup canonical.com" + action = await k8s.units[0].run(exec_cmd) + result = await action.wait() + assert result.results["return-code"] == 0, "DNS Test failed" + + output = json.loads(result.results["stdout"]) + + #TODO test this output, https://charmhub.io/microk8s/docs/how-to-advanced-dns + log.info("DNS test passed.") From e16b6e814241401829f5d7ed5ee05c81ce6a0606 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 27 Feb 2024 12:09:54 +0100 Subject: [PATCH 06/34] cleanup --- tests/integration/conftest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e9c41da5..3a6f79c6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -295,14 +295,13 @@ async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model): log.info(f"Deploying Coredns ") #TODO: check what k8s_alias is and what it should be - k8s_alias = coredns_model - with ops_test.model_context(k8s_alias) as model: + with ops_test.model_context(coredns_model ) as model: await asyncio.gather( model.deploy(entity_url="coredns", trust=True, channel="edge", ), ) await model.block_until( - lambda: all("coredns" in model.applications), + lambda: "coredns" in model.applications, timeout=60, ) await model.wait_for_idle(status="active", timeout=5 * 60) @@ -314,6 +313,11 @@ async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model): await ops_test.model.wait_for_idle(status="active", timeout=5 * 60) + yield + + with ops_test.model_context(coredns_model) as m: + log.info("Removing Coredns charm...") + log.info(f"Removing coredns ...") cmd = "remove-application coredns --destroy-storage --force" rc, stdout, stderr = await ops_test.juju(*shlex.split(cmd)) From bbd57c225640bdca866ea40196167409245dc467 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 27 Feb 2024 12:56:46 +0100 Subject: [PATCH 07/34] use opstest for cross model relation --- tests/integration/conftest.py | 76 ++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3a6f79c6..206f6e2f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -270,7 +270,7 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): coredns_model_result = await coredns_model(ops_test) # Deploying Coredns charm - await manage_coredns_lifecycle(ops_test, coredns_model_result) + await manage_coredns_lifecycle(ops_test, coredns_model_result, cluster_model) @pytest_asyncio.fixture(name="_grafana_agent", scope="module") @@ -285,9 +285,53 @@ async def grafana_agent(kubernetes_cluster: Model): await kubernetes_cluster.remove_application("grafana-agent") +@pytest.fixture(scope="module") +async def coredns_model(k8s_cloud, ops_test: OpsTest): + """ + This fixture sets up a Coredns model on the specified Kubernetes (k8s) cloud for testing purposes. + It adds the k8s model, performs necessary operations, and removes the model after the test. + """ + log.info("Creating Coredns model ...") + + model_name = "coredns-model" + await ops_test.juju( + "add-model", + f"--controller={ops_test.controller_name}", + model_name, + k8s_cloud, + "--no-switch", #TODO: does not switch to the new model, does that bite me in the next call to juju? + ) + + model = await ops_test.track_model( + model_name, + model_name=model_name, + cloud_name=k8s_cloud, + # credential_name=k8s_cloud, #TODO: is this necessary? + keep=False, + ) + model_uuid = model.info.uuid + + yield model, model_name + + timeout = 5 * 60 + await ops_test.forget_model(model_name, timeout=timeout, allow_failure=False) + + async def model_removed(): + _, stdout, stderr = await ops_test.juju("models", "--format", "yaml") + if _ != 0: + return False + model_list = yaml.safe_load(stdout)["models"] + which = [m for m in model_list if m["model-uuid"] == model_uuid] + return len(which) == 0 + + log.info("Removing Coredns model") + await juju.utils.block_until_with_coroutine(model_removed, timeout=timeout) + # Update client's model cache + await ops_test.juju("models") + log.info("Coredns model removed ...") @pytest.fixture(scope="module") -async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model): +async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model: str, cluster_model: str): """ This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. It waits for the deployment to complete and ensures that the Coredns application is active. @@ -309,9 +353,8 @@ async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model): coredns_app = model.applications["coredns"] # Consume and relate Coredns - await consume_core_dns(ops_test, cluster_model="your_cluster_model_name", k8s_model="k8s-model") - - await ops_test.model.wait_for_idle(status="active", timeout=5 * 60) + # TODO: Should this be moved out into kubernetes cluster function? + await integrate_coredns(ops_test, coredns_model=coredns_model, cluster_model=cluster_model) yield @@ -325,21 +368,18 @@ async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model): assert rc == 0 await m.block_until(lambda: "coredns" not in m.applications, timeout=60 * 10) - -async def consume_core_dns(ops_test: OpsTest, cluster_model: str, k8s_model: str = "k8s-model"): +async def integrate_coredns(ops_test: OpsTest, coredns_model: str = "coredns-model", cluster_model: str = "main"): """ - This function consumes and relates Coredns in the specified Kubernetes (k8s) model with a cluster model. + This function offers Coredns in the specified Kubernetes (k8s) model. """ - log.info("Consuming Coredns...") - # TODO: find out if ops test has something that does this? - - # Juju consume command - consume_cmd = f"juju consume -m {cluster_model} {k8s_model}.coredns" - rc, stdout, stderr = await ops_test.juju(*shlex.split(consume_cmd)) - log.info(f"{(stdout or stderr)}") - assert rc == 0, "Failed to consume Coredns in the cluster model." - # TODO: or wait for the relation to be established? - await ops_test.model.wait_for_idle(status="active", timeout=5 * 60) + log.info("Offering Coredns...") + with ops_test.model_context(coredns_model) as model: + await model.offer("coredns:dns-provider") + offers = await model.list_offers() + await model.block_until( + lambda: all(offer.application_name == 'coredns' #TODO check if this name is correct + for offer in offers.results)) + log.info("Coredns offered...") # Juju relate command relate_cmd = f"juju relate -m {cluster_model} coredns k8s" From 528abf06739f9712b54275426d13ec6dbe8902ce Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 27 Feb 2024 14:16:50 +0100 Subject: [PATCH 08/34] use future endpoint for setting dns bits --- .../k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 7 ++++++- tests/integration/conftest.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index 61386b90..d5251c4c 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -722,7 +722,12 @@ def configure_dns(self, dns_domain: str, dns_ip: str): dns_domain (str): The domain name for the DNS. dns_ip (str): The IP address for the DNS. """ - # TODO: Implement the DNS configuration. + endpoint = "/1.0/k8sd/cluster/config" + body = { + "dns.dns-ip": dns_ip, + "dns.cluster-domain": dns_domain, + "dns.enabled": False} + self._send_request(endpoint, "POST", EmptyResponse, body) def get_cluster_status(self) -> GetClusterStatusResponse: """Retrieve cluster status. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 206f6e2f..b370b02f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -387,6 +387,21 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model: str = "coredns-mod log.info(f"{(stdout or stderr)}") assert rc == 0, "Failed to relate Coredns in the cluster model with k8s" + status = await model_2.get_status() + if 'coredns' not in status.remote_applications: + raise Exception("Expected coredns") + log.info("Coredns consumed...") + + log.info("Relating Coredns...") + await model_2.relate("coredns:dns-provider admin/{}.coredns".format(coredns_model)) + if 'coredns' not in status.remote_applications: + raise Exception("Expected coredns") + log.info("Coredns related...") + + # TODO cleanup + # await model.remove_offer("admin/{}.ubuntu".format(model.name), force=True) #TODO: when do we remove the offer? + + @pytest_asyncio.fixture(scope="module") async def cos_model( ops_test: OpsTest, kubernetes_cluster, _grafana_agent # pylint: disable=W0613 From 8d41a889f1ddc3c1dd85e2f2d6ac34bd99e0e2c3 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 27 Feb 2024 17:33:07 +0100 Subject: [PATCH 09/34] Adams suggestions --- charms/worker/k8s/src/charm.py | 41 ++++++++++------------------------ 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index 88339726..1c71e599 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -423,43 +423,26 @@ def _ensure_cluster_config(self): def _configure_components(self): """Enable necessary components for the Kubernetes cluster.""" - if self._dns_charm_integrated(): - status.add(ops.MaintenanceStatus("Disabling DNS")) + dns_settings = self._dns_charm_integrated() + status.add(ops.MaintenanceStatus(f"Configuring DNS")) + + if dns_settings: self.api_manager.configure_component("dns", False) - self._configure_dns() + self.api_manager.configure_dns(dns_settings.get("dns_domain"), dns_settings.get("dns_ip")) else: - status.add(ops.MaintenanceStatus("Enabling DNS")) self.api_manager.configure_component("dns", True) - def dns_charm_integrated(self) -> bool: + def _dns_charm_integrated(self) -> dict: """Check if the DNS charm is integrated. Returns: - bool: True if the DNS charm is integrated, False otherwise. + dict[str, str]: The DNS settings if the DNS charm is integrated. + None: If the DNS charm is not integrated it returns None. """ - dns_relation = self.model.get_relation("dns-provider") - if not dns_relation: - return False - return True - - def configure_dns(self): - """Configure DNS with dns config from the dns-provider relation.""" - if isinstance(self.unit.status, ops.BlockedStatus): - return - - dns_relation = self.model.get_relation("dns-provider") - if not dns_relation: - return - - dns_ip = self.kube_dns.address - dns_domain = self.kube_dns.domain - # port = self.kube_dns.port or 53 - - if not dns_ip or not dns_domain: - return - - self.unit.status = ops.MaintenanceStatus("configuring DNS") - self.api_manager.configure_dns(dns_domain, dns_ip) + dns_settings = {"dns_domain": self.kube_dns.domain, "dns_ip": self.kube_dns.address} + if all(p is not None for p in dns_settings.values()): # check for any `None` + return dns_settings + return None def _get_scrape_jobs(self): """Retrieve the Prometheus Scrape Jobs. From 3d788d5fcb8b5e8acdc5fcef6a670877fe5bb956 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 28 Feb 2024 08:56:47 +0100 Subject: [PATCH 10/34] return statement --- charms/worker/k8s/src/charm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index 1c71e599..b5f387ea 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -432,7 +432,10 @@ def _configure_components(self): else: self.api_manager.configure_component("dns", True) - def _dns_charm_integrated(self) -> dict: + status.add(ops.MaintenanceStatus("Enabling Network")) + self.api_manager.configure_component("network", True) + + def _dns_charm_integrated(self) -> Optional[dict]: """Check if the DNS charm is integrated. Returns: From 1e17afc56254b981da1cd9e4ca58536ffd02f8c8 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 5 Mar 2024 14:46:43 +0100 Subject: [PATCH 11/34] update dns config --- charms/worker/k8s/src/charm.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index b5f387ea..e8905447 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -436,16 +436,26 @@ def _configure_components(self): self.api_manager.configure_component("network", True) def _dns_charm_integrated(self) -> Optional[dict]: - """Check if the DNS charm is integrated. + """Check if the DNS charm is integrated.""" + status.add(ops.MaintenanceStatus("Configuring DNS and Network")) + + dns_config = self._get_dns_config() + network_config = NetworkConfig(enabled=True) + + user_cluster_config = UserFacingClusterConfig(dns=dns_config, network=network_config) + update_request = UpdateClusterConfigRequest(config=user_cluster_config) + self.api_manager.update_cluster_config(update_request) + + def _get_dns_config(self) -> Optional[dict]: + """Get DNS config either for the enabled built-in dns or an integrated charm. Returns: - dict[str, str]: The DNS settings if the DNS charm is integrated. - None: If the DNS charm is not integrated it returns None. + DNSConfig: A DNSConfig object with the cluster domain, service IP and whether the default dns is enabled or not. """ - dns_settings = {"dns_domain": self.kube_dns.domain, "dns_ip": self.kube_dns.address} - if all(p is not None for p in dns_settings.values()): # check for any `None` - return dns_settings - return None + if self.kube_dns.domain is not None and self.kube_dns.address is not None: + return DNSConfig(enabled=False, cluster_domain=self.kube_dns.domain, service_ip=self.kube_dns.address) + + return DNSConfig(enabled=True) def _get_scrape_jobs(self): """Retrieve the Prometheus Scrape Jobs. From 16de0182fb17b3d15a2fbcb867c89f9bd8ca5cbd Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 5 Mar 2024 15:36:25 +0100 Subject: [PATCH 12/34] use adams pytest fixtures --- tests/integration/conftest.py | 77 ++++++++++++++--------------------- tests/integration/test_k8s.py | 5 +-- 2 files changed, 33 insertions(+), 49 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b370b02f..69954e1b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -263,14 +263,7 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): async with deploy_model(request, ops_test, model, bundle, raise_on_blocked) as the_model: yield the_model - # Creating Kubernetes cloud - await configure_k8s_cloud(ops_test) - - # Creating Coredns model - coredns_model_result = await coredns_model(ops_test) - - # Deploying Coredns charm - await manage_coredns_lifecycle(ops_test, coredns_model_result, cluster_model) + ops_test.add_k8s(skip_storage=True) @pytest_asyncio.fixture(name="_grafana_agent", scope="module") @@ -331,55 +324,47 @@ async def model_removed(): log.info("Coredns model removed ...") @pytest.fixture(scope="module") -async def manage_coredns_lifecycle(ops_test: OpsTest, coredns_model: str, cluster_model: str): +async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.Model): """ This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. It waits for the deployment to complete and ensures that the Coredns application is active. """ log.info(f"Deploying Coredns ") - #TODO: check what k8s_alias is and what it should be - with ops_test.model_context(coredns_model ) as model: - await asyncio.gather( - model.deploy(entity_url="coredns", trust=True, channel="edge", ), - ) - - await model.block_until( - lambda: "coredns" in model.applications, - timeout=60, - ) - await model.wait_for_idle(status="active", timeout=5 * 60) - - coredns_app = model.applications["coredns"] - - # Consume and relate Coredns - # TODO: Should this be moved out into kubernetes cluster function? - await integrate_coredns(ops_test, coredns_model=coredns_model, cluster_model=cluster_model) - - yield - - with ops_test.model_context(coredns_model) as m: - log.info("Removing Coredns charm...") - - log.info(f"Removing coredns ...") - cmd = "remove-application coredns --destroy-storage --force" - rc, stdout, stderr = await ops_test.juju(*shlex.split(cmd)) - log.info(f"{(stdout or stderr)})") - assert rc == 0 - await m.block_until(lambda: "coredns" not in m.applications, timeout=60 * 10) + coredns_alias = "coredns-model" + # k8s_cloud = await ops_test.add_k8s(skip_storage=False, kubeconfig=) + k8s_model = await ops_test.track_model( + coredns_alias, cloud_name=k8s_cloud, keep=ops_test.ModelKeep.NEVER + ) + await k8s_model.deploy("coredns", trust=True) + await k8s_model.wait_for_idle(apps=["coredns"], status="active") + yield k8s_model + await ops_test.forget_model(coredns_alias) -async def integrate_coredns(ops_test: OpsTest, coredns_model: str = "coredns-model", cluster_model: str = "main"): +@pytest.fixture(scope="module") +async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster): """ This function offers Coredns in the specified Kubernetes (k8s) model. """ log.info("Offering Coredns...") - with ops_test.model_context(coredns_model) as model: - await model.offer("coredns:dns-provider") - offers = await model.list_offers() - await model.block_until( - lambda: all(offer.application_name == 'coredns' #TODO check if this name is correct - for offer in offers.results)) - log.info("Coredns offered...") + await coredns_model.create_offer("coredns:dns-provider") + await coredns_model.block_until(lambda: 'coredns' in coredns_model.application_offers) + log.info("Coredns offered...") + + log.info("Consuming Coredns...") + saas = await kubernetes_cluster.consume("{coredns_model.name}.coredns") + assert "coredns" in kubernetes_cluster.remote_applications + log.info("Coredns consumed...") + + log.info("Relating Coredns...") + await kubernetes_cluster.integrate("k8s:dns-provider", "coredns") + + yield + + # Now let's clean up + await kubernetes_cluster.remove_relation("k8s:dns-provider", "coredns") + await kubernetes_cluster.remove_saas(saas) + await coredns_model.remove_offer(f"{coredns_model.name}.{saas}", force=True) # Juju relate command relate_cmd = f"juju relate -m {cluster_model} coredns k8s" diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 96ec4135..a7c07086 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -175,7 +175,7 @@ async def test_prometheus(traefik_address: str, cos_model: model.Model): ] for query in queries: await prometheus.check_metrics(query) -async def test_coredns_integration(kubernetes_cluster): +async def test_coredns_integration(kubernetes_cluster, integrate_coredns): """Test the coredns integration.""" k8s = kubernetes_cluster.applications["k8s"] @@ -186,8 +186,7 @@ async def test_coredns_integration(kubernetes_cluster): assert dns_relation, "No DNS relation found" assert dns_relation.data["domain"] == "cluster.local", "Domain not set to cluster.local" - -async def test_dns(kubernetes_cluster): +async def test_dns(kubernetes_cluster, integrate_coredns): """ This function performs a DNS test on the specified Kubernetes (k8s) unit in the cluster model. The test is performed by running a pod in the k8s unit and From 181845f0e05beef747588f89680bfd55c045bf67 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 5 Mar 2024 16:23:36 +0100 Subject: [PATCH 13/34] add pytest operator with add-k8s in requirements --- tests/integration/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 69954e1b..7de2ec64 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -262,8 +262,6 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): bundle.switch(charm.app_name, path) async with deploy_model(request, ops_test, model, bundle, raise_on_blocked) as the_model: yield the_model - - ops_test.add_k8s(skip_storage=True) @pytest_asyncio.fixture(name="_grafana_agent", scope="module") @@ -327,7 +325,6 @@ async def model_removed(): async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.Model): """ This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. - It waits for the deployment to complete and ensures that the Coredns application is active. """ log.info(f"Deploying Coredns ") From 37ce5828cf2532ba033bb531dc30a3af559b38f2 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 5 Mar 2024 17:22:12 +0100 Subject: [PATCH 14/34] k8s creds --- tests/integration/conftest.py | 53 +++++------------------------------ 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7de2ec64..7af3c703 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -276,60 +276,20 @@ async def grafana_agent(kubernetes_cluster: Model): await kubernetes_cluster.remove_application("grafana-agent") -@pytest.fixture(scope="module") -async def coredns_model(k8s_cloud, ops_test: OpsTest): - """ - This fixture sets up a Coredns model on the specified Kubernetes (k8s) cloud for testing purposes. - It adds the k8s model, performs necessary operations, and removes the model after the test. - """ - log.info("Creating Coredns model ...") - - model_name = "coredns-model" - await ops_test.juju( - "add-model", - f"--controller={ops_test.controller_name}", - model_name, - k8s_cloud, - "--no-switch", #TODO: does not switch to the new model, does that bite me in the next call to juju? - ) - - model = await ops_test.track_model( - model_name, - model_name=model_name, - cloud_name=k8s_cloud, - # credential_name=k8s_cloud, #TODO: is this necessary? - keep=False, - ) - model_uuid = model.info.uuid - - yield model, model_name - - timeout = 5 * 60 - await ops_test.forget_model(model_name, timeout=timeout, allow_failure=False) - - async def model_removed(): - _, stdout, stderr = await ops_test.juju("models", "--format", "yaml") - if _ != 0: - return False - model_list = yaml.safe_load(stdout)["models"] - which = [m for m in model_list if m["model-uuid"] == model_uuid] - return len(which) == 0 - - log.info("Removing Coredns model") - await juju.utils.block_until_with_coroutine(model_removed, timeout=timeout) - # Update client's model cache - await ops_test.juju("models") - log.info("Coredns model removed ...") @pytest.fixture(scope="module") -async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.Model): +async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model): """ This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. """ log.info(f"Deploying Coredns ") coredns_alias = "coredns-model" - # k8s_cloud = await ops_test.add_k8s(skip_storage=False, kubeconfig=) + + k8s = kubernetes_cluster.applications["k8s"].units[0] + client_config = k8s.config.load_config() + + k8s_cloud = await ops_test.add_k8s(skip_storage=False, kubeconfig=client_config) k8s_model = await ops_test.track_model( coredns_alias, cloud_name=k8s_cloud, keep=ops_test.ModelKeep.NEVER ) @@ -338,6 +298,7 @@ async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.Model): yield k8s_model await ops_test.forget_model(coredns_alias) + @pytest.fixture(scope="module") async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster): """ From b4c96a4152a025c34582cf4748d4c45696b50ffd Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 10:11:14 +0100 Subject: [PATCH 15/34] get kubeconfig --- tests/integration/conftest.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7af3c703..dd4ad8da 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -287,7 +287,7 @@ async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model): coredns_alias = "coredns-model" k8s = kubernetes_cluster.applications["k8s"].units[0] - client_config = k8s.config.load_config() + client_config = await get_kubeconfig(k8s) k8s_cloud = await ops_test.add_k8s(skip_storage=False, kubeconfig=client_config) k8s_model = await ops_test.track_model( @@ -299,6 +299,22 @@ async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model): await ops_test.forget_model(coredns_alias) +async def get_kubeconfig(k8s): + """Return Node list + + Args: + k8s: any k8s unit + + Returns: + kubeconfig + """ + action = await k8s.run("k8s kubectl config view --raw") + result = await action.wait() + assert result.results["return-code"] == 0, "Failed to get kubeconfig with kubectl" + log.info("Parsing node list...") + return result.results["stdout"] + + @pytest.fixture(scope="module") async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster): """ From 95073b18a3f03b58601b141bb573280298370dca Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 10:55:57 +0100 Subject: [PATCH 16/34] fix test --- tests/integration/test_k8s.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index a7c07086..05903bf6 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -178,8 +178,10 @@ async def test_prometheus(traefik_address: str, cos_model: model.Model): async def test_coredns_integration(kubernetes_cluster, integrate_coredns): """Test the coredns integration.""" k8s = kubernetes_cluster.applications["k8s"] + k8s_unit = k8s.units[0] - dns_relation = k8s.model.get_relation("dns-provider") + dns_relation = k8s_unit.get_relation("dns-provider") + log.info("DNS relation: %s", dns_relation) #TODO make sure this works # Check if the DNS relation is set, and the domain is set to cluster.local From 2314eec2299a8b5d803afe85a64b4bc2bbbd1dfa Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 11:18:01 +0100 Subject: [PATCH 17/34] tweak test --- tests/integration/test_k8s.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 05903bf6..e1dfec09 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -180,6 +180,12 @@ async def test_coredns_integration(kubernetes_cluster, integrate_coredns): k8s = kubernetes_cluster.applications["k8s"] k8s_unit = k8s.units[0] + coredns = integrate_coredns.applications["coredns"] + coredns_unit = coredns.units[0] + log.info("Coredns: %s", coredns) + log.info("coredns offers %s", coredns.application_offers) + log.info("K8s: %s", k8s_unit) + dns_relation = k8s_unit.get_relation("dns-provider") log.info("DNS relation: %s", dns_relation) From b3e1063df17bbb0e90788dc974d664fcf58d7da3 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 11:27:53 +0100 Subject: [PATCH 18/34] add log --- tests/integration/conftest.py | 2 +- tests/integration/test_k8s.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index dd4ad8da..bec1bbbd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -333,7 +333,7 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster log.info("Relating Coredns...") await kubernetes_cluster.integrate("k8s:dns-provider", "coredns") - yield + yield coredns_model # Now let's clean up await kubernetes_cluster.remove_relation("k8s:dns-provider", "coredns") diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index e1dfec09..06716567 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -207,7 +207,8 @@ async def test_dns(kubernetes_cluster, integrate_coredns): exec_cmd = f"juju exec --unit {k8s_unit} -- k8s kubectl run --rm -it --image alpine --restart=Never test-dns -- nslookup canonical.com" action = await k8s.units[0].run(exec_cmd) result = await action.wait() - assert result.results["return-code"] == 0, "DNS Test failed" + log.info("DNS test result: %s", result) + assert result.results["return-code"] == 0, "DNS Test failed." output = json.loads(result.results["stdout"]) From c61854c282687da542ab0acb85461f1522c44eec Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 12:08:38 +0100 Subject: [PATCH 19/34] try sth --- tests/integration/conftest.py | 2 ++ tests/integration/test_k8s.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bec1bbbd..c7681c53 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -289,6 +289,8 @@ async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model): k8s = kubernetes_cluster.applications["k8s"].units[0] client_config = await get_kubeconfig(k8s) + log.info("Adding k8s cloud") + log.info("Kubeconfig: %s", client_config) k8s_cloud = await ops_test.add_k8s(skip_storage=False, kubeconfig=client_config) k8s_model = await ops_test.track_model( coredns_alias, cloud_name=k8s_cloud, keep=ops_test.ModelKeep.NEVER diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 06716567..a93e4e88 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -175,7 +175,7 @@ async def test_prometheus(traefik_address: str, cos_model: model.Model): ] for query in queries: await prometheus.check_metrics(query) -async def test_coredns_integration(kubernetes_cluster, integrate_coredns): +async def test_coredns_integration(kubernetes_cluster: model.Model, integrate_coredns: model.Model): """Test the coredns integration.""" k8s = kubernetes_cluster.applications["k8s"] k8s_unit = k8s.units[0] @@ -194,7 +194,7 @@ async def test_coredns_integration(kubernetes_cluster, integrate_coredns): assert dns_relation, "No DNS relation found" assert dns_relation.data["domain"] == "cluster.local", "Domain not set to cluster.local" -async def test_dns(kubernetes_cluster, integrate_coredns): +async def test_dns(kubernetes_cluster: model.Model, integrate_coredns: model.Model): """ This function performs a DNS test on the specified Kubernetes (k8s) unit in the cluster model. The test is performed by running a pod in the k8s unit and From e7d589f6b2dfd6b6c137474725568e83f9620647 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 14:08:48 +0100 Subject: [PATCH 20/34] change back to fixture --- tests/integration/test_k8s.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index a93e4e88..82d25990 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -175,6 +175,8 @@ async def test_prometheus(traefik_address: str, cos_model: model.Model): ] for query in queries: await prometheus.check_metrics(query) + + async def test_coredns_integration(kubernetes_cluster: model.Model, integrate_coredns: model.Model): """Test the coredns integration.""" k8s = kubernetes_cluster.applications["k8s"] From 47464249acd3f15c159318a470f75600ee87c114 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 15:36:02 +0100 Subject: [PATCH 21/34] fixture test --- tests/integration/conftest.py | 4 ++-- tests/integration/test_k8s.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c7681c53..60c5a798 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -278,7 +278,7 @@ async def grafana_agent(kubernetes_cluster: Model): @pytest.fixture(scope="module") -async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model): +async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model.Model): """ This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. """ @@ -335,7 +335,7 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster log.info("Relating Coredns...") await kubernetes_cluster.integrate("k8s:dns-provider", "coredns") - yield coredns_model + yield # Now let's clean up await kubernetes_cluster.remove_relation("k8s:dns-provider", "coredns") diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 82d25990..a0b3751f 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -177,6 +177,11 @@ async def test_prometheus(traefik_address: str, cos_model: model.Model): await prometheus.check_metrics(query) +async def test_fixtures(kubernetes_cluster: model.Model, integrate_coredns: model.Model): + """Test the coredns integration.""" + log.info("Testing coredns integration...") + + async def test_coredns_integration(kubernetes_cluster: model.Model, integrate_coredns: model.Model): """Test the coredns integration.""" k8s = kubernetes_cluster.applications["k8s"] From 2be211463467c8044c986bab7e36cb227b139820 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 6 Mar 2024 16:14:41 +0100 Subject: [PATCH 22/34] use mark fixture --- tests/integration/test_k8s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index a0b3751f..8e0e9b39 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -187,7 +187,7 @@ async def test_coredns_integration(kubernetes_cluster: model.Model, integrate_co k8s = kubernetes_cluster.applications["k8s"] k8s_unit = k8s.units[0] - coredns = integrate_coredns.applications["coredns"] + coredns = coredns_model.applications["coredns"] coredns_unit = coredns.units[0] log.info("Coredns: %s", coredns) log.info("coredns offers %s", coredns.application_offers) From 06a9dc9006b31dd65006f60e42f4f5dfefa03b13 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 6 Mar 2024 10:20:46 -0600 Subject: [PATCH 23/34] improvements from pair programming --- tests/integration/conftest.py | 44 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 60c5a798..b29b39a5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,6 +8,8 @@ import logging import shlex from dataclasses import dataclass, field +from kubernetes import config as k8s_config +from kubernetes.client import Configuration from itertools import chain from pathlib import Path from typing import List, Mapping, Optional @@ -277,8 +279,23 @@ async def grafana_agent(kubernetes_cluster: Model): await kubernetes_cluster.remove_application("grafana-agent") -@pytest.fixture(scope="module") -async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model.Model): +@pytest_asyncio.fixture(scope="module") +async def cluster_kubeconfig(ops_test: OpsTest, kubernetes_cluster: juju.model.Model): + """ + Fixture to pull the kubeconfig out of the kubernetes cluster + """ + k8s = kubernetes_cluster.applications["k8s"].units[0] + action = await k8s.run("k8s config") + result = await action.wait() + assert result.results["return-code"] == 0, "Failed to get kubeconfig with kubectl" + + kubeconfig_path = ops_test.tmp_path / "kubeconfig" + kubeconfig_path.write_text(result.results["stdout"]) + yield kubeconfig_path + + +@pytest_asyncio.fixture(scope="module") +async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): """ This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. """ @@ -286,12 +303,11 @@ async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model.Model) coredns_alias = "coredns-model" - k8s = kubernetes_cluster.applications["k8s"].units[0] - client_config = await get_kubeconfig(k8s) + config = type.__call__(Configuration) + k8s_config.load_config(client_configuration=config, config_file=str(cluster_kubeconfig)) log.info("Adding k8s cloud") - log.info("Kubeconfig: %s", client_config) - k8s_cloud = await ops_test.add_k8s(skip_storage=False, kubeconfig=client_config) + k8s_cloud = await ops_test.add_k8s(skip_storage=True, kubeconfig=config) k8s_model = await ops_test.track_model( coredns_alias, cloud_name=k8s_cloud, keep=ops_test.ModelKeep.NEVER ) @@ -301,22 +317,6 @@ async def coredns_model(ops_test: OpsTest, kubernetes_cluster: juju.model.Model) await ops_test.forget_model(coredns_alias) -async def get_kubeconfig(k8s): - """Return Node list - - Args: - k8s: any k8s unit - - Returns: - kubeconfig - """ - action = await k8s.run("k8s kubectl config view --raw") - result = await action.wait() - assert result.results["return-code"] == 0, "Failed to get kubeconfig with kubectl" - log.info("Parsing node list...") - return result.results["stdout"] - - @pytest.fixture(scope="module") async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster): """ From c6f95f8e4c40f46722a6c499065e5379c643908d Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 7 Mar 2024 14:58:03 +0100 Subject: [PATCH 24/34] fix fixtures --- CONTRIBUTING.md | 2 +- tests/integration/conftest.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40bbddeb..7f6fc98e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -122,7 +122,7 @@ The COS integration tests are optional as these are slow/heavy tests. Currently, #### Useful arguments `--keep-models`: Doesn't delete the model once the integration tests are finished -`--model`: Rerun the test with a given model name -- if it already exist, the integration tests will use it +`--model`: Rerun the test with a given model name -- if it already exist, the integration tests will use it. The model is not destroyed after the tests are finished. `-k regex-pattern`: run a specific set of matching tests names ignore other passing tests Remember that cloud costs could be incurred for every machine -- so be sure to clean up your models on clouds if you instruct pytest-operator to not clean up the models. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b29b39a5..a1bb8adf 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -314,11 +314,14 @@ async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): await k8s_model.deploy("coredns", trust=True) await k8s_model.wait_for_idle(apps=["coredns"], status="active") yield k8s_model + + # Now let's clean up + await k8s_model.remove_offer("coredns:dns-provider", force=True) await ops_test.forget_model(coredns_alias) -@pytest.fixture(scope="module") -async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster): +@pytest_asyncio.fixture(scope="module") +async def integrate_coredns(ops_test: OpsTest, coredns_model: juju.model.Model, kubernetes_cluster: juju.model.Model): """ This function offers Coredns in the specified Kubernetes (k8s) model. """ @@ -328,12 +331,19 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model, kubernetes_cluster log.info("Coredns offered...") log.info("Consuming Coredns...") - saas = await kubernetes_cluster.consume("{coredns_model.name}.coredns") - assert "coredns" in kubernetes_cluster.remote_applications + model_owner = untag("user-", coredns_model.info.owner_tag) + + await coredns_model.wait_for_idle(status="active") + await kubernetes_cluster.wait_for_idle(status="active") + + offer_url = f"{model_owner}/{coredns_model.name}.coredns" + saas = await kubernetes_cluster.consume(offer_url) + log.info("Coredns consumed...") log.info("Relating Coredns...") await kubernetes_cluster.integrate("k8s:dns-provider", "coredns") + assert "coredns" in kubernetes_cluster.remote_applications yield From 82e809a522b1b73d4224f7e1005a4f283b2726ac Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 7 Mar 2024 15:34:07 +0100 Subject: [PATCH 25/34] cleanup --- tests/integration/conftest.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a1bb8adf..4bd21a9d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -315,8 +315,7 @@ async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): await k8s_model.wait_for_idle(apps=["coredns"], status="active") yield k8s_model - # Now let's clean up - await k8s_model.remove_offer("coredns:dns-provider", force=True) + # the cluster is consuming this model: remove saas first await ops_test.forget_model(coredns_alias) @@ -348,29 +347,10 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model: juju.model.Model, yield # Now let's clean up - await kubernetes_cluster.remove_relation("k8s:dns-provider", "coredns") - await kubernetes_cluster.remove_saas(saas) await coredns_model.remove_offer(f"{coredns_model.name}.{saas}", force=True) + await kubernetes_cluster.remove_application("coredns") + await kubernetes_cluster.remove_saas(saas) - # Juju relate command - relate_cmd = f"juju relate -m {cluster_model} coredns k8s" - rc, stdout, stderr = await ops_test.juju(*shlex.split(relate_cmd)) - log.info(f"{(stdout or stderr)}") - assert rc == 0, "Failed to relate Coredns in the cluster model with k8s" - - status = await model_2.get_status() - if 'coredns' not in status.remote_applications: - raise Exception("Expected coredns") - log.info("Coredns consumed...") - - log.info("Relating Coredns...") - await model_2.relate("coredns:dns-provider admin/{}.coredns".format(coredns_model)) - if 'coredns' not in status.remote_applications: - raise Exception("Expected coredns") - log.info("Coredns related...") - - # TODO cleanup - # await model.remove_offer("admin/{}.ubuntu".format(model.name), force=True) #TODO: when do we remove the offer? @pytest_asyncio.fixture(scope="module") From bf40711cd4337db538dd48a76945ed023c21dbfe Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 7 Mar 2024 16:21:52 +0100 Subject: [PATCH 26/34] correct cleanup order --- tests/integration/conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4bd21a9d..d00f5da6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -316,7 +316,7 @@ async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): yield k8s_model # the cluster is consuming this model: remove saas first - await ops_test.forget_model(coredns_alias) + await ops_test.forget_model(coredns_alias, timeout=40) @pytest_asyncio.fixture(scope="module") @@ -347,10 +347,11 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model: juju.model.Model, yield # Now let's clean up - await coredns_model.remove_offer(f"{coredns_model.name}.{saas}", force=True) - await kubernetes_cluster.remove_application("coredns") + cluster = kubernetes_cluster + await kubernetes_cluster.applications["k8s"].destroy_relation("k8s:dns-provider", "coredns") + await kubernetes_cluster.wait_for_idle(status="active") await kubernetes_cluster.remove_saas(saas) - + await coredns_model.remove_offer(f"{coredns_model.name}.{saas}", force=True) @pytest_asyncio.fixture(scope="module") From ed3674ebe0076eb757ff6d6fbc8f67461067ccbf Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 8 Mar 2024 10:48:57 +0100 Subject: [PATCH 27/34] update test --- tests/integration/test_k8s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 8e0e9b39..cb4fdc9d 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -211,7 +211,7 @@ async def test_dns(kubernetes_cluster: model.Model, integrate_coredns: model.Mod k8s = kubernetes_cluster.applications["k8s"] k8s_unit = k8s.units[0] # Do we need to switch models? - exec_cmd = f"juju exec --unit {k8s_unit} -- k8s kubectl run --rm -it --image alpine --restart=Never test-dns -- nslookup canonical.com" + exec_cmd = f"k8s kubectl run --rm -it --image alpine --restart=Never test-dns -- nslookup canonical.com" action = await k8s.units[0].run(exec_cmd) result = await action.wait() log.info("DNS test result: %s", result) From 236cf5edc468fe399c670be348f8dfad6627f29c Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 12 Mar 2024 14:33:15 +0100 Subject: [PATCH 28/34] rm empty test --- tests/integration/test_k8s.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index cb4fdc9d..10a03409 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -201,12 +201,14 @@ async def test_coredns_integration(kubernetes_cluster: model.Model, integrate_co assert dns_relation, "No DNS relation found" assert dns_relation.data["domain"] == "cluster.local", "Domain not set to cluster.local" +@pytest.mark.skip(reason="Test skipped until cluster configs are propagated properly") async def test_dns(kubernetes_cluster: model.Model, integrate_coredns: model.Model): """ This function performs a DNS test on the specified Kubernetes (k8s) unit in the cluster model. The test is performed by running a pod in the k8s unit and checking if it can resolve the domain name (See: https://charmhub.io/microk8s/docs/how-to-advanced-dns). """ + #TODO: Validate the DNS test works once cluster configs are propagated properly log.info("Running DNS test...") k8s = kubernetes_cluster.applications["k8s"] k8s_unit = k8s.units[0] From 4e8d2ea1a459745d0dc8928f9ca017378fe495a1 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 12 Mar 2024 15:20:50 +0100 Subject: [PATCH 29/34] linting --- .../k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 5 +--- charms/worker/k8s/src/charm.py | 10 +++++--- tests/integration/conftest.py | 25 ++++++++----------- tests/integration/cos_substrate.py | 3 ++- tests/integration/grafana.py | 1 + tests/integration/prometheus.py | 2 ++ tests/integration/test_k8s.py | 15 +++++------ 7 files changed, 32 insertions(+), 29 deletions(-) diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index d5251c4c..5a99f5c5 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -723,10 +723,7 @@ def configure_dns(self, dns_domain: str, dns_ip: str): dns_ip (str): The IP address for the DNS. """ endpoint = "/1.0/k8sd/cluster/config" - body = { - "dns.dns-ip": dns_ip, - "dns.cluster-domain": dns_domain, - "dns.enabled": False} + body = {"dns.dns-ip": dns_ip, "dns.cluster-domain": dns_domain, "dns.enabled": False} self._send_request(endpoint, "POST", EmptyResponse, body) def get_cluster_status(self) -> GetClusterStatusResponse: diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index e8905447..8b1ea018 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -441,7 +441,7 @@ def _dns_charm_integrated(self) -> Optional[dict]: dns_config = self._get_dns_config() network_config = NetworkConfig(enabled=True) - + user_cluster_config = UserFacingClusterConfig(dns=dns_config, network=network_config) update_request = UpdateClusterConfigRequest(config=user_cluster_config) self.api_manager.update_cluster_config(update_request) @@ -453,8 +453,12 @@ def _get_dns_config(self) -> Optional[dict]: DNSConfig: A DNSConfig object with the cluster domain, service IP and whether the default dns is enabled or not. """ if self.kube_dns.domain is not None and self.kube_dns.address is not None: - return DNSConfig(enabled=False, cluster_domain=self.kube_dns.domain, service_ip=self.kube_dns.address) - + return DNSConfig( + enabled=False, + cluster_domain=self.kube_dns.domain, + service_ip=self.kube_dns.address, + ) + return DNSConfig(enabled=True) def _get_scrape_jobs(self): diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d00f5da6..cfe959d7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,13 +8,10 @@ import logging import shlex from dataclasses import dataclass, field -from kubernetes import config as k8s_config -from kubernetes.client import Configuration from itertools import chain from pathlib import Path from typing import List, Mapping, Optional -import juju.utils -import shlex + import pytest import pytest_asyncio import yaml @@ -297,9 +294,10 @@ async def cluster_kubeconfig(ops_test: OpsTest, kubernetes_cluster: juju.model.M @pytest_asyncio.fixture(scope="module") async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): """ - This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. + This fixture deploys Coredns on the specified + Kubernetes (k8s) model for testing purposes. """ - log.info(f"Deploying Coredns ") + log.info("Deploying Coredns ") coredns_alias = "coredns-model" @@ -326,30 +324,29 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model: juju.model.Model, """ log.info("Offering Coredns...") await coredns_model.create_offer("coredns:dns-provider") - await coredns_model.block_until(lambda: 'coredns' in coredns_model.application_offers) + await coredns_model.block_until(lambda: "coredns" in coredns_model.application_offers) log.info("Coredns offered...") log.info("Consuming Coredns...") model_owner = untag("user-", coredns_model.info.owner_tag) - + await coredns_model.wait_for_idle(status="active") - await kubernetes_cluster.wait_for_idle(status="active") + await kubernetes_cluster.wait_for_idle(status="active") offer_url = f"{model_owner}/{coredns_model.name}.coredns" saas = await kubernetes_cluster.consume(offer_url) log.info("Coredns consumed...") - + log.info("Relating Coredns...") await kubernetes_cluster.integrate("k8s:dns-provider", "coredns") assert "coredns" in kubernetes_cluster.remote_applications - + yield - + # Now let's clean up - cluster = kubernetes_cluster await kubernetes_cluster.applications["k8s"].destroy_relation("k8s:dns-provider", "coredns") - await kubernetes_cluster.wait_for_idle(status="active") + await kubernetes_cluster.wait_for_idle(status="active") await kubernetes_cluster.remove_saas(saas) await coredns_model.remove_offer(f"{coredns_model.name}.{saas}", force=True) diff --git a/tests/integration/cos_substrate.py b/tests/integration/cos_substrate.py index 0e6c2764..2d799168 100644 --- a/tests/integration/cos_substrate.py +++ b/tests/integration/cos_substrate.py @@ -253,7 +253,8 @@ def execute_command(self, container, command: List[str]): return None def get_kubeconfig(self, container) -> str: - """Get kubeconfig from a container. + """ + Get kubeconfig from a container. Args: container: Container instance. diff --git a/tests/integration/grafana.py b/tests/integration/grafana.py index 8ee725c1..184f0ede 100644 --- a/tests/integration/grafana.py +++ b/tests/integration/grafana.py @@ -35,6 +35,7 @@ def __init__( def _get_with_auth(self, url: str) -> str: """Send GET request with basic authentication. + Raises AssertionError: If the response status code is not 200. Args: url (str): The URL to send the request to. diff --git a/tests/integration/prometheus.py b/tests/integration/prometheus.py index e0716813..8602860e 100644 --- a/tests/integration/prometheus.py +++ b/tests/integration/prometheus.py @@ -28,6 +28,7 @@ def __init__( def _get_url(self, url): """Send GET request to the provided URL. + Raises AssertionError: If the response status code is not 200. Args: url (str): The URL to send the request to. @@ -66,6 +67,7 @@ async def health(self) -> str: async def check_metrics(self, query: str): """Query Prometheus for metrics. + Raises AssertionError: If the query fails or if data is not yet available. Args: query (str): The Prometheus query to execute. diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 10a03409..dbaa5ab5 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -205,21 +205,22 @@ async def test_coredns_integration(kubernetes_cluster: model.Model, integrate_co async def test_dns(kubernetes_cluster: model.Model, integrate_coredns: model.Model): """ This function performs a DNS test on the specified Kubernetes (k8s) unit in the cluster model. - The test is performed by running a pod in the k8s unit and - checking if it can resolve the domain name (See: https://charmhub.io/microk8s/docs/how-to-advanced-dns). + The test is performed by running a pod in the k8s unit and + checking if it can resolve the domain name + (See: https://charmhub.io/microk8s/docs/how-to-advanced-dns). """ - #TODO: Validate the DNS test works once cluster configs are propagated properly + # TODO: Validate the DNS test works once cluster configs are propagated properly log.info("Running DNS test...") k8s = kubernetes_cluster.applications["k8s"] - k8s_unit = k8s.units[0] # Do we need to switch models? - exec_cmd = f"k8s kubectl run --rm -it --image alpine --restart=Never test-dns -- nslookup canonical.com" + exec_cmd = "k8s kubectl run --rm -it --image alpine \ + --restart=Never test-dns -- nslookup canonical.com" action = await k8s.units[0].run(exec_cmd) result = await action.wait() log.info("DNS test result: %s", result) assert result.results["return-code"] == 0, "DNS Test failed." output = json.loads(result.results["stdout"]) - - #TODO test this output, https://charmhub.io/microk8s/docs/how-to-advanced-dns + assert "canonical.com" in output, "Canonical.com not found in DNS result." + # TODO: test this output, https://charmhub.io/microk8s/docs/how-to-advanced-dns log.info("DNS test passed.") From 910fdf9b4c326375f42a43bbb60209b7f59d6de5 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 12 Mar 2024 16:07:08 +0100 Subject: [PATCH 30/34] lint --- charms/worker/k8s/src/charm.py | 3 +- charms/worker/pyproject.toml | 82 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 charms/worker/pyproject.toml diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index 8b1ea018..c052693b 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -450,7 +450,8 @@ def _get_dns_config(self) -> Optional[dict]: """Get DNS config either for the enabled built-in dns or an integrated charm. Returns: - DNSConfig: A DNSConfig object with the cluster domain, service IP and whether the default dns is enabled or not. + DNSConfig: A DNSConfig object with the cluster domain, + service IP and whether the default dns is enabled or not. """ if self.kube_dns.domain is not None and self.kube_dns.address is not None: return DNSConfig( diff --git a/charms/worker/pyproject.toml b/charms/worker/pyproject.toml new file mode 100644 index 00000000..1f0b158e --- /dev/null +++ b/charms/worker/pyproject.toml @@ -0,0 +1,82 @@ +[tool.bandit] +exclude_dirs = ["/venv/", "tests"] +skips = ["B404","B603"] +[tool.bandit.assert_used] +skips = ["*/*test.py", "*/test_*.py", "*tests/*.py"] + +# Testing tools configuration +[tool.coverage.run] +branch = true + +# Formatting tools configuration +[tool.black] +line-length = 99 +target-version = ["py38"] + +[tool.coverage.report] +show_missing = true + +# Linting tools configuration +[tool.flake8] +max-line-length = 99 +max-doc-length = 99 +max-complexity = 10 +exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] +select = ["E", "W", "F", "C", "N", "R", "D", "H"] +# Ignore W503, E501 because using black creates errors with this +# Ignore D107 Missing docstring in __init__ +# Ignore N805 first argument should be named self. Pydantic validators do not comply. +ignore = ["W503", "E501", "D107", "N805"] + +# D100, D101, D102, D103: Ignore missing docstrings in tests +per-file-ignores = ["tests/*:D100,D101,D102,D103,D104,D205,D212,D415"] +docstring-convention = "google" + +[tool.isort] +line_length = 99 +profile = "black" + +[tool.mypy] +ignore_missing_imports = true +explicit_package_bases = true +namespace_packages = true + +[tool.pylint] +# Ignore too-few-public-methods due to pydantic models +# Ignore no-self-argument due to pydantic validators +disable = "wrong-import-order,redefined-outer-name,too-few-public-methods,no-self-argument,fixme,too-many-instance-attributes" +# Ignore Pydantic check: https://github.com/pydantic/pydantic/issues/1961 +extension-pkg-whitelist = "pydantic" # wokeignore:rule=whitelist + +[tool.pytest.ini_options] +minversion = "6.0" +log_cli_level = "INFO" + +# Linting tools configuration +[tool.ruff] +line-length = 99 +select = ["E", "W", "F", "C", "N", "D", "I001"] +extend-ignore = [ + "D203", + "D204", + "D213", + "D215", + "D400", + "D404", + "D406", + "D407", + "D408", + "D409", + "D413", +] +ignore = ["E501", "D107"] +extend-exclude = ["__pycache__", "*.egg_info"] +per-file-ignores = {"tests/*" = ["D100","D101","D102","D103","D104"]} + +[tool.ruff.mccabe] +max-complexity = 10 + +[tool.codespell] +skip = "build,lib,venv,icon.svg,.tox,.git,.mypy_cache,.ruff_cache,.coverage" +[tool.pyright] +extraPaths = ["./lib"] \ No newline at end of file From 95e5d07dc7759615f875439fa7ac686dcd1e3a16 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Sat, 23 Mar 2024 17:40:58 +0100 Subject: [PATCH 31/34] try kubeconfig from action --- tests/integration/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index cfe959d7..efbf0775 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -282,12 +282,11 @@ async def cluster_kubeconfig(ops_test: OpsTest, kubernetes_cluster: juju.model.M Fixture to pull the kubeconfig out of the kubernetes cluster """ k8s = kubernetes_cluster.applications["k8s"].units[0] - action = await k8s.run("k8s config") + action = await k8s.run_action("get-kubeconfig") result = await action.wait() assert result.results["return-code"] == 0, "Failed to get kubeconfig with kubectl" - kubeconfig_path = ops_test.tmp_path / "kubeconfig" - kubeconfig_path.write_text(result.results["stdout"]) + kubeconfig_path.write_text(result.results["kubeconfig"]) yield kubeconfig_path From 11c9b70773a7d5d6968accb9f81ea99f4c2bd57e Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 25 Mar 2024 11:34:47 +0100 Subject: [PATCH 32/34] undo kubeconfig --- CONTRIBUTING.md | 2 +- tests/integration/conftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f6fc98e..195fd98a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,7 +113,7 @@ Sometimes you will want to debug certain situations, and having the models torn Running the integration tests with extra arguments can be accomplished with ```shell -tox run -e integration-tests -- --positional --arguments +tox run -e integration -- --positional --arguments ``` #### COS Integration diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index efbf0775..bb9cff6c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -282,11 +282,11 @@ async def cluster_kubeconfig(ops_test: OpsTest, kubernetes_cluster: juju.model.M Fixture to pull the kubeconfig out of the kubernetes cluster """ k8s = kubernetes_cluster.applications["k8s"].units[0] - action = await k8s.run_action("get-kubeconfig") + action = await k8s.run("k8s config") result = await action.wait() assert result.results["return-code"] == 0, "Failed to get kubeconfig with kubectl" kubeconfig_path = ops_test.tmp_path / "kubeconfig" - kubeconfig_path.write_text(result.results["kubeconfig"]) + kubeconfig_path.write_text(result.results["stdout"]) yield kubeconfig_path From 9f1d8f7e6bb81d45479c6d53ab8ac6e55e4b6a2e Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 25 Apr 2024 17:01:10 -0500 Subject: [PATCH 33/34] Post rebase fixup --- charms/worker/k8s/src/charm.py | 31 ++----------------------------- tests/integration/conftest.py | 25 +++++++++---------------- tests/integration/test_k8s.py | 34 ++++------------------------------ 3 files changed, 15 insertions(+), 75 deletions(-) diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index c052693b..eda75920 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -51,7 +51,6 @@ UserFacingDatastoreConfig, ) from charms.kubernetes_libs.v0.etcd import EtcdReactiveRequires -from charms.interface_kube_dns import KubeDnsRequires from charms.node_base import LabelMaker from charms.reconciler import Reconciler from cos_integration import COSIntegration @@ -384,11 +383,10 @@ def _enable_functionalities(self): """Enable necessary components for the Kubernetes cluster.""" status.add(ops.MaintenanceStatus("Updating K8s features")) log.info("Enabling K8s features") - dns_config = DNSConfig(enabled=True) + dns_config = self._get_dns_config() network_config = NetworkConfig(enabled=True) user_cluster_config = UserFacingClusterConfig(dns=dns_config, network=network_config) update_request = UpdateClusterConfigRequest(config=user_cluster_config) - self.api_manager.update_cluster_config(update_request) @on_error( @@ -421,32 +419,7 @@ def _ensure_cluster_config(self): self.api_manager.update_cluster_config(update_request) - def _configure_components(self): - """Enable necessary components for the Kubernetes cluster.""" - dns_settings = self._dns_charm_integrated() - status.add(ops.MaintenanceStatus(f"Configuring DNS")) - - if dns_settings: - self.api_manager.configure_component("dns", False) - self.api_manager.configure_dns(dns_settings.get("dns_domain"), dns_settings.get("dns_ip")) - else: - self.api_manager.configure_component("dns", True) - - status.add(ops.MaintenanceStatus("Enabling Network")) - self.api_manager.configure_component("network", True) - - def _dns_charm_integrated(self) -> Optional[dict]: - """Check if the DNS charm is integrated.""" - status.add(ops.MaintenanceStatus("Configuring DNS and Network")) - - dns_config = self._get_dns_config() - network_config = NetworkConfig(enabled=True) - - user_cluster_config = UserFacingClusterConfig(dns=dns_config, network=network_config) - update_request = UpdateClusterConfigRequest(config=user_cluster_config) - self.api_manager.update_cluster_config(update_request) - - def _get_dns_config(self) -> Optional[dict]: + def _get_dns_config(self) -> DNSConfig: """Get DNS config either for the enabled built-in dns or an integrated charm. Returns: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bb9cff6c..0019f232 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -277,16 +277,13 @@ async def grafana_agent(kubernetes_cluster: Model): @pytest_asyncio.fixture(scope="module") -async def cluster_kubeconfig(ops_test: OpsTest, kubernetes_cluster: juju.model.Model): - """ - Fixture to pull the kubeconfig out of the kubernetes cluster - """ +async def cluster_kubeconfig(ops_test: OpsTest, kubernetes_cluster: Model): + """Fixture to pull the kubeconfig out of the kubernetes cluster.""" k8s = kubernetes_cluster.applications["k8s"].units[0] - action = await k8s.run("k8s config") - result = await action.wait() - assert result.results["return-code"] == 0, "Failed to get kubeconfig with kubectl" + action = await k8s.run_action("get-kubeconfig") + action = await action.wait() kubeconfig_path = ops_test.tmp_path / "kubeconfig" - kubeconfig_path.write_text(result.results["stdout"]) + kubeconfig_path.write_text(action.results["kubeconfig"]) yield kubeconfig_path @@ -296,8 +293,6 @@ async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): This fixture deploys Coredns on the specified Kubernetes (k8s) model for testing purposes. """ - log.info("Deploying Coredns ") - coredns_alias = "coredns-model" config = type.__call__(Configuration) @@ -308,6 +303,7 @@ async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): k8s_model = await ops_test.track_model( coredns_alias, cloud_name=k8s_cloud, keep=ops_test.ModelKeep.NEVER ) + log.info("Deploying Coredns ") await k8s_model.deploy("coredns", trust=True) await k8s_model.wait_for_idle(apps=["coredns"], status="active") yield k8s_model @@ -317,10 +313,8 @@ async def coredns_model(ops_test: OpsTest, cluster_kubeconfig: Path): @pytest_asyncio.fixture(scope="module") -async def integrate_coredns(ops_test: OpsTest, coredns_model: juju.model.Model, kubernetes_cluster: juju.model.Model): - """ - This function offers Coredns in the specified Kubernetes (k8s) model. - """ +async def integrate_coredns(coredns_model: Model, kubernetes_cluster: Model): + """This function offers Coredns in the specified Kubernetes (k8s) model.""" log.info("Offering Coredns...") await coredns_model.create_offer("coredns:dns-provider") await coredns_model.block_until(lambda: "coredns" in coredns_model.application_offers) @@ -334,14 +328,13 @@ async def integrate_coredns(ops_test: OpsTest, coredns_model: juju.model.Model, offer_url = f"{model_owner}/{coredns_model.name}.coredns" saas = await kubernetes_cluster.consume(offer_url) - log.info("Coredns consumed...") log.info("Relating Coredns...") await kubernetes_cluster.integrate("k8s:dns-provider", "coredns") assert "coredns" in kubernetes_cluster.remote_applications - yield + yield coredns_model # Now let's clean up await kubernetes_cluster.applications["k8s"].destroy_relation("k8s:dns-provider", "coredns") diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index dbaa5ab5..1038164f 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -177,50 +177,24 @@ async def test_prometheus(traefik_address: str, cos_model: model.Model): await prometheus.check_metrics(query) -async def test_fixtures(kubernetes_cluster: model.Model, integrate_coredns: model.Model): - """Test the coredns integration.""" - log.info("Testing coredns integration...") - - -async def test_coredns_integration(kubernetes_cluster: model.Model, integrate_coredns: model.Model): - """Test the coredns integration.""" - k8s = kubernetes_cluster.applications["k8s"] - k8s_unit = k8s.units[0] - - coredns = coredns_model.applications["coredns"] - coredns_unit = coredns.units[0] - log.info("Coredns: %s", coredns) - log.info("coredns offers %s", coredns.application_offers) - log.info("K8s: %s", k8s_unit) - - dns_relation = k8s_unit.get_relation("dns-provider") - log.info("DNS relation: %s", dns_relation) - - #TODO make sure this works - # Check if the DNS relation is set, and the domain is set to cluster.local - assert dns_relation, "No DNS relation found" - assert dns_relation.data["domain"] == "cluster.local", "Domain not set to cluster.local" - -@pytest.mark.skip(reason="Test skipped until cluster configs are propagated properly") -async def test_dns(kubernetes_cluster: model.Model, integrate_coredns: model.Model): +@pytest.mark.usefixtures("integrate_coredns") +async def test_dns(kubernetes_cluster: model.Model): """ This function performs a DNS test on the specified Kubernetes (k8s) unit in the cluster model. The test is performed by running a pod in the k8s unit and checking if it can resolve the domain name (See: https://charmhub.io/microk8s/docs/how-to-advanced-dns). """ - # TODO: Validate the DNS test works once cluster configs are propagated properly log.info("Running DNS test...") k8s = kubernetes_cluster.applications["k8s"] - # Do we need to switch models? exec_cmd = "k8s kubectl run --rm -it --image alpine \ --restart=Never test-dns -- nslookup canonical.com" action = await k8s.units[0].run(exec_cmd) result = await action.wait() - log.info("DNS test result: %s", result) + log.info("DNS test result: %s", result.results) assert result.results["return-code"] == 0, "DNS Test failed." - output = json.loads(result.results["stdout"]) + output = result.results["stdout"] assert "canonical.com" in output, "Canonical.com not found in DNS result." # TODO: test this output, https://charmhub.io/microk8s/docs/how-to-advanced-dns log.info("DNS test passed.") From 16b97d56a6bf7e85f93cacff73a4b428609f463e Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 26 Apr 2024 10:31:33 -0500 Subject: [PATCH 34/34] Remove merge mistakes --- .../worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 11 ----------- tests/integration/cos_substrate.py | 3 +-- tests/integration/grafana.py | 1 - tests/integration/prometheus.py | 2 -- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index 5a99f5c5..d928ceae 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -715,17 +715,6 @@ def update_cluster_config(self, config: UpdateClusterConfigRequest): body = config.dict(exclude_none=True, by_alias=True) self._send_request(endpoint, "PUT", EmptyResponse, body) - def configure_dns(self, dns_domain: str, dns_ip: str): - """Configure the DNS for the k8s cluster. - - Args: - dns_domain (str): The domain name for the DNS. - dns_ip (str): The IP address for the DNS. - """ - endpoint = "/1.0/k8sd/cluster/config" - body = {"dns.dns-ip": dns_ip, "dns.cluster-domain": dns_domain, "dns.enabled": False} - self._send_request(endpoint, "POST", EmptyResponse, body) - def get_cluster_status(self) -> GetClusterStatusResponse: """Retrieve cluster status. diff --git a/tests/integration/cos_substrate.py b/tests/integration/cos_substrate.py index 2d799168..0e6c2764 100644 --- a/tests/integration/cos_substrate.py +++ b/tests/integration/cos_substrate.py @@ -253,8 +253,7 @@ def execute_command(self, container, command: List[str]): return None def get_kubeconfig(self, container) -> str: - """ - Get kubeconfig from a container. + """Get kubeconfig from a container. Args: container: Container instance. diff --git a/tests/integration/grafana.py b/tests/integration/grafana.py index 184f0ede..8ee725c1 100644 --- a/tests/integration/grafana.py +++ b/tests/integration/grafana.py @@ -35,7 +35,6 @@ def __init__( def _get_with_auth(self, url: str) -> str: """Send GET request with basic authentication. - Raises AssertionError: If the response status code is not 200. Args: url (str): The URL to send the request to. diff --git a/tests/integration/prometheus.py b/tests/integration/prometheus.py index 8602860e..e0716813 100644 --- a/tests/integration/prometheus.py +++ b/tests/integration/prometheus.py @@ -28,7 +28,6 @@ def __init__( def _get_url(self, url): """Send GET request to the provided URL. - Raises AssertionError: If the response status code is not 200. Args: url (str): The URL to send the request to. @@ -67,7 +66,6 @@ async def health(self) -> str: async def check_metrics(self, query: str): """Query Prometheus for metrics. - Raises AssertionError: If the query fails or if data is not yet available. Args: query (str): The Prometheus query to execute.