Add external-load-balancer relation

charms/worker/k8s/charmcraft.yaml

@@ -389,6 +389,12 @@ config:
       description: |
         Labels can be used to organize and to select subsets of nodes in the
         cluster. Declare node labels in key=value format, separated by spaces.      
+    external-load-balancer-port:
+      description: |
+        Port exposed by the external load balancer to direct traffic to this charm.
+        The external load balancer should be related to this charm on the `external-load-balancer` relation.
+      type: int
+      default: 443
 
 resources:
   snap-installation:
@@ -454,3 +460,5 @@ requires:
     interface: external_cloud_provider
   gcp:
     interface: gcp-integration
+  external-load-balancer:
+    interface: loadbalancer

charms/worker/k8s/requirements.txt

@@ -17,3 +17,4 @@ websocket-client==1.8.0
 poetry-core==1.9.1
 lightkube==0.16.0
 httpx==0.27.2
+loadbalancer_interface == 1.2.0

charms/worker/k8s/src/charm.py

@@ -18,6 +18,7 @@
 import hashlib
 import logging
 import os
+import re
 import shlex
 import socket
 import subprocess
@@ -67,21 +68,30 @@
 from inspector import ClusterInspector
 from kube_control import configure as configure_kube_control
 from literals import (
+    APISERVER_PORT,
     CLUSTER_RELATION,
     CLUSTER_WORKER_RELATION,
     CONTAINERD_RELATION,
     COS_RELATION,
     COS_TOKENS_RELATION,
     COS_TOKENS_WORKER_RELATION,
     DEPENDENCIES,
+    ENDPOINT_HAS_PORT_REGEX,
     ETC_KUBERNETES,
     ETCD_RELATION,
+    EXTERNAL_LOAD_BALANCER_RELATION,
+    EXTERNAL_LOAD_BALANCER_PORT_CONFIG,
+    EXTERNAL_LOAD_BALANCER_REQUEST_NAME,
+    EXTERNAL_LOAD_BALANCER_RESPONSE_NAME,
+    HTTP_SCHEME,
+    HTTPS_SCHEME,
     K8SD_PORT,
     K8SD_SNAP_SOCKET,
     KUBECONFIG,
     KUBECTL_PATH,
     SUPPORTED_DATASTORES,
 )
+from loadbalancer_interface import LBProvider
 from ops.interface_kube_control import KubeControlProvides
 from pydantic import SecretStr
 from snap import management as snap_management
@@ -94,7 +104,7 @@
 log = logging.getLogger(__name__)
 
 
-def _get_public_address() -> str:
+def _get_juju_public_address() -> str:
     """Get public address from juju.
 
     Returns:
@@ -192,6 +202,7 @@ def __init__(self, *args):
             self.etcd = EtcdReactiveRequires(self)
             self.kube_control = KubeControlProvides(self, endpoint="kube-control")
             self.framework.observe(self.on.get_kubeconfig_action, self._get_external_kubeconfig)
+            self.external_load_balancer = LBProvider(self, EXTERNAL_LOAD_BALANCER_RELATION)
 
     def _k8s_info(self, event: ops.EventBase):
         """Send cluster information on the kubernetes-info relation.
@@ -362,9 +373,13 @@ def _check_k8sd_ready(self):
     def _get_extra_sans(self):
         """Retrieve the certificate extra SANs."""
         extra_sans_str = str(self.config.get("kube-apiserver-extra-sans") or "")
-        configured_sans = {san for san in extra_sans_str.strip().split() if san}
-        all_sans = configured_sans | set([_get_public_address()])
-        return sorted(all_sans)
+        extra_sans = {san for san in extra_sans_str.strip().split() if san}
+        if public_address := self._get_public_address():
+            log.info("Public address %s found, adding it to extra SANs", public_address)
+            extra_sans.add(public_address)
+        else:
+            log.info("No public address found, skipping adding public address to extra SANs")
+        return sorted(extra_sans)
 
     def _assemble_bootstrap_config(self):
         """Assemble the bootstrap configuration for the Kubernetes cluster.
@@ -383,6 +398,64 @@ def _assemble_bootstrap_config(self):
         config.extra_args.craft(self.config, bootstrap_config, cluster_name)
         return bootstrap_config
 
+    @on_error(ops.WaitingStatus("Waiting for external load balancer"), ReconcilerError)
+    def _configure_external_load_balancer(self):
+        """Configure the external load balancer for the application.
+
+        This method checks if the external load balancer is available and then
+        proceeds to configure it by sending a request with the necessary parameters.
+        It waits for a response from the external load balancer and handles any errors that
+        may occur during the process.
+
+        Raises:
+            ReconcilerError: If there is an error configuring the external load balancer
+                            or if a response is not received within the timeout period.
+        """
+        if not self.is_control_plane:
+            log.info("External load balancer is only configured for control-plane units.")
+            return
+
+        if not self.external_load_balancer.is_available:
+            log.info("External load balancer relation is not available. Skipping setup.")
+            return
+
+        status.add(ops.MaintenanceStatus("Configuring external loadBalancer"))
+
+        req = self.external_load_balancer.get_request(EXTERNAL_LOAD_BALANCER_REQUEST_NAME)
+        req.protocol = req.protocols.tcp
+        req.port_mapping = {self.config.get(EXTERNAL_LOAD_BALANCER_PORT_CONFIG): APISERVER_PORT}
+        req.public = True
+        if not req.health_checks:
+            req.add_health_check(protocol=req.protocols.http, port=APISERVER_PORT, path="/livez")
+        self.external_load_balancer.send_request(req)
+
+        # wait for response
+        retry_interval_seconds = 3
+        retries = 10
+        for _ in range(retries):
+            res = self.external_load_balancer.get_response(EXTERNAL_LOAD_BALANCER_RESPONSE_NAME)
+            if res is None:
+                log.info(
+                    "Waiting for external load balancer response. Retrying in %s seconds",
+                    retry_interval_seconds,
+                )
+                sleep(retry_interval_seconds)
+                continue
+            if res.error_message:
+                log.error("Error from external load balancer: %s", res.error_message)
+                raise ReconcilerError(
+                    "Failed to configure external load balancer. Check logs for details"
+                )
+
+            log.info("External load balancer response received: %s", res)
+            return
+
+        log.error(
+            "Timed out waiting for external load balancer response after %s seconds",
+            retry_interval_seconds * retries,
+        )
+        raise ReconcilerError("External load balancer response not received")
+
     @on_error(
         ops.WaitingStatus("Waiting to bootstrap k8s snap"),
         ReconcilerError,
@@ -887,6 +960,7 @@ def _reconcile(self, event: ops.EventBase):
         self._update_kubernetes_version()
         if self.lead_control_plane:
             self._k8s_info(event)
+            self._configure_external_load_balancer()
             self._bootstrap_k8s_snap()
             self._ensure_cluster_config()
             self._create_cluster_tokens()
@@ -1059,14 +1133,81 @@ def _get_external_kubeconfig(self, event: ops.ActionEvent):
         try:
             server = event.params.get("server")
             if not server:
-                log.info("No server requested, use public-address")
-                server = f"{_get_public_address()}:6443"
+                log.info("No server requested, use public address")
+                server = self._get_public_address()
+                if not server:
+                    log.info("No public address found")
+                else:
+                    log.info("Found public address: %s", server)
+                    port = str(APISERVER_PORT)
+                    if self.is_control_plane and self.external_load_balancer.is_available:
+                        log.info("Using external load balancer port as the public port")
+                        port = str(self.config.get(EXTERNAL_LOAD_BALANCER_PORT_CONFIG))
+                    server = self._format_kube_api_url(server, port)
+                    log.info("Formatted server address: %s", server)
             log.info("Requesting kubeconfig for server=%s", server)
             resp = self.api_manager.get_kubeconfig(server)
             event.set_results({"kubeconfig": resp})
         except (InvalidResponseError, K8sdConnectionError) as e:
             event.fail(f"Failed to retrieve kubeconfig: {e}")
 
+    def _get_public_address(self) -> Optional[str]:
+        """Get public address either from external load balancer or from juju.
+
+        Returns:
+            str: public ip address of the unit
+            None: if the public address is not available
+        """
+        if self.is_control_plane and self.external_load_balancer.is_available:
+            log.info("Using external load balancer address as the public address")
+            return self._get_external_load_balancer_address()
+        log.info("Using juju public address as the public address")
+        return _get_juju_public_address()
+
+    def _get_external_load_balancer_address(self) -> Optional[str]:
+        """Get the external load balancer address.
+
+        Returns:
+            None: If the external load balancer address is not available.
+            str: The external load balancer address.
+        """
+        if not self.is_control_plane:
+            log.error("External load balancer address is only available for control-plane units")
+            return None
+
+        response = self.external_load_balancer.get_response(EXTERNAL_LOAD_BALANCER_RESPONSE_NAME)
+
+        if not response:
+            log.error("No response from external load balancer when trying to get address")
+            return None
+        if response.error:
+            log.error(
+                "Error from external load balancer when trying to get address: %s",
+                response.error,
+            )
+            return None
+
+        return response.address
+
+    def _format_kube_api_url(self, addr: str, port: str) -> str:
+        """Format the given Kubernetes API address to ensure it includes the protocol and port.
+
+        Args:
+            addr (str): The Kubernetes API address.
+            port (str): The Kubernetes API port.
+
+        Returns:
+            str: The formatted Kubernetes API address with protocol and port.
+        """
+        addr = addr.lower()
+        if not addr.startswith(HTTPS_SCHEME) and not addr.startswith(HTTP_SCHEME):
+            log.info("Adding %s to addr %s", HTTPS_SCHEME, addr)
+            addr = HTTPS_SCHEME + addr
+        if re.search(ENDPOINT_HAS_PORT_REGEX, addr) is None:
+            log.info("Adding port %s to addr %s", port, addr)
+            addr = f"{addr}:{port}"
+        return addr
+
 
 if __name__ == "__main__":  # pragma: nocover
     ops.main(K8sCharm)

charms/worker/k8s/src/kube_control.py

@@ -8,6 +8,7 @@
 import charms.contextual_status as status
 import ops
 import yaml
+from literals import APISERVER_PORT
 from protocols import K8sCharmProtocol
 
 # Log messages can be retrieved using juju debug-log
@@ -24,7 +25,7 @@ def configure(charm: K8sCharmProtocol):
         return
 
     status.add(ops.MaintenanceStatus("Configuring Kube Control"))
-    ca_cert, endpoints = "", [f"https://{binding.network.bind_address}:6443"]
+    ca_cert, endpoints = "", [f"https://{binding.network.bind_address}:{APISERVER_PORT}"]
     labels = str(charm.model.config["node-labels"])
     taints = str(charm.model.config["bootstrap-node-taints"])
     if charm._internal_kubeconfig.exists():

charms/worker/k8s/src/literals.py

@@ -18,6 +18,13 @@
 K8SD_SNAP_SOCKET = "/var/snap/k8s/common/var/lib/k8sd/state/control.socket"
 K8SD_PORT = 6400
 SUPPORTED_DATASTORES = ["dqlite", "etcd"]
+HTTP_SCHEME = "http://"
+HTTPS_SCHEME = "https://"
+EXTERNAL_LOAD_BALANCER_REQUEST_NAME = "api-server-external"
+EXTERNAL_LOAD_BALANCER_RESPONSE_NAME = EXTERNAL_LOAD_BALANCER_REQUEST_NAME
+EXTERNAL_LOAD_BALANCER_PORT_CONFIG = "external-load-balancer-port"
+ENDPOINT_HAS_PORT_REGEX = r":\d+$"
+APISERVER_PORT = 6443
 
 # Features
 SUPPORT_SNAP_INSTALLATION_OVERRIDE = True
@@ -31,6 +38,7 @@
 COS_RELATION = "cos-agent"
 ETCD_RELATION = "etcd"
 UPGRADE_RELATION = "upgrade"
+EXTERNAL_LOAD_BALANCER_RELATION = "external-load-balancer"
 
 # Kubernetes services
 K8S_COMMON_SERVICES = [

charms/worker/k8s/tests/unit/test_base.py

@@ -57,6 +57,7 @@ def mock_reconciler_handlers(harness):
     }
     if harness.charm.is_control_plane:
         handler_names |= {
+            "_configure_external_load_balancer",
             "_bootstrap_k8s_snap",
             "_create_cluster_tokens",
             "_create_cos_tokens",
@@ -210,8 +211,8 @@ def test_configure_boostrap_extra_sans(harness):
     public_addr = "11.12.13.14"
     harness.update_config({"kube-apiserver-extra-sans": " ".join(cfg_extra_sans)})
 
-    with mock.patch("charm._get_public_address") as mock_get_public_addr:
-        mock_get_public_addr.return_value = public_addr
+    with mock.patch("charm._get_juju_public_address") as mock_get_juju_public_addr:
+        mock_get_juju_public_addr.return_value = public_addr
 
         bs_config = harness.charm._assemble_bootstrap_config()
 

charms/worker/k8s/tests/unit/test_config_options.py

@@ -86,7 +86,7 @@ def test_configure_common_extra_args(harness):
     harness.update_config({"kubelet-extra-args": "v=3 foo=bar flag"})
     harness.update_config({"kube-proxy-extra-args": "v=4 foo=baz flog"})
 
-    with mock.patch("charm._get_public_address"):
+    with mock.patch("charm._get_juju_public_address"):
         bootstrap_config = harness.charm._assemble_bootstrap_config()
     assert bootstrap_config.extra_node_kubelet_args == {
         "--v": "3",
@@ -115,7 +115,7 @@ def test_configure_controller_extra_args(harness):
     harness.update_config({"kube-controller-manager-extra-args": "v=4 foo=baz flog"})
     harness.update_config({"kube-scheduler-extra-args": "v=5 foo=bat blog"})
 
-    with mock.patch("charm._get_public_address"):
+    with mock.patch("charm._get_juju_public_address"):
         bootstrap_config = harness.charm._assemble_bootstrap_config()
     assert bootstrap_config.extra_node_kube_apiserver_args == {
         "--v": "3",