diff --git a/config.yaml b/config.yaml index 20edb91..0eb7fb5 100644 --- a/config.yaml +++ b/config.yaml @@ -4,6 +4,60 @@ options: default: "1.28/edge" description: | Snap channel to install Kubernetes worker services from + ingress: + type: boolean + default: true + description: | + Deploy nginx-ingress-controller to handle Ingress resources. When set to + true, the unit will open ports 80 and 443 to make the nginx-ingress-controller + endpoint accessible. + ingress-default-ssl-certificate: + type: string + default: "" + description: | + SSL certificate to be used by the default HTTPS server. If one of the + flag ingress-default-ssl-certificate or ingress-default-ssl-key is not + provided ingress will use a self-signed certificate. This parameter is + specific to nginx-ingress-controller. + ingress-default-ssl-key: + type: string + default: "" + description: | + Private key to be used by the default HTTPS server. If one of the flag + ingress-default-ssl-certificate or ingress-default-ssl-key is not + provided ingress will use a self-signed certificate. This parameter is + specific to nginx-ingress-controller. + ingress-ssl-chain-completion: + type: boolean + default: false + description: | + Enable chain completion for TLS certificates used by the nginx ingress + controller. Set this to true if you would like the ingress controller + to attempt auto-retrieval of intermediate certificates. The default + (false) is recommended for all production kubernetes installations, and + any environment which does not have outbound Internet access. + ingress-ssl-passthrough: + type: boolean + default: false + description: | + Enable ssl passthrough on ingress server. This allows passing the ssl + connection through to the workloads and not terminating it at the ingress + controller. + ingress-use-forwarded-headers: + type: boolean + default: false + description: | + If true, NGINX passes the incoming X-Forwarded-* headers to upstreams. Use this + option when NGINX is behind another L7 proxy / load balancer that is setting + these headers. + + If false, NGINX ignores incoming X-Forwarded-* headers, filling them with the + request information it sees. Use this option if NGINX is exposed directly to + the internet, or it's behind a L3/packet-based load balancer that doesn't alter + the source IP in the packets. + + Reference: https://github.com/kubernetes/ingress-nginx/blob/a9c706be12a8be418c49ab1f60a02f52f9b14e55/ + docs/user-guide/nginx-configuration/configmap.md#use-forwarded-headers. kubelet-extra-args: type: string default: "" @@ -28,6 +82,21 @@ options: For more information about KubeletConfiguration, see upstream docs: https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ + labels: + type: string + default: "" + description: | + Labels can be used to organize and to select subsets of nodes in the + cluster. Declare node labels in key=value format, separated by spaces. + nginx-image: + type: string + default: "auto" + description: | + Container image to use for the nginx ingress controller. Using "auto" will select + an image based on architecture. + + Example: + quay.io/kubernetes-ingress-controller/nginx-ingress-controller-amd64:0.32.0 proxy-extra-args: type: string default: "" diff --git a/requirements.txt b/requirements.txt index 7959f98..f548767 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,12 @@ charm-lib-interface-external-cloud-provider @ git+https://github.com/charmed-kub charm-lib-interface-kubernetes-cni @ git+https://github.com/charmed-kubernetes/charm-lib-interface-kubernetes-cni charm-lib-interface-tokens @ git+https://github.com/charmed-kubernetes/charm-lib-interface-tokens charm-lib-kubernetes-snaps @ git+https://github.com/charmed-kubernetes/charm-lib-kubernetes-snaps +charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@main#subdirectory=ops charm-lib-reconciler @ git+https://github.com/charmed-kubernetes/charm-lib-reconciler cosl == 0.0.7 +jinja2 ops >= 2.2.0 ops.interface_kube_control @ git+https://github.com/juju-solutions/interface-kube-control#subdirectory=ops ops.interface_tls_certificates @ git+https://github.com/juju-solutions/interface-tls-certificates#subdirectory=ops pydantic==1.* +tenacity diff --git a/src/charm.py b/src/charm.py index 1d2cc26..ffe6621 100755 --- a/src/charm.py +++ b/src/charm.py @@ -5,8 +5,10 @@ """Charmed Machine Operator for Kubernetes Worker.""" import logging +import os import shlex import subprocess +from base64 import b64encode from dataclasses import dataclass from pathlib import Path from socket import gethostname @@ -22,7 +24,10 @@ from charms.interface_external_cloud_provider import ExternalCloudProvider from charms.interface_kubernetes_cni import KubernetesCniProvides from charms.interface_tokens import TokensRequirer +from charms.node_base import LabelMaker from charms.reconciler import BlockedStatus, Reconciler +from jinja2 import Environment, FileSystemLoader +from kubectl import kubectl from ops.interface_kube_control import KubeControlRequirer from ops.interface_tls_certificates import CertificatesRequires from ops.model import MaintenanceStatus, ModelError, WaitingStatus @@ -79,6 +84,7 @@ def __init__(self, *args): ) self.external_cloud_provider = ExternalCloudProvider(self, "kube-control") self.kube_control = KubeControlRequirer(self) + self.label_maker = LabelMaker(self, kubeconfig_path="/root/.kube/config") self.tokens = TokensRequirer(self) self.reconciler = Reconciler(self, self.reconcile) @@ -165,6 +171,78 @@ def _configure_kubeproxy(self, event): external_cloud_provider=self.external_cloud_provider, ) + def _configure_labels(self): + """Configure labels.""" + if not os.path.exists("/root/.kube/config"): + log.info("Waiting for kubeconfig before configuring labels") + return + + status.add(MaintenanceStatus("Configuring node labels")) + + if self.label_maker.active_labels() is not None: + self.label_maker.apply_node_labels() + + def _configure_nginx_ingress_controller(self): + """Configure nginx-ingress-controller.""" + if not os.path.exists("/root/.kube/config"): + log.info("Waiting for kubeconfig before configuring ingress") + return + + status.add(MaintenanceStatus("Configuring ingress")) + + manifest_dir = "/root/cdk/addons" + manifest_path = manifest_dir + "/ingress-daemon-set.yaml" + + if self.config["ingress"]: + image = self.config["nginx-image"] + if image == "" or image == "auto": + registry = self.kube_control.get_registry_location() or "registry.k8s.io" + image = f"{registry}/ingress-nginx/controller:v1.6.4" + + context = { + "daemonset_api_version": "apps/v1", + "default_ssl_certificate_option": None, + "enable_ssl_passthrough": self.config["ingress-ssl-passthrough"], + "ingress_image": image, + "ingress_uid": "101", + "juju_application": self.app.name, + "ssl_chain_completion": self.config["ingress-ssl-chain-completion"], + "use_forwarded_headers": "true" + if self.config["ingress-use-forwarded-headers"] + else "false", + } + + ssl_cert = self.config["ingress-default-ssl-certificate"] + ssl_key = self.config["ingress-default-ssl-key"] + if ssl_cert and ssl_key: + context.update( + { + "default_ssl_certificate": b64encode(ssl_cert.encode("utf-8")).decode( + "utf-8" + ), + "default_ssl_certificate_option": "- --default-ssl-certificate=$(POD_NAMESPACE)/default-ssl-certificate", + "default_ssl_key": b64encode(ssl_key.encode("utf-8")).decode("utf-8"), + } + ) + + env = Environment(loader=FileSystemLoader("templates")) + template = env.get_template("ingress-daemon-set.yaml") + output = template.render(context) + os.makedirs(manifest_dir, exist_ok=True) + with open(manifest_path, "w") as f: + f.write(output) + kubectl("apply", "-f", manifest_path) + + self.unit.open_port("tcp", 80) + self.unit.open_port("tcp", 443) + else: + self.unit.close_port("tcp", 80) + self.unit.close_port("tcp", 443) + + if os.path.exists(manifest_path): + kubectl("delete", "--ignore-not-found", "-f", manifest_path) + os.remove(manifest_path) + def _create_kubeconfigs(self, event): """Generate kubeconfig files for the cluster components.""" status.add(MaintenanceStatus("Generating Kubeconfig")) @@ -176,7 +254,7 @@ def _create_kubeconfigs(self, event): if not self._check_kubecontrol_integration(event): return - node_user = f"system:node:{self._get_node_name()}" + node_user = f"system:node:{self.get_node_name()}" credentials = self.kube_control.get_auth_credentials(node_user) if not credentials: status.add(WaitingStatus("Waiting for kube-control credentials")) @@ -221,11 +299,15 @@ def _create_kubeconfigs(self, event): token=credentials.get("proxy_token"), ) + def get_cloud_name(self) -> str: + """Return cloud name.""" + return self.external_cloud_provider.name + def _get_metrics_endpoints(self) -> list: """Return the metrics endpoints for K8s components.""" log.info("Building Prometheus scraping jobs.") - cos_user = f"system:cos:{self._get_node_name()}" + cos_user = f"system:cos:{self.get_node_name()}" token = self.tokens.get_token(cos_user) if not token: @@ -286,8 +368,9 @@ def create_scrape_job(config: JobConfig): def _get_unit_number(self) -> int: return int(self.unit.name.split("/")[1]) - def _get_node_name(self) -> str: - fqdn = self.external_cloud_provider.name == "aws" + def get_node_name(self) -> str: + """Return node name.""" + fqdn = self.get_cloud_name() == "aws" return kubernetes_snaps.get_node_name(fqdn) def _install_cni_binaries(self): @@ -326,7 +409,7 @@ def _request_kubelet_and_proxy_credentials(self): """Request authorization for kubelet and kube-proxy.""" status.add(MaintenanceStatus("Requesting kubelet and kube-proxy credentials")) - node_user = f"system:node:{self._get_node_name()}" + node_user = f"system:node:{self.get_node_name()}" self.kube_control.set_auth_request(node_user) def _request_monitoring_token(self, event): @@ -334,7 +417,7 @@ def _request_monitoring_token(self, event): if not self._check_tokens_integration(event): return - cos_user = f"system:cos:{self._get_node_name()}" + cos_user = f"system:cos:{self.get_node_name()}" self.tokens.request_token(cos_user, OBSERVABILITY_GROUP) def reconcile(self, event): @@ -352,6 +435,8 @@ def reconcile(self, event): self._configure_kernel_parameters() self._configure_kubelet(event) self._configure_kubeproxy(event) + self._configure_nginx_ingress_controller() + self._configure_labels() def _request_certificates(self): """Request client and server certificates.""" diff --git a/src/kubectl.py b/src/kubectl.py new file mode 100644 index 0000000..94f427d --- /dev/null +++ b/src/kubectl.py @@ -0,0 +1,25 @@ +"""kubectl.""" + +import logging +from subprocess import CalledProcessError, check_output + +from tenacity import retry, stop_after_delay, wait_exponential + +log = logging.getLogger(__name__) + + +@retry(stop=stop_after_delay(60), wait=wait_exponential()) +def kubectl(*args): + """Run a kubectl cli command with a config file. + + Returns stdout and throws an error if the command fails. + """ + command = ["kubectl", "--kubeconfig=/root/.kube/config"] + list(args) + log.info("Executing {}".format(command)) + try: + return check_output(command).decode("utf-8") + except CalledProcessError as e: + log.error( + f"Command failed: {command}\nreturncode: {e.returncode}\nstdout: {e.output.decode()}" + ) + raise diff --git a/templates/ingress-daemon-set.yaml b/templates/ingress-daemon-set.yaml new file mode 100644 index 0000000..52cd190 --- /dev/null +++ b/templates/ingress-daemon-set.yaml @@ -0,0 +1,370 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx-{{ juju_application }} + labels: + cdk-{{ juju_application }}-ingress: "true" + +{%- if default_ssl_certificate_option %} +--- +kind: Secret +apiVersion: v1 +type: Opaque +metadata: + name: default-ssl-certificate + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" +data: + tls.crt: {{ default_ssl_certificate }} + tls.key: {{ default_ssl_key }} +{%- endif %} + +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-configuration + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" +data: + use-forwarded-headers: "{{ use_forwarded_headers }}" + +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: tcp-services + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" + +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: udp-services + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-ingress-serviceaccount-{{ juju_application }} + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nginx-ingress-clusterrole-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "networking.k8s.io" # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "networking.k8s.io" # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - "networking.k8s.io" # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: nginx-ingress-role-{{ juju_application }} + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - "networking.k8s.io" # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - "networking.k8s.io" # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - "networking.k8s.io" # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resourceNames: + - ingress-controller-leader + resources: + - configmaps + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: nginx-ingress-role-nisa-binding-{{ juju_application }} + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-role-{{ juju_application }} +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount-{{ juju_application }} + namespace: ingress-nginx-{{ juju_application }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nginx-ingress-clusterrole-nisa-binding-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-ingress-clusterrole-{{ juju_application }} +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount-{{ juju_application }} + namespace: ingress-nginx-{{ juju_application }} + +--- +apiVersion: {{ daemonset_api_version }} +kind: DaemonSet +metadata: + name: nginx-ingress-controller-{{ juju_application }} + namespace: ingress-nginx-{{ juju_application }} + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + juju-application: nginx-ingress-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" + cdk-restart-on-ca-change: "true" +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + spec: + serviceAccountName: nginx-ingress-serviceaccount-{{ juju_application }} + nodeSelector: + juju-application: {{ juju_application }} + terminationGracePeriodSeconds: 60 + # hostPort doesn't work with CNI, so we have to use hostNetwork instead + # see https://github.com/kubernetes/kubernetes/issues/23920 + hostNetwork: true + containers: + - name: nginx-ingress-controller{{ juju_application }} + image: {{ ingress_image }} + args: + - /nginx-ingress-controller + - --configmap=$(POD_NAMESPACE)/nginx-configuration + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --annotations-prefix=nginx.ingress.kubernetes.io + - --enable-ssl-chain-completion={{ ssl_chain_completion }} + - --enable-ssl-passthrough={{ enable_ssl_passthrough }} +{%- if default_ssl_certificate_option %} + {{ default_ssl_certificate_option }} +{%- endif %} + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: {{ ingress_uid }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/name: ingress-nginx-{{ juju_application }} + app.kubernetes.io/part-of: ingress-nginx-{{ juju_application }} + cdk-{{ juju_application }}-ingress: "true" + name: nginx-ingress-controller + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +spec: + controller: k8s.io/ingress-nginx