Integrate kube-state-metrics and CR config into tilt.

Tiltfile

@@ -354,6 +354,10 @@ def deploy_observability():
         k8s_yaml(read_file("./.tiltbuild/yaml/prometheus.observability.yaml"), allow_duplicates = True)
         k8s_resource(workload = "prometheus-server", new_name = "prometheus", port_forwards = "9090", extra_pod_selectors = [{"app": "prometheus"}], labels = ["observability"])
 
+    if "kube-state-metrics" in settings.get("deploy_observability", []):
+        k8s_yaml(read_file("./.tiltbuild/yaml/kube-state-metrics.observability.yaml"), allow_duplicates = True)
+        k8s_resource(workload = "kube-state-metrics", new_name = "kube-state-metrics", extra_pod_selectors = [{"app": "kube-state-metrics"}], labels = ["observability"])
+
     if "visualizer" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/visualizer.observability.yaml"), allow_duplicates = True)
         k8s_resource(

docs/book/src/developer/tilt.md

@@ -173,9 +173,20 @@ kustomize_substitutions:
 {{#/tabs }}
 
 **deploy_observability** ([string], default=[]): If set, installs on the dev cluster one of more observability
-tools. Supported values are `grafana`, `loki`, `visualizer`, `promtail` and/or `prometheus` (Note: the UI for `grafana`, `prometheus`, and `visualizer` will be accessible via a link in the tilt console).
+tools. 
 Important! This feature requires the `helm` command to be available in the user's path.
 
+Supported values are:
+
+  * `grafana`*: To create dashboards and query `loki` as well as `prometheus`.
+  * `kube-state-metrics`: For exposing metrics for kubernetes and CAPI resources to `prometheus`.
+  * `loki`: To receive and store logs.
+  * `prometheus`*: For collecting metrics from Kubernetes..
+  * `promtail`: For providing pod logs to `loki`.
+  * `visualizer`*: Visualize all the Cluster API custom resources for each cluster, provide quick access to the specs and status of any resource.
+
+\*: Note: the UI will be accessible via a link in the tilt console
+
 **debug** (Map{string: Map} default{}): A map of named configurations for the provider. The key is the name of the provider.
 
 Supported settings:

hack/observability/kube-state-metrics/chart/kustomization.yaml

@@ -0,0 +1,6 @@
+helmCharts:
+  - name: kube-state-metrics
+    repo: https://prometheus-community.github.io/helm-charts
+    namespace: observability
+    releaseName: kube-state-metrics
+    valuesFile: values.yaml

hack/observability/kube-state-metrics/chart/values.yaml

@@ -0,0 +1,82 @@
+image:
+  # TODO(chrischdi): drop to default to released image as soon as it got released
+  # repository: registry.k8s.io/kube-state-metrics/kube-state-metrics
+  # tag: v2.5.0
+  # custom image which includes changes > v2.5.0 required for custom resource metrics
+  repository: chrischdi/kube-state-metrics
+  tag: v2.5.0-fe097b6c
+  pullPolicy: IfNotPresent
+
+# Add the CR configuration from the config map.
+volumeMounts:
+ - mountPath: /etc/config
+   name: config-volume
+
+volumes:
+ - configMap:
+     name: kube-state-metrics-crd-config
+   name: config-volume
+
+extraArgs:
+- "--custom-resource-state-config-file=/etc/config/crd-config.yaml"
+
+rbac:
+  extraRules:
+  - apiGroups:
+    - cluster.x-k8s.io
+    resources:
+    - clusters
+    - machinedeployments
+    - machinesets
+    - machines
+    - machinehealthchecks
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - controlplane.cluster.x-k8s.io
+    resources:
+    - kubeadmcontrolplanes
+    verbs:
+    - get
+    - list
+    - watch
+
+collectors:
+  # CAPI CRs
+  - clusters
+  - machinedeployments
+  - machinesets
+  - machines
+  - machinehealthchecks
+  - kubeadmcontrolplanes
+  # We need to define all default collectors too, otherwise the helm chart does not include this resources in rbac
+  - certificatesigningrequests
+  - configmaps
+  - cronjobs
+  - daemonsets
+  - deployments
+  - endpoints
+  - horizontalpodautoscalers
+  - ingresses
+  - jobs
+  - limitranges
+  - mutatingwebhookconfigurations
+  - namespaces
+  - networkpolicies
+  - nodes
+  - persistentvolumeclaims
+  - persistentvolumes
+  - poddisruptionbudgets
+  - pods
+  - replicasets
+  - replicationcontrollers
+  - resourcequotas
+  - secrets
+  - services
+  - statefulsets
+  - storageclasses
+  - validatingwebhookconfigurations
+  - volumeattachments
+  # - verticalpodautoscalers # not a default resource, see also: https://github.com/kubernetes/kube-state-metrics#enabling-verticalpodautoscalers