Integrate kube-state-metrics and CR config into tilt.

Tiltfile

@@ -354,6 +354,10 @@ def deploy_observability():
         k8s_yaml(read_file("./.tiltbuild/yaml/prometheus.observability.yaml"), allow_duplicates = True)
         k8s_resource(workload = "prometheus-server", new_name = "prometheus", port_forwards = "9090", extra_pod_selectors = [{"app": "prometheus"}], labels = ["observability"])
 
+    if "kube-state-metrics" in settings.get("deploy_observability", []):
+        k8s_yaml(read_file("./.tiltbuild/yaml/kube-state-metrics.observability.yaml"), allow_duplicates = True)
+        k8s_resource(workload = "kube-state-metrics", new_name = "kube-state-metrics", extra_pod_selectors = [{"app": "kube-state-metrics"}], labels = ["observability"])
+
     if "visualizer" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/visualizer.observability.yaml"), allow_duplicates = True)
         k8s_resource(

hack/observability/kube-state-metrics/chart/kustomization.yaml

@@ -0,0 +1,6 @@
+helmCharts:
+  - name: kube-state-metrics
+    repo: https://prometheus-community.github.io/helm-charts
+    namespace: observability
+    releaseName: kube-state-metrics
+    valuesFile: values.yaml

hack/observability/kube-state-metrics/chart/values.yaml

@@ -0,0 +1,82 @@
+image:
+  # TODO(chrischdi): drop to default to released image as soon as it got released
+  # repository: registry.k8s.io/kube-state-metrics/kube-state-metrics
+  # tag: v2.5.0
+  # custom image which includes changes > v2.5.0 required for custom resource metrics
+  repository: chrischdi/kube-state-metrics
+  tag: v2.5.0-fe097b6c
+  pullPolicy: IfNotPresent
+
+# Add the CR configuration from the config map.
+volumeMounts:
+ - mountPath: /etc/config
+   name: config-volume
+
+volumes:
+ - configMap:
+     name: kube-state-metrics-crd-config
+   name: config-volume
+
+extraArgs:
+- "--custom-resource-state-config-file=/etc/config/crd-config.yaml"
+
+rbac:
+  extraRules:
+  - apiGroups:
+    - cluster.x-k8s.io
+    resources:
+    - clusters
+    - machinedeployments
+    - machinesets
+    - machines
+    - machinehealthchecks
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - controlplane.cluster.x-k8s.io
+    resources:
+    - kubeadmcontrolplanes
+    verbs:
+    - get
+    - list
+    - watch
+
+collectors:
+  # CAPI CRs
+  - clusters
+  - machinedeployments
+  - machinesets
+  - machines
+  - machinehealthchecks
+  - kubeadmcontrolplanes
+  # We need to define all default collectors too, otherwise the helm chart does not include this resources in rbac
+  - certificatesigningrequests
+  - configmaps
+  - cronjobs
+  - daemonsets
+  - deployments
+  - endpoints
+  - horizontalpodautoscalers
+  - ingresses
+  - jobs
+  - limitranges
+  - mutatingwebhookconfigurations
+  - namespaces
+  - networkpolicies
+  - nodes
+  - persistentvolumeclaims
+  - persistentvolumes
+  - poddisruptionbudgets
+  - pods
+  - replicasets
+  - replicationcontrollers
+  - resourcequotas
+  - secrets
+  - services
+  - statefulsets
+  - storageclasses
+  - validatingwebhookconfigurations
+  - volumeattachments
+  # - verticalpodautoscalers # not a default resource, see also: https://github.com/kubernetes/kube-state-metrics#enabling-verticalpodautoscalers