Adding checksum annotations to the clustershield deployment

Imagine the following scenario: the clustershield helm chart is re-deployed and the contents of either its configmap, secrets, or webhook change... and the clustershield deployment itself does not change. The clustershield deployment won't restart because it does not know that anything has changed - even though the things it depended on change. This problem is especially pronounced for clustershield because with the default behavior it will auto-generate the certificates used for communication between the webhook and the deployment. If the webhook certificate changes and the deployment doesn't restart, the webhook won't be able to communicate with the pods in the deployment. The canonical way to address this is to put checksums of the depended-upon templates into the deployment as annotations. Now when resources the deployment depend upon change (like the auto-genereted certificates) the clustershield pods will restart. This is the same solution implemented in the nodeanalyzer daemonset.
sysdiglabs · Sep 13, 2024 · e572d38 · e572d38
1 parent ac04bc3
commit e572d38
Show file tree

Hide file tree

Showing 6 changed files with 95 additions and 22 deletions.
diff --git a/charts/cluster-shield/Chart.yaml b/charts/cluster-shield/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: cluster-shield
 description: Cluster Shield Helm Chart for Kubernetes
 type: application
-version: 1.3.1
-appVersion: "1.3.1"
+version: 1.3.2
+appVersion: "1.3.2"
 maintainers:
   - name: AlbertoBarba
     email: [email protected]

diff --git a/charts/cluster-shield/templates/deployment.yaml b/charts/cluster-shield/templates/deployment.yaml
@@ -15,10 +15,13 @@ spec:
       {{- include "cluster-shield.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+        checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote  }}
+        checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum | quote  }}
+        checksum/webhook: {{ include (print $.Template.BasePath "/validatingwebhookconfiguration.yaml") . | sha256sum | quote  }}
+        {{- if .Values.podAnnotations }}
+        {{- toYaml .Values.podAnnotations | nindent 8 }}
+        {{- end }}
       labels:
         {{- include "cluster-shield.labels" . | nindent 8 }}
   {{- with .Values.podLabels }}

diff --git a/charts/cluster-shield/tests/custom_ca_test.yaml b/charts/cluster-shield/tests/custom_ca_test.yaml
@@ -2,6 +2,8 @@ suite: Test Custom CA
 templates:
   - templates/deployment.yaml
   - templates/secrets.yaml
+  - templates/configmap.yaml
+  - templates/validatingwebhookconfiguration.yaml
 values:
   - ../ci/base-values.yaml
 release:

diff --git a/charts/cluster-shield/tests/deployment_test.yaml b/charts/cluster-shield/tests/deployment_test.yaml
@@ -1,6 +1,9 @@
 suite: Test Deployment
 templates:
   - templates/deployment.yaml
+  - templates/secrets.yaml
+  - templates/configmap.yaml
+  - templates/validatingwebhookconfiguration.yaml
 values:
   - ../ci/base-values.yaml
 release:
@@ -33,6 +36,7 @@ tests:
       - contains:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].args
           content: run-all-namespaced
+    template: templates/deployment.yaml
 
   - it: Test run-all mode
     set:
@@ -57,6 +61,7 @@ tests:
       - contains:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].args
           content: run-all
+    template: templates/deployment.yaml
 
   - it: Test GKE Autopilot
     set:
@@ -80,6 +85,7 @@ tests:
       - contains:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].args
           content: run-all
+    template: templates/deployment.yaml
 
   - it: Test accessKeySecret is mounted when provided
     set:
@@ -106,6 +112,7 @@ tests:
           content:
             name: secret-test-existing-access-key-secret
             mountPath: /etc/sysdig/secret-files/test-existing-access-key-secret
+    template: templates/deployment.yaml
 
   - it: Test env var is set when secureAPITokenSecret provided
     set:
@@ -132,6 +139,7 @@ tests:
           content:
             name: secret-test-existing-secure-api-token-secret
             mountPath: /etc/sysdig/secret-files/test-existing-secure-api-token-secret
+    template: templates/deployment.yaml
 
   - it: Test Both accessKeySecret and secureAPITokenSecret are mounted when provided
     set:
@@ -170,6 +178,7 @@ tests:
           content:
             name: secret-test-existing-secure-api-token-secret
             mountPath: /etc/sysdig/secret-files/test-existing-secure-api-token-secret
+    template: templates/deployment.yaml
 
   - it: Test it has only one mount when accessKeySecret and secureAPITokenSecret are the same
     set:
@@ -198,6 +207,7 @@ tests:
           content:
             name: secret-test-existing-credentials-secret
             mountPath: /etc/sysdig/secret-files/test-existing-credentials-secret
+    template: templates/deployment.yaml
 
   - it: Test readinessProbe default values
     asserts:
@@ -213,6 +223,7 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].readinessProbe.periodSeconds
           value: 5
+    template: templates/deployment.yaml
 
   - it: Test readinessProbe custom values
     set:
@@ -233,6 +244,7 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].readinessProbe.periodSeconds
           value: 10
+    template: templates/deployment.yaml
 
   - it: Test livenessProbe default values
     asserts:
@@ -248,6 +260,7 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].livenessProbe.periodSeconds
           value: 5
+    template: templates/deployment.yaml
 
   - it: Test livenessProbe custom values
     set:
@@ -268,6 +281,7 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].livenessProbe.periodSeconds
           value: 10
+    template: templates/deployment.yaml
 
   - it: Test default container ports
     set:
@@ -293,6 +307,7 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].ports[?(@.name == "admission")].containerPort
           value: 8443
+    template: templates/deployment.yaml
 
   - it: Test custom container ports
     set:
@@ -321,11 +336,13 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].ports[?(@.name == "admission")].containerPort
           value: 9876
+    template: templates/deployment.yaml
 
   - it: Test without specifying priorityClassName
     asserts:
       - isNull:
           path: spec.template.spec.priorityClassName
+    template: templates/deployment.yaml
 
   - it: Test specifying priorityClassName
     set:
@@ -336,6 +353,7 @@ tests:
       - equal:
           path: spec.template.spec.priorityClassName
           value: fake-priority-class-name
+    template: templates/deployment.yaml
 
   - it: Test specifying createPriorityClass
     set:
@@ -346,6 +364,7 @@ tests:
       - equal:
           path: spec.template.spec.priorityClassName
           value: shield-release-cluster-shield
+    template: templates/deployment.yaml
 
   - it: Test specifying both priorityClassName and createPriorityClass
     set:
@@ -357,12 +376,14 @@ tests:
       - equal:
           path: spec.template.spec.priorityClassName
           value: fake-priority-class-name
+    template: templates/deployment.yaml
 
   - it: Default service account name
     asserts:
       - equal:
           path: spec.template.spec.serviceAccountName
           value: shield-release-cluster-shield
+    template: templates/deployment.yaml
 
   - it: Custom service account name
     set:
@@ -372,6 +393,7 @@ tests:
       - equal:
           path: spec.template.spec.serviceAccountName
           value: custom-service-account
+    template: templates/deployment.yaml
 
   - it: Use image.registry, image.repository and image.tag to build the pullString
     set:
@@ -383,6 +405,7 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].image
           value: test-registry.io/sysdig/cluster-shield:0.0.0
+    template: templates/deployment.yaml
 
   - it: Use global.imageRegistry,image.repository and image.tag to build the pullString
     set:
@@ -396,11 +419,13 @@ tests:
       - equal:
           path: spec.template.spec.containers[?(@.name == "cluster-shield")].image
           value: test-global-registry.io/sysdig/cluster-shield:0.0.0
+    template: templates/deployment.yaml
 
   - it: Default DNS Policy
     asserts:
       - notExists:
           path: spec.template.spec.dnsPolicy
+    template: templates/deployment.yaml
 
   - it: Test hostNetwork
     set:
@@ -412,6 +437,7 @@ tests:
       - equal:
           path: spec.template.spec.dnsPolicy
           value: ClusterFirstWithHostNet
+    template: templates/deployment.yaml
 
   - it: Custom DNS Policy
     set:
@@ -420,6 +446,7 @@ tests:
       - equal:
           path: spec.template.spec.dnsPolicy
           value: None
+    template: templates/deployment.yaml
 
   - it: Custom DNS Policy with hostNetwork
     set:
@@ -432,23 +459,55 @@ tests:
       - equal:
           path: spec.template.spec.dnsPolicy
           value: None
+    template: templates/deployment.yaml
 
-  - it: Mount existing TLS Secret
+  - it: Checksum Annotations
+    asserts:
+      - isNotNull:
+          path: spec.template.metadata.annotations.checksum/configmap
+      - isNotNull:
+          path: spec.template.metadata.annotations.checksum/secrets
+      - isNotNull:
+          path: spec.template.metadata.annotations.checksum/webhook
+    template: templates/deployment.yaml
+
+  - it: Custom Annotations
     set:
-      cluster_shield:
-        features:
-          admission_control:
-            enabled: true
-          audit:
-            enabled: true
-          container_vulnerability_management:
-            enabled: true
-      existingTLSSecret:
-        name: my-custom-tls-secret
+      podAnnotations:
+        hello: world
     asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: cert
-            secret:
-              secretName: my-custom-tls-secret
+      - isNotNull:
+          path: spec.template.metadata.annotations.checksum/configmap
+      - isNotNull:
+          path: spec.template.metadata.annotations.checksum/secrets
+      - isNotNull:
+          path: spec.template.metadata.annotations.checksum/webhook
+      - equal:
+          path: spec.template.metadata.annotations.hello
+          value: world
+    template: templates/deployment.yaml
+
+  # Sadly can't run this, as it fails with
+  # "The TLS Secret 'my-custom-tls-secret' does not exist"
+  # This is because, now that we added the other templates, the chart
+  # wants to perform a lookup() on this secret - and we have no cluster.
+  # - it: Mount existing TLS Secret
+  #   set:
+  #     cluster_shield:
+  #       features:
+  #         admission_control:
+  #           enabled: true
+  #         audit:
+  #           enabled: true
+  #         container_vulnerability_management:
+  #           enabled: true
+  #     existingTLSSecret:
+  #       name: my-custom-tls-secret
+  #   asserts:
+  #     - contains:
+  #         path: spec.template.spec.volumes
+  #         content:
+  #           name: cert
+  #           secret:
+  #             secretName: my-custom-tls-secret
+  #   template: templates/deployment.yaml
diff --git a/charts/cluster-shield/tests/image_pull_secrets_test.yaml b/charts/cluster-shield/tests/image_pull_secrets_test.yaml
@@ -1,6 +1,9 @@
 suite: Test image pull secrets
 templates:
   - templates/deployment.yaml
+  - templates/secrets.yaml
+  - templates/configmap.yaml
+  - templates/validatingwebhookconfiguration.yaml
 values:
   - ../ci/base-values.yaml
 release:
@@ -11,6 +14,7 @@ tests:
     asserts:
       - isNull:
           path: spec.template.spec.imagePullSecrets
+    template: templates/deployment.yaml
 
   - it: Test with specific secrets
     set:
@@ -21,6 +25,7 @@ tests:
           path: spec.template.spec.imagePullSecrets
           value:
             - name: existing-specific-secret
+    template: templates/deployment.yaml
 
   - it: Test with global secrets
     set:
@@ -33,6 +38,7 @@ tests:
           path: spec.template.spec.imagePullSecrets
           value:
             - name: existing-global-secret
+    template: templates/deployment.yaml
 
   - it: Test with global and specific secrets
     set:
@@ -47,3 +53,4 @@ tests:
           path: spec.template.spec.imagePullSecrets
           value:
             - name: existing-specific-secret
+    template: templates/deployment.yaml
diff --git a/charts/cluster-shield/tests/proxy_settings_test.yaml b/charts/cluster-shield/tests/proxy_settings_test.yaml
@@ -2,6 +2,8 @@ suite: Test Deployment
 templates:
   - templates/deployment.yaml
   - templates/secrets.yaml
+  - templates/configmap.yaml
+  - templates/validatingwebhookconfiguration.yaml
 values:
   - ../ci/base-values.yaml
 release: