From e572d386bff8244036ac5e0bab036f952522ca9c Mon Sep 17 00:00:00 2001 From: Mike Yoder Date: Fri, 13 Sep 2024 11:46:15 -0700 Subject: [PATCH] Adding checksum annotations to the clustershield deployment Imagine the following scenario: the clustershield helm chart is re-deployed and the contents of either its configmap, secrets, or webhook change... and the clustershield deployment itself does not change. The clustershield deployment won't restart because it does not know that anything has changed - even though the things it depended on change. This problem is especially pronounced for clustershield because with the default behavior it will auto-generate the certificates used for communication between the webhook and the deployment. If the webhook certificate changes and the deployment doesn't restart, the webhook won't be able to communicate with the pods in the deployment. The canonical way to address this is to put checksums of the depended-upon templates into the deployment as annotations. Now when resources the deployment depend upon change (like the auto-genereted certificates) the clustershield pods will restart. This is the same solution implemented in the nodeanalyzer daemonset. --- charts/cluster-shield/Chart.yaml | 4 +- .../cluster-shield/templates/deployment.yaml | 9 +- .../cluster-shield/tests/custom_ca_test.yaml | 2 + .../cluster-shield/tests/deployment_test.yaml | 93 +++++++++++++++---- .../tests/image_pull_secrets_test.yaml | 7 ++ .../tests/proxy_settings_test.yaml | 2 + 6 files changed, 95 insertions(+), 22 deletions(-) diff --git a/charts/cluster-shield/Chart.yaml b/charts/cluster-shield/Chart.yaml index fab0feace..a387a5882 100644 --- a/charts/cluster-shield/Chart.yaml +++ b/charts/cluster-shield/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: cluster-shield description: Cluster Shield Helm Chart for Kubernetes type: application -version: 1.3.1 -appVersion: "1.3.1" +version: 1.3.2 +appVersion: "1.3.2" maintainers: - name: AlbertoBarba email: alberto.barba@sysdig.com diff --git a/charts/cluster-shield/templates/deployment.yaml b/charts/cluster-shield/templates/deployment.yaml index 71edbc84d..2f5d0ec64 100644 --- a/charts/cluster-shield/templates/deployment.yaml +++ b/charts/cluster-shield/templates/deployment.yaml @@ -15,10 +15,13 @@ spec: {{- include "cluster-shield.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} - {{- end }} + checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }} + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum | quote }} + checksum/webhook: {{ include (print $.Template.BasePath "/validatingwebhookconfiguration.yaml") . | sha256sum | quote }} + {{- if .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} labels: {{- include "cluster-shield.labels" . | nindent 8 }} {{- with .Values.podLabels }} diff --git a/charts/cluster-shield/tests/custom_ca_test.yaml b/charts/cluster-shield/tests/custom_ca_test.yaml index ba012fa35..8e0f0eaf3 100644 --- a/charts/cluster-shield/tests/custom_ca_test.yaml +++ b/charts/cluster-shield/tests/custom_ca_test.yaml @@ -2,6 +2,8 @@ suite: Test Custom CA templates: - templates/deployment.yaml - templates/secrets.yaml + - templates/configmap.yaml + - templates/validatingwebhookconfiguration.yaml values: - ../ci/base-values.yaml release: diff --git a/charts/cluster-shield/tests/deployment_test.yaml b/charts/cluster-shield/tests/deployment_test.yaml index 80a60d0da..d62c5d3b9 100644 --- a/charts/cluster-shield/tests/deployment_test.yaml +++ b/charts/cluster-shield/tests/deployment_test.yaml @@ -1,6 +1,9 @@ suite: Test Deployment templates: - templates/deployment.yaml + - templates/secrets.yaml + - templates/configmap.yaml + - templates/validatingwebhookconfiguration.yaml values: - ../ci/base-values.yaml release: @@ -33,6 +36,7 @@ tests: - contains: path: spec.template.spec.containers[?(@.name == "cluster-shield")].args content: run-all-namespaced + template: templates/deployment.yaml - it: Test run-all mode set: @@ -57,6 +61,7 @@ tests: - contains: path: spec.template.spec.containers[?(@.name == "cluster-shield")].args content: run-all + template: templates/deployment.yaml - it: Test GKE Autopilot set: @@ -80,6 +85,7 @@ tests: - contains: path: spec.template.spec.containers[?(@.name == "cluster-shield")].args content: run-all + template: templates/deployment.yaml - it: Test accessKeySecret is mounted when provided set: @@ -106,6 +112,7 @@ tests: content: name: secret-test-existing-access-key-secret mountPath: /etc/sysdig/secret-files/test-existing-access-key-secret + template: templates/deployment.yaml - it: Test env var is set when secureAPITokenSecret provided set: @@ -132,6 +139,7 @@ tests: content: name: secret-test-existing-secure-api-token-secret mountPath: /etc/sysdig/secret-files/test-existing-secure-api-token-secret + template: templates/deployment.yaml - it: Test Both accessKeySecret and secureAPITokenSecret are mounted when provided set: @@ -170,6 +178,7 @@ tests: content: name: secret-test-existing-secure-api-token-secret mountPath: /etc/sysdig/secret-files/test-existing-secure-api-token-secret + template: templates/deployment.yaml - it: Test it has only one mount when accessKeySecret and secureAPITokenSecret are the same set: @@ -198,6 +207,7 @@ tests: content: name: secret-test-existing-credentials-secret mountPath: /etc/sysdig/secret-files/test-existing-credentials-secret + template: templates/deployment.yaml - it: Test readinessProbe default values asserts: @@ -213,6 +223,7 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].readinessProbe.periodSeconds value: 5 + template: templates/deployment.yaml - it: Test readinessProbe custom values set: @@ -233,6 +244,7 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].readinessProbe.periodSeconds value: 10 + template: templates/deployment.yaml - it: Test livenessProbe default values asserts: @@ -248,6 +260,7 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].livenessProbe.periodSeconds value: 5 + template: templates/deployment.yaml - it: Test livenessProbe custom values set: @@ -268,6 +281,7 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].livenessProbe.periodSeconds value: 10 + template: templates/deployment.yaml - it: Test default container ports set: @@ -293,6 +307,7 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].ports[?(@.name == "admission")].containerPort value: 8443 + template: templates/deployment.yaml - it: Test custom container ports set: @@ -321,11 +336,13 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].ports[?(@.name == "admission")].containerPort value: 9876 + template: templates/deployment.yaml - it: Test without specifying priorityClassName asserts: - isNull: path: spec.template.spec.priorityClassName + template: templates/deployment.yaml - it: Test specifying priorityClassName set: @@ -336,6 +353,7 @@ tests: - equal: path: spec.template.spec.priorityClassName value: fake-priority-class-name + template: templates/deployment.yaml - it: Test specifying createPriorityClass set: @@ -346,6 +364,7 @@ tests: - equal: path: spec.template.spec.priorityClassName value: shield-release-cluster-shield + template: templates/deployment.yaml - it: Test specifying both priorityClassName and createPriorityClass set: @@ -357,12 +376,14 @@ tests: - equal: path: spec.template.spec.priorityClassName value: fake-priority-class-name + template: templates/deployment.yaml - it: Default service account name asserts: - equal: path: spec.template.spec.serviceAccountName value: shield-release-cluster-shield + template: templates/deployment.yaml - it: Custom service account name set: @@ -372,6 +393,7 @@ tests: - equal: path: spec.template.spec.serviceAccountName value: custom-service-account + template: templates/deployment.yaml - it: Use image.registry, image.repository and image.tag to build the pullString set: @@ -383,6 +405,7 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].image value: test-registry.io/sysdig/cluster-shield:0.0.0 + template: templates/deployment.yaml - it: Use global.imageRegistry,image.repository and image.tag to build the pullString set: @@ -396,11 +419,13 @@ tests: - equal: path: spec.template.spec.containers[?(@.name == "cluster-shield")].image value: test-global-registry.io/sysdig/cluster-shield:0.0.0 + template: templates/deployment.yaml - it: Default DNS Policy asserts: - notExists: path: spec.template.spec.dnsPolicy + template: templates/deployment.yaml - it: Test hostNetwork set: @@ -412,6 +437,7 @@ tests: - equal: path: spec.template.spec.dnsPolicy value: ClusterFirstWithHostNet + template: templates/deployment.yaml - it: Custom DNS Policy set: @@ -420,6 +446,7 @@ tests: - equal: path: spec.template.spec.dnsPolicy value: None + template: templates/deployment.yaml - it: Custom DNS Policy with hostNetwork set: @@ -432,23 +459,55 @@ tests: - equal: path: spec.template.spec.dnsPolicy value: None + template: templates/deployment.yaml - - it: Mount existing TLS Secret + - it: Checksum Annotations + asserts: + - isNotNull: + path: spec.template.metadata.annotations.checksum/configmap + - isNotNull: + path: spec.template.metadata.annotations.checksum/secrets + - isNotNull: + path: spec.template.metadata.annotations.checksum/webhook + template: templates/deployment.yaml + + - it: Custom Annotations set: - cluster_shield: - features: - admission_control: - enabled: true - audit: - enabled: true - container_vulnerability_management: - enabled: true - existingTLSSecret: - name: my-custom-tls-secret + podAnnotations: + hello: world asserts: - - contains: - path: spec.template.spec.volumes - content: - name: cert - secret: - secretName: my-custom-tls-secret + - isNotNull: + path: spec.template.metadata.annotations.checksum/configmap + - isNotNull: + path: spec.template.metadata.annotations.checksum/secrets + - isNotNull: + path: spec.template.metadata.annotations.checksum/webhook + - equal: + path: spec.template.metadata.annotations.hello + value: world + template: templates/deployment.yaml + + # Sadly can't run this, as it fails with + # "The TLS Secret 'my-custom-tls-secret' does not exist" + # This is because, now that we added the other templates, the chart + # wants to perform a lookup() on this secret - and we have no cluster. + # - it: Mount existing TLS Secret + # set: + # cluster_shield: + # features: + # admission_control: + # enabled: true + # audit: + # enabled: true + # container_vulnerability_management: + # enabled: true + # existingTLSSecret: + # name: my-custom-tls-secret + # asserts: + # - contains: + # path: spec.template.spec.volumes + # content: + # name: cert + # secret: + # secretName: my-custom-tls-secret + # template: templates/deployment.yaml diff --git a/charts/cluster-shield/tests/image_pull_secrets_test.yaml b/charts/cluster-shield/tests/image_pull_secrets_test.yaml index 29d0e9ca6..4aa482d5d 100644 --- a/charts/cluster-shield/tests/image_pull_secrets_test.yaml +++ b/charts/cluster-shield/tests/image_pull_secrets_test.yaml @@ -1,6 +1,9 @@ suite: Test image pull secrets templates: - templates/deployment.yaml + - templates/secrets.yaml + - templates/configmap.yaml + - templates/validatingwebhookconfiguration.yaml values: - ../ci/base-values.yaml release: @@ -11,6 +14,7 @@ tests: asserts: - isNull: path: spec.template.spec.imagePullSecrets + template: templates/deployment.yaml - it: Test with specific secrets set: @@ -21,6 +25,7 @@ tests: path: spec.template.spec.imagePullSecrets value: - name: existing-specific-secret + template: templates/deployment.yaml - it: Test with global secrets set: @@ -33,6 +38,7 @@ tests: path: spec.template.spec.imagePullSecrets value: - name: existing-global-secret + template: templates/deployment.yaml - it: Test with global and specific secrets set: @@ -47,3 +53,4 @@ tests: path: spec.template.spec.imagePullSecrets value: - name: existing-specific-secret + template: templates/deployment.yaml diff --git a/charts/cluster-shield/tests/proxy_settings_test.yaml b/charts/cluster-shield/tests/proxy_settings_test.yaml index 510d902f3..bf673f1f2 100644 --- a/charts/cluster-shield/tests/proxy_settings_test.yaml +++ b/charts/cluster-shield/tests/proxy_settings_test.yaml @@ -2,6 +2,8 @@ suite: Test Deployment templates: - templates/deployment.yaml - templates/secrets.yaml + - templates/configmap.yaml + - templates/validatingwebhookconfiguration.yaml values: - ../ci/base-values.yaml release: