[kube-prometheus-stack] WIP: upgrade CRDs using helm

charts/kube-prometheus-stack/Chart.yaml

@@ -23,7 +23,7 @@ name: kube-prometheus-stack
 sources:
   - https://github.com/prometheus-community/helm-charts
   - https://github.com/prometheus-operator/kube-prometheus
-version: 68.1.0
+version: 69.0.0
 appVersion: v0.79.2
 kubeVersion: ">=1.19.0-0"
 home: https://github.com/prometheus-operator/kube-prometheus

charts/kube-prometheus-stack/charts/crds/templates/_helpers.tpl

@@ -0,0 +1,20 @@
+{{/* Shortened name suffixed with upgrade-crd */}}
+{{- define "kube-prometheus-stack.upgradeCRD.name" -}}
+{{- print (include "kube-prometheus-stack.fullname" .) "-upgrade" -}}
+{{- end -}}
+
+{{- define "kube-prometheus-stack.upgradeCRD.labels" -}}
+{{- include "kube-prometheus-stack.labels" . }}
+app: {{ template "kube-prometheus-stack.name" . }}-operator
+app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator
+app.kubernetes.io/component: crds-upgrade
+{{- end -}}
+
+{{/* Create the name of upgradeCRD service account to use */}}
+{{- define "kube-prometheus-stack.upgradeCRD.serviceAccountName" -}}
+{{- if .Values.upgradeJob.serviceAccount.create -}}
+    {{ default (include "kube-prometheus-stack.upgradeCRD.name" .) .Values.upgradeJob.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.upgradeJob.serviceAccount.name }}
+{{- end -}}
+{{- end -}}

charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrole.yaml

@@ -0,0 +1,35 @@
+{{- if .Values.upgradeJob.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }}
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }}
+rules:
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - "customresourcedefinitions"
+    verbs:
+      - create
+      - patch
+      - update
+      - get
+      - list
+    resourceNames:
+      - alertmanagerconfigs.monitoring.coreos.com
+      - alertmanagers.monitoring.coreos.com
+      - podmonitors.monitoring.coreos.com
+      - probes.monitoring.coreos.com
+      - prometheusagents.monitoring.coreos.com
+      - prometheuses.monitoring.coreos.com
+      - prometheusrules.monitoring.coreos.com
+      - scrapeconfigs.monitoring.coreos.com
+      - servicemonitors.monitoring.coreos.com
+      - thanosrulers.monitoring.coreos.com
+{{- end }}

charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrolebinding.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.upgradeJob.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }}
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    namespace: {{ template "kube-prometheus-stack.namespace" . }}
+    name: {{ template "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }}
+roleRef:
+  kind: ClusterRole
+  name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/kube-prometheus-stack/charts/crds/templates/upgrade/crds.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.upgradeJob.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }}
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-2"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }}
+binaryData:
+  crds.tar.xz: {{ .Files.Get "files/crds.tar.xz" | b64enc }}
+{{- end }}

charts/kube-prometheus-stack/charts/crds/templates/upgrade/job.yaml

@@ -0,0 +1,143 @@
+{{- if .Values.upgradeJob.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }}
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "5"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    {{- with .Values.upgradeJob.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }}
+    {{- with .Values.upgradeJob.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      {{- with .Values.upgradeJob.podLabels }}
+      labels:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.upgradeJob.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+    {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+      {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }}
+    {{- end }}
+      serviceAccountName: {{ include "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }}
+      initContainers:
+        - name: busybox
+          {{- $busyboxRegistry := .Values.global.imageRegistry | default .Values.upgradeJob.image.busybox.registry -}}
+          {{- if .Values.upgradeJob.image.sha }}
+          image: "{{ $busyboxRegistry }}/{{ .Values.upgradeJob.image.busybox.repository }}:{{ .Values.upgradeJob.image.busybox.tag }}@sha256:{{ .Values.upgradeJob.image.busybox.sha }}"
+          {{- else }}
+          image: "{{ $busyboxRegistry }}/{{ .Values.upgradeJob.image.busybox.repository }}:{{ .Values.upgradeJob.image.busybox.tag }}"
+          {{- end }}
+          imagePullPolicy: "{{ .Values.upgradeJob.image.busybox.pullPolicy }}"
+          workingDir: /tmp/
+          command:
+            - tar
+          args:
+            - xvJf
+            - /crds/crds.tar.xz
+          {{- with .Values.upgradeJob.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.upgradeJob.containerSecurityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - mountPath: /crds/
+              name: crds
+            - mountPath: /tmp/
+              name: tmp
+            {{- with .Values.upgradeJob.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with .Values.upgradeJob.env }}
+          env:
+            {{- range $key, $value := . }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+          {{- end }}
+      containers:
+        - name: kubectl
+          {{- $kubectlRegistry := .Values.global.imageRegistry | default .Values.upgradeJob.image.kubectl.registry -}}
+          {{- if .Values.upgradeJob.image.kubectl.sha }}
+          image: "{{ $kubectlRegistry }}/{{ .Values.upgradeJob.image.kubectl.repository }}:{{ .Values.upgradeJob.image.kubectl.tag | default .Capabilities.KubeVersion.Version }}@sha256:{{ .Values.upgradeJob.image.kubectl.sha }}"
+          {{- else }}
+          image: "{{ $kubectlRegistry }}/{{ .Values.upgradeJob.image.kubectl.repository }}:{{ .Values.upgradeJob.image.kubectl.tag | default .Capabilities.KubeVersion.Version }}"
+          {{- end }}
+          imagePullPolicy: "{{ .Values.upgradeJob.image.kubectl.pullPolicy }}"
+          command:
+            - kubectl
+          args:
+            - apply
+            - --server-side
+            - --recursive
+            - --filename
+            - /tmp/
+          {{- with .Values.upgradeJob.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.upgradeJob.containerSecurityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - mountPath: /tmp/
+              name: tmp
+            {{- with .Values.upgradeJob.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with .Values.upgradeJob.env }}
+          env:
+            {{- range $key, $value := . }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+          {{- end }}
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: crds
+          configMap:
+            name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }}
+        {{- with .Values.upgradeJob.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      restartPolicy: OnFailure
+      {{- with .Values.upgradeJob.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.upgradeJob.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.upgradeJob.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.upgradeJob.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.upgradeJob.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/kube-prometheus-stack/charts/crds/templates/upgrade/serviceaccount.yaml

@@ -0,0 +1,20 @@
+{{- if and .Values.upgradeJob.enabled .Values.upgradeJob.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.upgradeJob.serviceAccount.automountServiceAccountToken }}
+metadata:
+  name: {{ include "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }}
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-4"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    {{- with .Values.upgradeJob.serviceAccount.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }}
+    {{- with .Values.upgradeJob.serviceAccount.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- end }}

charts/kube-prometheus-stack/ci/lint.sh

@@ -8,11 +8,24 @@ set -euo pipefail
     cd "${SCRIPT_DIR}/../"
 
     ./hack/update_crds.sh
-    if ! git diff "$GITHUB_SHA" --color=always --exit-code; then
+    if ! git diff "$GITHUB_SHA" --color=always --exit-code -- ':!*.tar.xz'; then
       echo "Please run ./hack/update_crds.sh"
       exit 1
     fi
 
+    cd "${SCRIPT_DIR}/../charts/crds/crds/"
+
+    rm ./*.yaml
+
+    tar xJf "${SCRIPT_DIR}/../charts/crds/files/crds.tar.xz"
+
+    if ! git diff "$GITHUB_SHA" --color=always --exit-code -- ':!*.tar.xz'; then
+      echo "Suspicious crds.tar.xz. Please run ./hack/update_crds.sh"
+      exit 1
+    fi
+
+    cd "${SCRIPT_DIR}/../"
+
     cd hack
 
     export PIP_DISABLE_PIP_VERSION_CHECK=1

charts/kube-prometheus-stack/hack/update_crds.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 if [[ $(uname -s) = "Darwin" ]]; then
@@ -36,3 +38,9 @@ for line in "${FILES[@]}"; do
       exit 1
     fi
 done
+
+cd "${SCRIPT_DIR}/../charts/crds/crds/"
+
+# COPYFILE_DISABLE is used to prevent tar from including BSD metadata in the tarball
+# ref: https://unix.stackexchange.com/a/282142/433641
+env COPYFILE_DISABLE=1 tar cJf ../files/crds.tar.xz crd-*

charts/kube-prometheus-stack/templates/_helpers.tpl

@@ -72,7 +72,6 @@ The longest name that gets created adds and extra 37 characters, so truncation s
 {{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}}
 {{- end }}
 
-
 {{/* Create chart name and version as used by the chart label. */}}
 {{- define "kube-prometheus-stack.chartref" -}}
 {{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}}
@@ -126,6 +125,7 @@ heritage: {{ $.Release.Service | quote }}
 {{- else -}}
     {{ default "default" .Values.alertmanager.serviceAccount.name }}
 {{- end -}}
+
 {{- end -}}
 
 {{/* Create the name of thanosRuler service account to use */}}