diff --git a/charts/kube-prometheus-stack/charts/crds/files/crds.tar.xz b/charts/kube-prometheus-stack/charts/crds/files/crds.tar.xz new file mode 100644 index 000000000000..4c1e865f6249 Binary files /dev/null and b/charts/kube-prometheus-stack/charts/crds/files/crds.tar.xz differ diff --git a/charts/kube-prometheus-stack/charts/crds/templates/_helpers.tpl b/charts/kube-prometheus-stack/charts/crds/templates/_helpers.tpl new file mode 100644 index 000000000000..e382875dd6dd --- /dev/null +++ b/charts/kube-prometheus-stack/charts/crds/templates/_helpers.tpl @@ -0,0 +1,20 @@ +{{/* Shortened name suffixed with upgrade-crd */}} +{{- define "kube-prometheus-stack.upgradeCRD.name" -}} +{{- print (include "kube-prometheus-stack.fullname" .) "-upgrade" -}} +{{- end -}} + +{{- define "kube-prometheus-stack.upgradeCRD.labels" -}} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: crds-upgrade +{{- end -}} + +{{/* Create the name of upgradeCRD service account to use */}} +{{- define "kube-prometheus-stack.upgradeCRD.serviceAccountName" -}} +{{- if .Values.upgradeJob.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.upgradeCRD.name" .) .Values.upgradeJob.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.upgradeJob.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrole.yaml b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrole.yaml new file mode 100644 index 000000000000..2940b84222b1 --- /dev/null +++ b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrole.yaml @@ -0,0 +1,35 @@ +{{- if .Values.upgradeJob.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }} +rules: + - apiGroups: + - "apiextensions.k8s.io" + resources: + - "customresourcedefinitions" + verbs: + - create + - patch + - update + - get + - list + resourceNames: + - alertmanagerconfigs.monitoring.coreos.com + - alertmanagers.monitoring.coreos.com + - podmonitors.monitoring.coreos.com + - probes.monitoring.coreos.com + - prometheusagents.monitoring.coreos.com + - prometheuses.monitoring.coreos.com + - prometheusrules.monitoring.coreos.com + - scrapeconfigs.monitoring.coreos.com + - servicemonitors.monitoring.coreos.com + - thanosrulers.monitoring.coreos.com +{{- end }} diff --git a/charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrolebinding.yaml b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrolebinding.yaml new file mode 100644 index 000000000000..572c866e569b --- /dev/null +++ b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/clusterrolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.upgradeJob.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "-3" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }} +roleRef: + kind: ClusterRole + name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/kube-prometheus-stack/charts/crds/templates/upgrade/crds.yaml b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/crds.yaml new file mode 100644 index 000000000000..c09dd168c907 --- /dev/null +++ b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/crds.yaml @@ -0,0 +1,15 @@ +{{- if .Values.upgradeJob.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "-2" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }} +binaryData: + crds.tar.xz: {{ .Files.Get "files/crds.tar.xz" | b64enc }} +{{- end }} diff --git a/charts/kube-prometheus-stack/charts/crds/templates/upgrade/job.yaml b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/job.yaml new file mode 100644 index 000000000000..552c9017454a --- /dev/null +++ b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/job.yaml @@ -0,0 +1,143 @@ +{{- if .Values.upgradeJob.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.upgradeJob.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }} + {{- with .Values.upgradeJob.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + backoffLimit: 3 + template: + metadata: + {{- with .Values.upgradeJob.podLabels }} + labels: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.upgradeJob.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + serviceAccountName: {{ include "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }} + initContainers: + - name: busybox + {{- $busyboxRegistry := .Values.global.imageRegistry | default .Values.upgradeJob.image.busybox.registry -}} + {{- if .Values.upgradeJob.image.sha }} + image: "{{ $busyboxRegistry }}/{{ .Values.upgradeJob.image.busybox.repository }}:{{ .Values.upgradeJob.image.busybox.tag }}@sha256:{{ .Values.upgradeJob.image.busybox.sha }}" + {{- else }} + image: "{{ $busyboxRegistry }}/{{ .Values.upgradeJob.image.busybox.repository }}:{{ .Values.upgradeJob.image.busybox.tag }}" + {{- end }} + imagePullPolicy: "{{ .Values.upgradeJob.image.busybox.pullPolicy }}" + workingDir: /tmp/ + command: + - tar + args: + - xvJf + - /crds/crds.tar.xz + {{- with .Values.upgradeJob.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.upgradeJob.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /crds/ + name: crds + - mountPath: /tmp/ + name: tmp + {{- with .Values.upgradeJob.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.upgradeJob.env }} + env: + {{- range $key, $value := . }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + containers: + - name: kubectl + {{- $kubectlRegistry := .Values.global.imageRegistry | default .Values.upgradeJob.image.kubectl.registry -}} + {{- if .Values.upgradeJob.image.kubectl.sha }} + image: "{{ $kubectlRegistry }}/{{ .Values.upgradeJob.image.kubectl.repository }}:{{ .Values.upgradeJob.image.kubectl.tag | default .Capabilities.KubeVersion.Version }}@sha256:{{ .Values.upgradeJob.image.kubectl.sha }}" + {{- else }} + image: "{{ $kubectlRegistry }}/{{ .Values.upgradeJob.image.kubectl.repository }}:{{ .Values.upgradeJob.image.kubectl.tag | default .Capabilities.KubeVersion.Version }}" + {{- end }} + imagePullPolicy: "{{ .Values.upgradeJob.image.kubectl.pullPolicy }}" + command: + - kubectl + args: + - apply + - --server-side + - --recursive + - --filename + - /tmp/ + {{- with .Values.upgradeJob.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.upgradeJob.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp/ + name: tmp + {{- with .Values.upgradeJob.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.upgradeJob.env }} + env: + {{- range $key, $value := . }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + volumes: + - name: tmp + emptyDir: {} + - name: crds + configMap: + name: {{ template "kube-prometheus-stack.upgradeCRD.name" . }} + {{- with .Values.upgradeJob.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + {{- with .Values.upgradeJob.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.upgradeJob.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.upgradeJob.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.upgradeJob.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.upgradeJob.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kube-prometheus-stack/charts/crds/templates/upgrade/serviceaccount.yaml b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/serviceaccount.yaml new file mode 100644 index 000000000000..93e861c6c929 --- /dev/null +++ b/charts/kube-prometheus-stack/charts/crds/templates/upgrade/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.upgradeJob.enabled .Values.upgradeJob.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.upgradeJob.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ include "kube-prometheus-stack.upgradeCRD.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback + "helm.sh/hook-weight": "-4" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + {{- with .Values.upgradeJob.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "kube-prometheus-stack.upgradeCRD.labels" . | nindent 4 }} + {{- with .Values.upgradeJob.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kube-prometheus-stack/ci/lint.sh b/charts/kube-prometheus-stack/ci/lint.sh index f1704f76ab6e..686f9acf0825 100644 --- a/charts/kube-prometheus-stack/ci/lint.sh +++ b/charts/kube-prometheus-stack/ci/lint.sh @@ -8,11 +8,24 @@ set -euo pipefail cd "${SCRIPT_DIR}/../" ./hack/update_crds.sh - if ! git diff "$GITHUB_SHA" --color=always --exit-code; then + if ! git diff "$GITHUB_SHA" --color=always --exit-code -- ':!*.tar.xz'; then echo "Please run ./hack/update_crds.sh" exit 1 fi + cd "${SCRIPT_DIR}/../charts/crds/crds/" + + rm ./*.yaml + + tar xJf "${SCRIPT_DIR}/../charts/crds/files/crds.tar.xz" + + if ! git diff "$GITHUB_SHA" --color=always --exit-code -- ':!*.tar.xz'; then + echo "Suspicious crds.tar.xz. Please run ./hack/update_crds.sh" + exit 1 + fi + + cd "${SCRIPT_DIR}/../" + cd hack export PIP_DISABLE_PIP_VERSION_CHECK=1 diff --git a/charts/kube-prometheus-stack/hack/update_crds.sh b/charts/kube-prometheus-stack/hack/update_crds.sh index 22d7124516c7..a9a47afc8840 100755 --- a/charts/kube-prometheus-stack/hack/update_crds.sh +++ b/charts/kube-prometheus-stack/hack/update_crds.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) if [[ $(uname -s) = "Darwin" ]]; then @@ -36,3 +38,9 @@ for line in "${FILES[@]}"; do exit 1 fi done + +cd "${SCRIPT_DIR}/../charts/crds/crds/" + +# COPYFILE_DISABLE is used to prevent tar from including BSD metadata in the tarball +# ref: https://unix.stackexchange.com/a/282142/433641 +env COPYFILE_DISABLE=1 tar cJf ../files/crds.tar.xz crd-* diff --git a/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kube-prometheus-stack/templates/_helpers.tpl index b3a5af703dc2..b6d2e512075a 100644 --- a/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -72,7 +72,6 @@ The longest name that gets created adds and extra 37 characters, so truncation s {{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} {{- end }} - {{/* Create chart name and version as used by the chart label. */}} {{- define "kube-prometheus-stack.chartref" -}} {{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} @@ -126,6 +125,7 @@ heritage: {{ $.Release.Service | quote }} {{- else -}} {{ default "default" .Values.alertmanager.serviceAccount.name }} {{- end -}} + {{- end -}} {{/* Create the name of thanosRuler service account to use */}} diff --git a/charts/kube-prometheus-stack/values.yaml b/charts/kube-prometheus-stack/values.yaml index 5097cba32d47..49a2787f0fed 100644 --- a/charts/kube-prometheus-stack/values.yaml +++ b/charts/kube-prometheus-stack/values.yaml @@ -32,6 +32,126 @@ commonLabels: {} ## crds: enabled: true + ## The CRD upgrade job mitigates the limitation of helm not being able to upgrade CRDs. + ## The job will apply the CRDs to the cluster before the operator is deployed, using helm hooks. + ## It deploy a corresponding clusterrole, clusterrolebinding and serviceaccount to apply the CRDs. + ## This feature is in preview, off by default and may change in the future. + upgradeJob: + enabled: false + image: + busybox: + registry: docker.io + repository: busybox + tag: "latest" + sha: "" + pullPolicy: IfNotPresent + kubectl: + registry: registry.k8s.io + repository: kubectl + tag: "" # defaults to the Kubernetes version + sha: "" + pullPolicy: IfNotPresent + + env: {} + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Assign custom affinity rules to the upgrade-crd job + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + # ## Labels to add to the upgrade-crd job + # ## + labels: {} + + ## Annotations to add to the upgrade-crd job + ## + annotations: {} + + ## Labels to add to the upgrade-crd pod + ## + podLabels: {} + + ## Annotations to add to the upgrade-crd pod + ## + podAnnotations: {} + + ## Service account for upgrade crd job to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + labels: {} + automountServiceAccountToken: true + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault ## custom Rules to override "for" and "severity" in defaultRules ##