Skip to content

Commit

Permalink
[bitnami/kafka] feat: ✨ Add autoscaling support (experimental)
Browse files Browse the repository at this point in the history
Signed-off-by: Javier Salmeron Garcia <[email protected]>
  • Loading branch information
javsalgar committed Apr 5, 2024
1 parent 56a341f commit 4132e01
Show file tree
Hide file tree
Showing 6 changed files with 794 additions and 484 deletions.
1,000 changes: 516 additions & 484 deletions bitnami/kafka/README.md

Large diffs are not rendered by default.

51 changes: 51 additions & 0 deletions bitnami/kafka/templates/broker/hpa.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
{{- /*
Copyright VMware, Inc.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- $replicaCount := int .Values.broker.replicaCount }}
{{- if and (gt $replicaCount 0) .Values.broker.autoscaling.hpa.enabled }}
apiVersion: {{ include "common.capabilities.hpa.apiVersion" ( dict "context" $ ) }}
kind: HorizontalPodAutoscaler
metadata:
name: {{ printf "%s-broker" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: broker
app.kubernetes.io/part-of: kafka
{{- if or .Values.broker.autoscaling.hpa.annotations .Values.commonAnnotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.broker.autoscaling.hpa.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
apiVersion: {{ template "common.capabilities.statefulset.apiVersion" . }}
kind: StatefulSet
name: {{ printf "%s-broker" (include "common.names.fullname" .) }}
minReplicas: {{ .Values.broker.autoscaling.hpa.minReplicas }}
maxReplicas: {{ .Values.broker.autoscaling.hpa.maxReplicas }}
metrics:
{{- if .Values.broker.autoscaling.hpa.targetCPU }}
- type: Resource
resource:
name: cpu
{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
targetAverageUtilization: {{ .Values.broker.autoscaling.hpa.targetCPU }}
{{- else }}
target:
type: Utilization
averageUtilization: {{ .Values.broker.autoscaling.hpa.targetCPU }}
{{- end }}
{{- end }}
{{- if .Values.broker.autoscaling.hpa.targetMemory }}
- type: Resource
resource:
name: memory
{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
targetAverageUtilization: {{ .Values.broker.autoscaling.hpa.targetMemory }}
{{- else }}
target:
type: Utilization
averageUtilization: {{ .Values.broker.autoscaling.hpa.targetMemory }}
{{- end }}
{{- end }}
{{- end }}
45 changes: 45 additions & 0 deletions bitnami/kafka/templates/broker/vpa.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{{- /*
Copyright VMware, Inc.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- $replicaCount := int .Values.broker.replicaCount }}
{{- if and (gt $replicaCount 0) (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler") .Values.broker.autoscaling.vpa.enabled }}
apiVersion: {{ include "common.capabilities.vpa.apiVersion" . }}
kind: VerticalPodAutoscaler
metadata:
name: {{ printf "%s-broker" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: broker
app.kubernetes.io/part-of: kafka
{{- if or .Values.broker.autoscaling.vpa.annotations .Values.commonAnnotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.broker.autoscaling.vpa.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
spec:
resourcePolicy:
containerPolicies:
- containerName: kafka
{{- with .Values.broker.autoscaling.vpa.controlledResources }}
controlledResources:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.broker.autoscaling.vpa.maxAllowed }}
maxAllowed:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.broker.autoscaling.vpa.minAllowed }}
minAllowed:
{{- toYaml . | nindent 8 }}
{{- end }}
targetRef:
apiVersion: {{ (include "common.capabilities.statefulset.apiVersion" .) }}
kind: StatefulSet
name: {{ printf "%s-broker" (include "common.names.fullname" .) }}
{{- if .Values.broker.autoscaling.vpa.updatePolicy }}
updatePolicy:
{{- with .Values.broker.autoscaling.vpa.updatePolicy.updateMode }}
updateMode: {{ . }}
{{- end }}
{{- end }}
{{- end }}
51 changes: 51 additions & 0 deletions bitnami/kafka/templates/controller-eligible/hpa.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
{{- /*
Copyright VMware, Inc.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- $replicaCount := int .Values.controller.replicaCount }}
{{- if and .Values.kraft.enabled (gt $replicaCount 0) .Values.controller.autoscaling.hpa.enabled }}
apiVersion: {{ include "common.capabilities.hpa.apiVersion" ( dict "context" $ ) }}
kind: HorizontalPodAutoscaler
metadata:
name: {{ printf "%s-controller" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: controller
app.kubernetes.io/part-of: kafka
{{- if or .Values.controller.autoscaling.hpa.annotations .Values.commonAnnotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.controller.autoscaling.hpa.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
apiVersion: {{ template "common.capabilities.statefulset.apiVersion" . }}
kind: StatefulSet
name: {{ printf "%s-controller" (include "common.names.fullname" .) }}
minReplicas: {{ .Values.controller.autoscaling.hpa.minReplicas }}
maxReplicas: {{ .Values.controller.autoscaling.hpa.maxReplicas }}
metrics:
{{- if .Values.controller.autoscaling.hpa.targetCPU }}
- type: Resource
resource:
name: cpu
{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
targetAverageUtilization: {{ .Values.controller.autoscaling.hpa.targetCPU }}
{{- else }}
target:
type: Utilization
averageUtilization: {{ .Values.controller.autoscaling.hpa.targetCPU }}
{{- end }}
{{- end }}
{{- if .Values.controller.autoscaling.hpa.targetMemory }}
- type: Resource
resource:
name: memory
{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
targetAverageUtilization: {{ .Values.controller.autoscaling.hpa.targetMemory }}
{{- else }}
target:
type: Utilization
averageUtilization: {{ .Values.controller.autoscaling.hpa.targetMemory }}
{{- end }}
{{- end }}
{{- end }}
45 changes: 45 additions & 0 deletions bitnami/kafka/templates/controller-eligible/vpa.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{{- /*
Copyright VMware, Inc.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- $replicaCount := int .Values.controller.replicaCount }}
{{- if and .Values.kraft.enabled (gt $replicaCount 0) (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler") .Values.controller.autoscaling.vpa.enabled }}
apiVersion: {{ include "common.capabilities.vpa.apiVersion" . }}
kind: VerticalPodAutoscaler
metadata:
name: {{ printf "%s-controller" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: controller
app.kubernetes.io/part-of: kafka
{{- if or .Values.controller.autoscaling.vpa.annotations .Values.commonAnnotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.controller.autoscaling.vpa.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
spec:
resourcePolicy:
containerPolicies:
- containerName: kafka
{{- with .Values.controller.autoscaling.vpa.controlledResources }}
controlledResources:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.controller.autoscaling.vpa.maxAllowed }}
maxAllowed:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.controller.autoscaling.vpa.minAllowed }}
minAllowed:
{{- toYaml . | nindent 8 }}
{{- end }}
targetRef:
apiVersion: {{ (include "common.capabilities.statefulset.apiVersion" .) }}
kind: StatefulSet
name: {{ printf "%s-controller" (include "common.names.fullname" .) }}
{{- if .Values.controller.autoscaling.vpa.updatePolicy }}
updatePolicy:
{{- with .Values.controller.autoscaling.vpa.updatePolicy.updateMode }}
updateMode: {{ . }}
{{- end }}
{{- end }}
{{- end }}
86 changes: 86 additions & 0 deletions bitnami/kafka/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,49 @@ controller:
## containerPort: 1234
##
initContainers: []
## @section Experimental: Kafka Controller Autoscaling configuration
## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/
##
autoscaling:
vpa:
## @param controller.autoscaling.vpa.enabled Enable VPA
##
enabled: false
## @param controller.autoscaling.vpa.annotations Annotations for VPA resource
##
annotations: {}
## @param controller.autoscaling.vpa.controlledResources VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory
##
controlledResources: []
## @param controller.autoscaling.vpa.maxAllowed VPA Max allowed resources for the pod
## cpu: 200m
## memory: 100Mi
maxAllowed: {}
## @param controller.autoscaling.vpa.minAllowed VPA Min allowed resources for the pod
## cpu: 200m
## memory: 100Mi
minAllowed: {}
updatePolicy:
## @param controller.autoscaling.vpa.updatePolicy.updateMode Autoscaling update policy Specifies whether recommended updates are applied when a Pod is started and whether recommended updates are applied during the life of a Pod
## Possible values are "Off", "Initial", "Recreate", and "Auto".
##
updateMode: Auto
hpa:
## @param controller.autoscaling.hpa.enabled Enable HPA for Kafka Controller
##
enabled: false
## @param controller.autoscaling.hpa.minReplicas Minimum number of Kafka Controller replicas
##
minReplicas: ""
## @param controller.autoscaling.hpa.maxReplicas Maximum number of Kafka Controller replicas
##
maxReplicas: ""
## @param controller.autoscaling.hpa.targetCPU Target CPU utilization percentage
##
targetCPU: ""
## @param controller.autoscaling.hpa.targetMemory Target Memory utilization percentage
##
targetMemory: ""
## Kafka Pod Disruption Budget
## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/
## @param controller.pdb.create Deploy a pdb object for the Kafka pod
Expand Down Expand Up @@ -1211,6 +1254,49 @@ broker:
create: false
minAvailable: ""
maxUnavailable: 1
## @section Experimental: Kafka Broker Autoscaling configuration
## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/
##
autoscaling:
vpa:
## @param broker.autoscaling.vpa.enabled Enable VPA
##
enabled: false
## @param broker.autoscaling.vpa.annotations Annotations for VPA resource
##
annotations: {}
## @param broker.autoscaling.vpa.controlledResources VPA List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory
##
controlledResources: []
## @param broker.autoscaling.vpa.maxAllowed VPA Max allowed resources for the pod
## cpu: 200m
## memory: 100Mi
maxAllowed: {}
## @param broker.autoscaling.vpa.minAllowed VPA Min allowed resources for the pod
## cpu: 200m
## memory: 100Mi
minAllowed: {}
updatePolicy:
## @param broker.autoscaling.vpa.updatePolicy.updateMode Autoscaling update policy Specifies whether recommended updates are applied when a Pod is started and whether recommended updates are applied during the life of a Pod
## Possible values are "Off", "Initial", "Recreate", and "Auto".
##
updateMode: Auto
hpa:
## @param broker.autoscaling.hpa.enabled Enable HPA for Kafka Broker
##
enabled: false
## @param broker.autoscaling.hpa.minReplicas Minimum number of Kafka Broker replicas
##
minReplicas: ""
## @param broker.autoscaling.hpa.maxReplicas Maximum number of Kafka Broker replicas
##
maxReplicas: ""
## @param broker.autoscaling.hpa.targetCPU Target CPU utilization percentage
##
targetCPU: ""
## @param broker.autoscaling.hpa.targetMemory Target Memory utilization percentage
##
targetMemory: ""
## Enable persistence using Persistent Volume Claims
## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/
##
Expand Down

0 comments on commit 4132e01

Please sign in to comment.