[EKS] Change Kubernetes control plane to EKS (the eks branch)

cluster/config-defaults.yaml

@@ -733,8 +733,8 @@ tracing_coredns_local_zone_traces_endpoint: ""
 # AMI id given the image name and the Image AWS account owner.
 #
 # [0]: https://github.com/zalando-incubator/cluster-lifecycle-manager/blob/8a9bd1cb2d094038a9e23e646421f8146b48886a/provisioner/template.go#L116
-kuberuntu_image_v1_31_jammy_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.0-amd64-master-347" "861068367966" }}
-kuberuntu_image_v1_31_jammy_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.0-arm64-master-347" "861068367966" }}
+kuberuntu_image_v1_31_jammy_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.1-amd64-master-357" "861068367966" }}
+kuberuntu_image_v1_31_jammy_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.1-arm64-master-357" "861068367966" }}
 
 # Which distro from the previous config items should be used. Valid options are only `jammy` for now. Can be set for each node pool.
 kuberuntu_distro_master: "jammy"
@@ -1150,8 +1150,19 @@ control_plane_graceful_shutdown: "true"
 #  fs.inotify.max_user_watches = 100000
 sysctl_settings: ""
 
-
-
 # scheduling_controls
 teapot_admission_controller_scheduling_controls_enabled: "false"
 teapot_admission_controller_scheduling_controls_default_architecture: "amd64"
+
+eks: "false"
+eks_control_plane_logging: "false"
+eks_ip_family: "ipv4"
+# prefix delegation can only be configured for ipv4. For ipv6 it can only be
+# true.
+aws_vpc_cni_prefix_delegation: "true"
+eks_zalando_iam_aws_proxy_cpu: "100m"
+eks_zalando_iam_aws_proxy_memory: "512Mi"
+eks_zalando_iam_aws_proxy_hpa_max_replicas: "10"
+eks_zalando_iam_aws_proxy_hpa_cpu_target: "80"
+eks_zalando_iam_aws_proxy_hpa_memory_target: "80"
+eks_okta_identity_provider: "true"

cluster/manifests/01-aws-node/daemonset.yaml

@@ -0,0 +1,258 @@
+{{- if eq .Cluster.Provider "zalando-eks" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/instance: aws-vpc-cni
+    app.kubernetes.io/name: aws-node
+    app.kubernetes.io/version: v1.18.1
+    k8s-app: aws-node
+    application: kubernetes
+    component: aws-node
+  name: aws-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-node
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: aws-vpc-cni
+        app.kubernetes.io/name: aws-node
+        k8s-app: aws-node
+        application: kubernetes
+        component: aws-node
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+              - key: kubernetes.io/arch
+                operator: In
+                values:
+                - amd64
+                - arm64
+              - key: eks.amazonaws.com/compute-type
+                operator: NotIn
+                values:
+                - fargate
+      containers:
+      - env:
+        - name: ADDITIONAL_ENI_TAGS
+          value: '{}'
+        - name: ANNOTATE_POD_IP
+          value: "false"
+        - name: AWS_VPC_CNI_NODE_PORT_SUPPORT
+          value: "true"
+        - name: AWS_VPC_ENI_MTU
+          value: "9001"
+        - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG
+          value: "false"
+        - name: AWS_VPC_K8S_CNI_EXTERNALSNAT
+          value: "false"
+        - name: AWS_VPC_K8S_CNI_LOGLEVEL
+          value: DEBUG
+        - name: AWS_VPC_K8S_CNI_LOG_FILE
+          value: /host/var/log/aws-routed-eni/ipamd.log
+        - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT
+          value: prng
+        - name: AWS_VPC_K8S_CNI_VETHPREFIX
+          value: eni
+        - name: AWS_VPC_K8S_PLUGIN_LOG_FILE
+          value: /var/log/aws-routed-eni/plugin.log
+        - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL
+          value: DEBUG
+        - name: CLUSTER_NAME
+          value: "{{ .Cluster.ID | eksID }}"
+        - name: DISABLE_INTROSPECTION
+          value: "false"
+        - name: DISABLE_METRICS
+          value: "false"
+        - name: DISABLE_NETWORK_RESOURCE_PROVISIONING
+          value: "false"
+        - name: ENABLE_IPv4
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}true{{else}}false{{end}}"
+        - name: ENABLE_IPv6
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}"
+        - name: ENABLE_POD_ENI
+          value: "false"
+        - name: ENABLE_PREFIX_DELEGATION
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}{{.Cluster.ConfigItems.aws_vpc_cni_prefix_delegation}}{{else}}true{{end}}"
+        - name: ENABLE_SUBNET_DISCOVERY
+          value: "true"
+        - name: NETWORK_POLICY_ENFORCING_MODE
+          value: standard
+        - name: VPC_CNI_VERSION
+          value: v1.18.1
+        - name: VPC_ID
+          value: "{{ .Cluster.ConfigItems.vpc_id }}"
+        - name: WARM_ENI_TARGET
+          value: "1"
+        - name: WARM_PREFIX_TARGET
+          value: "1"
+        - name: MY_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: MY_POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni:v1.18.1-eksbuild.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - /app/grpc-health-probe
+            - -addr=:50051
+            - -connect-timeout=5s
+            - -rpc-timeout=5s
+          failureThreshold: 3
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 10
+        name: aws-node
+        ports:
+        - containerPort: 61678
+          hostPort: 61678
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - /app/grpc-health-probe
+            - -addr=:50051
+            - -connect-timeout=5s
+            - -rpc-timeout=5s
+          failureThreshold: 3
+          initialDelaySeconds: 1
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 10
+        resources:
+          requests:
+            cpu: 25m
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+        - mountPath: /host/etc/cni/net.d
+          name: cni-net-dir
+        - mountPath: /host/var/log/aws-routed-eni
+          name: log-dir
+        - mountPath: /var/run/aws-node
+          name: run-dir
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+      - args:
+        - --enable-ipv6={{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}
+        - --enable-network-policy=false
+        - --enable-cloudwatch-logs=false
+        - --enable-policy-event-logs=false
+        - --metrics-bind-addr=:8162
+        - --health-probe-bind-addr=:8163
+        - --conntrack-cache-cleanup-period=300
+        env:
+        - name: MY_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon/aws-network-policy-agent:v1.1.1-eksbuild.2
+        imagePullPolicy: IfNotPresent
+        name: aws-eks-nodeagent
+        resources:
+          requests:
+            cpu: 25m
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+        - mountPath: /sys/fs/bpf
+          name: bpf-pin-path
+        - mountPath: /var/log/aws-routed-eni
+          name: log-dir
+        - mountPath: /var/run/aws-node
+          name: run-dir
+      dnsPolicy: ClusterFirst
+      hostNetwork: true
+      initContainers:
+      - env:
+        - name: DISABLE_TCP_EARLY_DEMUX
+          value: "false"
+        - name: ENABLE_IPv6
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}"
+        image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni-init:v1.18.1-eksbuild.3
+        imagePullPolicy: IfNotPresent
+        name: aws-vpc-cni-init
+        resources:
+          requests:
+            cpu: 25m
+        securityContext:
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: aws-node
+      serviceAccountName: aws-node
+      terminationGracePeriodSeconds: 10
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /sys/fs/bpf
+          type: ""
+        name: bpf-pin-path
+      - hostPath:
+          path: /opt/cni/bin
+          type: ""
+        name: cni-bin-dir
+      - hostPath:
+          path: /etc/kubernetes/cni/net.d #/etc/cni/net.d
+          type: ""
+        name: cni-net-dir
+      - hostPath:
+          path: /var/log/aws-routed-eni
+          type: DirectoryOrCreate
+        name: log-dir
+      - hostPath:
+          path: /var/run/aws-node
+          type: DirectoryOrCreate
+        name: run-dir
+      - hostPath:
+          path: /run/xtables.lock
+          type: ""
+        name: xtables-lock
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 10%
+    type: RollingUpdate
+{{- end }}

cluster/manifests/01-aws-node/sa.yaml

@@ -0,0 +1,11 @@
+{{- if eq .Cluster.Provider "zalando-eks"}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-node
+  namespace: kube-system
+  labels:
+    application: kubernetes
+    component: aws-node
+{{- end}}

cluster/manifests/coredns-local/configmap-local.yaml → cluster/manifests/01-coredns-local/configmap-local.yaml

@@ -71,7 +71,11 @@ data:
 {{ end }}
         template IN A {
             match "^.*[.]ingress[.]cluster[.]local"
+            {{- if eq .Cluster.Provider "zalando-eks" }}
+            answer "{{"{{"}} .Name {{"}}"}} 60 IN AAAA {{ nthAddressFromCIDR .Cluster.ConfigItems.service_cidr 50 }}"
+            {{- else}}
             answer "{{"{{"}} .Name {{"}}"}} 60 IN A 10.5.99.99"
+            {{- end}}
             fallthrough
         }
         template IN AAAA {
@@ -83,7 +87,7 @@ data:
 
     # Defines that this server is authority for reverse
     # lookups for these ranges.
-    cluster.local:9254 10.2.0.0/15:9254 10.5.0.0/16:9254 {{ if eq .Cluster.ConfigItems.tracing_coredns_route_traces_to_local_zone "true"}}{{ range $src := split .Cluster.ConfigItems.tracing_coredns_global_traces_endpoint  "," }}{{ $src }}:9254 {{ end }} {{ end }} {
+    cluster.local:9254 {{if eq .Cluster.Provider "zalando-eks"}}in-addr.arpa:9254 ip6.arpa:9254{{else}}10.2.0.0/15:9254 10.5.0.0/16:9254{{end}} {{ if eq .Cluster.ConfigItems.tracing_coredns_route_traces_to_local_zone "true"}}{{ range $src := split .Cluster.ConfigItems.tracing_coredns_global_traces_endpoint  "," }}{{ $src }}:9254 {{ end }} {{ end }} {
         errors
         {{ if eq .Cluster.ConfigItems.tracing_coredns_route_traces_to_local_zone "true"}}
           {{- with $cluster := .Cluster }}
@@ -94,6 +98,9 @@ data:
         {{ end }}
         kubernetes {
             pods insecure
+            {{- if eq .Cluster.Provider "zalando-eks"}}
+            fallthrough in-addr.arpa ip6.arpa
+            {{- end}}
         }
         cache 30
 {{ if eq .Cluster.ConfigItems.coredns_log_svc_names "true"}}
@@ -118,7 +125,7 @@ data:
 {{ else }}
         forward . /etc/resolv.conf
 {{ end }}
-        pprof 127.0.0.1:9156
+        pprof :9156
         cache 30
         reload
     }