Add Cilium kube-proxy replacement support

Cilium can take ownership of all kube-proxy responsibilities. In Cilium
v1.13 there are partial and strict modes, however the API changes in
v1.14 to a binary setting so this support is for v1.13 strict mode only.

pkg/api/v1alpha1/cluster_types.go

@@ -834,6 +834,10 @@ type CiliumConfig struct {
 	// +optional
 	SkipUpgrade *bool `json:"skipUpgrade,omitempty"`
 
+	// EnableKubeProxyReplacement replaces the default kube-proxy with Ciliums builtin functionality.
+	// +optional
+	EnableKubeProxyReplacement *bool `json:"enableKubeProxyReplacement,omitempty"`
+
 	// RoutingMode indicates the routing tunnel mode to use for Cilium. Accepted values are overlay (geneve tunnel with overlay)
 	// or direct (tunneling disabled with direct routing)
 	// Defaults to overlay.
@@ -863,6 +867,12 @@ func (n *CiliumConfig) IsManaged() bool {
 	return n.SkipUpgrade == nil || !*n.SkipUpgrade
 }
 
+// IsKubeProxyReplacementEnabled checks if the EnableKubeProxyReplacement flag is true. Nil
+// indicates false.
+func (n *CiliumConfig) IsKubeProxyReplacementEnabled() bool {
+	return n.EnableKubeProxyReplacement != nil && *n.EnableKubeProxyReplacement
+}
+
 // KindnetdConfig contains configuration specific to the Kindnetd CNI.
 type KindnetdConfig struct{}
 

pkg/networking/cilium/templater.go

@@ -245,6 +245,13 @@ func templateValues(spec *cluster.Spec, versionsBundle *cluster.VersionsBundle)
 		}
 	}
 
+	if spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.IsKubeProxyReplacementEnabled() {
+		// See https://docs.cilium.io/en/v1.13/network/kubernetes/kubeproxy-free/#kube-proxy-hybrid-modes.
+		// When upgrading to Cilium> v1.13 this needs to be changed to 'true' as Cilium has simplified the
+		// API.
+		val["kubeProxyReplacement"] = "strict"
+	}
+
 	return val
 }
 

pkg/providers/cloudstack/config/template-cp.yaml

@@ -306,6 +306,12 @@ spec:
           timeAdded: {{ .TimeAdded }}
 {{- end }}
 {{- end }}
+{{- end }}
+{{- with .kubeadmSkipPhases }}
+      skipPhases:
+      {{- range . }}
+      - {{ . }}
+      {{- end }}
 {{- end }}
     joinConfiguration:
       nodeRegistration:

pkg/providers/cloudstack/template.go

@@ -191,6 +191,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 		"externalEtcdVersion":                        versionsBundle.KubeDistro.EtcdVersion,
 		"etcdImage":                                  versionsBundle.KubeDistro.EtcdImage.VersionedImage(),
 		"eksaSystemNamespace":                        constants.EksaSystemNamespace,
+		"kubeadmSkipPhases":                          []string{},
 	}
 
 	auditPolicy, err := common.GetAuditPolicy(clusterSpec.Cluster.Spec.KubernetesVersion)
@@ -246,6 +247,13 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 		values["encryptionProviderConfig"] = conf
 	}
 
+	cni := clusterSpec.Cluster.Spec.ClusterNetwork
+	if cni.CNIConfig != nil && cni.CNIConfig.Cilium != nil {
+		if cni.CNIConfig.Cilium.IsKubeProxyReplacementEnabled() {
+			values["kubeadmSkipPhases"] = append(values["kubeadmSkipPhases"].([]string), "addon/kube-proxy")
+		}
+	}
+
 	return values, nil
 }
 

pkg/providers/cloudstack/template_test.go

@@ -169,3 +169,21 @@ func TestTemplateBuilder_CertSANs(t *testing.T) {
 		test.AssertContentToFile(t, string(data), tc.Output)
 	}
 }
+
+func TestTemplateBuilder_KubeProxyReplacement(t *testing.T) {
+	input := "testdata/cluster_cilium_kube_proxy_replacement.yaml"
+	output := "testdata/expected_cluster_cilium_kube_proxy_replacement.yaml"
+
+	g := NewWithT(t)
+	clusterSpec := test.NewFullClusterSpec(t, input)
+
+	bldr := cloudstack.NewTemplateBuilder(time.Now)
+
+	data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec, func(values map[string]interface{}) {
+		values["controlPlaneTemplateName"] = clusterapi.ControlPlaneMachineTemplateName(clusterSpec.Cluster)
+	})
+	g.Expect(err).ToNot(HaveOccurred())
+
+	test.AssertContentToFile(t, string(data), output)
+
+}

pkg/providers/cloudstack/testdata/cluster_cilium_kube_proxy_replacement.yaml

@@ -0,0 +1,69 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: Cluster
+metadata:
+  name: test
+  namespace: test
+spec:
+  clusterNetwork:
+    cniConfig:
+      cilium:
+        enableKubeProxyReplacement: true
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    services:
+      cidrBlocks:
+        - 10.96.0.0/12
+  controlPlaneConfiguration:
+    count: 1
+    endpoint:
+      host: 0.0.0.0
+    machineGroupRef:
+      kind: CloudStackMachineConfig
+      name: test
+  datacenterRef:
+    kind: CloudStackDatacenterConfig
+    name: test
+  kubernetesVersion: "1.21"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: CloudStackDatacenterConfig
+metadata:
+  name: test
+  namespace: test
+spec:
+  availabilityZones:
+    - account: "admin"
+      domain: "domain1"
+      name: "default-az-0"
+      credentialsRef: "global"
+      zone:
+        name: "zone1"
+        network:
+          name: "net1"
+      managementApiEndpoint: "http://127.16.0.1:8080/client/api"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: CloudStackMachineConfig
+metadata:
+  name: test
+  namespace: test
+spec:
+  computeOffering:
+    name: "m4-large"
+  users:
+  - name: "mySshUsername"
+    sshAuthorizedKeys: # The key below was manually generated and not used in any production systems
+    - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+  template:
+    name: "kubernetes_1_21"
+  diskOffering:
+    name: "Small"
+    mountPath: "/data-small"
+    device: "/dev/vdb"
+    filesystem: "ext4"
+    label: "data_disk"
+  symlinks:
+    /var/log/kubernetes: /data-small/var/log/kubernetes
+  affinityGroupIds:
+  - control-plane-anti-affinity