Bring back cloud-provider flag for kubelet (#348)

Signed-off-by: Waleed Malik <[email protected]>
kubermatic · Jan 16, 2024 · 34aa7a2 · 34aa7a2
1 parent d9581af
commit 34aa7a2
Show file tree

Hide file tree

Showing 11 changed files with 39 additions and 26 deletions.
diff --git a/deploy/osps/default/osp-amzn2.yaml b/deploy/osps/default/osp-amzn2.yaml
@@ -522,10 +522,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/deploy/osps/default/osp-centos.yaml b/deploy/osps/default/osp-centos.yaml
@@ -561,10 +561,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/deploy/osps/default/osp-flatcar-cloud-init.yaml b/deploy/osps/default/osp-flatcar-cloud-init.yaml
@@ -541,10 +541,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/deploy/osps/default/osp-flatcar.yaml b/deploy/osps/default/osp-flatcar.yaml
@@ -654,10 +654,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/deploy/osps/default/osp-rhel.yaml b/deploy/osps/default/osp-rhel.yaml
@@ -564,10 +564,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/deploy/osps/default/osp-rockylinux.yaml b/deploy/osps/default/osp-rockylinux.yaml
@@ -568,10 +568,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/deploy/osps/default/osp-ubuntu.yaml b/deploy/osps/default/osp-ubuntu.yaml
@@ -570,10 +570,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/pkg/controllers/osc/osc_reconciler_test.go b/pkg/controllers/osc/osc_reconciler_test.go
@@ -113,10 +113,11 @@ func init() {
 }
 
 type testConfig struct {
-	namespace        string
-	containerRuntime string
-	clusterDNSIPs    []net.IP
-	featureGates     map[string]bool
+	namespace             string
+	containerRuntime      string
+	clusterDNSIPs         []net.IP
+	featureGates          map[string]bool
+	externalCloudProvider bool
 }
 
 func TestReconciler_Reconcile(t *testing.T) {
@@ -149,9 +150,10 @@ func TestReconciler_Reconcile(t *testing.T) {
 			provisioningSecretFile: "secret-ubuntu-aws-containerd-provisioning.yaml",
 			bootstrapSecretFile:    "secret-ubuntu-aws-containerd-bootstrap.yaml",
 			config: testConfig{
-				namespace:        "kube-system",
-				containerRuntime: "containerd",
-				clusterDNSIPs:    []net.IP{net.IPv4(10, 0, 0, 0)},
+				namespace:             "kube-system",
+				containerRuntime:      "containerd",
+				externalCloudProvider: true,
+				clusterDNSIPs:         []net.IP{net.IPv4(10, 0, 0, 0)},
 			},
 			cloudProvider:     "aws",
 			cloudProviderSpec: runtime.RawExtension{Raw: []byte(`{"availabilityZone": "eu-central-1b", "vpcId": "e-123f", "subnetID": "test-subnet"}`)},
@@ -851,15 +853,16 @@ func buildReconciler(fakeClient controllerruntimeclient.Client, config testConfi
 		Client:       fakeClient,
 		workerClient: fakeClient,
 
-		log:                  testUtil.DefaultLogger,
-		generator:            generator.NewDefaultCloudConfigGenerator(""),
-		namespace:            config.namespace,
-		caCert:               dummyCACert,
-		containerRuntime:     config.containerRuntime,
-		clusterDNSIPs:        config.clusterDNSIPs,
-		kubeletFeatureGates:  featureGates,
-		bootstrappingManager: bootstrappingManager,
-		nodeHTTPProxy:        "http://test-http-proxy.com",
-		nodeNoProxy:          "http://test-no-proxy.com",
+		log:                   testUtil.DefaultLogger,
+		generator:             generator.NewDefaultCloudConfigGenerator(""),
+		namespace:             config.namespace,
+		caCert:                dummyCACert,
+		containerRuntime:      config.containerRuntime,
+		clusterDNSIPs:         config.clusterDNSIPs,
+		kubeletFeatureGates:   featureGates,
+		bootstrappingManager:  bootstrappingManager,
+		externalCloudProvider: config.externalCloudProvider,
+		nodeHTTPProxy:         "http://test-http-proxy.com",
+		nodeNoProxy:           "http://test-no-proxy.com",
 	}
 }
diff --git a/pkg/controllers/osc/testdata/osc-ubuntu-aws-containerd.yaml b/pkg/controllers/osc/testdata/osc-ubuntu-aws-containerd.yaml
@@ -397,6 +397,8 @@ spec:
               --kubeconfig=/var/lib/kubelet/kubeconfig \
               --config=/etc/kubernetes/kubelet.conf \
               --cert-dir=/etc/kubernetes/pki \
+              --cloud-provider=external \
+              --hostname-override=${KUBELET_HOSTNAME} \
               --exit-on-lock-contention \
               --lock-file=/tmp/kubelet.lock \
               --container-runtime-endpoint=unix:///run/containerd/containerd.sock \

diff --git a/pkg/controllers/osc/testdata/osp-rhel-aws-cloud-init-modules.yaml b/pkg/controllers/osc/testdata/osp-rhel-aws-cloud-init-modules.yaml
@@ -571,10 +571,11 @@ spec:
                 --kubeconfig=/var/lib/kubelet/kubeconfig \
                 --config=/etc/kubernetes/kubelet.conf \
                 --cert-dir=/etc/kubernetes/pki \
-                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 {{- if .ExternalCloudProvider }}
                 --cloud-provider=external \
                 {{- else if .InTreeCCMAvailable }}
+                {{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}}
+                {{- if or (semverCompare "<1.29" .KubeVersion) (.DisableCloudProvidersFeatureGateDisabled) }}
                 --cloud-provider={{- .CloudProviderName }} \
                 --cloud-config=/etc/kubernetes/cloud-config \
                 {{- end }}

diff --git a/pkg/controllers/osc/testdata/secret-ubuntu-aws-containerd-provisioning.yaml b/pkg/controllers/osc/testdata/secret-ubuntu-aws-containerd-provisioning.yaml