[release/v1.5] Fix support for ca-bundle (#399) (#417)

* Fix support for ca-bundle * Add support for custom ca-bundle in all the OSPs --------- Signed-off-by: Waleed Malik <[email protected]> Signed-off-by: Artiom Diomin <[email protected]> Co-authored-by: Waleed Malik <[email protected]>
kubermatic · Sep 20, 2024 · 9c72405 · 9c72405
1 parent 56a6a21
commit 9c72405
Show file tree

Hide file tree

Showing 10 changed files with 93 additions and 7 deletions.
diff --git a/cmd/osm-controller/main.go b/cmd/osm-controller/main.go
@@ -89,6 +89,7 @@ type options struct {
 
 	overrideBootstrapKubeletAPIServer string
 	bootstrapTokenServiceAccountName  string
+	kubernetesCABundleFile            string
 	caBundleFile                      string
 }
 
@@ -142,7 +143,12 @@ func main() {
 	flag.BoolVar(&opt.enableLeaderElection, "leader-elect", true, "Enable leader election for controller manager.")
 	flag.StringVar(&opt.overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf")
 	flag.StringVar(&opt.bootstrapTokenServiceAccountName, "bootstrap-token-service-account-name", "", "When set use the service account token from this SA as bootstrap token instead of creating a temporary one. Passed in namespace/name format")
-	flag.StringVar(&opt.caBundleFile, "ca-bundle", "", "Path to a file containing all PEM-encoded CA certificates. Will be used for Kubernetes CA certificates.")
+
+	flag.StringVar(&opt.kubernetesCABundleFile, "kubernetes-ca-bundle", "", "Path to a file containing all PEM-encoded CA certificates. Will be used for Kubernetes CA certificates.")
+	flag.StringVar(&opt.caBundleFile, "host-ca-bundle", "", "Path to a file containing all PEM-encoded CA certificates. Will be propagated to the machine and used instead of the host's certificates if set.")
+
+	var noopFlag string
+	flag.StringVar(&noopFlag, "ca-bundle", "", "DEPRECATED: This flag is no-op and will have no effect. Use `host-ca-bundle` or `kubernetes-ca-bundle` instead.")
 
 	flag.Parse()
 
@@ -165,14 +171,22 @@ func main() {
 	}
 
 	var (
-		err          error
-		customCACert string
+		err                    error
+		kubernetesCustomCACert string
+		hostCACert             string
 	)
 
+	if len(opt.kubernetesCABundleFile) > 0 {
+		kubernetesCustomCACert, err = retrieveCustomCACertificate(opt.kubernetesCABundleFile)
+		if err != nil {
+			log.Fatalf("-kubernetes-ca-bundle is invalid: %s", err.Error())
+		}
+	}
+
 	if len(opt.caBundleFile) > 0 {
-		customCACert, err = retrieveCustomCACertificate(opt.caBundleFile)
+		hostCACert, err = retrieveCustomCACertificate(opt.caBundleFile)
 		if err != nil {
-			log.Fatalf("-ca-bundle is invalid: %s", err.Error())
+			log.Fatalf("-host-ca-bundle is invalid: %s", err.Error())
 		}
 	}
 
@@ -262,8 +276,8 @@ func main() {
 		}
 	}
 
-	caCert := customCACert
-	if opt.caBundleFile == "" {
+	caCert := kubernetesCustomCACert
+	if opt.kubernetesCABundleFile == "" {
 		caCert, err = certificate.GetCACert(opt.kubeconfig, mgr.GetConfig())
 		if err != nil {
 			log.Fatal("failed to load CA certificate", zap.Error(err))
@@ -289,6 +303,7 @@ func main() {
 		mgr.GetClient(),
 		bootstrappingManager,
 		caCert,
+		hostCACert,
 		opt.namespace,
 		opt.workerCount,
 		parsedClusterDNSIPs,

diff --git a/deploy/osps/default/osp-amzn2.yaml b/deploy/osps/default/osp-amzn2.yaml
@@ -46,6 +46,14 @@ spec:
 
         source /etc/environment
 
+      configureHostCABundle: |-
+        {{- if .HostCACert }}
+        cat <<EOF | tee /usr/local/share/ca-certificates/custom.crt
+        {{ .HostCACert }}
+        EOF
+        update-ca-certificates
+        {{- end }}
+
     files:
       - path: /opt/bin/supervise.sh
         permissions: 755
@@ -76,6 +84,7 @@ spec:
 
               {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
               {{- template "configureProxyScript" }}
+              {{- template "configureHostCABundle" }}
 
               yum install -y curl jq
 

diff --git a/deploy/osps/default/osp-centos.yaml b/deploy/osps/default/osp-centos.yaml
@@ -54,6 +54,14 @@ spec:
 
         source /etc/environment
 
+      configureHostCABundle: |-
+        {{- if .HostCACert }}
+        cat <<EOF | tee /etc/pki/ca-trust/source/anchors/custom.crt
+        {{ .HostCACert }}
+        EOF
+        update-ca-trust
+        {{- end }}
+
     files:
       - path: /opt/bin/supervise.sh
         permissions: 755
@@ -84,6 +92,7 @@ spec:
 
               {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
               {{- template "configureProxyScript" }}
+              {{- template "configureHostCABundle" }}
 
               source /etc/os-release
               if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then

diff --git a/deploy/osps/default/osp-flatcar-cloud-init.yaml b/deploy/osps/default/osp-flatcar-cloud-init.yaml
@@ -47,6 +47,14 @@ spec:
 
         source /etc/environment
 
+      configureHostCABundle: |-
+        {{- if .HostCACert }}
+        cat <<EOF | tee /etc/ssl/certs/custom.pem
+        {{ .HostCACert }}
+        EOF
+        update-ca-certificates
+        {{- end }}
+
       safeDownloadBinariesScript: |-
         {{- /* setup some common directories */}}
         opt_bin=/opt/bin
@@ -209,6 +217,7 @@ spec:
 
               {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
               {{- template "configureProxyScript" }}
+              {{- template "configureHostCABundle" }}
 
               {{- if not .FlatcarConfig.DisableAutoUpdate }}
               cat << EOF | tee /etc/polkit-1/rules.d/60-noreboot_norestart.rules

diff --git a/deploy/osps/default/osp-flatcar.yaml b/deploy/osps/default/osp-flatcar.yaml
@@ -52,6 +52,7 @@ spec:
         {{- end }}
 
         source /etc/environment
+
       configureVCloudNetworkStatic: |-
         function query_ovf () {
           PATH=$PATH:/usr/share/oem/bin/
@@ -102,6 +103,14 @@ spec:
 
         fi
 
+      configureHostCABundle: |-
+        {{- if .HostCACert }}
+        cat <<EOF | tee /etc/ssl/certs/custom.pem
+        {{ .HostCACert }}
+        EOF
+        update-ca-certificates
+        {{- end }}
+
     units:
       - name: bootstrap.service
         enable: true
@@ -148,6 +157,7 @@ spec:
 
               {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
               {{- template "configureProxyScript" }}
+              {{- template "configureHostCABundle" }}
 
               curl -s -k -v --header 'Authorization: Bearer {{ .Token }}'	{{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /usr/share/oem/config.ign
 

diff --git a/deploy/osps/default/osp-rhel.yaml b/deploy/osps/default/osp-rhel.yaml
@@ -50,6 +50,14 @@ spec:
 
         source /etc/environment
 
+      configureHostCABundle: |-
+        {{- if .HostCACert }}
+        cat <<EOF | tee /etc/pki/ca-trust/source/whitelist/custom.crt
+        {{ .HostCACert }}
+        EOF
+        update-ca-trust
+        {{- end }}
+
     files:
       - path: /opt/bin/supervise.sh
         permissions: 755
@@ -80,6 +88,7 @@ spec:
 
               {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
               {{- template "configureProxyScript" }}
+              {{- template "configureHostCABundle" }}
 
               yum install -y curl jq
 

diff --git a/deploy/osps/default/osp-rockylinux.yaml b/deploy/osps/default/osp-rockylinux.yaml
@@ -53,6 +53,14 @@ spec:
 
         source /etc/environment
 
+      configureHostCABundle: |-
+        {{- if .HostCACert }}
+        cat <<EOF | tee /etc/pki/ca-trust/source/anchors/custom.crt
+        {{ .HostCACert }}
+        EOF
+        update-ca-trust
+        {{- end }}
+
     files:
       - path: /opt/bin/supervise.sh
         permissions: 755
@@ -83,6 +91,7 @@ spec:
 
               {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
               {{- template "configureProxyScript" }}
+              {{- template "configureHostCABundle" }}
 
               yum install -y curl jq
 

diff --git a/deploy/osps/default/osp-ubuntu.yaml b/deploy/osps/default/osp-ubuntu.yaml
@@ -66,6 +66,14 @@ spec:
 
         source /etc/environment
 
+      configureHostCABundle: |-
+        {{- if .HostCACert }}
+        cat <<EOF | sudo tee /usr/local/share/ca-certificates/custom.crt
+        {{ .HostCACert }}
+        EOF
+        sudo update-ca-certificates
+        {{- end }}
+
     files:
       - path: /opt/bin/supervise.sh
         permissions: 755
@@ -96,6 +104,7 @@ spec:
 
               {{- /* Configure proxy as the first step to ensure that all the phases of provisioning respect the proxy environment. */}}
               {{- template "configureProxyScript" }}
+              {{- template "configureHostCABundle" }}
 
               export DEBIAN_FRONTEND=noninteractive
               apt update && apt install -y curl jq

diff --git a/pkg/controllers/osc/osc_controller.go b/pkg/controllers/osc/osc_controller.go
@@ -74,6 +74,7 @@ type Reconciler struct {
 	generator                     generator.CloudConfigGenerator
 	clusterDNSIPs                 []net.IP
 	caCert                        string
+	hostCACert                    string
 	nodeHTTPProxy                 string
 	nodeNoProxy                   string
 	nodeRegistryCredentialsSecret string
@@ -88,6 +89,7 @@ func Add(
 	client ctrlruntimeclient.Client,
 	bootstrappingManager bootstrap.Bootstrap,
 	caCert string,
+	hostCACert string,
 	namespace string,
 	workerCount int,
 	clusterDNSIPs []net.IP,
@@ -108,6 +110,7 @@ func Add(
 		recorder:                      mgr.GetEventRecorderFor(ControllerName),
 		bootstrappingManager:          bootstrappingManager,
 		caCert:                        caCert,
+		hostCACert:                    hostCACert,
 		namespace:                     namespace,
 		generator:                     generator,
 		clusterDNSIPs:                 clusterDNSIPs,
@@ -258,6 +261,7 @@ func (r *Reconciler) reconcileOperatingSystemConfigs(ctx context.Context, md *cl
 		oscName,
 		r.namespace,
 		r.caCert,
+		r.hostCACert,
 		r.clusterDNSIPs,
 		r.containerRuntime,
 		r.externalCloudProvider,

diff --git a/pkg/controllers/osc/resources/operating_system_config.go b/pkg/controllers/osc/resources/operating_system_config.go
@@ -70,6 +70,7 @@ func GenerateOperatingSystemConfig(
 	oscName string,
 	namespace string,
 	caCert string,
+	hostCACert string,
 	clusterDNSIPs []net.IP,
 	containerRuntime string,
 	externalCloudProvider bool,
@@ -176,6 +177,7 @@ func GenerateOperatingSystemConfig(
 		KubeVersion:                kubeletVersionStr,
 		ClusterDNSIPs:              clusterDNSIPs,
 		KubernetesCACert:           caCert,
+		HostCACert:                 hostCACert,
 		InTreeCCMAvailable:         inTreeCCM,
 		CloudConfig:                cloudConfig,
 		ContainerRuntime:           containerRuntime,
@@ -268,6 +270,7 @@ type filesData struct {
 	CNIVersion                 string
 	ClusterDNSIPs              []net.IP
 	KubernetesCACert           string
+	HostCACert                 string
 	ServerAddress              string
 	CloudConfig                string
 	ContainerRuntime           string