Add SANs configurability for API server (#6921)

config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml

@@ -124,6 +124,12 @@ spec:
                 type: object
               controlPlaneConfiguration:
                 properties:
+                  certSans:
+                    description: CertSANs is a slice of domain names or IPs to be
+                      added as Subject Name Alternatives of the Kube API Servers Certificate.
+                    items:
+                      type: string
+                    type: array
                   count:
                     description: Count defines the number of desired control plane
                       nodes. Defaults to 1.

pkg/api/v1alpha1/cluster.go

@@ -188,6 +188,7 @@ var clusterConfigValidations = []func(*Cluster) error{
 	validateControlPlaneLabels,
 	validatePackageControllerConfiguration,
 	validateEksaVersion,
+	validateControlPlaneCertSANs,
 }
 
 // GetClusterConfig parses a Cluster object from a multiobject yaml file in disk
@@ -419,6 +420,26 @@ func validateControlPlaneEndpoint(clusterConfig *Cluster) error {
 	return nil
 }
 
+var domainNameRegex = regexp.MustCompile(`(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]`)
+
+func validateControlPlaneCertSANs(cfg *Cluster) error {
+	var invalid []string
+	for _, san := range cfg.Spec.ControlPlaneConfiguration.CertSANs {
+		isDomain := domainNameRegex.MatchString(san)
+		isIP := net.ParseIP(san)
+
+		if !isDomain && isIP == nil {
+			invalid = append(invalid, san)
+		}
+	}
+
+	if len(invalid) > 0 {
+		return fmt.Errorf("invalid ControlPlaneConfiguration.CertSANs; must be an IP or domain name: [%v]", strings.Join(invalid, ", "))
+	}
+
+	return nil
+}
+
 func validateWorkerNodeGroups(clusterConfig *Cluster) error {
 	workerNodeGroupConfigs := clusterConfig.Spec.WorkerNodeGroupConfigurations
 	if len(workerNodeGroupConfigs) <= 0 {

pkg/api/v1alpha1/cluster_types.go

@@ -282,6 +282,9 @@ type ControlPlaneConfiguration struct {
 	// SkipLoadBalancerDeployment skip deploying control plane load balancer.
 	// Make sure your infrastructure can handle control plane load balancing when you set this field to true.
 	SkipLoadBalancerDeployment bool `json:"skipLoadBalancerDeployment,omitempty"`
+	// CertSANs is a slice of domain names or IPs to be added as Subject Name Alternatives of the
+	// Kube API Servers Certificate.
+	CertSANs []string `json:"certSans,omitempty"`
 }
 
 // MachineHealthCheck allows to configure timeouts for machine health checks. Machine Health Checks are responsible for remediating unhealthy Machines.
@@ -335,7 +338,8 @@ func (n *ControlPlaneConfiguration) Equal(o *ControlPlaneConfiguration) bool {
 		return false
 	}
 	return n.Count == o.Count && n.MachineGroupRef.Equal(o.MachineGroupRef) &&
-		TaintsSliceEqual(n.Taints, o.Taints) && MapEqual(n.Labels, o.Labels)
+		TaintsSliceEqual(n.Taints, o.Taints) && MapEqual(n.Labels, o.Labels) &&
+		SliceEqual(n.CertSANs, o.CertSANs)
 }
 
 type Endpoint struct {

pkg/api/v1alpha1/cluster_types_test.go

@@ -3169,3 +3169,64 @@ func TestCNIConfigIsManaged(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateCluster(t *testing.T) {
+	for _, tc := range []struct {
+		Name           string
+		Cluster        *v1alpha1.Cluster
+		ExpectContains []string
+	}{
+		{
+			Name: "APIServerCertSAN_DomainName",
+			Cluster: baseCluster(func(c *v1alpha1.Cluster) {
+				c.Spec.ControlPlaneConfiguration.CertSANs = []string{"domain.com"}
+			}),
+		},
+		{
+			Name: "APIServerCertSAN_DomainName",
+			Cluster: baseCluster(func(c *v1alpha1.Cluster) {
+				c.Spec.ControlPlaneConfiguration.CertSANs = []string{"domain%com"}
+			}),
+			ExpectContains: []string{"domain%com"},
+		},
+		{
+			Name: "APIServerCertSAN_IP",
+			Cluster: baseCluster(func(c *v1alpha1.Cluster) {
+				c.Spec.ControlPlaneConfiguration.CertSANs = []string{"11.11.11.11"}
+			}),
+		},
+		{
+			Name: "APIServerCertSAN_Empty",
+			Cluster: baseCluster(func(c *v1alpha1.Cluster) {
+				c.Spec.ControlPlaneConfiguration.CertSANs = []string{""}
+			}),
+			ExpectContains: []string{""},
+		},
+		{
+			Name: "APIServerCertSAN_Multi",
+			Cluster: baseCluster(func(c *v1alpha1.Cluster) {
+				c.Spec.ControlPlaneConfiguration.CertSANs = []string{"11.11.11.11", "domain.com"}
+			}),
+		},
+		{
+			Name: "APIServerCertSAN_Multi",
+			Cluster: baseCluster(func(c *v1alpha1.Cluster) {
+				c.Spec.ControlPlaneConfiguration.CertSANs = []string{"11.11.11.11", "domain%com"}
+			}),
+			ExpectContains: []string{"domain%com"},
+		},
+	} {
+		t.Run(tc.Name, func(t *testing.T) {
+			g := NewWithT(t)
+			err := tc.Cluster.Validate()
+			if len(tc.ExpectContains) > 0 {
+				g.Expect(err).To(HaveOccurred())
+				for _, str := range tc.ExpectContains {
+					g.Expect(err.Error()).To(ContainSubstring(str))
+				}
+			} else {
+				g.Expect(err).To(Succeed())
+			}
+		})
+	}
+}

pkg/clusterapi/apibuilder.go

@@ -159,6 +159,7 @@ func KubeadmControlPlane(clusterSpec *cluster.Spec, infrastructureObject APIObje
 							ExtraArgs:    map[string]string{},
 							ExtraVolumes: []bootstrapv1.HostPathMount{},
 						},
+						CertSANs: clusterSpec.Cluster.Spec.ControlPlaneConfiguration.CertSANs,
 					},
 					ControllerManager: bootstrapv1.ControlPlaneComponent{
 						ExtraArgs:    ControllerManagerArgs(clusterSpec),
@@ -222,6 +223,7 @@ func KubeadmConfigTemplate(clusterSpec *cluster.Spec, workerNodeGroupConfig anyw
 							ControlPlaneComponent: bootstrapv1.ControlPlaneComponent{
 								ExtraArgs: map[string]string{},
 							},
+							CertSANs: clusterSpec.Cluster.Spec.ControlPlaneConfiguration.CertSANs,
 						},
 					},
 					JoinConfiguration: &bootstrapv1.JoinConfiguration{

pkg/clusterapi/apibuilder_test.go

@@ -96,6 +96,7 @@ func newApiBuilerTest(t *testing.T) apiBuilerTest {
 						"key1": "val1",
 						"key2": "val2",
 					},
+					CertSANs: []string{"foo.bar", "11.11.11.11"},
 				},
 				KubernetesVersion: "1.21",
 			},
@@ -311,6 +312,7 @@ func wantKubeadmControlPlane() *controlplanev1.KubeadmControlPlane {
 							ExtraArgs:    map[string]string{},
 							ExtraVolumes: []bootstrapv1.HostPathMount{},
 						},
+						CertSANs: []string{"foo.bar", "11.11.11.11"},
 					},
 					ControllerManager: bootstrapv1.ControlPlaneComponent{
 						ExtraArgs:    tlsCipherSuitesArgs(),
@@ -392,6 +394,7 @@ func wantKubeadmConfigTemplate() *bootstrapv1.KubeadmConfigTemplate {
 							ControlPlaneComponent: bootstrapv1.ControlPlaneComponent{
 								ExtraArgs: map[string]string{},
 							},
+							CertSANs: []string{"foo.bar", "11.11.11.11"},
 						},
 					},
 					JoinConfiguration: &bootstrapv1.JoinConfiguration{

pkg/clusterapi/identity_test.go

@@ -105,6 +105,7 @@ func TestConfigureAWSIAMAuthInKubeadmControlPlane(t *testing.T) {
 										},
 									},
 								},
+								CertSANs: []string{"foo.bar", "11.11.11.11"},
 							},
 							ControllerManager: bootstrapv1.ControlPlaneComponent{
 								ExtraArgs:    tlsCipherSuitesArgs(),
@@ -302,6 +303,7 @@ func TestConfigureOIDCInKubeadmControlPlane(t *testing.T) {
 									},
 									ExtraVolumes: []bootstrapv1.HostPathMount{},
 								},
+								CertSANs: []string{"foo.bar", "11.11.11.11"},
 							},
 							ControllerManager: bootstrapv1.ControlPlaneComponent{
 								ExtraArgs:    tlsCipherSuitesArgs(),
@@ -426,6 +428,7 @@ func TestConfigurePodIamAuthInKubeadmControlPlane(t *testing.T) {
 									},
 									ExtraVolumes: []bootstrapv1.HostPathMount{},
 								},
+								CertSANs: []string{"foo.bar", "11.11.11.11"},
 							},
 							ControllerManager: bootstrapv1.ControlPlaneComponent{
 								ExtraArgs:    tlsCipherSuitesArgs(),

pkg/providers/cloudstack/config/template-cp.yaml

@@ -87,6 +87,10 @@ spec:
         imageRepository: {{.corednsRepository}}
         imageTag: {{.corednsVersion}}
       apiServer:
+        {{- with .apiServerCertSANs }}
+        certSANs:
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
         extraArgs:
           cloud-provider: external
           audit-policy-file: /etc/kubernetes/audit-policy.yaml

pkg/providers/cloudstack/template.go

@@ -143,6 +143,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 		"controlPlaneEndpointHost":                   host,
 		"controlPlaneEndpointPort":                   port,
 		"controlPlaneReplicas":                       clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Count,
+		"apiServerCertSANs":                          clusterSpec.Cluster.Spec.ControlPlaneConfiguration.CertSANs,
 		"kubernetesRepository":                       versionsBundle.KubeDistro.Kubernetes.Repository,
 		"kubernetesVersion":                          versionsBundle.KubeDistro.Kubernetes.Tag,
 		"etcdRepository":                             versionsBundle.KubeDistro.Etcd.Repository,

pkg/providers/cloudstack/template_test.go

@@ -141,3 +141,31 @@ func TestTemplateBuilderGenerateCAPISpecWorkersInvalidEndpoint(t *testing.T) {
 	_, err := templateBuilder.GenerateCAPISpecWorkers(clusterSpec, machineTemplateNames, kubeadmConfigTemplateNames)
 	g.Expect(err).To(MatchError(ContainSubstring("building template map for MD host 1.1.1.1:: is invalid: address 1.1.1.1::: too many colons in address")))
 }
+
+func TestTemplateBuilder_CertSANs(t *testing.T) {
+	for _, tc := range []struct {
+		Input  string
+		Output string
+	}{
+		{
+			Input:  "testdata/cluster_api_server_cert_san_domain_name.yaml",
+			Output: "testdata/expected_cluster_api_server_cert_san_domain_name.yaml",
+		},
+		{
+			Input:  "testdata/cluster_api_server_cert_san_ip.yaml",
+			Output: "testdata/expected_cluster_api_server_cert_san_ip.yaml",
+		},
+	} {
+		g := NewWithT(t)
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+
+		bldr := cloudstack.NewTemplateBuilder(time.Now)
+
+		data, err := bldr.GenerateCAPISpecControlPlane(clusterSpec, func(values map[string]interface{}) {
+			values["controlPlaneTemplateName"] = clusterapi.ControlPlaneMachineTemplateName(clusterSpec.Cluster)
+		})
+		g.Expect(err).ToNot(HaveOccurred())
+
+		test.AssertContentToFile(t, string(data), tc.Output)
+	}
+}

pkg/providers/cloudstack/testdata/cluster_api_server_cert_san_domain_name.yaml

@@ -0,0 +1,69 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: Cluster
+metadata:
+  name: test
+  namespace: test
+spec:
+  clusterNetwork:
+    cniConfig:
+      cilium: {}
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    services:
+      cidrBlocks:
+        - 10.96.0.0/12
+  controlPlaneConfiguration:
+    count: 1
+    endpoint:
+      host: 0.0.0.0
+    certSANs: ["foo.bar"]
+    machineGroupRef:
+      kind: CloudStackMachineConfig
+      name: test
+  datacenterRef:
+    kind: CloudStackDatacenterConfig
+    name: test
+  kubernetesVersion: "1.21"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: CloudStackDatacenterConfig
+metadata:
+  name: test
+  namespace: test
+spec:
+  availabilityZones:
+    - account: "admin"
+      domain: "domain1"
+      name: "default-az-0"
+      credentialsRef: "global"
+      zone:
+        name: "zone1"
+        network:
+          name: "net1"
+      managementApiEndpoint: "http://127.16.0.1:8080/client/api"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: CloudStackMachineConfig
+metadata:
+  name: test
+  namespace: test
+spec:
+  computeOffering:
+    name: "m4-large"
+  users:
+  - name: "mySshUsername"
+    sshAuthorizedKeys: # The key below was manually generated and not used in any production systems
+    - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+  template:
+    name: "kubernetes_1_21"
+  diskOffering:
+    name: "Small"
+    mountPath: "/data-small"
+    device: "/dev/vdb"
+    filesystem: "ext4"
+    label: "data_disk"
+  symlinks:
+    /var/log/kubernetes: /data-small/var/log/kubernetes
+  affinityGroupIds:
+  - control-plane-anti-affinity