aws · eks-distro-bot · Sep 22, 2023 · Aug 10, 2023 · Aug 16, 2023 · Aug 23, 2023
diff --git a/pkg/api/v1alpha1/cluster_defaults_test.go b/pkg/api/v1alpha1/cluster_defaults_test.go
@@ -301,8 +301,8 @@ func TestSetClusterDefaults(t *testing.T) {
 									KMS: &KMS{
 										Name:                "test-config",
 										SocketListenAddress: "unix:///kms/socket/path",
-										CacheSize:           defaultKMSCacheSize,
-										Timeout:             &defaultKMSTimeout,
+										CacheSize:           DefaultKMSCacheSize,
+										Timeout:             &DefaultKMSTimeout,
 									},
 								},
 							},

diff --git a/pkg/api/v1alpha1/etcdencryption.go b/pkg/api/v1alpha1/etcdencryption.go
@@ -11,8 +11,10 @@ import (
 )
 
 var (
-	defaultKMSCacheSize = ptr.Int32(1000)
-	defaultKMSTimeout   = metav1.Duration{Duration: time.Second * 3}
+	// DefaultKMSCacheSize is the default cache size for KMS provider (1000).
+	DefaultKMSCacheSize = ptr.Int32(1000)
+	// DefaultKMSTimeout is the default timeout for KMS provider (3s).
+	DefaultKMSTimeout = metav1.Duration{Duration: time.Second * 3}
 )
 
 // ValidateEtcdEncryptionConfig validates the etcd encryption configuration.
@@ -85,10 +87,10 @@ func setEtcdEncryptionConfigDefaults(cluster *Cluster) error {
 func setKMSConfigDefauts(kms *KMS) {
 	if kms != nil {
 		if kms.CacheSize == nil {
-			kms.CacheSize = defaultKMSCacheSize
+			kms.CacheSize = DefaultKMSCacheSize
 		}
 		if kms.Timeout == nil {
-			kms.Timeout = &defaultKMSTimeout
+			kms.Timeout = &DefaultKMSTimeout
 		}
 	}
 }
diff --git a/pkg/clusterapi/extraargs.go b/pkg/clusterapi/extraargs.go
@@ -43,6 +43,16 @@ func AwsIamAuthExtraArgs(awsiam *v1alpha1.AWSIamConfig) ExtraArgs {
 	return args
 }
 
+// EtcdEncryptionExtraArgs takes a list of EtcdEncryption configs and returns the relevant API server extra args if it's not nil or empty.
+func EtcdEncryptionExtraArgs(config *[]v1alpha1.EtcdEncryption) ExtraArgs {
+	args := ExtraArgs{}
+	if config == nil || len(*config) == 0 {
+		return args
+	}
+	args.AddIfNotEmpty("encryption-provider-config", "/etc/kubernetes/enc/encryption-config.yaml")
+	return args
+}
+
 // FeatureGatesExtraArgs takes a list of features with the value and returns it in the proper format
 // Example FeatureGatesExtraArgs("ServiceLoadBalancerClass=true").
 func FeatureGatesExtraArgs(features ...string) ExtraArgs {

diff --git a/pkg/clusterapi/extraargs_test.go b/pkg/clusterapi/extraargs_test.go
@@ -519,6 +519,109 @@ func TestNodeCIDRMaskExtraArgs(t *testing.T) {
 	}
 }
 
+func TestEtcdEncryptionExtraArgs(t *testing.T) {
+	tests := []struct {
+		name           string
+		etcdEncryption *[]v1alpha1.EtcdEncryption
+		want           clusterapi.ExtraArgs
+	}{
+		{
+			name:           "nil config",
+			etcdEncryption: nil,
+			want:           clusterapi.ExtraArgs{},
+		},
+		{
+			name:           "empty config",
+			etcdEncryption: &[]v1alpha1.EtcdEncryption{},
+			want:           clusterapi.ExtraArgs{},
+		},
+		{
+			name: "one config",
+			etcdEncryption: &[]v1alpha1.EtcdEncryption{
+				{
+					Providers: []v1alpha1.EtcdEncryptionProvider{
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config1",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock",
+							},
+						},
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config2",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock",
+							},
+						},
+					},
+					Resources: []string{
+						"secrets",
+						"crd1.anywhere.eks.amazonsaws.com",
+					},
+				},
+			},
+			want: clusterapi.ExtraArgs{
+				"encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml",
+			},
+		},
+		{
+			name: "multiple configs",
+			etcdEncryption: &[]v1alpha1.EtcdEncryption{
+				{
+					Providers: []v1alpha1.EtcdEncryptionProvider{
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config1",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock",
+							},
+						},
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config2",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock",
+							},
+						},
+					},
+					Resources: []string{
+						"secrets",
+						"crd1.anywhere.eks.amazonsaws.com",
+					},
+				},
+				{
+					Providers: []v1alpha1.EtcdEncryptionProvider{
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config3",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket2-new.sock",
+							},
+						},
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config4",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket2-old.sock",
+							},
+						},
+					},
+					Resources: []string{
+						"configmaps",
+						"crd2.anywhere.eks.amazonsaws.com",
+					},
+				},
+			},
+			want: clusterapi.ExtraArgs{
+				"encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(*testing.T) {
+			if got := clusterapi.EtcdEncryptionExtraArgs(tt.etcdEncryption); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("EtcdEncryptionExtraArgs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestFeatureGatesExtraArgs(t *testing.T) {
 	tests := []struct {
 		testName string

diff --git a/pkg/providers/cloudstack/cloudstack_test.go b/pkg/providers/cloudstack/cloudstack_test.go
@@ -2600,3 +2600,72 @@ func TestValidateNewSpecMachineConfigNotFound(t *testing.T) {
 	err := provider.ValidateNewSpec(context.TODO(), &types.Cluster{}, newClusterSpec)
 	assert.ErrorContains(t, err, "not found")
 }
+
+func TestProviderGenerateCAPISpecForUpgradeEtcdEncryption(t *testing.T) {
+	tests := []struct {
+		testName          string
+		clusterconfigFile string
+		wantCPFile        string
+		wantMDFile        string
+	}{
+		{
+			testName:          "etcd-encryption",
+			clusterconfigFile: "cluster_etcd_encryption.yaml",
+			wantCPFile:        "testdata/expected_results_encryption_config_cp.yaml",
+			wantMDFile:        "testdata/expected_results_minimal_md.yaml",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.testName, func(t *testing.T) {
+			mockCtrl := gomock.NewController(t)
+			setupContext(t)
+			ctx := context.Background()
+			kubectl := mocks.NewMockProviderKubectlClient(mockCtrl)
+			cluster := &types.Cluster{
+				Name: "test",
+			}
+			bootstrapCluster := &types.Cluster{
+				Name: "bootstrap-test",
+			}
+			clusterSpec := givenClusterSpec(t, tt.clusterconfigFile)
+			cloudstackDatacenter := &v1alpha1.CloudStackDatacenterConfig{
+				Spec: v1alpha1.CloudStackDatacenterConfigSpec{},
+			}
+			cloudstackMachineConfig := &v1alpha1.CloudStackMachineConfig{
+				Spec: v1alpha1.CloudStackMachineConfigSpec{
+					Users: []v1alpha1.UserConfiguration{
+						{
+							Name:              "capv",
+							SshAuthorizedKeys: []string{"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ=="},
+						},
+					},
+				},
+			}
+
+			kubectl.EXPECT().GetMachineDeployment(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(workerNodeGroup1MachineDeployment(), nil)
+			kubectl.EXPECT().GetEksaCluster(ctx, cluster, clusterSpec.Cluster.Name).Return(clusterSpec.Cluster, nil)
+			kubectl.EXPECT().GetEksaCloudStackDatacenterConfig(ctx, cluster.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackDatacenter, nil)
+			kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil)
+			kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations[0].MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil)
+			datacenterConfig := givenDatacenterConfig(t, tt.clusterconfigFile)
+			validator := givenWildcardValidator(mockCtrl, clusterSpec)
+			provider := newProviderWithKubectl(t, datacenterConfig, clusterSpec.Cluster, kubectl, validator)
+			if provider == nil {
+				t.Fatalf("provider object is nil")
+			}
+
+			err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec)
+			if err != nil {
+				t.Fatalf("failed to setup and validate: %v", err)
+			}
+
+			cp, md, err := provider.GenerateCAPISpecForUpgrade(context.Background(), bootstrapCluster, cluster, clusterSpec, clusterSpec.DeepCopy())
+			if err != nil {
+				t.Fatalf("failed to generate cluster api spec contents: %v", err)
+			}
+
+			test.AssertContentToFile(t, string(cp), tt.wantCPFile)
+			test.AssertContentToFile(t, string(md), tt.wantMDFile)
+		})
+	}
+}
diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml
@@ -119,6 +119,16 @@ spec:
           name: awsiamcert
           readOnly: false
 {{- end}}
+{{- if .encryptionProviderConfig }}
+        - hostPath: /etc/kubernetes/enc
+          mountPath: /etc/kubernetes/enc
+          name: encryption-config
+          readOnly: false
+        - hostPath: /var/run/kmsplugin/
+          mountPath: /var/run/kmsplugin/
+          name: kms-plugin
+          readOnly: false
+{{- end }}
       controllerManager:
         extraArgs:
           cloud-provider: external
@@ -133,6 +143,12 @@ spec:
 {{ .schedulerExtraArgs.ToYaml | indent 10 }}
 {{- end }}
     files:
+{{- if .encryptionProviderConfig }}
+    - content: |
+{{ .encryptionProviderConfig | indent 8}}
+      owner: root:root
+      path: /etc/kubernetes/enc/encryption-config.yaml
+{{- end }}
 {{- if .cloudstackKubeVip}}
     - content: |
         apiVersion: v1

diff --git a/pkg/providers/cloudstack/template.go b/pkg/providers/cloudstack/template.go
@@ -116,7 +116,9 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 	apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig).
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
+		Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)).
 		Append(sharedExtraArgs)
+
 	controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
 		Append(clusterapi.NodeCIDRMaskExtraArgs(&clusterSpec.Cluster.Spec.ClusterNetwork))
 
@@ -236,6 +238,14 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 		values["maxSurge"] = clusterSpec.Cluster.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy.RollingUpdate.MaxSurge
 	}
 
+	if clusterSpec.Cluster.Spec.EtcdEncryption != nil && len(*clusterSpec.Cluster.Spec.EtcdEncryption) != 0 {
+		conf, err := common.GenerateKMSEncryptionConfiguration(clusterSpec.Cluster.Spec.EtcdEncryption)
+		if err != nil {
+			return nil, err
+		}
+		values["encryptionProviderConfig"] = conf
+	}
+
 	return values, nil
 }
 

diff --git a/pkg/providers/cloudstack/testdata/cluster_etcd_encryption.yaml b/pkg/providers/cloudstack/testdata/cluster_etcd_encryption.yaml
@@ -0,0 +1,81 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: Cluster
+metadata:
+  name: test
+  namespace: test-namespace
+spec:
+  clusterNetwork:
+    cni: cilium
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    services:
+      cidrBlocks:
+        - 10.96.0.0/12
+  controlPlaneConfiguration:
+    count: 3
+    endpoint:
+      host: 1.2.3.4
+    machineGroupRef:
+      kind: CloudStackMachineConfig
+      name: test
+  datacenterRef:
+    kind: CloudStackDatacenterConfig
+    name: test
+  kubernetesVersion: "1.21"
+  etcdEncryption:
+  - providers:
+    - kms:
+        name: config1
+        socketListenAddress: unix:///var/run/kmsplugin/socket1-new.sock
+    - kms:
+        name: config2
+        socketListenAddress: unix:///var/run/kmsplugin/socket1-old.sock
+    resources:
+    - secrets
+    - resource1.anywhere.eks.amazonsaws.com
+  - providers:
+    - kms:
+        name: config3
+        socketListenAddress: unix:///var/run/kmsplugin/socket2-new.sock
+    - kms:
+        name: config4
+        socketListenAddress: unix:///var/run/kmsplugin/socket2-old.sock
+    resources:
+    - configmaps
+    - resource2.anywhere.eks.amazonsaws.com
+  workerNodeGroupConfigurations:
+  - count: 3
+    machineGroupRef:
+      kind: CloudStackMachineConfig
+      name: test
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: CloudStackDatacenterConfig
+metadata:
+  name: test
+  namespace: test-namespace
+spec:
+  account: "admin"
+  domain: "domain1"
+  zones:
+  - name: "zone1"
+    network:
+      name: "net1"
+  managementApiEndpoint: "http://127.16.0.1:8080/client/api"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: CloudStackMachineConfig
+metadata:
+  name: test
+  namespace: test-namespace
+spec:
+  computeOffering:
+    name: "m4-large"
+  users:
+  - name: "mySshUsername"
+    sshAuthorizedKeys:
+    - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
+  template:
+    name: "centos7-k8s-118"
+---