Add support for ETCD encryption in Cloudstack

aws · Sep 21, 2023 · 2da13d7 · 2da13d7
1 parent b128b35
commit 2da13d7
Show file tree

Hide file tree

Showing 16 changed files with 921 additions and 11 deletions.
diff --git a/cmd/eksctl-anywhere/cmd/upgradecluster.go b/cmd/eksctl-anywhere/cmd/upgradecluster.go
@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-anywhere/cmd/eksctl-anywhere/cmd/flags"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
@@ -154,6 +155,9 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command) error {
 		return err
 	}
 
+	b, _ := yaml.Marshal(clusterSpec.Cluster)
+	fmt.Println(string(b))
+
 	workloadCluster := &types.Cluster{
 		Name:           clusterSpec.Cluster.Name,
 		KubeconfigFile: getKubeconfigPath(clusterSpec.Cluster.Name, uc.wConfig),

diff --git a/config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml b/config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml
@@ -248,8 +248,7 @@ spec:
                               timeout:
                                 description: Timeout for kube-apiserver to wait for
                                   KMS plugin. Default is 3s.
-                                format: int64
-                                type: integer
+                                type: string
                             required:
                             - name
                             - socketListenAddress

diff --git a/config/manifest/eksa-components.yaml b/config/manifest/eksa-components.yaml
@@ -3884,8 +3884,7 @@ spec:
                               timeout:
                                 description: Timeout for kube-apiserver to wait for
                                   KMS plugin. Default is 3s.
-                                format: int64
-                                type: integer
+                                type: string
                             required:
                             - name
                             - socketListenAddress

diff --git a/pkg/api/v1alpha1/etcdencryption.go b/pkg/api/v1alpha1/etcdencryption.go
@@ -5,13 +5,14 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/aws/eks-anywhere/pkg/utils/ptr"
 )
 
 var (
 	defaultKMSCacheSize = ptr.Int32(1000)
-	defaultKMSTimeout   = time.Second * 3
+	defaultKMSTimeout   = metav1.Duration{Duration: time.Second * 3}
 )
 
 // ValidateEtcdEncryptionConfig validates the etcd encryption configuration.

diff --git a/pkg/api/v1alpha1/etcdencryption_types.go b/pkg/api/v1alpha1/etcdencryption_types.go
@@ -1,8 +1,6 @@
 package v1alpha1
 
-import (
-	"time"
-)
+import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 // EtcdEncryption defines the configuration for ETCD encryption.
 type EtcdEncryption struct {
@@ -28,5 +26,5 @@ type KMS struct {
 	// SocketListenAddress defines a UNIX socket address that the KMS provider listens on.
 	SocketListenAddress string `json:"socketListenAddress"`
 	// Timeout for kube-apiserver to wait for KMS plugin. Default is 3s.
-	Timeout *time.Duration `json:"timeout,omitempty"`
+	Timeout *metav1.Duration `json:"timeout,omitempty"`
 }
diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/clusterapi/extraargs.go b/pkg/clusterapi/extraargs.go
@@ -43,6 +43,16 @@ func AwsIamAuthExtraArgs(awsiam *v1alpha1.AWSIamConfig) ExtraArgs {
 	return args
 }
 
+// EtcdEncryptionExtraArgs takes a list of EtcdEncryption configs and returns the relevant API server extra args if it's not nil or empty.
+func EtcdEncryptionExtraArgs(config *[]v1alpha1.EtcdEncryption) ExtraArgs {
+	args := ExtraArgs{}
+	if config == nil || len(*config) == 0 {
+		return args
+	}
+	args.AddIfNotEmpty("encryption-provider-config", "/etc/kubernetes/enc/encryption-config.yaml")
+	return args
+}
+
 // FeatureGatesExtraArgs takes a list of features with the value and returns it in the proper format
 // Example FeatureGatesExtraArgs("ServiceLoadBalancerClass=true").
 func FeatureGatesExtraArgs(features ...string) ExtraArgs {

diff --git a/pkg/clusterapi/extraargs_test.go b/pkg/clusterapi/extraargs_test.go
@@ -519,6 +519,109 @@ func TestNodeCIDRMaskExtraArgs(t *testing.T) {
 	}
 }
 
+func TestEtcdEncryptionExtraArgs(t *testing.T) {
+	tests := []struct {
+		name           string
+		etcdEncryption *[]v1alpha1.EtcdEncryption
+		want           clusterapi.ExtraArgs
+	}{
+		{
+			name:           "nil config",
+			etcdEncryption: nil,
+			want:           clusterapi.ExtraArgs{},
+		},
+		{
+			name:           "empty config",
+			etcdEncryption: &[]v1alpha1.EtcdEncryption{},
+			want:           clusterapi.ExtraArgs{},
+		},
+		{
+			name: "one config",
+			etcdEncryption: &[]v1alpha1.EtcdEncryption{
+				{
+					Providers: []v1alpha1.EtcdEncryptionProvider{
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config1",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock",
+							},
+						},
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config2",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock",
+							},
+						},
+					},
+					Resources: []string{
+						"secrets",
+						"crd1.anywhere.eks.amazonsaws.com",
+					},
+				},
+			},
+			want: clusterapi.ExtraArgs{
+				"encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml",
+			},
+		},
+		{
+			name: "multiple configs",
+			etcdEncryption: &[]v1alpha1.EtcdEncryption{
+				{
+					Providers: []v1alpha1.EtcdEncryptionProvider{
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config1",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock",
+							},
+						},
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config2",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock",
+							},
+						},
+					},
+					Resources: []string{
+						"secrets",
+						"crd1.anywhere.eks.amazonsaws.com",
+					},
+				},
+				{
+					Providers: []v1alpha1.EtcdEncryptionProvider{
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config3",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket2-new.sock",
+							},
+						},
+						{
+							KMS: &v1alpha1.KMS{
+								Name:                "config4",
+								SocketListenAddress: "unix:///var/run/kmsplugin/socket2-old.sock",
+							},
+						},
+					},
+					Resources: []string{
+						"configmaps",
+						"crd2.anywhere.eks.amazonsaws.com",
+					},
+				},
+			},
+			want: clusterapi.ExtraArgs{
+				"encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(*testing.T) {
+			if got := clusterapi.EtcdEncryptionExtraArgs(tt.etcdEncryption); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("EtcdEncryptionExtraArgs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestFeatureGatesExtraArgs(t *testing.T) {
 	tests := []struct {
 		testName string

diff --git a/pkg/providers/cloudstack/cloudstack_test.go b/pkg/providers/cloudstack/cloudstack_test.go
@@ -2600,3 +2600,72 @@ func TestValidateNewSpecMachineConfigNotFound(t *testing.T) {
 	err := provider.ValidateNewSpec(context.TODO(), &types.Cluster{}, newClusterSpec)
 	assert.ErrorContains(t, err, "not found")
 }
+
+func TestProviderGenerateCAPISpecForUpgradeEtcdEncryption(t *testing.T) {
+	tests := []struct {
+		testName          string
+		clusterconfigFile string
+		wantCPFile        string
+		wantMDFile        string
+	}{
+		{
+			testName:          "etcd-encryption",
+			clusterconfigFile: "cluster_etcd_encryption.yaml",
+			wantCPFile:        "testdata/expected_results_encryption_config_cp.yaml",
+			wantMDFile:        "testdata/expected_results_minimal_md.yaml",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.testName, func(t *testing.T) {
+			mockCtrl := gomock.NewController(t)
+			setupContext(t)
+			ctx := context.Background()
+			kubectl := mocks.NewMockProviderKubectlClient(mockCtrl)
+			cluster := &types.Cluster{
+				Name: "test",
+			}
+			bootstrapCluster := &types.Cluster{
+				Name: "bootstrap-test",
+			}
+			clusterSpec := givenClusterSpec(t, tt.clusterconfigFile)
+			cloudstackDatacenter := &v1alpha1.CloudStackDatacenterConfig{
+				Spec: v1alpha1.CloudStackDatacenterConfigSpec{},
+			}
+			cloudstackMachineConfig := &v1alpha1.CloudStackMachineConfig{
+				Spec: v1alpha1.CloudStackMachineConfigSpec{
+					Users: []v1alpha1.UserConfiguration{
+						{
+							Name:              "capv",
+							SshAuthorizedKeys: []string{"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ=="},
+						},
+					},
+				},
+			}
+
+			kubectl.EXPECT().GetMachineDeployment(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(workerNodeGroup1MachineDeployment(), nil)
+			kubectl.EXPECT().GetEksaCluster(ctx, cluster, clusterSpec.Cluster.Name).Return(clusterSpec.Cluster, nil)
+			kubectl.EXPECT().GetEksaCloudStackDatacenterConfig(ctx, cluster.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackDatacenter, nil)
+			kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil)
+			kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations[0].MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil)
+			datacenterConfig := givenDatacenterConfig(t, tt.clusterconfigFile)
+			validator := givenWildcardValidator(mockCtrl, clusterSpec)
+			provider := newProviderWithKubectl(t, datacenterConfig, clusterSpec.Cluster, kubectl, validator)
+			if provider == nil {
+				t.Fatalf("provider object is nil")
+			}
+
+			err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec)
+			if err != nil {
+				t.Fatalf("failed to setup and validate: %v", err)
+			}
+
+			cp, md, err := provider.GenerateCAPISpecForUpgrade(context.Background(), bootstrapCluster, cluster, clusterSpec, clusterSpec.DeepCopy())
+			if err != nil {
+				t.Fatalf("failed to generate cluster api spec contents: %v", err)
+			}
+
+			test.AssertContentToFile(t, string(cp), tt.wantCPFile)
+			test.AssertContentToFile(t, string(md), tt.wantMDFile)
+		})
+	}
+}
diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml
@@ -119,6 +119,16 @@ spec:
           name: awsiamcert
           readOnly: false
 {{- end}}
+{{- if .encryptionProviderConfig }}
+        - hostPath: /etc/kubernetes/enc
+          mountPath: /etc/kubernetes/enc
+          name: encryption-config
+          readOnly: false
+        - hostPath: /var/run/kmsplugin/
+          mountPath: /var/run/kmsplugin/
+          name: kms-plugin
+          readOnly: false
+{{- end }}
       controllerManager:
         extraArgs:
           cloud-provider: external
@@ -133,6 +143,12 @@ spec:
 {{ .schedulerExtraArgs.ToYaml | indent 10 }}
 {{- end }}
     files:
+{{- if .encryptionProviderConfig }}
+    - content: |
+{{ .encryptionProviderConfig | indent 8}}
+      owner: root:root
+      path: /etc/kubernetes/enc/encryption-config.yaml
+{{- end }}
 {{- if .cloudstackKubeVip}}
     - content: |
         apiVersion: v1

diff --git a/pkg/providers/cloudstack/template.go b/pkg/providers/cloudstack/template.go
@@ -116,7 +116,9 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 	apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig).
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
+		Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)).
 		Append(sharedExtraArgs)
+
 	controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
 		Append(clusterapi.NodeCIDRMaskExtraArgs(&clusterSpec.Cluster.Spec.ClusterNetwork))
 
@@ -236,6 +238,14 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 		values["maxSurge"] = clusterSpec.Cluster.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy.RollingUpdate.MaxSurge
 	}
 
+	if clusterSpec.Cluster.Spec.EtcdEncryption != nil && len(*clusterSpec.Cluster.Spec.EtcdEncryption) != 0 {
+		conf, err := common.GenerateKMSEncryptionConfiguration(clusterSpec.Cluster.Spec.EtcdEncryption)
+		if err != nil {
+			return nil, err
+		}
+		values["encryptionProviderConfig"] = conf
+	}
+
 	return values, nil
 }