Fix etcd-encryption for bottlerocket (#6874)

* Fix etcd-encryption for bottlerocket + e2e * Block etcd encrytion for unsupported providers
aws · Oct 23, 2023 · 4210d96 · 4210d96
1 parent 3528d9e
commit 4210d96
Show file tree

Hide file tree

Showing 12 changed files with 83 additions and 17 deletions.
diff --git a/cmd/eksctl-anywhere/cmd/upgradecluster.go b/cmd/eksctl-anywhere/cmd/upgradecluster.go
@@ -97,6 +97,10 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command, args []strin
 		}
 	}
 
+	if clusterConfig.Spec.EtcdEncryption != nil && clusterConfig.Spec.DatacenterRef.Kind != v1alpha1.CloudStackDatacenterKind && clusterConfig.Spec.DatacenterRef.Kind != v1alpha1.VSphereDatacenterKind {
+		return fmt.Errorf("etcdEncryption is currently not supported for the current provider: %s", clusterConfig.Spec.DatacenterRef.Kind)
+	}
+
 	if err := v1alpha1.ValidateEtcdEncryptionConfig(clusterConfig.Spec.EtcdEncryption); err != nil {
 		return err
 	}

diff --git a/pkg/api/v1alpha1/cluster_webhook.go b/pkg/api/v1alpha1/cluster_webhook.go
@@ -107,6 +107,10 @@ func (r *Cluster) ValidateUpdate(old runtime.Object) error {
 
 	allErrs = append(allErrs, ValidateWorkerKubernetesVersionSkew(r, oldCluster)...)
 
+	if r.Spec.EtcdEncryption != nil && r.Spec.DatacenterRef.Kind != CloudStackDatacenterKind && r.Spec.DatacenterRef.Kind != VSphereDatacenterKind {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec.etcdEncryption"), r.Spec.EtcdEncryption, fmt.Sprintf("etcdEncryption is currently not supported for the provider: %s", r.Spec.DatacenterRef.Kind)))
+	}
+
 	if err := ValidateEtcdEncryptionConfig(r.Spec.EtcdEncryption); err != nil {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("spec.etcdEncryption"), r.Spec.EtcdEncryption, err.Error()))
 	}

diff --git a/pkg/api/v1alpha1/cluster_webhook_test.go b/pkg/api/v1alpha1/cluster_webhook_test.go
@@ -1214,6 +1214,31 @@ func TestClusterCreateEtcdEncryption(t *testing.T) {
 	g.Expect(cluster.ValidateCreate()).To(MatchError(ContainSubstring("etcdEncryption is not supported during cluster creation")))
 }
 
+func TestClusterUpdateEtcdEncryptionUnsupported(t *testing.T) {
+	features.ClearCache()
+	workerConfiguration := append([]v1alpha1.WorkerNodeGroupConfiguration{}, v1alpha1.WorkerNodeGroupConfiguration{Count: ptr.Int(5)})
+	cluster := &v1alpha1.Cluster{
+		Spec: v1alpha1.ClusterSpec{
+			WorkerNodeGroupConfigurations: workerConfiguration,
+			KubernetesVersion:             v1alpha1.Kube119,
+			ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{
+				Count: 3, Endpoint: &v1alpha1.Endpoint{Host: "1.1.1.1/1"},
+			},
+			ExternalEtcdConfiguration: &v1alpha1.ExternalEtcdConfiguration{Count: 3},
+			EtcdEncryption:            &[]v1alpha1.EtcdEncryption{},
+			ManagementCluster: v1alpha1.ManagementCluster{
+				Name: "management-cluster",
+			},
+			DatacenterRef: v1alpha1.Ref{
+				Kind: v1alpha1.TinkerbellDatacenterKind,
+			},
+		},
+	}
+
+	g := NewWithT(t)
+	g.Expect(cluster.ValidateUpdate(cluster)).To(MatchError(ContainSubstring("etcdEncryption is currently not supported for the provider")))
+}
+
 func TestClusterUpdateEtcdEncryption(t *testing.T) {
 	features.ClearCache()
 	resources := []string{"secrets"}

diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml
@@ -120,10 +120,11 @@ spec:
           readOnly: false
 {{- end}}
 {{- if .encryptionProviderConfig }}
-        - hostPath: /etc/kubernetes/enc
-          mountPath: /etc/kubernetes/enc
+        - hostPath: /var/lib/kubeadm/encryption-config.yaml
+          mountPath: /etc/kubernetes/enc/encryption-config.yaml
           name: encryption-config
-          readOnly: false
+          pathType: File
+          readOnly: true
         - hostPath: /var/run/kmsplugin/
           mountPath: /var/run/kmsplugin/
           name: kms-plugin
@@ -147,7 +148,7 @@ spec:
     - content: |
 {{ .encryptionProviderConfig | indent 8}}
       owner: root:root
-      path: /etc/kubernetes/enc/encryption-config.yaml
+      path: /var/lib/kubeadm/encryption-config.yaml
 {{- end }}
 {{- if .cloudstackKubeVip}}
     - content: |

diff --git a/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml
@@ -91,10 +91,11 @@ spec:
           name: audit-log-dir
           pathType: DirectoryOrCreate
           readOnly: false
-        - hostPath: /etc/kubernetes/enc
-          mountPath: /etc/kubernetes/enc
+        - hostPath: /var/lib/kubeadm/encryption-config.yaml
+          mountPath: /etc/kubernetes/enc/encryption-config.yaml
           name: encryption-config
-          readOnly: false
+          pathType: File
+          readOnly: true
         - hostPath: /var/run/kmsplugin/
           mountPath: /var/run/kmsplugin/
           name: kms-plugin
@@ -148,7 +149,7 @@ spec:
           - configmaps
           - resource2.anywhere.eks.amazonsaws.com
       owner: root:root
-      path: /etc/kubernetes/enc/encryption-config.yaml
+      path: /var/lib/kubeadm/encryption-config.yaml
     - content: |
         apiVersion: v1
         kind: Pod

diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml
@@ -183,10 +183,11 @@ spec:
           readOnly: false
 {{- end}}
 {{- if .encryptionProviderConfig }}
-        - hostPath: /etc/kubernetes/enc
-          mountPath: /etc/kubernetes/enc
+        - hostPath: /var/lib/kubeadm/encryption-config.yaml
+          mountPath: /etc/kubernetes/enc/encryption-config.yaml
           name: encryption-config
-          readOnly: false
+          pathType: File
+          readOnly: true
         - hostPath: /var/run/kmsplugin/
           mountPath: /var/run/kmsplugin/
           name: kms-plugin
@@ -227,7 +228,7 @@ spec:
     - content: |
 {{ .encryptionProviderConfig | indent 8}}
       owner: root:root
-      path: /etc/kubernetes/enc/encryption-config.yaml
+      path: /var/lib/kubeadm/encryption-config.yaml
 {{- end }}
     - content: |
         apiVersion: v1

diff --git a/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp.yaml b/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp.yaml
@@ -110,10 +110,11 @@ spec:
           name: audit-log-dir
           pathType: DirectoryOrCreate
           readOnly: false
-        - hostPath: /etc/kubernetes/enc
-          mountPath: /etc/kubernetes/enc
+        - hostPath: /var/lib/kubeadm/encryption-config.yaml
+          mountPath: /etc/kubernetes/enc/encryption-config.yaml
           name: encryption-config
-          readOnly: false
+          pathType: File
+          readOnly: true
         - hostPath: /var/run/kmsplugin/
           mountPath: /var/run/kmsplugin/
           name: kms-plugin
@@ -167,7 +168,7 @@ spec:
           - configmaps
           - resource2.anywhere.eks.amazonsaws.com
       owner: root:root
-      path: /etc/kubernetes/enc/encryption-config.yaml
+      path: /var/lib/kubeadm/encryption-config.yaml
     - content: |
         apiVersion: v1
         kind: Pod

diff --git a/test/e2e/cloudstack_test.go b/test/e2e/cloudstack_test.go
@@ -4373,6 +4373,7 @@ func TestCloudStackKubernetes124EtcdEncryption(t *testing.T) {
 		),
 		framework.WithPodIamConfig(),
 	)
+	test.OSFamily = v1alpha1.RedHat
 	test.GenerateClusterConfig()
 	test.CreateCluster()
 	test.PostClusterCreateEtcdEncryptionSetup()

diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go
@@ -3201,7 +3201,7 @@ func TestVSphereKubernetes128UbuntuAirgappedRegistryMirror(t *testing.T) {
 	runAirgapConfigFlow(test, "195.18.0.1/16,196.18.0.1/16")
 }
 
-func TestVSphereKubernetes128EtcdEncryption(t *testing.T) {
+func TestVSphereKubernetesUbuntu128EtcdEncryption(t *testing.T) {
 	test := framework.NewClusterE2ETest(
 		t,
 		framework.NewVSphere(t, framework.WithUbuntu128()),
@@ -3212,6 +3212,7 @@ func TestVSphereKubernetes128EtcdEncryption(t *testing.T) {
 		),
 		framework.WithPodIamConfig(),
 	)
+	test.OSFamily = v1alpha1.Ubuntu
 	test.GenerateClusterConfig()
 	test.CreateCluster()
 	test.PostClusterCreateEtcdEncryptionSetup()
@@ -3221,6 +3222,26 @@ func TestVSphereKubernetes128EtcdEncryption(t *testing.T) {
 	test.DeleteCluster()
 }
 
+func TestVSphereKubernetesBottlerocket128EtcdEncryption(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewVSphere(t, framework.WithBottleRocket128()),
+		framework.WithClusterFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube128),
+			api.WithExternalEtcdTopology(1),
+			api.WithControlPlaneCount(1),
+		),
+		framework.WithPodIamConfig(),
+	)
+	test.OSFamily = v1alpha1.Bottlerocket
+	test.GenerateClusterConfig()
+	test.CreateCluster()
+	test.PostClusterCreateEtcdEncryptionSetup()
+	test.UpgradeClusterWithNewConfig([]framework.ClusterE2ETestOpt{framework.WithEtcdEncrytion()})
+	test.StopIfFailed()
+	test.DeleteCluster()
+}
+
 func ubuntu128ProviderWithLabels(t *testing.T) *framework.VSphere {
 	return framework.NewVSphere(t,
 		framework.WithVSphereWorkerNodeGroup(

diff --git a/test/framework/cluster.go b/test/framework/cluster.go
@@ -97,6 +97,7 @@ type ClusterE2ETest struct {
 	PackageConfig        *PackageConfig
 	GitWriter            filewriter.FileWriter
 	eksaBinaryLocation   string
+	OSFamily             v1alpha1.OSFamily
 	ExpectFailure        bool
 	// PersistentCluster avoids creating the clusters if it finds a kubeconfig
 	// in the corresponding cluster folder. Useful for local development of tests.

diff --git a/test/framework/config/aws-kms-encryption-provider.yaml b/test/framework/config/aws-kms-encryption-provider.yaml
@@ -57,8 +57,10 @@ spec:
         effect: "NoSchedule"
       - effect: NoSchedule
         key: node-role.kubernetes.io/control-plane
+{{- if .deployOnlyOnControlPlane }}
       nodeSelector:
         node-role.kubernetes.io/control-plane: ""
+{{- end }}
       serviceAccountName: {{ .serviceAccountName }}
       volumes:
       - hostPath:

diff --git a/test/framework/etcdencryption.go b/test/framework/etcdencryption.go
@@ -220,6 +220,10 @@ func (e *ClusterE2ETest) deployKMSProvider(ctx context.Context, envVars *etcdEnc
 		"serviceAccountName": "kms-encrypter-decrypter",
 	}
 
+	if e.OSFamily != v1alpha1.Bottlerocket {
+		values["deployOnlyOnControlPlane"] = "true"
+	}
+
 	manifest, err := templater.Execute(kmsProviderManifest, values)
 	if err != nil {
 		return fmt.Errorf("templating kms provider manifest: %v", err)