diff --git a/cmd/eksctl-anywhere/cmd/upgradecluster.go b/cmd/eksctl-anywhere/cmd/upgradecluster.go index 0a55d0cd091c..40452374d187 100644 --- a/cmd/eksctl-anywhere/cmd/upgradecluster.go +++ b/cmd/eksctl-anywhere/cmd/upgradecluster.go @@ -97,6 +97,10 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command, args []strin } } + if clusterConfig.Spec.EtcdEncryption != nil && clusterConfig.Spec.DatacenterRef.Kind != v1alpha1.CloudStackDatacenterKind && clusterConfig.Spec.DatacenterRef.Kind != v1alpha1.VSphereDatacenterKind { + return fmt.Errorf("etcdEncryption is currently not supported for the current provider: %s", clusterConfig.Spec.DatacenterRef.Kind) + } + if err := v1alpha1.ValidateEtcdEncryptionConfig(clusterConfig.Spec.EtcdEncryption); err != nil { return err } diff --git a/pkg/api/v1alpha1/cluster_webhook.go b/pkg/api/v1alpha1/cluster_webhook.go index 0339c7887758..7251f4dae13c 100644 --- a/pkg/api/v1alpha1/cluster_webhook.go +++ b/pkg/api/v1alpha1/cluster_webhook.go @@ -107,6 +107,10 @@ func (r *Cluster) ValidateUpdate(old runtime.Object) error { allErrs = append(allErrs, ValidateWorkerKubernetesVersionSkew(r, oldCluster)...) + if r.Spec.EtcdEncryption != nil && r.Spec.DatacenterRef.Kind != CloudStackDatacenterKind && r.Spec.DatacenterRef.Kind != VSphereDatacenterKind { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec.etcdEncryption"), r.Spec.EtcdEncryption, fmt.Sprintf("etcdEncryption is currently not supported for the provider: %s", r.Spec.DatacenterRef.Kind))) + } + if err := ValidateEtcdEncryptionConfig(r.Spec.EtcdEncryption); err != nil { allErrs = append(allErrs, field.Invalid(field.NewPath("spec.etcdEncryption"), r.Spec.EtcdEncryption, err.Error())) } diff --git a/pkg/api/v1alpha1/cluster_webhook_test.go b/pkg/api/v1alpha1/cluster_webhook_test.go index 891df2fc3f7c..87504de3009a 100644 --- a/pkg/api/v1alpha1/cluster_webhook_test.go +++ b/pkg/api/v1alpha1/cluster_webhook_test.go @@ -1214,6 +1214,31 @@ func TestClusterCreateEtcdEncryption(t *testing.T) { g.Expect(cluster.ValidateCreate()).To(MatchError(ContainSubstring("etcdEncryption is not supported during cluster creation"))) } +func TestClusterUpdateEtcdEncryptionUnsupported(t *testing.T) { + features.ClearCache() + workerConfiguration := append([]v1alpha1.WorkerNodeGroupConfiguration{}, v1alpha1.WorkerNodeGroupConfiguration{Count: ptr.Int(5)}) + cluster := &v1alpha1.Cluster{ + Spec: v1alpha1.ClusterSpec{ + WorkerNodeGroupConfigurations: workerConfiguration, + KubernetesVersion: v1alpha1.Kube119, + ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{ + Count: 3, Endpoint: &v1alpha1.Endpoint{Host: "1.1.1.1/1"}, + }, + ExternalEtcdConfiguration: &v1alpha1.ExternalEtcdConfiguration{Count: 3}, + EtcdEncryption: &[]v1alpha1.EtcdEncryption{}, + ManagementCluster: v1alpha1.ManagementCluster{ + Name: "management-cluster", + }, + DatacenterRef: v1alpha1.Ref{ + Kind: v1alpha1.TinkerbellDatacenterKind, + }, + }, + } + + g := NewWithT(t) + g.Expect(cluster.ValidateUpdate(cluster)).To(MatchError(ContainSubstring("etcdEncryption is currently not supported for the provider"))) +} + func TestClusterUpdateEtcdEncryption(t *testing.T) { features.ClearCache() resources := []string{"secrets"} diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml index e96266b475c3..a55be33d4c07 100644 --- a/pkg/providers/cloudstack/config/template-cp.yaml +++ b/pkg/providers/cloudstack/config/template-cp.yaml @@ -120,10 +120,11 @@ spec: readOnly: false {{- end}} {{- if .encryptionProviderConfig }} - - hostPath: /etc/kubernetes/enc - mountPath: /etc/kubernetes/enc + - hostPath: /var/lib/kubeadm/encryption-config.yaml + mountPath: /etc/kubernetes/enc/encryption-config.yaml name: encryption-config - readOnly: false + pathType: File + readOnly: true - hostPath: /var/run/kmsplugin/ mountPath: /var/run/kmsplugin/ name: kms-plugin @@ -147,7 +148,7 @@ spec: - content: | {{ .encryptionProviderConfig | indent 8}} owner: root:root - path: /etc/kubernetes/enc/encryption-config.yaml + path: /var/lib/kubeadm/encryption-config.yaml {{- end }} {{- if .cloudstackKubeVip}} - content: | diff --git a/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml index 51752b7b8e3e..a4f546f225ab 100644 --- a/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml +++ b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml @@ -91,10 +91,11 @@ spec: name: audit-log-dir pathType: DirectoryOrCreate readOnly: false - - hostPath: /etc/kubernetes/enc - mountPath: /etc/kubernetes/enc + - hostPath: /var/lib/kubeadm/encryption-config.yaml + mountPath: /etc/kubernetes/enc/encryption-config.yaml name: encryption-config - readOnly: false + pathType: File + readOnly: true - hostPath: /var/run/kmsplugin/ mountPath: /var/run/kmsplugin/ name: kms-plugin @@ -148,7 +149,7 @@ spec: - configmaps - resource2.anywhere.eks.amazonsaws.com owner: root:root - path: /etc/kubernetes/enc/encryption-config.yaml + path: /var/lib/kubeadm/encryption-config.yaml - content: | apiVersion: v1 kind: Pod diff --git a/pkg/providers/vsphere/config/template-cp.yaml b/pkg/providers/vsphere/config/template-cp.yaml index 2bbabfcf305e..e29a7cf7311f 100644 --- a/pkg/providers/vsphere/config/template-cp.yaml +++ b/pkg/providers/vsphere/config/template-cp.yaml @@ -183,10 +183,11 @@ spec: readOnly: false {{- end}} {{- if .encryptionProviderConfig }} - - hostPath: /etc/kubernetes/enc - mountPath: /etc/kubernetes/enc + - hostPath: /var/lib/kubeadm/encryption-config.yaml + mountPath: /etc/kubernetes/enc/encryption-config.yaml name: encryption-config - readOnly: false + pathType: File + readOnly: true - hostPath: /var/run/kmsplugin/ mountPath: /var/run/kmsplugin/ name: kms-plugin @@ -227,7 +228,7 @@ spec: - content: | {{ .encryptionProviderConfig | indent 8}} owner: root:root - path: /etc/kubernetes/enc/encryption-config.yaml + path: /var/lib/kubeadm/encryption-config.yaml {{- end }} - content: | apiVersion: v1 diff --git a/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp.yaml b/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp.yaml index ff9706bb6acb..81bd4d0f74b4 100644 --- a/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp.yaml +++ b/pkg/providers/vsphere/testdata/expected_results_ubuntu_etcd_encryption_cp.yaml @@ -110,10 +110,11 @@ spec: name: audit-log-dir pathType: DirectoryOrCreate readOnly: false - - hostPath: /etc/kubernetes/enc - mountPath: /etc/kubernetes/enc + - hostPath: /var/lib/kubeadm/encryption-config.yaml + mountPath: /etc/kubernetes/enc/encryption-config.yaml name: encryption-config - readOnly: false + pathType: File + readOnly: true - hostPath: /var/run/kmsplugin/ mountPath: /var/run/kmsplugin/ name: kms-plugin @@ -167,7 +168,7 @@ spec: - configmaps - resource2.anywhere.eks.amazonsaws.com owner: root:root - path: /etc/kubernetes/enc/encryption-config.yaml + path: /var/lib/kubeadm/encryption-config.yaml - content: | apiVersion: v1 kind: Pod diff --git a/test/e2e/cloudstack_test.go b/test/e2e/cloudstack_test.go index 3991ede217a8..1d4a9a016715 100644 --- a/test/e2e/cloudstack_test.go +++ b/test/e2e/cloudstack_test.go @@ -4373,6 +4373,7 @@ func TestCloudStackKubernetes124EtcdEncryption(t *testing.T) { ), framework.WithPodIamConfig(), ) + test.OSFamily = v1alpha1.RedHat test.GenerateClusterConfig() test.CreateCluster() test.PostClusterCreateEtcdEncryptionSetup() diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go index 492b0f969551..cccc2aeb6571 100644 --- a/test/e2e/vsphere_test.go +++ b/test/e2e/vsphere_test.go @@ -3201,7 +3201,7 @@ func TestVSphereKubernetes128UbuntuAirgappedRegistryMirror(t *testing.T) { runAirgapConfigFlow(test, "195.18.0.1/16,196.18.0.1/16") } -func TestVSphereKubernetes128EtcdEncryption(t *testing.T) { +func TestVSphereKubernetesUbuntu128EtcdEncryption(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewVSphere(t, framework.WithUbuntu128()), @@ -3212,6 +3212,7 @@ func TestVSphereKubernetes128EtcdEncryption(t *testing.T) { ), framework.WithPodIamConfig(), ) + test.OSFamily = v1alpha1.Ubuntu test.GenerateClusterConfig() test.CreateCluster() test.PostClusterCreateEtcdEncryptionSetup() @@ -3221,6 +3222,26 @@ func TestVSphereKubernetes128EtcdEncryption(t *testing.T) { test.DeleteCluster() } +func TestVSphereKubernetesBottlerocket128EtcdEncryption(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewVSphere(t, framework.WithBottleRocket128()), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + ), + framework.WithPodIamConfig(), + ) + test.OSFamily = v1alpha1.Bottlerocket + test.GenerateClusterConfig() + test.CreateCluster() + test.PostClusterCreateEtcdEncryptionSetup() + test.UpgradeClusterWithNewConfig([]framework.ClusterE2ETestOpt{framework.WithEtcdEncrytion()}) + test.StopIfFailed() + test.DeleteCluster() +} + func ubuntu128ProviderWithLabels(t *testing.T) *framework.VSphere { return framework.NewVSphere(t, framework.WithVSphereWorkerNodeGroup( diff --git a/test/framework/cluster.go b/test/framework/cluster.go index b92f6d13e70a..3fc17c477f79 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -97,6 +97,7 @@ type ClusterE2ETest struct { PackageConfig *PackageConfig GitWriter filewriter.FileWriter eksaBinaryLocation string + OSFamily v1alpha1.OSFamily ExpectFailure bool // PersistentCluster avoids creating the clusters if it finds a kubeconfig // in the corresponding cluster folder. Useful for local development of tests. diff --git a/test/framework/config/aws-kms-encryption-provider.yaml b/test/framework/config/aws-kms-encryption-provider.yaml index c6ce15bcea0f..fee300de3b9b 100644 --- a/test/framework/config/aws-kms-encryption-provider.yaml +++ b/test/framework/config/aws-kms-encryption-provider.yaml @@ -57,8 +57,10 @@ spec: effect: "NoSchedule" - effect: NoSchedule key: node-role.kubernetes.io/control-plane +{{- if .deployOnlyOnControlPlane }} nodeSelector: node-role.kubernetes.io/control-plane: "" +{{- end }} serviceAccountName: {{ .serviceAccountName }} volumes: - hostPath: diff --git a/test/framework/etcdencryption.go b/test/framework/etcdencryption.go index 6cd3185c90e1..dcd68e8ee246 100644 --- a/test/framework/etcdencryption.go +++ b/test/framework/etcdencryption.go @@ -220,6 +220,10 @@ func (e *ClusterE2ETest) deployKMSProvider(ctx context.Context, envVars *etcdEnc "serviceAccountName": "kms-encrypter-decrypter", } + if e.OSFamily != v1alpha1.Bottlerocket { + values["deployOnlyOnControlPlane"] = "true" + } + manifest, err := templater.Execute(kmsProviderManifest, values) if err != nil { return fmt.Errorf("templating kms provider manifest: %v", err)