diff --git a/test/e2e/cloudstack_test.go b/test/e2e/cloudstack_test.go index 26f704d8220f..3b6d386ed40c 100644 --- a/test/e2e/cloudstack_test.go +++ b/test/e2e/cloudstack_test.go @@ -68,6 +68,16 @@ func TestCloudStackKubernetes127AWSIamAuth(t *testing.T) { runAWSIamAuthFlow(test) } +func TestCloudStackKubernetes128AWSIamAuth(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithAWSIam(), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + ) + runAWSIamAuthFlow(test) +} + // TODO: Add TestCloudStackKubernetes123to124AWSIamAuthUpgrade func TestCloudStackKubernetes123to124AWSIamAuthUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -133,6 +143,22 @@ func TestCloudStackKubernetes126to127AWSIamAuthUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes127to128AWSIamAuthUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithAWSIam(), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + ) + runUpgradeFlowWithAWSIamAuth( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), + provider.WithProviderUpgrade(provider.Redhat128Template()), + ) +} + // Curated packages test func TestCloudStackKubernetes123RedhatCuratedPackagesSimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) @@ -199,6 +225,19 @@ func TestCloudStackKubernetes127RedhatCuratedPackagesSimpleFlow(t *testing.T) { runCuratedPackageInstallSimpleFlow(test) } +func TestCloudStackKubernetes128RedhatCuratedPackagesSimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithPackageConfig(t, packageBundleURI(v1alpha1.Kube128), + "my-packages-test", EksaPackageControllerHelmURI, + EksaPackageControllerHelmVersion, EksaPackageControllerHelmValues, nil), + ) + runCuratedPackageInstallSimpleFlow(test) +} + func TestCloudStackKubernetes123RedhatCuratedPackagesEmissarySimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) test := framework.NewClusterE2ETest( @@ -264,6 +303,19 @@ func TestCloudStackKubernetes127RedhatCuratedPackagesEmissarySimpleFlow(t *testi runCuratedPackageEmissaryInstallSimpleFlow(test) } +func TestCloudStackKubernetes128RedhatCuratedPackagesEmissarySimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithPackageConfig(t, packageBundleURI(v1alpha1.Kube128), + "my-packages-test", EksaPackageControllerHelmURI, + EksaPackageControllerHelmVersion, EksaPackageControllerHelmValues, nil), + ) + runCuratedPackageEmissaryInstallSimpleFlow(test) +} + func TestCloudStackKubernetes123RedhatCuratedPackagesHarborSimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) test := framework.NewClusterE2ETest( @@ -329,6 +381,19 @@ func TestCloudStackKubernetes127RedhatCuratedPackagesHarborSimpleFlow(t *testing runCuratedPackageHarborInstallSimpleFlowLocalStorageProvisioner(test) } +func TestCloudStackKubernetes128RedhatCuratedPackagesHarborSimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithPackageConfig(t, packageBundleURI(v1alpha1.Kube128), + "my-packages-test", EksaPackageControllerHelmURI, + EksaPackageControllerHelmVersion, EksaPackageControllerHelmValues, nil), + ) + runCuratedPackageHarborInstallSimpleFlowLocalStorageProvisioner(test) +} + func TestCloudStackKubernetes123RedhatWorkloadClusterCuratedPackagesSimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -364,6 +429,13 @@ func TestCloudStackKubernetes127RedhatWorkloadClusterCuratedPackagesSimpleFlow(t runCuratedPackageRemoteClusterInstallSimpleFlow(test) } +func TestCloudStackKubernetes128RedhatWorkloadClusterCuratedPackagesSimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat128()) + test := SetupSimpleMultiCluster(t, provider, v1alpha1.Kube128) + runCuratedPackageRemoteClusterInstallSimpleFlow(test) +} + func TestCloudStackKubernetes123RedhatWorkloadClusterCuratedPackagesEmissarySimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -399,6 +471,13 @@ func TestCloudStackKubernetes127RedhatWorkloadClusterCuratedPackagesEmissarySimp runCuratedPackageEmissaryRemoteClusterInstallSimpleFlow(test) } +func TestCloudStackKubernetes128RedhatWorkloadClusterCuratedPackagesEmissarySimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat128()) + test := SetupSimpleMultiCluster(t, provider, v1alpha1.Kube128) + runCuratedPackageEmissaryRemoteClusterInstallSimpleFlow(test) +} + func TestCloudStackKubernetes123RedhatCuratedPackagesCertManagerSimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) framework.CheckCertManagerCredentials(t) @@ -439,6 +518,14 @@ func TestCloudStackKubernetes127RedhatCuratedPackagesCertManagerSimpleFlow(t *te runCertManagerRemoteClusterInstallSimpleFlow(test) } +func TestCloudStackKubernetes128RedhatCuratedPackagesCertManagerSimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + framework.CheckCertManagerCredentials(t) + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat128()) + test := SetupSimpleMultiCluster(t, provider, v1alpha1.Kube128) + runCertManagerRemoteClusterInstallSimpleFlow(test) +} + func TestCloudStackKubernetes123RedhatCuratedPackagesAdotSimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) test := framework.NewClusterE2ETest(t, @@ -499,6 +586,18 @@ func TestCloudStackKubernetes127RedhatCuratedPackagesAdotSimpleFlow(t *testing.T runCuratedPackagesAdotInstallSimpleFlow(test) } +func TestCloudStackKubernetes128RedhatCuratedPackagesAdotSimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + test := framework.NewClusterE2ETest(t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithPackageConfig(t, packageBundleURI(v1alpha1.Kube128), + "my-packages-test", EksaPackageControllerHelmURI, + EksaPackageControllerHelmVersion, EksaPackageControllerHelmValues, nil), + ) + runCuratedPackagesAdotInstallSimpleFlow(test) +} + func TestCloudStackKubernetes123RedhatCuratedPackagesAdotUpdateFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) test := framework.NewClusterE2ETest(t, @@ -559,6 +658,18 @@ func TestCloudStackKubernetes127RedhatCuratedPackagesAdotUpdateFlow(t *testing.T runCuratedPackagesAdotInstallUpdateFlow(test) } +func TestCloudStackKubernetes128RedhatCuratedPackagesAdotUpdateFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + test := framework.NewClusterE2ETest(t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithPackageConfig(t, packageBundleURI(v1alpha1.Kube128), + "my-packages-test", EksaPackageControllerHelmURI, + EksaPackageControllerHelmVersion, EksaPackageControllerHelmValues, nil), + ) + runCuratedPackagesAdotInstallUpdateFlow(test) +} + func TestCloudStackKubernetes123RedHatCuratedPackagesClusterAutoscalerSimpleFlow(t *testing.T) { minNodes := 1 maxNodes := 2 @@ -634,6 +745,21 @@ func TestCloudStackKubernetes127RedHatCuratedPackagesClusterAutoscalerSimpleFlow runAutoscalerWithMetricsServerSimpleFlow(test) } +func TestCloudStackKubernetes128RedHatCuratedPackagesClusterAutoscalerSimpleFlow(t *testing.T) { + minNodes := 1 + maxNodes := 2 + framework.CheckCuratedPackagesCredentials(t) + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128), api.WithWorkerNodeAutoScalingConfig(minNodes, maxNodes)), + framework.WithPackageConfig(t, packageBundleURI(v1alpha1.Kube128), + EksaPackageControllerHelmChartName, EksaPackageControllerHelmURI, + EksaPackageControllerHelmVersion, EksaPackageControllerHelmValues, nil), + ) + runAutoscalerWithMetricsServerSimpleFlow(test) +} + func TestCloudStackKubernetes123RedhatCuratedPackagesPrometheusSimpleFlow(t *testing.T) { framework.CheckCuratedPackagesCredentials(t) test := framework.NewClusterE2ETest(t, @@ -694,6 +820,18 @@ func TestCloudStackKubernetes127RedhatCuratedPackagesPrometheusSimpleFlow(t *tes runCuratedPackagesPrometheusInstallSimpleFlow(test) } +func TestCloudStackKubernetes128RedhatCuratedPackagesPrometheusSimpleFlow(t *testing.T) { + framework.CheckCuratedPackagesCredentials(t) + test := framework.NewClusterE2ETest(t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithPackageConfig(t, packageBundleURI(v1alpha1.Kube128), + "my-packages-test", EksaPackageControllerHelmURI, + EksaPackageControllerHelmVersion, EksaPackageControllerHelmValues, nil), + ) + runCuratedPackagesPrometheusInstallSimpleFlow(test) +} + // Download artifacts func TestCloudStackDownloadArtifacts(t *testing.T) { test := framework.NewClusterE2ETest( @@ -768,6 +906,18 @@ func TestCloudStackKubernetes127GithubFlux(t *testing.T) { runFluxFlow(test) } +func TestCloudStackKubernetes128GithubFlux(t *testing.T) { + test := framework.NewClusterE2ETest(t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithFluxGithub(), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + ) + runFluxFlow(test) +} + func TestCloudStackKubernetes123GitFlux(t *testing.T) { test := framework.NewClusterE2ETest(t, framework.NewCloudStack(t, framework.WithCloudStackRedhat123()), @@ -828,6 +978,18 @@ func TestCloudStackKubernetes127GitFlux(t *testing.T) { runFluxFlow(test) } +func TestCloudStackKubernetes128GitFlux(t *testing.T) { + test := framework.NewClusterE2ETest(t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithFluxGit(), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + ) + runFluxFlow(test) +} + // TODO: Add 1.23 to 1.24 flux upgrade tests func TestCloudStackKubernetes123To124GitFluxUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -901,6 +1063,24 @@ func TestCloudStackKubernetes126To127GitFluxUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes127To128GitFluxUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest(t, + provider, + framework.WithFluxGit(), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + ) + runUpgradeFlowWithFlux( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), + provider.WithProviderUpgrade(provider.Redhat128Template()), + ) +} + func TestCloudStackKubernetes123InstallGitFluxDuringUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) test := framework.NewClusterE2ETest(t, @@ -986,6 +1166,23 @@ func TestCloudStackKubernetes127InstallGitFluxDuringUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes128InstallGitFluxDuringUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat128()) + test := framework.NewClusterE2ETest(t, + provider, + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + ) + runUpgradeFlowWithFlux( + test, + v1alpha1.Kube128, + framework.WithFluxGit(), + framework.WithClusterUpgrade(api.WithGitOpsRef(framework.DefaultFluxConfigName, v1alpha1.FluxConfigKind)), + ) +} + // Labels func TestCloudStackKubernetes123LabelsAndNodeNameRedhat(t *testing.T) { test := framework.NewClusterE2ETest( @@ -1097,6 +1294,28 @@ func TestCloudStackKubernetes127LabelsAndNodeNameRedhat(t *testing.T) { test.DeleteCluster() } +func TestCloudStackKubernetes128LabelsAndNodeNameRedhat(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, + framework.WithCloudStackRedhat128(), + ), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithControlPlaneLabel(constants.FailureDomainLabelName, constants.CloudstackFailureDomainPlaceholder), + api.WithWorkerNodeGroup(constants.DefaultWorkerNodeGroupName, + api.WithCount(1), + api.WithLabel(constants.FailureDomainLabelName, constants.CloudstackFailureDomainPlaceholder), + ), + ), + ) + test.GenerateClusterConfig() + test.CreateCluster() + test.ValidateControlPlaneNodes(framework.ValidateControlPlaneFailureDomainLabels, framework.ValidateControlPlaneNodeNameMatchCAPIMachineName) + test.ValidateWorkerNodes(framework.ValidateWorkerNodeFailureDomainLabels, framework.ValidateWorkerNodeNameMatchCAPIMachineName) + test.DeleteCluster() +} + func TestCloudStackKubernetes123RedhatLabelsUpgradeFlow(t *testing.T) { provider := redhat123ProviderWithLabels(t) @@ -1227,6 +1446,32 @@ func TestCloudStackKubernetes127RedhatLabelsUpgradeFlow(t *testing.T) { ) } +func TestCloudStackKubernetes128RedhatLabelsUpgradeFlow(t *testing.T) { + provider := redhat128ProviderWithLabels(t) + + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + api.RemoveAllWorkerNodeGroups(), // This gives us a blank slate + ), + ) + + runLabelsUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithWorkerNodeGroup(worker0, api.WithLabel(key1, val1)), + api.WithWorkerNodeGroup(worker1, api.WithLabel(key2, val2)), + api.WithWorkerNodeGroup(worker2), + api.WithControlPlaneLabel(cpKey1, cpVal1), + ), + ) +} + func redhat123ProviderWithLabels(t *testing.T) *framework.CloudStack { return framework.NewCloudStack(t, framework.WithCloudStackWorkerNodeGroup( @@ -1327,6 +1572,26 @@ func redhat127ProviderWithLabels(t *testing.T) *framework.CloudStack { ) } +func redhat128ProviderWithLabels(t *testing.T) *framework.CloudStack { + return framework.NewCloudStack(t, + framework.WithCloudStackWorkerNodeGroup( + worker0, + framework.WithWorkerNodeGroup(worker0, api.WithCount(2), + api.WithLabel(key1, val2)), + ), + framework.WithCloudStackWorkerNodeGroup( + worker1, + framework.WithWorkerNodeGroup(worker1, api.WithCount(1)), + ), + framework.WithCloudStackWorkerNodeGroup( + worker2, + framework.WithWorkerNodeGroup(worker2, api.WithCount(1), + api.WithLabel(key2, val2)), + ), + framework.WithCloudStackRedhat128(), + ) +} + // Multicluster func TestCloudStackKubernetes123MulticlusterWorkloadCluster(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -1468,6 +1733,34 @@ func TestCloudStackKubernetes127MulticlusterWorkloadCluster(t *testing.T) { runWorkloadClusterFlow(test) } +func TestCloudStackKubernetes128MulticlusterWorkloadCluster(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat128()) + test := framework.NewMulticlusterE2ETest( + t, + framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(1), + api.WithStackedEtcdTopology(), + ), + ), + framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(1), + api.WithStackedEtcdTopology(), + ), + ), + ) + runWorkloadClusterFlow(test) +} + // WL cluster with prev release version from management cluster func TestCloudStackKubernetes123MulticlusterWorkloadClusterPrevVersion(t *testing.T) { prevLatestRel := prevLatestMinorRelease(t) @@ -1659,6 +1952,46 @@ func TestCloudStackUpgradeKubernetes127MulticlusterWorkloadClusterWithGithubFlux ) } +func TestCloudStackUpgradeKubernetes128MulticlusterWorkloadClusterWithGithubFlux(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewMulticlusterE2ETest( + t, + framework.NewClusterE2ETest( + t, + provider, + framework.WithFluxGithub(), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(1), + api.WithStackedEtcdTopology(), + ), + ), + framework.NewClusterE2ETest( + t, + provider, + framework.WithFluxGithub(), + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube127), + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(1), + api.WithStackedEtcdTopology(), + ), + ), + ) + runWorkloadClusterFlowWithGitOps( + test, + framework.WithClusterUpgradeGit( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithControlPlaneCount(3), + api.WithWorkerNodeCount(3), + ), + provider.WithProviderUpgradeGit( + provider.Redhat128Template(), + ), + ) +} + func TestCloudStackKubernetes123WithOIDCManagementClusterUpgradeFromLatestSideEffects(t *testing.T) { cloudstack := framework.NewCloudStack(t) runTestManagementClusterUpgradeSideEffects(t, cloudstack, framework.RedHat8, anywherev1.Kube123) @@ -1684,6 +2017,11 @@ func TestCloudStackKubernetes127WithOIDCManagementClusterUpgradeFromLatestSideEf runTestManagementClusterUpgradeSideEffects(t, cloudstack, framework.RedHat8, anywherev1.Kube127) } +func TestCloudStackKubernetes128WithOIDCManagementClusterUpgradeFromLatestSideEffects(t *testing.T) { + cloudstack := framework.NewCloudStack(t) + runTestManagementClusterUpgradeSideEffects(t, cloudstack, framework.RedHat8, anywherev1.Kube128) +} + // OIDC func TestCloudStackKubernetes123OIDC(t *testing.T) { test := framework.NewClusterE2ETest( @@ -1750,6 +2088,19 @@ func TestCloudStackKubernetes127OIDC(t *testing.T) { runOIDCFlow(test) } +func TestCloudStackKubernetes128OIDC(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithOIDC(), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + ) + runOIDCFlow(test) +} + // TODO: Add TestCloudStackKubernetes123To124OIDCUpgrade func TestCloudStackKubernetes123To124OIDCUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -1893,6 +2244,19 @@ func TestCloudStackKubernetes127RedhatProxyConfig(t *testing.T) { runProxyConfigFlow(test) } +func TestCloudStackKubernetes128RedhatProxyConfig(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithProxy(framework.CloudstackProxyRequiredEnvVars), + ) + runProxyConfigFlow(test) +} + // Proxy config multicluster func TestCloudStackKubernetes123RedhatProxyConfigAPI(t *testing.T) { cloudstack := framework.NewCloudStack(t) @@ -2114,6 +2478,50 @@ func TestCloudStackKubernetes127RedhatProxyConfigAPI(t *testing.T) { test.DeleteManagementCluster() } +func TestCloudStackKubernetes128RedhatProxyConfigAPI(t *testing.T) { + cloudstack := framework.NewCloudStack(t) + managementCluster := framework.NewClusterE2ETest( + t, + cloudstack, + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithControlPlaneCount(1), + api.WithWorkerNodeCount(1), + ), + cloudstack.WithRedhat128(), + ) + + test := framework.NewMulticlusterE2ETest(t, managementCluster) + test.WithWorkloadClusters( + framework.NewClusterE2ETest( + t, + cloudstack, + framework.WithClusterName(test.NewWorkloadClusterName()), + framework.WithProxy(framework.CloudstackProxyRequiredEnvVars), + ).WithClusterConfig( + api.ClusterToConfigFiller( + api.WithManagementCluster(managementCluster.ClusterName), + ), + cloudstack.WithRedhat128(), + ), + ) + + test.CreateManagementCluster() + + // Create workload clusters + test.RunConcurrentlyInWorkloadClusters(func(wc *framework.WorkloadCluster) { + wc.ApplyClusterManifest() + wc.WaitForKubeconfig() + wc.ValidateClusterState() + + wc.DeleteClusterWithKubectl() + wc.ValidateClusterDelete() + }) + + test.ManagementCluster.StopIfFailed() + test.DeleteManagementCluster() +} + // Registry mirror func TestCloudStackKubernetes123RedhatRegistryMirrorInsecureSkipVerify(t *testing.T) { test := framework.NewClusterE2ETest( @@ -2161,20 +2569,33 @@ func TestCloudStackKubernetes126RedhatRegistryMirrorInsecureSkipVerify(t *testin framework.WithClusterFiller(api.WithControlPlaneCount(1)), framework.WithClusterFiller(api.WithWorkerNodeCount(1)), framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube126)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube126)), + framework.WithRegistryMirrorInsecureSkipVerify(constants.CloudStackProviderName), + ) + runRegistryMirrorConfigFlow(test) +} + +func TestCloudStackKubernetes127RedhatRegistryMirrorInsecureSkipVerify(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat127()), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), framework.WithRegistryMirrorInsecureSkipVerify(constants.CloudStackProviderName), ) runRegistryMirrorConfigFlow(test) } -func TestCloudStackKubernetes127RedhatRegistryMirrorInsecureSkipVerify(t *testing.T) { +func TestCloudStackKubernetes128RedhatRegistryMirrorInsecureSkipVerify(t *testing.T) { test := framework.NewClusterE2ETest( t, - framework.NewCloudStack(t, framework.WithCloudStackRedhat127()), + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), framework.WithClusterFiller(api.WithControlPlaneCount(1)), framework.WithClusterFiller(api.WithWorkerNodeCount(1)), framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), framework.WithRegistryMirrorInsecureSkipVerify(constants.CloudStackProviderName), ) runRegistryMirrorConfigFlow(test) @@ -2245,6 +2666,19 @@ func TestCloudStackKubernetes127RedhatRegistryMirrorAndCert(t *testing.T) { runRegistryMirrorConfigFlow(test) } +func TestCloudStackKubernetes128RedhatRegistryMirrorAndCert(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithRegistryMirrorEndpointAndCert(constants.CloudStackProviderName), + ) + runRegistryMirrorConfigFlow(test) +} + func TestCloudStackKubernetes123RedhatAuthenticatedRegistryMirror(t *testing.T) { test := framework.NewClusterE2ETest( t, @@ -2310,6 +2744,19 @@ func TestCloudStackKubernetes127RedhatAuthenticatedRegistryMirror(t *testing.T) runRegistryMirrorConfigFlow(test) } +func TestCloudStackKubernetes128RedhatAuthenticatedRegistryMirror(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithAuthenticatedRegistryMirror(constants.CloudStackProviderName), + ) + runRegistryMirrorConfigFlow(test) +} + // Simpleflow func TestCloudStackKubernetes123SimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( @@ -2420,6 +2867,17 @@ func TestCloudStackKubernetes127ThreeReplicasFiveWorkersSimpleFlow(t *testing.T) runSimpleFlow(test) } +func TestCloudStackKubernetes128ThreeReplicasFiveWorkersSimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithControlPlaneCount(3)), + framework.WithClusterFiller(api.WithWorkerNodeCount(5)), + ) + runSimpleFlow(test) +} + func TestCloudStackKubernetes123MultiEndpointSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, @@ -2470,6 +2928,16 @@ func TestCloudStackKubernetes127MultiEndpointSimpleFlow(t *testing.T) { runSimpleFlow(test) } +func TestCloudStackKubernetes128MultiEndpointSimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128(), + framework.WithCloudStackFillers(framework.UpdateAddCloudStackAz2())), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + ) + runSimpleFlow(test) +} + func TestCloudStackKubernetes123DifferentNamespaceSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, @@ -2530,6 +2998,18 @@ func TestCloudStackKubernetes127DifferentNamespaceSimpleFlow(t *testing.T) { runSimpleFlow(test) } +func TestCloudStackKubernetes128DifferentNamespaceSimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128(), + framework.WithCloudStackFillers(api.WithCloudStackConfigNamespace(clusterNamespace), + api.WithCloudStackConfigNamespaceForAllMachinesAndDatacenter(clusterNamespace))), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithClusterNamespace(clusterNamespace)), + ) + runSimpleFlow(test) +} + // Cilium Policy func TestCloudStackKubernetes123CiliumAlwaysPolicyEnforcementModeSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( @@ -2581,6 +3061,16 @@ func TestCloudStackKubernetes127CiliumAlwaysPolicyEnforcementModeSimpleFlow(t *t runSimpleFlow(test) } +func TestCloudStackKubernetes128CiliumAlwaysPolicyEnforcementModeSimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithCiliumPolicyEnforcementMode(v1alpha1.CiliumPolicyModeAlways)), + ) + runSimpleFlow(test) +} + // TODO: Add TestCloudStackKubernetes123RedhatTo124UpgradeCiliumPolicyEnforcementMode func TestCloudStackKubernetes123RedhatTo124UpgradeCiliumPolicyEnforcementMode(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -2704,6 +3194,15 @@ func TestCloudStackKubernetes127StackedEtcdRedhat(t *testing.T) { runStackedEtcdFlow(test) } +func TestCloudStackKubernetes128StackedEtcdRedhat(t *testing.T) { + test := framework.NewClusterE2ETest(t, + framework.NewCloudStack(t, framework.WithCloudStackRedhat128()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithStackedEtcdTopology())) + runStackedEtcdFlow(test) +} + // Taints func TestCloudStackKubernetes123RedhatTaintsUpgradeFlow(t *testing.T) { provider := redhat123ProviderWithTaints(t) @@ -2835,6 +3334,32 @@ func TestCloudStackKubernetes127RedhatTaintsUpgradeFlow(t *testing.T) { ) } +func TestCloudStackKubernetes128RedhatTaintsUpgradeFlow(t *testing.T) { + provider := redhat128ProviderWithTaints(t) + + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + api.RemoveAllWorkerNodeGroups(), // This gives us a blank slate + ), + ) + + runTaintsUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.WithWorkerNodeGroup(worker0, api.WithTaint(framework.NoExecuteTaint())), + api.WithWorkerNodeGroup(worker1, api.WithTaint(framework.NoExecuteTaint())), + api.WithWorkerNodeGroup(worker2, api.WithNoTaints()), + api.WithControlPlaneTaints([]corev1.Taint{framework.PreferNoScheduleTaint()}), + ), + ) +} + func redhat123ProviderWithTaints(t *testing.T) *framework.CloudStack { return framework.NewCloudStack(t, framework.WithCloudStackWorkerNodeGroup( @@ -2925,6 +3450,24 @@ func redhat127ProviderWithTaints(t *testing.T) *framework.CloudStack { ) } +func redhat128ProviderWithTaints(t *testing.T) *framework.CloudStack { + return framework.NewCloudStack(t, + framework.WithCloudStackWorkerNodeGroup( + worker0, + framework.NoScheduleWorkerNodeGroup(worker0, 2), + ), + framework.WithCloudStackWorkerNodeGroup( + worker1, + framework.WithWorkerNodeGroup(worker1, api.WithCount(1)), + ), + framework.WithCloudStackWorkerNodeGroup( + worker2, + framework.PreferNoScheduleWorkerNodeGroup(worker2, 1), + ), + framework.WithCloudStackRedhat128(), + ) +} + // Upgrade func TestCloudStackKubernetes123RedhatAndRemoveWorkerNodeGroups(t *testing.T) { provider := framework.NewCloudStack(t, @@ -3126,6 +3669,46 @@ func TestCloudStackKubernetes127RedhatAndRemoveWorkerNodeGroups(t *testing.T) { ) } +func TestCloudStackKubernetes128RedhatAndRemoveWorkerNodeGroups(t *testing.T) { + provider := framework.NewCloudStack(t, + framework.WithCloudStackWorkerNodeGroup( + "worker-1", + framework.WithWorkerNodeGroup("workers-1", api.WithCount(2)), + ), + framework.WithCloudStackWorkerNodeGroup( + "worker-2", + framework.WithWorkerNodeGroup("workers-2", api.WithCount(1)), + ), + framework.WithCloudStackRedhat128(), + ) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller( + api.WithKubernetesVersion(v1alpha1.Kube128), + api.WithExternalEtcdTopology(1), + api.WithControlPlaneCount(1), + api.RemoveAllWorkerNodeGroups(), // This gives us a blank slate + ), + ) + + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade( + api.RemoveWorkerNodeGroup("workers-2"), + api.WithWorkerNodeGroup("workers-1", api.WithCount(1)), + ), + provider.WithNewCloudStackWorkerNodeGroup( + "worker-1", + framework.WithWorkerNodeGroup( + "workers-3", + api.WithCount(1), + ), + ), + ) +} + // TODO: 1.23 to 1.24 upgrade tests func TestCloudStackKubernetes123To124RedhatUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -3199,6 +3782,24 @@ func TestCloudStackKubernetes126To127RedhatUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes127To128RedhatUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithStackedEtcdTopology()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + ) + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterFiller(api.WithStackedEtcdTopology()), + framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), + provider.WithProviderUpgrade(provider.Redhat128Template()), + ) +} + func TestCloudStackKubernetes123To124RedhatUnstackedUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) test := framework.NewClusterE2ETest( @@ -3267,6 +3868,23 @@ func TestCloudStackKubernetes126To127RedhatUnstackedUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes127To128RedhatUnstackedUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + ) + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), + provider.WithProviderUpgrade(provider.Redhat128Template()), + ) +} + func TestCloudStackKubernetes123RedhatTo124StackedEtcdUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) test := framework.NewClusterE2ETest( @@ -3339,6 +3957,24 @@ func TestCloudStackKubernetes126RedhatTo127StackedEtcdUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes127RedhatTo128StackedEtcdUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithClusterFiller(api.WithStackedEtcdTopology()), + ) + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), + provider.WithProviderUpgrade(provider.Redhat128Template()), + ) +} + // TODO: investigate these tests further as they pass even without the expected behavior(upgrade should fail the first time and continue from the checkpoint on second upgrade)wq func TestCloudStackKubernetes123RedhatTo124UpgradeWithCheckpoint(t *testing.T) { var clusterOpts []framework.ClusterE2ETestOpt @@ -3460,6 +4096,36 @@ func TestCloudStackKubernetes126RedhatTo127UpgradeWithCheckpoint(t *testing.T) { ) } +func TestCloudStackKubernetes127RedhatTo128UpgradeWithCheckpoint(t *testing.T) { + var clusterOpts []framework.ClusterE2ETestOpt + var clusterOpts2 []framework.ClusterE2ETestOpt + + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + ) + + clusterOpts = append(clusterOpts, framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), framework.ExpectFailure(true), + provider.WithProviderUpgrade(provider.Redhat127Template()), framework.WithEnvVar(features.CheckpointEnabledEnvVar, "true"), framework.WithEnvVar(framework.CleanupVmsVar, "false")) + + commandOpts := []framework.CommandOpt{framework.WithExternalEtcdWaitTimeout("10m")} + + clusterOpts2 = append(clusterOpts, framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), framework.ExpectFailure(false), + provider.WithProviderUpgrade(provider.Redhat128Template()), framework.WithEnvVar(features.CheckpointEnabledEnvVar, "true"), framework.WithEnvVar(framework.CleanupVmsVar, "true")) + + runUpgradeFlowWithCheckpoint( + test, + v1alpha1.Kube128, + clusterOpts, + clusterOpts2, + commandOpts, + ) +} + func TestCloudStackKubernetes123RedhatUpgradeFromLatestMinorReleaseAlwaysNetworkPolicy(t *testing.T) { release := latestMinorRelease(t) provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -3584,6 +4250,22 @@ func TestCloudStackKubernetes127RedhatControlPlaneNodeUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes128RedhatControlPlaneNodeUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat128()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(3)), + ) + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade(api.WithControlPlaneCount(3)), + ) +} + func TestCloudStackKubernetes123RedhatWorkerNodeUpgrade(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) test := framework.NewClusterE2ETest( @@ -3664,6 +4346,22 @@ func TestCloudStackKubernetes127RedhatWorkerNodeUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes128RedhatWorkerNodeUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat128()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(3)), + ) + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade(api.WithWorkerNodeCount(5)), + ) +} + func TestCloudStackKubernetes123UpgradeFromLatestMinorRelease(t *testing.T) { release := latestMinorRelease(t) provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) @@ -3779,6 +4477,25 @@ func TestCloudStackKubernetes126To127RedhatMultipleFieldsUpgrade(t *testing.T) { ) } +func TestCloudStackKubernetes127To128RedhatMultipleFieldsUpgrade(t *testing.T) { + provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat127()) + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), + ) + runSimpleUpgradeFlow( + test, + v1alpha1.Kube128, + framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + provider.WithProviderUpgrade( + provider.Redhat128Template(), + framework.UpdateLargerCloudStackComputeOffering(), + ), + ) +} + func TestCloudStackKubernetes123AddRemoveAz(t *testing.T) { provider := framework.NewCloudStack(t, framework.WithCloudStackRedhat123()) test := framework.NewClusterE2ETest(