diff --git a/internal/pkg/api/cluster.go b/internal/pkg/api/cluster.go index 4d3ce5a89cb4..165ef3289549 100644 --- a/internal/pkg/api/cluster.go +++ b/internal/pkg/api/cluster.go @@ -129,6 +129,33 @@ func WithServiceCidr(svcCidr string) ClusterFiller { } } +// WithWorkerKubernetesVersion sets the kubernetes version field for the given worker group. +func WithWorkerKubernetesVersion(name string, version *anywherev1.KubernetesVersion) ClusterFiller { + return func(c *anywherev1.Cluster) { + pos := -1 + for i, wng := range c.Spec.WorkerNodeGroupConfigurations { + if wng.Name == name { + wng.KubernetesVersion = version + pos = i + c.Spec.WorkerNodeGroupConfigurations[pos] = wng + break + } + } + // Append the worker node group if not already found in existing configuration + if pos == -1 { + c.Spec.WorkerNodeGroupConfigurations = append(c.Spec.WorkerNodeGroupConfigurations, workerNodeWithKubernetesVersion(name, version)) + } + } +} + +func workerNodeWithKubernetesVersion(name string, version *anywherev1.KubernetesVersion) anywherev1.WorkerNodeGroupConfiguration { + return anywherev1.WorkerNodeGroupConfiguration{ + Name: name, + Count: ptr.Int(1), + KubernetesVersion: version, + } +} + func WithWorkerNodeCount(r int) ClusterFiller { return func(c *anywherev1.Cluster) { if len(c.Spec.WorkerNodeGroupConfigurations) == 0 { diff --git a/internal/pkg/api/tinkerbell.go b/internal/pkg/api/tinkerbell.go index a2a83e1a7c69..c5873441db3f 100644 --- a/internal/pkg/api/tinkerbell.go +++ b/internal/pkg/api/tinkerbell.go @@ -54,6 +54,29 @@ func WithTinkerbellOSImageURL(value string) TinkerbellFiller { } } +// WithTinkerbellCPMachineConfigOSImageURL sets the OSImageURL & OSFamily for control-plane machine config. +func WithTinkerbellCPMachineConfigOSImageURL(imageURL string, OSFamily anywherev1.OSFamily) TinkerbellFiller { + return func(config TinkerbellConfig) { + clusterName := config.clusterName + cpName := providers.GetControlPlaneNodeName(clusterName) + cpMachineConfig := config.machineConfigs[cpName] + cpMachineConfig.Spec.OSImageURL = imageURL + cpMachineConfig.Spec.OSFamily = OSFamily + config.machineConfigs[cpName] = cpMachineConfig + } +} + +// WithTinkerbellWorkerMachineConfigOSImageURL sets the OSImageURL & OSFamily for worker machine config. +func WithTinkerbellWorkerMachineConfigOSImageURL(imageURL string, OSFamily anywherev1.OSFamily) TinkerbellFiller { + return func(config TinkerbellConfig) { + clusterName := config.clusterName + workerMachineConfig := config.machineConfigs[clusterName] + workerMachineConfig.Spec.OSImageURL = imageURL + workerMachineConfig.Spec.OSFamily = OSFamily + config.machineConfigs[clusterName] = workerMachineConfig + } +} + // WithHookImagesURLPath modify HookImagesURL, it's useful for airgapped testing. func WithHookImagesURLPath(value string) TinkerbellFiller { return func(config TinkerbellConfig) { diff --git a/test/e2e/tinkerbell_test.go b/test/e2e/tinkerbell_test.go index 6d627a94c486..692ce2736d12 100644 --- a/test/e2e/tinkerbell_test.go +++ b/test/e2e/tinkerbell_test.go @@ -119,6 +119,48 @@ func TestTinkerbellKubernetes127UbuntuTo128Upgrade(t *testing.T) { ) } +func TestTinkerbellKubernetes127UbuntuTo128UpgradeCPOnly(t *testing.T) { + provider := framework.NewTinkerbell(t, framework.WithUbuntu127TinkerbellForCP(), framework.WithUbuntu127TinkerbellForWorker()) + kube127 := v1alpha1.Kube127 + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(api.WithKubernetesVersion(kube127)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithClusterFiller(api.WithWorkerKubernetesVersion(worker0, &kube127)), + framework.WithControlPlaneHardware(2), + framework.WithWorkerHardware(1), + ) + runSimpleUpgradeFlowWorkerNodeVersionForBareMetal( + test, + framework.WithClusterUpgrade(api.WithKubernetesVersion(v1alpha1.Kube128)), + provider.WithProviderUpgrade(framework.Ubuntu128ImageForCP()), + ) +} + +func TestTinkerbellKubernetes127UbuntuTo128UpgradeWorkerOnly(t *testing.T) { + provider := framework.NewTinkerbell(t, framework.WithUbuntu128TinkerbellForCP(), framework.WithUbuntu127TinkerbellForWorker()) + kube127 := v1alpha1.Kube127 + kube128 := v1alpha1.Kube128 + test := framework.NewClusterE2ETest( + t, + provider, + framework.WithClusterFiller(), + framework.WithClusterFiller(api.WithKubernetesVersion(kube128)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithClusterFiller(api.WithWorkerKubernetesVersion(worker0, &kube127)), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(2), + ) + runSimpleUpgradeFlowWorkerNodeVersionForBareMetal( + test, + framework.WithClusterUpgrade(api.WithWorkerKubernetesVersion(worker0, &kube128)), + provider.WithProviderUpgrade(framework.Ubuntu128ImageForWorker()), + ) +} + func TestTinkerbellKubernetes125To126Ubuntu2204Upgrade(t *testing.T) { provider := framework.NewTinkerbell(t) test := framework.NewClusterE2ETest( diff --git a/test/e2e/upgrade.go b/test/e2e/upgrade.go index ef681efb0484..9ddf43fb22ce 100644 --- a/test/e2e/upgrade.go +++ b/test/e2e/upgrade.go @@ -51,6 +51,18 @@ func runSimpleUpgradeFlowForBareMetal(test *framework.ClusterE2ETest, updateVers test.ValidateHardwareDecommissioned() } +func runSimpleUpgradeFlowWorkerNodeVersionForBareMetal(test *framework.ClusterE2ETest, clusterOpts ...framework.ClusterE2ETestOpt) { + test.GenerateClusterConfig() + test.GenerateHardwareConfig() + test.PowerOffHardware() + test.CreateCluster(framework.WithControlPlaneWaitTimeout("20m")) + test.UpgradeClusterWithNewConfig(clusterOpts) + test.ValidateClusterState() + test.StopIfFailed() + test.DeleteCluster() + test.ValidateHardwareDecommissioned() +} + // runSimpleUpgradeFlowForBaremetalWithoutClusterConfigGeneration runs the Create, Upgrade and Delete cluster flows // for Baremetal that use the cluster config generated by the WithClusterConfig method when the test object is created, // and avoids regenerating a cluster config with defaults. diff --git a/test/framework/cluster/validations/cluster.go b/test/framework/cluster/validations/cluster.go index bf06de9daeac..ade844aab07a 100644 --- a/test/framework/cluster/validations/cluster.go +++ b/test/framework/cluster/validations/cluster.go @@ -98,6 +98,10 @@ func ValidateWorkerNodes(ctx context.Context, vc clusterf.StateValidationConfig) // deduce the worker node group configuration to node mapping via the machine deployment and machine set for _, w := range wn { workerGroupCount := 0 + k8sVersion := vc.ClusterSpec.Cluster.Spec.KubernetesVersion + if w.KubernetesVersion != nil { + k8sVersion = *w.KubernetesVersion + } ms, err := getWorkerNodeMachineSets(ctx, vc, w) if err != nil { return fmt.Errorf("failed to get machine sets when validating worker node: %v", err) @@ -105,7 +109,7 @@ func ValidateWorkerNodes(ctx context.Context, vc clusterf.StateValidationConfig) workerNodes := filterWorkerNodes(nodes.Items, ms, w) workerGroupCount += len(workerNodes) for _, node := range workerNodes { - if err := validateNodeReady(node, vc.ClusterSpec.Cluster.Spec.KubernetesVersion); err != nil { + if err := validateNodeReady(node, k8sVersion); err != nil { errorList = append(errorList, fmt.Errorf("failed to validate worker node ready %v", err)) } if err := api.ValidateWorkerNodeTaints(w, node); err != nil { diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index b7298e5e210a..7f3738e7afd9 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -34,6 +34,8 @@ const ( tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" tinkerbellSSHAuthorizedKey = "T_TINKERBELL_SSH_AUTHORIZED_KEY" TinkerbellCIEnvironment = "T_TINKERBELL_CI_ENVIRONMENT" + controlPlaneIdentifier = "cp" + workerIdentifier = "worker" ) var requiredTinkerbellEnvVars = []string{ @@ -148,7 +150,7 @@ func (t *Tinkerbell) WithKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersi return api.JoinClusterConfigFillers( api.ClusterToConfigFiller(api.WithKubernetesVersion(kubeVersion)), api.TinkerbellToConfigFiller( - imageForKubeVersionAndOS(kubeVersion, os), + imageForKubeVersionAndOS(kubeVersion, os, ""), api.WithOsFamilyForAllTinkerbellMachines(osFamiliesForOS[os]), ), ) @@ -168,10 +170,17 @@ func envVarForImage(os OS, kubeVersion anywherev1.KubernetesVersion) string { // withKubeVersionAndOS returns a cluster config filler that sets the cluster kube version and the right image for all // tinkerbell machine configs. -func withKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, release *releasev1.EksARelease) TinkerbellOpt { +func withKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, machineConfigType string, release *releasev1.EksARelease) TinkerbellOpt { + if machineConfigType == controlPlaneIdentifier || machineConfigType == workerIdentifier { + return func(t *Tinkerbell) { + t.fillers = append(t.fillers, + imageForKubeVersionAndOS(kubeVersion, os, machineConfigType), + ) + } + } return func(t *Tinkerbell) { t.fillers = append(t.fillers, - imageForKubeVersionAndOS(kubeVersion, os), + imageForKubeVersionAndOS(kubeVersion, os, ""), api.WithOsFamilyForAllTinkerbellMachines(osFamiliesForOS[os]), ) } @@ -179,52 +188,72 @@ func withKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, relea // WithUbuntu124Tinkerbell tink test with ubuntu 1.24. func WithUbuntu124Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, nil) + return withKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, "", nil) } // WithUbuntu125Tinkerbell tink test with ubuntu 1.25. func WithUbuntu125Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, nil) + return withKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, "", nil) } // WithUbuntu126Tinkerbell tink test with ubuntu 1.26. func WithUbuntu126Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, nil) + return withKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, "", nil) } // WithUbuntu127Tinkerbell tink test with ubuntu 1.27. func WithUbuntu127Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, nil) + return withKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, "", nil) } // WithUbuntu128Tinkerbell tink test with ubuntu 1.28. func WithUbuntu128Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, nil) + return withKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, "", nil) +} + +// WithUbuntu127TinkerbellForCP tink test with ubuntu 1.27 for Control Plane Machine Config. +func WithUbuntu127TinkerbellForCP() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, controlPlaneIdentifier, nil) +} + +// WithUbuntu128TinkerbellForCP tink test with ubuntu 1.28 for Control Plane Machine Config. +func WithUbuntu128TinkerbellForCP() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, controlPlaneIdentifier, nil) +} + +// WithUbuntu127TinkerbellForWorker tink test with ubuntu 1.27 for Worker Node Group Machine Config. +func WithUbuntu127TinkerbellForWorker() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, workerIdentifier, nil) +} + +// WithUbuntu128TinkerbellForWorker tink test with ubuntu 1.28 for Worker Node Group Machine Config. +func WithUbuntu128TinkerbellForWorker() TinkerbellOpt { + return withKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, workerIdentifier, nil) } // WithRedHat124Tinkerbell tink test with redhat 1.24. func WithRedHat124Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube124, RedHat8, nil) + return withKubeVersionAndOS(anywherev1.Kube124, RedHat8, "", nil) } // WithRedHat125Tinkerbell tink test with redhat 1.25. func WithRedHat125Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube125, RedHat8, nil) + return withKubeVersionAndOS(anywherev1.Kube125, RedHat8, "", nil) } // WithRedHat126Tinkerbell tink test with redhat 1.26. func WithRedHat126Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube126, RedHat8, nil) + return withKubeVersionAndOS(anywherev1.Kube126, RedHat8, "", nil) } // WithRedHat127Tinkerbell tink test with redhat 1.27. func WithRedHat127Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube127, RedHat8, nil) + return withKubeVersionAndOS(anywherev1.Kube127, RedHat8, "", nil) } // WithRedHat128Tinkerbell tink test with redhat 1.27. func WithRedHat128Tinkerbell() TinkerbellOpt { - return withKubeVersionAndOS(anywherev1.Kube128, RedHat8, nil) + return withKubeVersionAndOS(anywherev1.Kube128, RedHat8, "", nil) } func WithBottleRocketTinkerbell() TinkerbellOpt { @@ -271,46 +300,85 @@ func WithHookImagesURLPath(url string) TinkerbellOpt { } } -func imageForKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, operatingSystem OS) api.TinkerbellFiller { - return api.WithTinkerbellOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion))) +// imageForKubeVersionAndOS sets osImageURL on the appropriate field in the Machine Config based on the machineConfigType string provided else sets it at Data Center config. +func imageForKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, operatingSystem OS, machineConfigType string) api.TinkerbellFiller { + var tinkerbellFiller api.TinkerbellFiller + if machineConfigType == workerIdentifier { + tinkerbellFiller = api.WithTinkerbellWorkerMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion)), osFamiliesForOS[operatingSystem]) + } else if machineConfigType == controlPlaneIdentifier { + tinkerbellFiller = api.WithTinkerbellCPMachineConfigOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion)), osFamiliesForOS[operatingSystem]) + } else { + tinkerbellFiller = api.WithTinkerbellOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion))) + } + return tinkerbellFiller } // Ubuntu124Image represents an Ubuntu raw image corresponding to Kubernetes 1.24. func Ubuntu124Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004) + return imageForKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, "") } // Ubuntu125Image represents an Ubuntu raw image corresponding to Kubernetes 1.25. func Ubuntu125Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004) + return imageForKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, "") } // Ubuntu126Image represents an Ubuntu raw image corresponding to Kubernetes 1.26. func Ubuntu126Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004) + return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, "") } // Ubuntu127Image represents an Ubuntu raw image corresponding to Kubernetes 1.27. func Ubuntu127Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004) + return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, "") } // Ubuntu128Image represents an Ubuntu raw image corresponding to Kubernetes 1.28. func Ubuntu128Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004) + return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, "") +} + +// Ubuntu126ImageForCP represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for CP machine config. +func Ubuntu126ImageForCP() api.TinkerbellFiller { + return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, controlPlaneIdentifier) +} + +// Ubuntu127ImageForCP represents an Ubuntu raw image corresponding to Kubernetes 1.27 and is set for CP machine config. +func Ubuntu127ImageForCP() api.TinkerbellFiller { + return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, controlPlaneIdentifier) +} + +// Ubuntu128ImageForCP represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for CP machine config. +func Ubuntu128ImageForCP() api.TinkerbellFiller { + return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, controlPlaneIdentifier) +} + +// Ubuntu126ImageForWorker represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for worker machine config. +func Ubuntu126ImageForWorker() api.TinkerbellFiller { + return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, workerIdentifier) +} + +// Ubuntu127ImageForWorker represents an Ubuntu raw image corresponding to Kubernetes 1.27 and is set for worker machine config. +func Ubuntu127ImageForWorker() api.TinkerbellFiller { + return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, workerIdentifier) +} + +// Ubuntu128ImageForWorker represents an Ubuntu raw image corresponding to Kubernetes 1.28 and is set for worker machine config. +func Ubuntu128ImageForWorker() api.TinkerbellFiller { + return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2004, workerIdentifier) } // Ubuntu2204Kubernetes126Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.26. func Ubuntu2204Kubernetes126Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2204) + return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2204, "") } // Ubuntu2204Kubernetes127Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.27. func Ubuntu2204Kubernetes127Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2204) + return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2204, "") } // Ubuntu2204Kubernetes128Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.28. func Ubuntu2204Kubernetes128Image() api.TinkerbellFiller { - return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2204) + return imageForKubeVersionAndOS(anywherev1.Kube128, Ubuntu2204, "") }