🌱 Bump Kubernetes version in tests to 1.31.0-rc.1

.gitignore

@@ -24,7 +24,7 @@ test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml
-test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml
+test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor*.yaml
 
 # env vars file used in getting-started.md and manifests generation
 envvars.txt

Makefile

@@ -392,7 +392,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml"
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-fast-rollout.yaml"
 	# for PCI passthrough template
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml"
 	# for DHCP overrides
@@ -410,7 +410,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml"
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-fast-rollout-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers-supervisor.yaml"
 
 .PHONY: generate-e2e-templates-v1.10

test/e2e/cluster_upgrade_test.go

@@ -49,10 +49,8 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				WorkerMachineCount:    ptr.To[int64](5),
-				// Note: install-on-bootstrap will install Kubernetes on bootstrap if the correct Kubernetes version
-				// cannot be detected. This is required to install versions we don't have images for (e.g. ci/latest-1.30).
-				Flavor:               ptr.To(testSpecificSettingsGetter().FlavorForMode("install-on-bootstrap")),
-				PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc,
+				Flavor:                ptr.To(testSpecificSettingsGetter().FlavorForMode("fast-rollout")),
+				PostNamespaceCreated:  testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 			}
 		})
 	})

test/e2e/config/vsphere.yaml

@@ -169,7 +169,7 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-dhcp-overrides.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-hw-upgrade.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ignition.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-install-on-bootstrap.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-fast-rollout.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ipam.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-kcp-remediation.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-md-remediation.yaml"
@@ -187,7 +187,7 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor-runtimesdk.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-fast-rollout-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-ownerrefs-finalizers-supervisor.yaml"
           - sourcePath: "../data/shared/capv/main/metadata.yaml"
@@ -267,10 +267,10 @@ providers:
 
 variables:
   # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
-  KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.0"
-  KUBERNETES_VERSION: "v1.30.0"
-  KUBERNETES_VERSION_UPGRADE_FROM: "v1.29.0"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.30.0"
+  KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.1"
+  KUBERNETES_VERSION: "v1.31.0-rc.1"
+  KUBERNETES_VERSION_UPGRADE_FROM: "v1.30.0"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.31.0-rc.1"
   KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.31"
   CPI_IMAGE_K8S_VERSION: "v1.31.0-rc.1"
   CNI: "./data/cni/calico/calico.yaml"

test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml

@@ -12,3 +12,9 @@ patches:
   - target:
       kind: VSphereMachineTemplate
     path: ../commons/remove-storage-policy.yaml
+  - target:
+      kind: KubeadmControlPlane
+    path: ./patch-k8s-install-script-kcp.yaml
+  - target:
+      kind: KubeadmConfigTemplate
+    path: ./patch-k8s-install-script-kct.yaml

test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml → test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml

@@ -1,15 +1,25 @@
-# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl
 - op: add
-  path: /spec/topology/variables/-
+  path: /spec/kubeadmConfigSpec/files/-
   value:
-    name: preKubeadmScript
-    # Note: This file has to be escaped as it is run through envsubst by clusterctl
-    value: |
+    owner: root:root
+    path:  "/etc/pre-kubeadm-commands/20-k8s-install.sh"
+    permissions: "0755"
+    content: |
       #!/bin/bash
 
       set -o nounset
       set -o pipefail
       set -o errexit
+
+      DISTRO="ubuntu"
+      KUBE_BINARY_DIR="/usr/bin"
+
+      source /etc/lsb-release
+      if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then
+        # Overrides for flatcar
+        DISTRO="flatcar"
+        KUBE_BINARY_DIR="/opt/bin"
+      fi
 
       function retry {
         attempt=0
@@ -36,8 +46,9 @@
       LINE_SEPARATOR="*************************************************"
       echo "$${LINE_SEPARATOR}"
 
-      ## Variables (replaced by JSON patching)
-      KUBERNETES_VERSION=$${KUBERNETES_VERSION}
+      ## Variables (replaced by envtest patching)
+      ## Note: this does not work for upgrades.
+      KUBERNETES_VERSION=${KUBERNETES_VERSION}
       ##
 
       # Note: We assume if kubectl has the right version, everything else has as well
@@ -53,41 +64,21 @@
         # Let's just also download the control plane images for worker nodes. It's easier then optimizing it.
         declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
         CONTAINER_EXT="tar"
-        echo "* testing CI version $${KUBERNETES_VERSION}"
-        # Check for semver
-        if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        echo "* testing version $${KUBERNETES_VERSION}"
+        CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
+        # Set CI_URL to the released binaries for actually released versions.
+        if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then
           CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
-          VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
-          export DEBIAN_FRONTEND=noninteractive
-          # sometimes the network is not immediately available, so we have to retry the apt-get update
-          retry 10 5 "apt-get update"
-          apt-get install -y apt-transport-https ca-certificates curl gpg
-          mkdir -p -m 755 /etc/apt/keyrings
-          curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
-          echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
-          apt-get update
-          # replace . with \.
-          VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
-          PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
-          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
-            echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
-            apt-mark unhold "$${CI_PACKAGE}"
-            apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
-            apt-mark hold "$${CI_PACKAGE}"
-          done
-          systemctl restart kubelet
-        else
-          CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
-          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
-            # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
-            # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
-            echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
-            wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
-            chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
-            mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
-          done
-          systemctl restart kubelet
         fi
+        for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+          # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
+          # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
+          echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
+          wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
+          chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
+          mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}"
+        done
+        systemctl restart kubelet
         IMAGE_REGISTRY_PREFIX=registry.k8s.io
         # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io
         if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then

test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml → test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml

@@ -1,15 +1,28 @@
-# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl
 - op: add
-  path: /spec/topology/variables/-
+  path: /spec/template/spec/files
+  value: []
+- op: add
+  path: /spec/template/spec/files/-
   value:
-    name: preKubeadmScript
-    # Note: This file has to be escaped as it is run through envsubst by clusterctl
-    value: |
+    owner: root:root
+    path:  "/etc/pre-kubeadm-commands/20-k8s-install.sh"
+    permissions: "0755"
+    content: |
       #!/bin/bash
 
       set -o nounset
       set -o pipefail
       set -o errexit
+
+      DISTRO="ubuntu"
+      KUBE_BINARY_DIR="/usr/bin"
+
+      source /etc/lsb-release
+      if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then
+        # Overrides for flatcar
+        DISTRO="flatcar"
+        KUBE_BINARY_DIR="/opt/bin"
+      fi
 
       function retry {
         attempt=0
@@ -36,8 +49,9 @@
       LINE_SEPARATOR="*************************************************"
       echo "$${LINE_SEPARATOR}"
 
-      ## Variables (replaced by JSON patching)
-      KUBERNETES_VERSION=$${KUBERNETES_VERSION}
+      ## Variables (replaced by envtest patching)
+      ## Note: this does not work for upgrades.
+      KUBERNETES_VERSION=${KUBERNETES_VERSION}
       ##
 
       # Note: We assume if kubectl has the right version, everything else has as well
@@ -53,41 +67,21 @@
         # Let's just also download the control plane images for worker nodes. It's easier then optimizing it.
         declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
         CONTAINER_EXT="tar"
-        echo "* testing CI version $${KUBERNETES_VERSION}"
-        # Check for semver
-        if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        echo "* testing version $${KUBERNETES_VERSION}"
+        CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
+        # Set CI_URL to the released binaries for actually released versions.
+        if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then
           CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
-          VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
-          export DEBIAN_FRONTEND=noninteractive
-          # sometimes the network is not immediately available, so we have to retry the apt-get update
-          retry 10 5 "apt-get update"
-          apt-get install -y apt-transport-https ca-certificates curl gpg
-          mkdir -p -m 755 /etc/apt/keyrings
-          curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
-          echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
-          apt-get update
-          # replace . with \.
-          VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
-          PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
-          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
-            echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
-            apt-mark unhold "$${CI_PACKAGE}"
-            apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
-            apt-mark hold "$${CI_PACKAGE}"
-          done
-          systemctl restart kubelet
-        else
-          CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
-          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
-            # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
-            # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
-            echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
-            wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
-            chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
-            mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
-          done
-          systemctl restart kubelet
         fi
+        for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+          # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
+          # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
+          echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
+          wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
+          chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
+          mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}"
+        done
+        systemctl restart kubelet
         IMAGE_REGISTRY_PREFIX=registry.k8s.io
         # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io
         if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then

test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml

@@ -9,3 +9,6 @@ patches:
   - target:
       kind: ClusterClass
     path: ./name.yaml
+  - target:
+      kind: ClusterClass
+    path: ../clusterclass/patch-k8s-install-script.yaml

test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml

@@ -9,6 +9,9 @@ patches:
   - target:
       kind: ClusterClass
     path: ./patch-prekubeadmscript.yaml
+  - target:
+      kind: ClusterClass
+    path: ./patch-k8s-install-script.yaml
   - target:
       kind: ClusterClass
     path: ./patch-namingstrategy.yaml