🌱 Bump Kubernetes version in tests to 1.31.0-rc.1

.gitignore

@@ -24,7 +24,7 @@ test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml
-test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml
+test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor*.yaml
 
 # env vars file used in getting-started.md and manifests generation
 envvars.txt

Makefile

@@ -392,7 +392,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml"
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-fast-rollout.yaml"
 	# for PCI passthrough template
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml"
 	# for DHCP overrides
@@ -410,7 +410,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml"
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-fast-rollout-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers-supervisor.yaml"
 
 .PHONY: generate-e2e-templates-v1.10

test/e2e/cluster_upgrade_test.go

@@ -49,10 +49,8 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				WorkerMachineCount:    ptr.To[int64](5),
-				// Note: install-on-bootstrap will install Kubernetes on bootstrap if the correct Kubernetes version
-				// cannot be detected. This is required to install versions we don't have images for (e.g. ci/latest-1.30).
-				Flavor:               ptr.To(testSpecificSettingsGetter().FlavorForMode("install-on-bootstrap")),
-				PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc,
+				Flavor:                ptr.To(testSpecificSettingsGetter().FlavorForMode("fast-rollout")),
+				PostNamespaceCreated:  testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 			}
 		})
 	})

test/e2e/config/vsphere.yaml

@@ -169,7 +169,7 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-dhcp-overrides.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-hw-upgrade.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ignition.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-install-on-bootstrap.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-fast-rollout.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-ipam.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-kcp-remediation.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-md-remediation.yaml"
@@ -187,7 +187,7 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor-runtimesdk.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-fast-rollout-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-ownerrefs-finalizers-supervisor.yaml"
           - sourcePath: "../data/shared/capv/main/metadata.yaml"
@@ -267,12 +267,12 @@ providers:
 
 variables:
   # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
-  KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.0"
-  KUBERNETES_VERSION: "v1.30.0"
-  KUBERNETES_VERSION_UPGRADE_FROM: "v1.29.0"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.30.0"
+  KUBERNETES_VERSION_MANAGEMENT: "v1.31.0-rc.1"
+  KUBERNETES_VERSION: "v1.31.0-rc.1"
+  KUBERNETES_VERSION_UPGRADE_FROM: "v1.30.0"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.31.0-rc.1"
   KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.31"
-  CPI_IMAGE_K8S_VERSION: "v1.31.0-rc.1"
+  CPI_IMAGE_K8S_VERSION: "v1.31.0-rc.0"
   CNI: "./data/cni/calico/calico.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
   EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true"

test/e2e/data/infrastructure-vsphere-govmomi/main/base/kustomization.yaml

@@ -12,3 +12,9 @@ patches:
   - target:
       kind: VSphereMachineTemplate
     path: ../commons/remove-storage-policy.yaml
+  - target:
+      kind: KubeadmControlPlane
+    path: ./patch-k8s-install-script-kcp.yaml
+  - target:
+      kind: KubeadmConfigTemplate
+    path: ./patch-k8s-install-script-kct.yaml

test/e2e/data/infrastructure-vsphere-govmomi/main/install-on-bootstrap/inject-install-on-bootstrap.yaml → test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kcp.yaml

@@ -1,15 +1,25 @@
-# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl
 - op: add
-  path: /spec/topology/variables/-
+  path: /spec/kubeadmConfigSpec/files/-
   value:
-    name: preKubeadmScript
-    # Note: This file has to be escaped as it is run through envsubst by clusterctl
-    value: |
+    owner: root:root
+    path:  "/etc/pre-kubeadm-commands/20-k8s-install.sh"
+    permissions: "0755"
+    content: |
       #!/bin/bash
 
       set -o nounset
       set -o pipefail
       set -o errexit
+
+      DISTRO="ubuntu"
+      KUBE_BINARY_DIR="/usr/bin"
+
+      source /etc/lsb-release
+      if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then
+        # Overrides for flatcar
+        DISTRO="flatcar"
+        KUBE_BINARY_DIR="/opt/bin"
+      fi
 
       function retry {
         attempt=0
@@ -36,8 +46,8 @@
       LINE_SEPARATOR="*************************************************"
       echo "$${LINE_SEPARATOR}"
 
-      ## Variables (replaced by JSON patching)
-      KUBERNETES_VERSION=$${KUBERNETES_VERSION}
+      ## Variables (replaced by envtest patching)
+      KUBERNETES_VERSION=${KUBERNETES_VERSION}
       ##
 
       # Note: We assume if kubectl has the right version, everything else has as well
@@ -55,7 +65,7 @@
         CONTAINER_EXT="tar"
         echo "* testing CI version $${KUBERNETES_VERSION}"
         # Check for semver
-        if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
           CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
           VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
           export DEBIAN_FRONTEND=noninteractive
@@ -78,13 +88,16 @@
           systemctl restart kubelet
         else
           CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
+          if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then
+            CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
+          fi
           for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
             # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
             # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
             echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
             wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
             chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
-            mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
+            mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}"
           done
           systemctl restart kubelet
         fi

test/e2e/data/infrastructure-vsphere-govmomi/main/base/patch-k8s-install-script-kct.yaml

@@ -0,0 +1,125 @@
+- op: add
+  path: /spec/template/spec/files
+  value: []
+- op: add
+  path: /spec/template/spec/files/-
+  value:
+    owner: root:root
+    path:  "/etc/pre-kubeadm-commands/20-k8s-install.sh"
+    permissions: "0755"
+    content: |
+      #!/bin/bash
+
+      set -o nounset
+      set -o pipefail
+      set -o errexit
+
+      DISTRO="ubuntu"
+      KUBE_BINARY_DIR="/usr/bin"
+
+      source /etc/lsb-release
+      if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then
+        # Overrides for flatcar
+        DISTRO="flatcar"
+        KUBE_BINARY_DIR="/opt/bin"
+      fi
+
+      function retry {
+        attempt=0
+        max_attempts=$${1}
+        interval=$${2}
+        shift; shift
+        until [[ $${attempt} -ge "$${max_attempts}" ]] ; do
+          attempt=$((attempt+1))
+          set +e
+          eval "$*" && return || echo "failed $${attempt} times: $*"
+          set -e
+          sleep "$${interval}"
+        done
+        echo "error: reached max attempts at retry($*)"
+        return 1
+      }
+
+      [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+
+      # This test installs release packages or binaries that are a result of the CI and release builds.
+      # It runs '... --version' commands to verify that the binaries are correctly installed
+      # and finally uninstalls the packages.
+      # For the release packages it tests all versions in the support skew.
+      LINE_SEPARATOR="*************************************************"
+      echo "$${LINE_SEPARATOR}"
+
+      ## Variables (replaced by envtest patching)
+      KUBERNETES_VERSION=${KUBERNETES_VERSION}
+      ##
+
+      # Note: We assume if kubectl has the right version, everything else has as well
+      if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then
+        echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do"
+        exit 0
+      fi
+
+      if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
+        CI_DIR=/tmp/k8s-ci
+        mkdir -p "$${CI_DIR}"
+        declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
+        # Let's just also download the control plane images for worker nodes. It's easier then optimizing it.
+        declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
+        CONTAINER_EXT="tar"
+        echo "* testing CI version $${KUBERNETES_VERSION}"
+        # Check for semver
+        if [[ "$${DISTRO}" == "ubuntu" ]] && [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
+          VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
+          export DEBIAN_FRONTEND=noninteractive
+          # sometimes the network is not immediately available, so we have to retry the apt-get update
+          retry 10 5 "apt-get update"
+          apt-get install -y apt-transport-https ca-certificates curl gpg
+          mkdir -p -m 755 /etc/apt/keyrings
+          curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+          echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
+          apt-get update
+          # replace . with \.
+          VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
+          PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
+          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+            echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
+            apt-mark unhold "$${CI_PACKAGE}"
+            apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
+            apt-mark hold "$${CI_PACKAGE}"
+          done
+          systemctl restart kubelet
+        else
+          CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
+          if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then
+            CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
+          fi
+          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+            # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
+            # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
+            echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
+            wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
+            chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
+            mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}"
+          done
+          systemctl restart kubelet
+        fi
+        IMAGE_REGISTRY_PREFIX=registry.k8s.io
+        # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io
+        if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then
+          IMAGE_REGISTRY_PREFIX=k8s.gcr.io
+        fi
+        for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
+          echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+          wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+          $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
+          $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+          $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+        done
+      fi
+      echo "* checking binary versions"
+      echo "ctr version: " "$(ctr version)"
+      echo "kubeadm version: " "$(kubeadm version -o=short)"
+      echo "kubectl version: " "$(kubectl version --client=true)"
+      echo "kubelet version: " "$(kubelet --version)"
+      echo "$${LINE_SEPARATOR}"

test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml

@@ -9,3 +9,6 @@ patches:
   - target:
       kind: ClusterClass
     path: ./name.yaml
+  - target:
+      kind: ClusterClass
+    path: ../clusterclass/patch-k8s-install-script.yaml

test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/kustomization.yaml

@@ -9,6 +9,9 @@ patches:
   - target:
       kind: ClusterClass
     path: ./patch-prekubeadmscript.yaml
+  - target:
+      kind: ClusterClass
+    path: ./patch-k8s-install-script.yaml
   - target:
       kind: ClusterClass
     path: ./patch-namingstrategy.yaml