test: implement cluster upgrade with runtime SDK test

.gitignore

@@ -19,12 +19,12 @@ _artifacts/
 test/e2e/data/infrastructure-vsphere-govmomi/main/**/clusterclass-quick-start.yaml
 test/e2e/data/infrastructure-vsphere-govmomi/main/**/cluster-template*.yaml
 test/e2e/data/infrastructure-vsphere-govmomi/*/cluster-template*.yaml
-test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start.yaml
+test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml
 
 test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml
 test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml
-test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor.yaml
+test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml
 
 # env vars file used in getting-started.md and manifests generation
 envvars.txt

Makefile

@@ -388,8 +388,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	# generate clusterclass and cluster topology
 	cp "$(RELEASE_DIR)/main/clusterclass-template.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start-runtimesdk.yaml"
 	cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml"
 	# for PCI passthrough template
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml"
@@ -403,8 +405,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/base" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-supervisor.yaml"
 	cp "$(RELEASE_DIR)/main/clusterclass-template-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-runtimesdk-supervisor.yaml"
 	cp "$(RELEASE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology/cluster-template-topology-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml"
 

test/e2e/cluster_upgrade_runtimesdk_test.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"github.com/blang/semver/v4"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/utils/ptr"
+	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+)
+
+var _ = Describe("When upgrading a workload cluster using ClusterClass with RuntimeSDK [supervisor] [ClusterClass]", func() {
+	const specName = "k8s-upgrade-with-runtimesdk" // aligned to CAPI
+	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
+		capi_e2e.ClusterUpgradeWithRuntimeSDKSpec(ctx, func() capi_e2e.ClusterUpgradeWithRuntimeSDKSpecInput {
+			version, err := semver.ParseTolerant(e2eConfig.GetVariable(capi_e2e.KubernetesVersionUpgradeFrom))
+			Expect(err).ToNot(HaveOccurred(), "Invalid argument, KUBERNETES_VERSION_UPGRADE_FROM is not a valid version")
+			if version.LT(semver.MustParse("1.24.0")) {
+				Fail("This test only supports upgrades from Kubernetes >= v1.24.0")
+			}
+
+			return capi_e2e.ClusterUpgradeWithRuntimeSDKSpecInput{
+				E2EConfig:             e2eConfig,
+				ClusterctlConfigPath:  testSpecificSettingsGetter().ClusterctlConfigPath,
+				BootstrapClusterProxy: bootstrapClusterProxy,
+				ArtifactFolder:        artifactFolder,
+				SkipCleanup:           skipCleanup,
+				// // TODO(chrischdi): enable after https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/pull/3059 merged
+				// PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+				// 	// This check ensures that the resourceVersions are stable, i.e. it verifies there are no
+				// 	// continuous reconciles when everything should be stable.
+				// 	framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
+				// },
+				// "upgrades-runtimesdk" is the same as the "topology" flavor but with an additional RuntimeExtension.
+				Flavor:                    ptr.To(testSpecificSettingsGetter().FlavorForMode("topology-runtimesdk")),
+				ExtensionServiceNamespace: "capv-test-extension",
+				ExtensionServiceName:      "capv-test-extension-webhook-service",
+				PostNamespaceCreated:      testSpecificSettingsGetter().PostNamespaceCreatedFunc,
+			}
+		})
+	})
+})

test/e2e/config/vsphere.yaml

@@ -16,6 +16,8 @@ images:
     loadBehavior: mustLoad
   - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-{ARCH}:dev
     loadBehavior: mustLoad
+  - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-test-extension-{ARCH}:dev
+    loadBehavior: mustLoad
   - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.6-0-gde75746a
     loadBehavior: tryLoad
 
@@ -159,13 +161,17 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-pci.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-storage-policy.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology-runtimesdk.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start-runtimesdk.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-runtimesdk-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-runtimesdk-supervisor.yaml"
           - sourcePath: "../data/shared/capv/main/metadata.yaml"
       - name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series
         # Use manifest from source files
@@ -226,6 +232,19 @@ providers:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
 
+  - name: capv-test-extension
+    type: RuntimeExtensionProvider # test-extension isn't a provider, but we fake it is so it can be handled by the clusterctl machinery.
+    versions:
+      - name: v1.11.99
+        # Use manifest from source files
+        value: ../../../../cluster-api-provider-vsphere/test/infrastructure/test-extension/config/default
+        contract: v1beta1
+        files:
+          - sourcePath: "../data/shared/capv/main/metadata.yaml"
+        replacements:
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+
 variables:
   # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
   KUBERNETES_VERSION: "v1.30.0"
@@ -264,11 +283,13 @@ variables:
   KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml"
   NODE_DRAIN_TIMEOUT: "60s"
   CLUSTER_TOPOLOGY: "true"
+  EXP_RUNTIME_SDK: "true"
   # These IDs correspond to Tesla T4s, they are the decimal representation of the hex values.
   DEVICE_ID: 7864
   VENDOR_ID: 4318
   # CAPV feature flags
   EXP_NODE_ANTI_AFFINITY: "true"
+  EXP_MACHINE_SET_PREFLIGHT_CHECKS: "true"
   CAPI_DIAGNOSTICS_ADDRESS: ":8080"
   CAPI_INSECURE_DIAGNOSTICS: "true"
 

test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../clusterclass
+patches:
+  - target:
+      kind: ClusterClass
+    path: ./patch-test-extension.yaml

test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/patch-test-extension.yaml

@@ -0,0 +1,10 @@
+- op: replace
+  path: /spec/patches
+  value:
+  - name: test-patch
+    external:
+      generateExtension: generate-patches.capv-test-extension
+      validateExtension: validate-topology.capv-test-extension
+      discoverVariablesExtension: discover-variables.capv-test-extension
+- op: remove
+  path: /spec/variables

test/e2e/data/infrastructure-vsphere-govmomi/main/topology-runtimesdk/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../topology

test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../clusterclass
+patches:
+  - target:
+      kind: ClusterClass
+    path: ./patch-test-extension.yaml

test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/patch-test-extension.yaml

@@ -0,0 +1,10 @@
+- op: replace
+  path: /spec/patches
+  value:
+  - name: test-patch
+    external:
+      generateExtension: generate-patches.capv-test-extension
+      validateExtension: validate-topology.capv-test-extension
+      discoverVariablesExtension: discover-variables.capv-test-extension
+- op: remove
+  path: /spec/variables

test/e2e/data/infrastructure-vsphere-supervisor/main/topology-runtimesdk/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../topology