diff --git a/.gitignore b/.gitignore index 1e0ca8c622..7203c391c8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,12 +19,12 @@ _artifacts/ test/e2e/data/infrastructure-vsphere-govmomi/main/**/clusterclass-quick-start.yaml test/e2e/data/infrastructure-vsphere-govmomi/main/**/cluster-template*.yaml test/e2e/data/infrastructure-vsphere-govmomi/*/cluster-template*.yaml -test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start.yaml +test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml -test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor.yaml +test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml # env vars file used in getting-started.md and manifests generation envvars.txt diff --git a/Makefile b/Makefile index bb2d831a8d..2b73e32aba 100644 --- a/Makefile +++ b/Makefile @@ -388,8 +388,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai # generate clusterclass and cluster topology cp "$(RELEASE_DIR)/main/clusterclass-template.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start-runtimesdk.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml" # for PCI passthrough template "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml" @@ -403,8 +405,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/base" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-supervisor.yaml" cp "$(RELEASE_DIR)/main/clusterclass-template-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-runtimesdk-supervisor.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology/cluster-template-topology-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml" diff --git a/test/e2e/cluster_upgrade_runtimesdk_test.go b/test/e2e/cluster_upgrade_runtimesdk_test.go new file mode 100644 index 0000000000..89fd248fab --- /dev/null +++ b/test/e2e/cluster_upgrade_runtimesdk_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/blang/semver/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" +) + +var _ = Describe("When upgrading a workload cluster using ClusterClass with RuntimeSDK [supervisor] [ClusterClass]", func() { + const specName = "k8s-upgrade-with-runtimesdk" // aligned to CAPI + Setup(specName, func(testSpecificSettingsGetter func() testSettings) { + capi_e2e.ClusterUpgradeWithRuntimeSDKSpec(ctx, func() capi_e2e.ClusterUpgradeWithRuntimeSDKSpecInput { + version, err := semver.ParseTolerant(e2eConfig.GetVariable(capi_e2e.KubernetesVersionUpgradeFrom)) + Expect(err).ToNot(HaveOccurred(), "Invalid argument, KUBERNETES_VERSION_UPGRADE_FROM is not a valid version") + if version.LT(semver.MustParse("1.24.0")) { + Fail("This test only supports upgrades from Kubernetes >= v1.24.0") + } + + return capi_e2e.ClusterUpgradeWithRuntimeSDKSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: testSpecificSettingsGetter().ClusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + // // TODO(chrischdi): enable after https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/pull/3059 merged + // PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) { + // // This check ensures that the resourceVersions are stable, i.e. it verifies there are no + // // continuous reconciles when everything should be stable. + // framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName)) + // }, + // "upgrades-runtimesdk" is the same as the "topology" flavor but with an additional RuntimeExtension. + Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("topology-runtimesdk")), + ExtensionServiceNamespace: "capv-test-extension", + ExtensionServiceName: "capv-test-extension-webhook-service", + PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + } + }) + }) +}) diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index 0b29ace34f..7e3a469dab 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -16,6 +16,8 @@ images: loadBehavior: mustLoad - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-{ARCH}:dev loadBehavior: mustLoad + - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-test-extension-{ARCH}:dev + loadBehavior: mustLoad - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.6-0-gde75746a loadBehavior: tryLoad @@ -159,13 +161,17 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-pci.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-storage-policy.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology-runtimesdk.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start-runtimesdk.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-runtimesdk-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-runtimesdk-supervisor.yaml" - sourcePath: "../data/shared/capv/main/metadata.yaml" - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.10}" # supported release in the v1beta1 series # Use manifest from source files @@ -226,6 +232,19 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" + - name: capv-test-extension + type: RuntimeExtensionProvider # test-extension isn't a provider, but we fake it is so it can be handled by the clusterctl machinery. + versions: + - name: v1.11.99 + # Use manifest from source files + value: ../../../../cluster-api-provider-vsphere/test/infrastructure/test-extension/config/default + contract: v1beta1 + files: + - sourcePath: "../data/shared/capv/main/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + variables: # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml KUBERNETES_VERSION: "v1.30.0" @@ -264,11 +283,13 @@ variables: KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" NODE_DRAIN_TIMEOUT: "60s" CLUSTER_TOPOLOGY: "true" + EXP_RUNTIME_SDK: "true" # These IDs correspond to Tesla T4s, they are the decimal representation of the hex values. DEVICE_ID: 7864 VENDOR_ID: 4318 # CAPV feature flags EXP_NODE_ANTI_AFFINITY: "true" + EXP_MACHINE_SET_PREFLIGHT_CHECKS: "true" CAPI_DIAGNOSTICS_ADDRESS: ":8080" CAPI_INSECURE_DIAGNOSTICS: "true" diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..4d603d359e --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../clusterclass +patches: + - target: + kind: ClusterClass + path: ./patch-test-extension.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/patch-test-extension.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/patch-test-extension.yaml new file mode 100644 index 0000000000..a4a5379ef1 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/patch-test-extension.yaml @@ -0,0 +1,10 @@ +- op: replace + path: /spec/patches + value: + - name: test-patch + external: + generateExtension: generate-patches.capv-test-extension + validateExtension: validate-topology.capv-test-extension + discoverVariablesExtension: discover-variables.capv-test-extension +- op: remove + path: /spec/variables diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/topology-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/topology-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..0aecb2500d --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/topology-runtimesdk/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../topology diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..4d603d359e --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../clusterclass +patches: + - target: + kind: ClusterClass + path: ./patch-test-extension.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/patch-test-extension.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/patch-test-extension.yaml new file mode 100644 index 0000000000..a4a5379ef1 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/patch-test-extension.yaml @@ -0,0 +1,10 @@ +- op: replace + path: /spec/patches + value: + - name: test-patch + external: + generateExtension: generate-patches.capv-test-extension + validateExtension: validate-topology.capv-test-extension + discoverVariablesExtension: discover-variables.capv-test-extension +- op: remove + path: /spec/variables diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..0aecb2500d --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-runtimesdk/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../topology