Merge pull request #2250 from killianmuldoon/pr-refactor-test-data

🌱 Refactor test data to allow multiple versions
kubernetes-sigs · Aug 18, 2023 · 5661ea2 · 5661ea2
2 parents 4839ea8 + 96f02b7
commit 5661ea2
Show file tree

Hide file tree

Showing 41 changed files with 90 additions and 1,971 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,15 +8,16 @@
 /clusterctl
 /cluster-api-provider-vsphere*
 
-# Generated files
-/cluster-api-provider-vsphere
+# Examples
 /examples/provider-components/provider-components*.yaml
-test/e2e/data/infrastructure-vsphere/kustomization/base/cluster-template.yaml
-test/e2e/data/infrastructure-vsphere/kustomization/topology/cluster-template-topology.yaml
-test/e2e/data/infrastructure-vsphere/cluster-template*.yaml
-test/e2e/data/infrastructure-vsphere/clusterclass-quick-start.yaml
+
+# E2E test artifacts folder
 _artifacts/
 
+# E2E test templates
+test/e2e/data/infrastructure-vsphere/**/cluster-template*.yaml
+test/e2e/data/infrastructure-vsphere/main/clusterclass-quick-start.yaml
+
 # env vars file used in getting-started.md and manifests generation
 envvars.txt
 

diff --git a/Makefile b/Makefile
@@ -288,28 +288,30 @@ generate-doctoc:
 	TRACE=$(TRACE) ./hack/generate-doctoc.sh
 
 .PHONY: generate-e2e-templates
-generate-e2e-templates: $(KUSTOMIZE) ## Generate e2e cluster templates
-	$(MAKE) release-flavors
-
-	cp $(RELEASE_DIR)/cluster-template.yaml $(E2E_TEMPLATE_DIR)/kustomization/base/cluster-template.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/base > $(E2E_TEMPLATE_DIR)/cluster-template.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/hw-upgrade > $(E2E_TEMPLATE_DIR)/cluster-template-hw-upgrade.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/storage-policy > $(E2E_TEMPLATE_DIR)/cluster-template-storage-policy.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/remote-management > $(E2E_TEMPLATE_DIR)/cluster-template-remote-management.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/conformance > $(E2E_TEMPLATE_DIR)/cluster-template-conformance.yaml
+generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, main) ## Generate test templates for all branches
+
+.PHONY: generate-e2e-templates-main
+generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the main branch
+	$(MAKE) e2e-flavors-main
+	cp $(RELEASE_DIR)/main/cluster-template.yaml $(E2E_TEMPLATE_DIR)/main/base/cluster-template.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/base > $(E2E_TEMPLATE_DIR)/main/cluster-template.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/hw-upgrade > $(E2E_TEMPLATE_DIR)/main/cluster-template-hw-upgrade.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/storage-policy > $(E2E_TEMPLATE_DIR)/main/cluster-template-storage-policy.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/remote-management > $(E2E_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/conformance > $(E2E_TEMPLATE_DIR)/main/cluster-template-conformance.yaml
 	# Since CAPI uses different flavor names for KCP and MD remediation using MHC
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/mhc-remediation/kcp > $(E2E_TEMPLATE_DIR)/cluster-template-kcp-remediation.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/mhc-remediation/md > $(E2E_TEMPLATE_DIR)/cluster-template-md-remediation.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/node-drain > $(E2E_TEMPLATE_DIR)/cluster-template-node-drain.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/ignition > $(E2E_TEMPLATE_DIR)/cluster-template-ignition.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/mhc-remediation/kcp > $(E2E_TEMPLATE_DIR)/main/cluster-template-kcp-remediation.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/mhc-remediation/md > $(E2E_TEMPLATE_DIR)/main/cluster-template-md-remediation.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/node-drain > $(E2E_TEMPLATE_DIR)/main/cluster-template-node-drain.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/ignition > $(E2E_TEMPLATE_DIR)/main/cluster-template-ignition.yaml
 	# generate clusterclass and cluster topology
-	cp $(RELEASE_DIR)/cluster-template-topology.yaml $(E2E_TEMPLATE_DIR)/kustomization/topology/cluster-template-topology.yaml
-	cp $(RELEASE_DIR)/clusterclass-template.yaml $(E2E_TEMPLATE_DIR)/clusterclass-quick-start.yaml
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/topology > $(E2E_TEMPLATE_DIR)/cluster-template-topology.yaml
+	cp $(RELEASE_DIR)/main/cluster-template-topology.yaml $(E2E_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml
+	cp $(RELEASE_DIR)/main/clusterclass-template.yaml $(E2E_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/topology > $(E2E_TEMPLATE_DIR)/main/cluster-template-topology.yaml
 	# for PCI passthrough template
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/pci > $(E2E_TEMPLATE_DIR)/cluster-template-pci.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/pci > $(E2E_TEMPLATE_DIR)/main/cluster-template-pci.yaml
 	# for DHCP overrides
-	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/dhcp-overrides > $(E2E_TEMPLATE_DIR)/cluster-template-dhcp-overrides.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/dhcp-overrides > $(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml
 
 
 ## --------------------------------------
@@ -584,10 +586,19 @@ release-manifests: $(BUILD_DIR) $(MANIFEST_DIR) $(KUSTOMIZE) $(STAGE)-flavors ##
 release-flavors: $(RELEASE_DIR)
 	$(MAKE) generate-flavors FLAVOR_DIR=$(RELEASE_DIR)
 
-.PHONY: dev-flavors ## Create release flavor manifests
+.PHONY: dev-flavors ## Create dev flavor manifests
 dev-flavors: $(OVERRIDES_DIR)
 	$(MAKE) generate-flavors FLAVOR_DIR=$(OVERRIDES_DIR)
 
+.PHONY: e2e-flavors ## Create dev flavor manifests for e2e testing
+e2e-flavors: $(KUSTOMIZE) $(addprefix e2e-flavors-, main)
+
+.PHONY: e2e-flavors-main ## Create dev flavor manifests for e2e testing for main branch
+e2e-flavors-main: $(RELEASE_DIR)
+	mkdir -p $(RELEASE_DIR)/main
+	$(MAKE) generate-flavors FLAVOR_DIR=$(RELEASE_DIR)/main
+
+
 .PHONY: generate-flavors
 generate-flavors: $(FLAVOR_DIR)
 	go run ./packaging/flavorgen --output-dir $(FLAVOR_DIR)

diff --git a/docs/release/release-tasks.md b/docs/release/release-tasks.md
@@ -28,17 +28,17 @@ This comes down to changing occurrences of the old version to the new version, e
 
 1. Setup E2E tests for the new release:
    1. Goal is that our clusterctl upgrade tests are testing the right versions. For `v1.8` this means:
+      - v1beta1: `v1.8` (will change with each new release)
       - v1beta1: `v1.7` (will change with each new release)
-      - v1alpha4: `v0.8`
    2. Update providers in `vsphere-ci.yaml`, `vsphere-dev.yaml`, `integration-dev.yaml`:
       1. Add a new `v1.7.0` entry.
       2. Remove providers that are not used anymore in clusterctl upgrade tests.
       3. Change `v1.7.99` to `v1.8.99`.
    3. Adjust `metadata.yaml`'s:
       1. Add new release to the top-level `metadata.yaml`
-      2. Create a new `v1.7` `metadata.yaml` (`test/e2e/data/shared/v1.7/metadata.yaml`) by copying
-        `test/e2e/data/shared/main/metadata.yaml`
-      3. Add the new v1.8 release to the main `metadata.yaml` (`test/e2e/data/shared/main/metadata.yaml`).
+      2. Create a new `v1.7` `metadata.yaml` (`test/e2e/data/shared/v1.7/v1beta1_provider/metadata.yaml`) by copying
+        `test/e2e/data/shared/main/v1beta1_provider/metadata.yaml`
+      3. Add the new v1.8 release to the main `metadata.yaml` (`test/e2e/data/shared/main/v1beta1_provider/metadata.yaml`).
       4. Remove old `metadata.yaml`'s that are not used anymore in clusterctl upgrade tests.
    4. Adjust cluster templates in `test/e2e/data/infrastructure-vsphere`:
       1. Create a new `v1.7` folder. It should be created based on the `main` folder and only contain the templates

diff --git a/test/e2e/capi_clusterctl_upgrade_test.go b/test/e2e/capi_clusterctl_upgrade_test.go
diff --git a/test/e2e/config/vsphere-ci.yaml b/test/e2e/config/vsphere-ci.yaml
@@ -28,83 +28,49 @@ providers:
   - name: cluster-api
     type: CoreProvider
     versions:
-      - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
-        contract: v1alpha4
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/core-components.yaml"
-        type: "url"
-        files:
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
-        replacements:
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
       - name: v1.5.0
         # Use manifest from source files
         value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.0/core-components.yaml"
         type: "url"
         contract: v1beta1
         files:
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+          - sourcePath: "../data/shared/main/v1beta1/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
 
   - name: kubeadm
     type: BootstrapProvider
     versions:
-      - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/bootstrap-components.yaml"
-        type: "url"
-        contract: v1alpha4
-        files:
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
-        replacements:
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
       - name: v1.5.0
         # Use manifest from source files
         value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.0/bootstrap-components.yaml"
         type: "url"
         contract: v1beta1
         files:
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+          - sourcePath: "../data/shared/main/v1beta1/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
 
   - name: kubeadm
     type: ControlPlaneProvider
     versions:
-      - name: v0.4.7
-        value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/control-plane-components.yaml"
-        type: "url"
-        contract: v1alpha4
-        files:
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
-        replacements:
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
       - name: v1.5.0
         # Use manifest from source files
         value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.0/control-plane-components.yaml"
         type: "url"
         contract: v1beta1
         files:
-          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+          - sourcePath: "../data/shared/main/v1beta1/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
 
   - name: vsphere
     type: InfrastructureProvider
     versions:
-      - name: v0.8.2
-        value: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/v0.8.2/infrastructure-components.yaml
-        type: "url"
-        contract: v1alpha4
-        files:
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/capi-upgrades/v1alpha4/cluster-template.yaml"
-          - sourcePath: "../../../metadata.yaml"
-      - name: v1.7.0
+      - name: v1.9.99
         # Use manifest from source files
         value: ../../../../cluster-api-provider-vsphere/config/default
         contract: v1beta1
@@ -115,20 +81,20 @@ providers:
             new: "imagePullPolicy: IfNotPresent"
         files:
           # Add a cluster template
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-conformance.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-hw-upgrade.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-kcp-remediation.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-md-remediation.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-node-drain.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-pci.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-remote-management.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-storage-policy.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-topology.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-dhcp-overrides.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/clusterclass-quick-start.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-ignition.yaml"
-          - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-conformance.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-hw-upgrade.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-kcp-remediation.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-md-remediation.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-node-drain.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-pci.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-remote-management.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-storage-policy.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-topology.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-dhcp-overrides.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/clusterclass-quick-start.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ignition.yaml"
+          - sourcePath: "../data/shared/main/v1beta1_provider/metadata.yaml"
 
 variables:
   KUBERNETES_VERSION: "v1.28.0"