Add clusterctl upgrade test

Signed-off-by: killianmuldoon <[email protected]>

Makefile

@@ -296,7 +296,7 @@ generate-doctoc:
 	TRACE=$(TRACE) ./hack/generate-doctoc.sh
 
 .PHONY: generate-e2e-templates
-generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, main) ## Generate test templates for all branches
+generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.8 main) ## Generate test templates for all branches
 
 .PHONY: generate-e2e-templates-main
 generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the main branch
@@ -322,6 +322,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/dhcp-overrides > $(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/ownerreferences > $(E2E_TEMPLATE_DIR)/main/cluster-template-ownerreferences.yaml
 
+.PHONY: generate-e2e-templates-v1.8
+generate-e2e-templates-v1.8: $(KUSTOMIZE)
+	$(KUSTOMIZE) build $(E2E_TEMPLATE_DIR)/v1.8/cluster-template --load-restrictor LoadRestrictionsNone > $(E2E_TEMPLATE_DIR)/v1.8/cluster-template.yaml
+
 
 ## --------------------------------------
 ## Lint / Verify

hack/e2e.sh

@@ -18,6 +18,7 @@ set -o errexit  # exits immediately on any unexpected error (does not bypass tra
 set -o nounset  # will error if variables are used without first being defined
 set -o pipefail # any non-zero exit code in a piped command causes the pipeline to fail with that code
 
+export TRACE=1
 export PATH=${PWD}/hack/tools/bin:${PATH}
 REPO_ROOT=$(git rev-parse --show-toplevel)
 

test/e2e/clusterctl_upgrade_test.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+)
+
+var _ = Describe("Upgrading cluster from CAPV 1.8.X to main using clusterctl  [PR-Blocking]", func() {
+	capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
+		return capi_e2e.ClusterctlUpgradeSpecInput{
+			E2EConfig:                         e2eConfig,
+			ClusterctlConfigPath:              clusterctlConfigPath,
+			BootstrapClusterProxy:             bootstrapClusterProxy,
+			ArtifactFolder:                    artifactFolder,
+			SkipCleanup:                       skipCleanup,
+			InitWithProvidersContract:         "v1beta1",
+			MgmtFlavor:                        "remote-management",
+			InitWithBinary:                    "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.2/clusterctl-{OS}-{ARCH}",
+			InitWithCoreProvider:              "cluster-api:v1.5.2",
+			InitWithBootstrapProviders:        []string{"kubeadm:v1.5.2"},
+			InitWithControlPlaneProviders:     []string{"kubeadm:v1.5.2"},
+			InitWithInfrastructureProviders:   []string{"vsphere:v1.8.1"},
+			InitWithRuntimeExtensionProviders: []string{},
+			InitWithKubernetesVersion:         "v1.28.0",
+			WorkloadKubernetesVersion:         "v1.28.0",
+			WorkloadFlavor:                    "",
+		}
+	})
+})

test/e2e/data/infrastructure-vsphere/main/base/cluster-network-CIDR.yaml

@@ -0,0 +1,10 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+  namespace: '${NAMESPACE}'
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+        - 192.168.30.0/24

test/e2e/data/infrastructure-vsphere/main/base/cluster-resource-set-csi-insecure.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: csi-vsphere-config
+  namespace: '${NAMESPACE}'
+stringData:
+  data: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: csi-vsphere-config
+      namespace: kube-system
+    stringData:
+      csi-vsphere.conf: |+
+        [Global]
+        cluster-id = "${NAMESPACE}/${CLUSTER_NAME}"
+
+        [VirtualCenter "${VSPHERE_SERVER}"]
+        insecure-flag = "${VSPHERE_INSECURE_CSI}"
+        user = "${VSPHERE_USERNAME}"
+        password = "${VSPHERE_PASSWORD}"
+        datacenters = "${VSPHERE_DATACENTER}"
+
+        [Network]
+        public-network = "${VSPHERE_NETWORK}"
+
+    type: Opaque
+type: addons.cluster.x-k8s.io/resource-set

test/e2e/data/infrastructure-vsphere/main/base/cluster-resource-set-label.yaml

@@ -0,0 +1,7 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+  namespace: '${NAMESPACE}'
+  labels:
+    cni: "${CLUSTER_NAME}-crs-cni"

test/e2e/data/infrastructure-vsphere/main/base/cluster-resource-set.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "cni-${CLUSTER_NAME}-crs-cni"
+data: ${CNI_RESOURCES}
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name:  "${CLUSTER_NAME}-crs-cni"
+spec:
+  strategy: ApplyOnce
+  clusterSelector:
+    matchLabels:
+      cni: "${CLUSTER_NAME}-crs-cni"
+  resources:
+    - name: "cni-${CLUSTER_NAME}-crs-cni"
+      kind: ConfigMap

test/e2e/data/infrastructure-vsphere/main/base/remove-storage-policy.yaml

@@ -0,0 +1,2 @@
+- op: remove
+  path: /spec/template/spec/storagePolicyName

test/e2e/data/infrastructure-vsphere/main/base/vcpu.yaml

@@ -0,0 +1,13 @@
+# VSphereMachineTemplate object with the number of CPUs raised to 4
+# for the purposes of mitigating the CPU spikes caused by scaling up
+# the control plane (during upgrades and for HA control planes)
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      numCPUs: 4

test/e2e/data/infrastructure-vsphere/v1.8/bases/cluster-network-CIDR.yaml

@@ -0,0 +1,10 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+  namespace: '${NAMESPACE}'
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+        - 192.168.30.0/24

test/e2e/data/infrastructure-vsphere/v1.8/bases/cluster-resource-set-csi-insecure.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: csi-vsphere-config
+  namespace: '${NAMESPACE}'
+stringData:
+  data: |
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: csi-vsphere-config
+      namespace: kube-system
+    stringData:
+      csi-vsphere.conf: |+
+        [Global]
+        cluster-id = "${NAMESPACE}/${CLUSTER_NAME}"
+
+        [VirtualCenter "${VSPHERE_SERVER}"]
+        insecure-flag = "${VSPHERE_INSECURE_CSI}"
+        user = "${VSPHERE_USERNAME}"
+        password = "${VSPHERE_PASSWORD}"
+        datacenters = "${VSPHERE_DATACENTER}"
+
+        [Network]
+        public-network = "${VSPHERE_NETWORK}"
+
+    type: Opaque
+type: addons.cluster.x-k8s.io/resource-set

test/e2e/data/infrastructure-vsphere/v1.8/bases/cluster-resource-set-label.yaml

@@ -0,0 +1,7 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+  namespace: '${NAMESPACE}'
+  labels:
+    cni: "${CLUSTER_NAME}-crs-cni"

test/e2e/data/infrastructure-vsphere/v1.8/bases/cluster-resource-set.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "cni-${CLUSTER_NAME}-crs-cni"
+data: ${CNI_RESOURCES}
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name:  "${CLUSTER_NAME}-crs-cni"
+spec:
+  strategy: ApplyOnce
+  clusterSelector:
+    matchLabels:
+      cni: "${CLUSTER_NAME}-crs-cni"
+  resources:
+    - name: "cni-${CLUSTER_NAME}-crs-cni"
+      kind: ConfigMap

test/e2e/data/infrastructure-vsphere/v1.8/bases/remove-storage-policy.yaml

@@ -0,0 +1,2 @@
+- op: remove
+  path: /spec/template/spec/storagePolicyName

test/e2e/data/infrastructure-vsphere/v1.8/bases/vcpu.yaml

@@ -0,0 +1,13 @@
+# VSphereMachineTemplate object with the number of CPUs raised to 4
+# for the purposes of mitigating the CPU spikes caused by scaling up
+# the control plane (during upgrades and for HA control planes)
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      numCPUs: 4