Merge pull request #2612 from chrischdi/pr-change-ip-reservation

🌱 e2e: Claim ip addresses dynamically so tests can run in parallel

Makefile

@@ -77,13 +77,15 @@ endif
 #
 GINKGO_FOCUS ?=
 GINKGO_SKIP ?=
+GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 3h
 E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere-dev.yaml)
 INTEGRATION_CONF_FILE ?= $(abspath test/integration/integration-dev.yaml)
 E2E_TEMPLATE_DIR := $(abspath test/e2e/data/infrastructure-vsphere/)
 SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false
 GINKGO_NOCOLOR ?= false
+E2E_IPAM_KUBECONFIG ?=
 
 # to set multiple ginkgo skip flags, if any
 ifneq ($(strip $(GINKGO_SKIP)),)
@@ -524,12 +526,13 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
 	@echo Contents of $(TOOLS_BIN_DIR):
 	@ls $(TOOLS_BIN_DIR)
 	@echo
-	time $(GINKGO) -v --trace -focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) -timeout=$(GINKGO_TIMEOUT) \
+	time $(GINKGO) -v --trace -focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) -timeout=$(GINKGO_TIMEOUT) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" ./test/e2e -- \
 		--e2e.config="$(E2E_CONF_FILE)" \
 		--e2e.artifacts-folder="$(ARTIFACTS)" \
 		--e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
-		--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)"
+		--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \
+		--e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)"
 
 ## --------------------------------------
 ## Release

hack/e2e.sh

@@ -25,11 +25,6 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
 source "${REPO_ROOT}/hack/ensure-kubectl.sh"
 
 on_exit() {
-  # release IPClaim
-  echo "Releasing IP claims"
-  kubectl --kubeconfig="${KUBECONFIG}" delete "ipaddressclaim.ipam.cluster.x-k8s.io" "${CONTROL_PLANE_IPCLAIM_NAME}" || true
-  kubectl --kubeconfig="${KUBECONFIG}" delete "ipaddressclaim.ipam.cluster.x-k8s.io" "${WORKLOAD_IPCLAIM_NAME}" || true
-
   # kill the VPN
   docker kill vpn
 
@@ -62,6 +57,12 @@ export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
 export DOCKER_IMAGE_TAR="/tmp/images/image.tar"
 export GC_KIND="false"
 
+# Make tests run in-parallel
+export GINKGO_NODES=5
+# Set the kubeconfig to the IPAM cluster so the e2e tests can claim ip addresses
+# for kube-vip.
+export E2E_IPAM_KUBECONFIG="/root/ipam-conf/capv-services.conf"
+
 # Run the vpn client in container
 docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
   -w "${HOME}/.openvpn/" --cap-add=NET_ADMIN --net=host --device=/dev/net/tun \
@@ -70,63 +71,28 @@ docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
 # Tail the vpn logs
 docker logs vpn
 
-# Sleep to allow vpn container to start running
-sleep 30
-
-
-function kubectl_get_jsonpath() {
-  local OBJECT_KIND="${1}"
-  local OBJECT_NAME="${2}"
-  local JSON_PATH="${3}"
+# Wait until the VPN connection is active and we are able to reach the ipam cluster
+function wait_for_ipam_reachable() {
   local n=0
   until [ $n -ge 30 ]; do
-    OUTPUT=$(kubectl --kubeconfig="${KUBECONFIG}" get "${OBJECT_KIND}.ipam.cluster.x-k8s.io" "${OBJECT_NAME}" -o=jsonpath="${JSON_PATH}")
-    if [[ "${OUTPUT}" != "" ]]; then
+    kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s  cluster-info && RET=$? || RET=$?
+    if [[ "$RET" -eq 0 ]]; then
       break
     fi
     n=$((n + 1))
     sleep 1
   done
-
-  if [[ "${OUTPUT}" == "" ]]; then
-    echo "Received empty output getting ${JSON_PATH} from ${OBJECT_KIND}/${OBJECT_NAME}" 1>&2
-    return 1
-  else
-    echo "${OUTPUT}"
-    return 0
-  fi
 }
-
-function claim_ip() {
-  IPCLAIM_NAME="$1"
-  export IPCLAIM_NAME
-  envsubst < "${REPO_ROOT}/hack/ipclaim-template.yaml" | kubectl --kubeconfig="${KUBECONFIG}" create -f - 1>&2
-  IPADDRESS_NAME=$(kubectl_get_jsonpath ipaddressclaim "${IPCLAIM_NAME}" '{@.status.addressRef.name}')
-  kubectl --kubeconfig="${KUBECONFIG}" get "ipaddresses.ipam.cluster.x-k8s.io" "${IPADDRESS_NAME}" -o=jsonpath='{@.spec.address}'
-}
-
-export KUBECONFIG="/root/ipam-conf/capv-services.conf"
+wait_for_ipam_reachable
 
 make envsubst
 
-# Retrieve an IP to be used as the kube-vip IP
-CONTROL_PLANE_IPCLAIM_NAME="ip-claim-$(openssl rand -hex 20)"
-CONTROL_PLANE_ENDPOINT_IP=$(claim_ip "${CONTROL_PLANE_IPCLAIM_NAME}")
-export CONTROL_PLANE_ENDPOINT_IP
-echo "Acquired Control Plane IP: $CONTROL_PLANE_ENDPOINT_IP"
-
-# Retrieve an IP to be used for the workload cluster in v1a3/v1a4 -> v1b1 upgrade tests
-WORKLOAD_IPCLAIM_NAME="workload-ip-claim-$(openssl rand -hex 20)"
-WORKLOAD_CONTROL_PLANE_ENDPOINT_IP=$(claim_ip "${WORKLOAD_IPCLAIM_NAME}")
-export WORKLOAD_CONTROL_PLANE_ENDPOINT_IP
-echo "Acquired Workload Cluster Control Plane IP: $WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"
-
-# save the docker image locally
+# Save the docker image locally
 make e2e-image
 mkdir -p /tmp/images
 docker save gcr.io/k8s-staging-cluster-api/capv-manager:e2e -o "$DOCKER_IMAGE_TAR"
 
-# store the image on gcs
+# Store the image on gcs
 login
 E2E_IMAGE_SHA=$(docker inspect --format='{{index .Id}}' gcr.io/k8s-staging-cluster-api/capv-manager:e2e)
 export E2E_IMAGE_SHA

test/e2e/README.md

@@ -20,30 +20,31 @@ In order to run the e2e tests the following requirements must be met:
 
 The first step to running the e2e tests is setting up the required environment variables:
 
-| Environment variable         | Description                                                                         | Example                                                                          |
-|------------------------------|-------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|
-| `VSPHERE_SERVER`             | The IP address or FQDN of a vCenter 6.7u3 server                                    | `my.vcenter.com`                                                                 |
-| `VSPHERE_USERNAME`           | The username used to access the vSphere server                                      | `my-username`                                                                    |
-| `VSPHERE_PASSWORD`           | The password used to access the vSphere server                                      | `my-password`                                                                    |
-| `VSPHERE_DATACENTER`         | The unique name or inventory path of the datacenter in which VMs will be created    | `my-datacenter` or `/my-datacenter`                                              |
-| `VSPHERE_FOLDER`             | The unique name or inventory path of the folder in which VMs will be created        | `my-folder` or `/my-datacenter/vm/my-folder`                                     |
-| `VSPHERE_RESOURCE_POOL`      | The unique name or inventory path of the resource pool in which VMs will be created | `my-resource-pool` or `/my-datacenter/host/Cluster-1/Resources/my-resource-pool` |
-| `VSPHERE_DATASTORE`          | The unique name or inventory path of the datastore in which VMs will be created     | `my-datastore` or `/my-datacenter/datstore/my-datastore`                         |
-| `VSPHERE_NETWORK`            | The unique name or inventory path of the network to which VMs will be connected     | `my-network` or `/my-datacenter/network/my-network`                              |
-| `VSPHERE_SSH_PRIVATE_KEY`    | The file path of the private key used to ssh into the CAPV VMs                      | `/home/foo/bar-ssh.key`                                                          |
-| `VSPHERE_SSH_AUTHORIZED_KEY` | The public key that is added to the CAPV VMs                                        | `ssh-rsa ABCDEF...XYZ=`                                                          |
-| `VSPHERE_TLS_THUMBPRINT`     | The TLS thumbprint of the vSphere server's certificate which should be trusted      | `2A:3F:BC:CA:C0:96:35:D4:B7:A2:AA:3C:C1:33:D9:D7:BE:EC:31:55`                    |
-| `CONTROL_PLANE_ENDPOINT_IP`  | The IP that kube-vip should use as a control plane endpoint                         | `10.10.123.100`                                                                  |
-| `VSPHERE_STORAGE_POLICY`     | The name of an existing vSphere storage policy to be assigned to created VMs        | `my-test-sp`                                                                     |
+| Environment variable         | Description                                                                                                       | Example                                                                          |
+|------------------------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|
+| `VSPHERE_SERVER`             | The IP address or FQDN of a vCenter 6.7u3 server                                                                  | `my.vcenter.com`                                                                 |
+| `VSPHERE_USERNAME`           | The username used to access the vSphere server                                                                    | `my-username`                                                                    |
+| `VSPHERE_PASSWORD`           | The password used to access the vSphere server                                                                    | `my-password`                                                                    |
+| `VSPHERE_DATACENTER`         | The unique name or inventory path of the datacenter in which VMs will be created                                  | `my-datacenter` or `/my-datacenter`                                              |
+| `VSPHERE_FOLDER`             | The unique name or inventory path of the folder in which VMs will be created                                      | `my-folder` or `/my-datacenter/vm/my-folder`                                     |
+| `VSPHERE_RESOURCE_POOL`      | The unique name or inventory path of the resource pool in which VMs will be created                               | `my-resource-pool` or `/my-datacenter/host/Cluster-1/Resources/my-resource-pool` |
+| `VSPHERE_DATASTORE`          | The unique name or inventory path of the datastore in which VMs will be created                                   | `my-datastore` or `/my-datacenter/datstore/my-datastore`                         |
+| `VSPHERE_NETWORK`            | The unique name or inventory path of the network to which VMs will be connected                                   | `my-network` or `/my-datacenter/network/my-network`                              |
+| `VSPHERE_SSH_PRIVATE_KEY`    | The file path of the private key used to ssh into the CAPV VMs                                                    | `/home/foo/bar-ssh.key`                                                          |
+| `VSPHERE_SSH_AUTHORIZED_KEY` | The public key that is added to the CAPV VMs                                                                      | `ssh-rsa ABCDEF...XYZ=`                                                          |
+| `VSPHERE_TLS_THUMBPRINT`     | The TLS thumbprint of the vSphere server's certificate which should be trusted                                    | `2A:3F:BC:CA:C0:96:35:D4:B7:A2:AA:3C:C1:33:D9:D7:BE:EC:31:55`                    |
+| `CONTROL_PLANE_ENDPOINT_IP`  | The IP that kube-vip should use as a control plane endpoint. It will not be used if `E2E_IPAM_KUBECONFIG` is set. | `10.10.123.100`                                                                  |
+| `VSPHERE_STORAGE_POLICY`     | The name of an existing vSphere storage policy to be assigned to created VMs                                      | `my-test-sp`                                                                     |
 
 ### Flags
 
-| Flag                    | Description                                                                                              | Default Value |
-|-------------------------|----------------------------------------------------------------------------------------------------------|---------------|
-| `SKIP_RESOURCE_CLEANUP` | This flags skips cleanup of the resources created during the tests as well as the kind/bootstrap cluster | `false`       |
-| `USE_EXISTING_CLUSTER`  | This flag enables the usage of an existing K8S cluster as the management cluster to run tests against.   | `false`       |
-| `GINKGO_TEST_TIMEOUT`   | This sets the timeout for the E2E test suite.                                                            | `2h`          |
-| `GINKGO_FOCUS`          | This populates the `-focus` flag of the `ginkgo` run command.                                            | `""`          |
+| Flag                    | Description                                                                                                                                                                                                    | Default Value |
+|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
+| `SKIP_RESOURCE_CLEANUP` | This flags skips cleanup of the resources created during the tests as well as the kind/bootstrap cluster                                                                                                       | `false`       |
+| `USE_EXISTING_CLUSTER`  | This flag enables the usage of an existing K8S cluster as the management cluster to run tests against.                                                                                                         | `false`       |
+| `GINKGO_TEST_TIMEOUT`   | This sets the timeout for the E2E test suite.                                                                                                                                                                  | `2h`          |
+| `GINKGO_FOCUS`          | This populates the `-focus` flag of the `ginkgo` run command.                                                                                                                                                  | `""`          |
+| `E2E_IPAM_KUBECONFIG`   | This flag points to a kubeconfig where the in-cluster IPAM provider is running to dynamically claim IP addresses for tests. If this is set, the environment variable `CONTROL_PLANE_ENDPOINT_IP` gets ignored. | `""`          |
 
 ### Running the e2e tests
 

test/e2e/anti_affinity_test.go

@@ -34,6 +34,8 @@ import (
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/govmomi/clustermodules"
+	. "sigs.k8s.io/cluster-api-provider-vsphere/test/e2e/helper"
+	"sigs.k8s.io/cluster-api-provider-vsphere/test/e2e/ipam"
 )
 
 type AntiAffinitySpecInput struct {
@@ -46,6 +48,17 @@ type AntiAffinitySpecInput struct {
 var _ = Describe("Cluster creation with anti affined nodes", func() {
 	var namespace *corev1.Namespace
 
+	var (
+		testSpecificClusterctlConfigPath string
+		testSpecificIPAddressClaims      ipam.IPAddressClaims
+	)
+	BeforeEach(func() {
+		testSpecificClusterctlConfigPath, testSpecificIPAddressClaims = ipamHelper.ClaimIPs(ctx, clusterctlConfigPath)
+	})
+	defer AfterEach(func() {
+		Expect(ipamHelper.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
+	})
+
 	BeforeEach(func() {
 		Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil")
 		namespace = setupSpecNamespace("anti-affinity-e2e")
@@ -66,7 +79,7 @@ var _ = Describe("Cluster creation with anti affined nodes", func() {
 			},
 			Global: GlobalInput{
 				BootstrapClusterProxy: bootstrapClusterProxy,
-				ClusterctlConfigPath:  clusterctlConfigPath,
+				ClusterctlConfigPath:  testSpecificClusterctlConfigPath,
 				E2EConfig:             e2eConfig,
 				ArtifactFolder:        artifactFolder,
 			},

test/e2e/capi_machine_deployment_rollout_test.go

@@ -18,15 +18,29 @@ package e2e
 
 import (
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+
+	"sigs.k8s.io/cluster-api-provider-vsphere/test/e2e/ipam"
 )
 
 var _ = Describe("ClusterAPI Machine Deployment Tests", func() {
 	Context("Running the MachineDeployment rollout spec", func() {
+		var (
+			testSpecificClusterctlConfigPath string
+			testSpecificIPAddressClaims      ipam.IPAddressClaims
+		)
+		BeforeEach(func() {
+			testSpecificClusterctlConfigPath, testSpecificIPAddressClaims = ipamHelper.ClaimIPs(ctx, clusterctlConfigPath)
+		})
+		defer AfterEach(func() {
+			Expect(ipamHelper.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
+		})
+
 		capi_e2e.MachineDeploymentRolloutSpec(ctx, func() capi_e2e.MachineDeploymentRolloutSpecInput {
 			return capi_e2e.MachineDeploymentRolloutSpecInput{
 				E2EConfig:             e2eConfig,
-				ClusterctlConfigPath:  clusterctlConfigPath,
+				ClusterctlConfigPath:  testSpecificClusterctlConfigPath,
 				BootstrapClusterProxy: bootstrapClusterProxy,
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,

test/e2e/cluster_upgrade_test.go

@@ -18,15 +18,29 @@ package e2e
 
 import (
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	"k8s.io/utils/ptr"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+
+	"sigs.k8s.io/cluster-api-provider-vsphere/test/e2e/ipam"
 )
 
 var _ = Describe("When upgrading a workload cluster using ClusterClass [ClusterClass]", func() {
+	var (
+		testSpecificClusterctlConfigPath string
+		testSpecificIPAddressClaims      ipam.IPAddressClaims
+	)
+	BeforeEach(func() {
+		testSpecificClusterctlConfigPath, testSpecificIPAddressClaims = ipamHelper.ClaimIPs(ctx, clusterctlConfigPath)
+	})
+	defer AfterEach(func() {
+		Expect(ipamHelper.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
+	})
+
 	capi_e2e.ClusterUpgradeConformanceSpec(ctx, func() capi_e2e.ClusterUpgradeConformanceSpecInput {
 		return capi_e2e.ClusterUpgradeConformanceSpecInput{
 			E2EConfig:             e2eConfig,
-			ClusterctlConfigPath:  clusterctlConfigPath,
+			ClusterctlConfigPath:  testSpecificClusterctlConfigPath,
 			BootstrapClusterProxy: bootstrapClusterProxy,
 			ArtifactFolder:        artifactFolder,
 			SkipCleanup:           skipCleanup,

test/e2e/clusterclass_changes_test.go

@@ -18,14 +18,28 @@ package e2e
 
 import (
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	capie2e "sigs.k8s.io/cluster-api/test/e2e"
+
+	"sigs.k8s.io/cluster-api-provider-vsphere/test/e2e/ipam"
 )
 
 var _ = Describe("When testing ClusterClass changes [ClusterClass]", func() {
+	var (
+		testSpecificClusterctlConfigPath string
+		testSpecificIPAddressClaims      ipam.IPAddressClaims
+	)
+	BeforeEach(func() {
+		testSpecificClusterctlConfigPath, testSpecificIPAddressClaims = ipamHelper.ClaimIPs(ctx, clusterctlConfigPath)
+	})
+	defer AfterEach(func() {
+		Expect(ipamHelper.Cleanup(ctx, testSpecificIPAddressClaims)).To(Succeed())
+	})
+
 	capie2e.ClusterClassChangesSpec(ctx, func() capie2e.ClusterClassChangesSpecInput {
 		return capie2e.ClusterClassChangesSpecInput{
 			E2EConfig:             e2eConfig,
-			ClusterctlConfigPath:  clusterctlConfigPath,
+			ClusterctlConfigPath:  testSpecificClusterctlConfigPath,
 			BootstrapClusterProxy: bootstrapClusterProxy,
 			ArtifactFolder:        artifactFolder,
 			SkipCleanup:           skipCleanup,