test/e2e: use vSphere projects from Boskos

Signed-off-by: Stefan Büringer [email protected]

Makefile

@@ -83,7 +83,7 @@ GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 3h
 E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere.yaml)
 E2E_CONF_OVERRIDE_FILE ?= $(abspath test/e2e/config/config-overrides.yaml)
-E2E_IPAM_KUBECONFIG ?=
+E2E_VSPHERE_IP_POOL ?=
 E2E_TEMPLATE_DIR := $(abspath test/e2e/data/)
 E2E_GOVMOMI_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-govmomi
 E2E_SUPERVISOR_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-supervisor
@@ -616,7 +616,7 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
 		--e2e.artifacts-folder="$(ARTIFACTS)" \
 		--e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \
-		--e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)"
+		--e2e.ip-pool='$(E2E_VSPHERE_IP_POOL)'
 
 ## --------------------------------------
 ## Release

hack/e2e.sh

@@ -24,7 +24,7 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
 RE_VCSIM='\[vcsim\\]'
 
 # In CI, ARTIFACTS is set to a different directory. This stores the value of
-# ARTIFACTS i1n ORIGINAL_ARTIFACTS and replaces ARTIFACTS by a temporary directory
+# ARTIFACTS in ORIGINAL_ARTIFACTS and replaces ARTIFACTS by a temporary directory
 # which gets cleaned up from credentials at the end of the test.
 export ORIGINAL_ARTIFACTS=""
 export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
@@ -33,10 +33,25 @@ if [[ "${ARTIFACTS}" != "${REPO_ROOT}/_artifacts" ]]; then
   ARTIFACTS=$(mktemp -d)
 fi
 
-# shellcheck source=./hack/ensure-kubectl.sh
-source "${REPO_ROOT}/hack/ensure-kubectl.sh"
+# shellcheck source=./hack/ensure-go.sh
+source "${REPO_ROOT}/hack/ensure-go.sh"
+
+export BOSKOS_RESOURCE_OWNER=cluster-api-provider-vsphere
+if [[ "${JOB_NAME}" != "" ]]; then
+  export BOSKOS_RESOURCE_OWNER="${JOB_NAME}/${BUILD_ID}"
+fi
+export BOSKOS_RESOURCE_TYPE=vsphere-project-cluster-api-provider
 
 on_exit() {
+  # Only handle Boskos when we have to (not for vcsim)
+  if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
+    # Stop boskos heartbeat
+    [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
+
+    # If Boskos is being used then release the vsphere project.
+    [ -z "${BOSKOS_HOST:-}" ] || docker run -e VSPHERE_USERNAME -e VSPHERE_PASSWORD gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest release --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --vsphere-server="${VSPHERE_SERVER}" --vsphere-tls-thumbprint="${VSPHERE_TLS_THUMBPRINT}" --vsphere-folder="${BOSKOS_RESOURCE_FOLDER}" --vsphere-resource-pool="${BOSKOS_RESOURCE_POOL}"
+  fi
+
   # kill the VPN only when we started it (not vcsim)
   if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     docker kill vpn
@@ -83,18 +98,13 @@ export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key"
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere.yaml"
 export E2E_CONF_OVERRIDE_FILE=""
 export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.6-0-gde75746a}"
-export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
 export DOCKER_IMAGE_TAR="/tmp/images/image.tar"
 export GC_KIND="false"
 
 # Make tests run in-parallel
 export GINKGO_NODES=5
 
-# Set the kubeconfig to the IPAM cluster so the e2e tests can claim ip addresses
-# for kube-vip.
-export E2E_IPAM_KUBECONFIG="/root/ipam-conf/capv-services.conf"
-
-# Only run the vpn/check for IPAM when we need them (not vcsim)
+# Only run the vpn/check for IPAM when we need them (not for vcsim)
 if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
   # Run the vpn client in container
   docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
@@ -104,11 +114,11 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
   # Tail the vpn logs
   docker logs vpn
 
-  # Wait until the VPN connection is active and we are able to reach the ipam cluster
-  function wait_for_ipam_reachable() {
+  # Wait until the VPN connection is active.
+  function wait_for_vpn_up() {
     local n=0
     until [ $n -ge 30 ]; do
-      kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s  get inclusterippools.ipam.cluster.x-k8s.io && RET=$? || RET=$?
+      curl "https://${VSPHERE_SERVER}" --connect-timeout 2 -k && RET=$? || RET=$?
       if [[ "$RET" -eq 0 ]]; then
         break
       fi
@@ -117,7 +127,39 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     done
     return "$RET"
   }
-  wait_for_ipam_reachable
+  wait_for_vpn_up
+
+  # If BOSKOS_HOST is set then acquire a vsphere-project from Boskos.
+  if [ -n "${BOSKOS_HOST:-}" ]; then
+    # Check out the account from Boskos and store the produced environment
+    # variables in a temporary file.
+    account_env_var_file="$(mktemp)"
+    docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest acquire --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-type="${BOSKOS_RESOURCE_TYPE}" 1>"${account_env_var_file}"
+    checkout_account_status="${?}"
+
+    # If the checkout process was a success then load the account's
+    # environment variables into this process.
+    # shellcheck disable=SC1090
+    [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+    export BOSKOS_RESOURCE_NAME=${BOSKOS_RESOURCE_NAME}
+    export VSPHERE_FOLDER=${BOSKOS_RESOURCE_FOLDER}
+    export VSPHERE_RESOURCE_POOL=${BOSKOS_RESOURCE_POOL}
+    export E2E_VSPHERE_IP_POOL="${BOSKOS_RESOURCE_IP_POOL}"
+
+    # Always remove the account environment variable file. It contains
+    # sensitive information.
+    rm -f "${account_env_var_file}"
+
+    if [ ! "${checkout_account_status}" = "0" ]; then
+      echo "error getting vsphere project from Boskos" 1>&2
+      exit "${checkout_account_status}"
+    fi
+
+    # Run the heartbeat to tell boskos periodically that we are still
+    # using the checked out account.
+    docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest heartbeat --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" >>"${ARTIFACTS}/boskos-heartbeat.log" 2>&1 &
+    HEART_BEAT_PID=$!
+  fi
 fi
 
 make envsubst

test/e2e/README.md

@@ -33,18 +33,18 @@ The first step to running the e2e tests is setting up the required environment v
 | `VSPHERE_SSH_PRIVATE_KEY`    | The file path of the private key used to ssh into the CAPV VMs                                                    | `/home/foo/bar-ssh.key`                                                          |
 | `VSPHERE_SSH_AUTHORIZED_KEY` | The public key that is added to the CAPV VMs                                                                      | `ssh-rsa ABCDEF...XYZ=`                                                          |
 | `VSPHERE_TLS_THUMBPRINT`     | The TLS thumbprint of the vSphere server's certificate which should be trusted                                    | `2A:3F:BC:CA:C0:96:35:D4:B7:A2:AA:3C:C1:33:D9:D7:BE:EC:31:55`                    |
-| `CONTROL_PLANE_ENDPOINT_IP`  | The IP that kube-vip should use as a control plane endpoint. It will not be used if `E2E_IPAM_KUBECONFIG` is set. | `10.10.123.100`                                                                  |
+| `CONTROL_PLANE_ENDPOINT_IP`  | The IP that kube-vip should use as a control plane endpoint. It will not be used if `E2E_VSPHERE_IP_POOL` is set. | `10.10.123.100`                                                                  |
 | `VSPHERE_STORAGE_POLICY`     | The name of an existing vSphere storage policy to be assigned to created VMs                                      | `my-test-sp`                                                                     |
 
 ### Flags
 
-| Flag                    | Description                                                                                                                                                                                                    | Default Value |
-|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
-| `SKIP_RESOURCE_CLEANUP` | This flags skips cleanup of the resources created during the tests as well as the kind/bootstrap cluster                                                                                                       | `false`       |
-| `USE_EXISTING_CLUSTER`  | This flag enables the usage of an existing K8S cluster as the management cluster to run tests against.                                                                                                         | `false`       |
-| `GINKGO_TEST_TIMEOUT`   | This sets the timeout for the E2E test suite.                                                                                                                                                                  | `2h`          |
-| `GINKGO_FOCUS`          | This populates the `-focus` flag of the `ginkgo` run command.                                                                                                                                                  | `""`          |
-| `E2E_IPAM_KUBECONFIG`   | This flag points to a kubeconfig where the in-cluster IPAM provider is running to dynamically claim IP addresses for tests. If this is set, the environment variable `CONTROL_PLANE_ENDPOINT_IP` gets ignored. | `""`          |
+| Flag                    | Description                                                                                                                                                                                                                                                                                                                                             | Default Value |
+|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
+| `SKIP_RESOURCE_CLEANUP` | This flags skips cleanup of the resources created during the tests as well as the kind/bootstrap cluster                                                                                                                                                                                                                                                | `false`       |
+| `USE_EXISTING_CLUSTER`  | This flag enables the usage of an existing K8S cluster as the management cluster to run tests against.                                                                                                                                                                                                                                                  | `false`       |
+| `GINKGO_TEST_TIMEOUT`   | This sets the timeout for the E2E test suite.                                                                                                                                                                                                                                                                                                           | `2h`          |
+| `GINKGO_FOCUS`          | This populates the `-focus` flag of the `ginkgo` run command.                                                                                                                                                                                                                                                                                           | `""`          |
+| `E2E_VSPHERE_IP_POOL`   | This allows to configure the IPPool to use for the e2e test. Supports the addresses, gateway and prefix fields from the InClusterIPPool CRD https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/blob/main/api/v1alpha2/inclusterippool_types.go. If this is set, the environment variable `CONTROL_PLANE_ENDPOINT_IP` gets ignored. | `""`          |
 
 ### Running the e2e tests
 

test/e2e/config/vsphere.yaml

@@ -268,8 +268,8 @@ variables:
   VSPHERE_CONTENT_LIBRARY: "capv"
   VSPHERE_CONTENT_LIBRARY_ITEMS: "ubuntu-2204-kube-v1.28.0,ubuntu-2204-kube-v1.29.0,ubuntu-2204-kube-v1.30.0"
   VSPHERE_IMAGE_NAME: "ubuntu-2204-kube-v1.30.0"
-  VSPHERE_NETWORK: "sddc-cgw-network-6"
-  VSPHERE_DISTRIBUTED_PORT_GROUP: "/SDDC-Datacenter/network/sddc-cgw-network-6"
+  VSPHERE_NETWORK: "sddc-cgw-network-10"
+  VSPHERE_DISTRIBUTED_PORT_GROUP: "/SDDC-Datacenter/network/sddc-cgw-network-10"
   VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.30.0"
   FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3815.2.2-kube-v1.30.0"
   VSPHERE_INSECURE_CSI: "true"

test/e2e/e2e_setup_test.go

@@ -45,6 +45,7 @@ import (
 type setupOptions struct {
 	additionalIPVariableNames []string
 	gatewayIPVariableName     string
+	prefixVariableName        string
 }
 
 // SetupOption is a configuration option supplied to Setup.
@@ -65,6 +66,13 @@ func WithGateway(variableName string) SetupOption {
 	}
 }
 
+// WithPrefix instructs Setup to store the prefix from IPAM into the provided variableName.
+func WithPrefix(variableName string) SetupOption {
+	return func(o *setupOptions) {
+		o.prefixVariableName = variableName
+	}
+}
+
 type testSettings struct {
 	ClusterctlConfigPath     string
 	PostNamespaceCreatedFunc func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
@@ -90,7 +98,7 @@ func Setup(specName string, f func(testSpecificSettings func() testSettings), op
 		case VCenterTestTarget:
 			Byf("Getting IP for %s", strings.Join(append([]string{vsphereip.ControlPlaneEndpointIPVariable}, options.additionalIPVariableNames...), ","))
 			// get IPs from the in cluster address manager
-			testSpecificIPAddressClaims, testSpecificVariables = inClusterAddressManager.ClaimIPs(ctx, vsphereip.WithGateway(options.gatewayIPVariableName), vsphereip.WithIP(options.additionalIPVariableNames...))
+			testSpecificIPAddressClaims, testSpecificVariables = inClusterAddressManager.ClaimIPs(ctx, vsphereip.WithGateway(options.gatewayIPVariableName), vsphereip.WithPrefix(options.prefixVariableName), vsphereip.WithIP(options.additionalIPVariableNames...))
 		case VCSimTestTarget:
 			c := bootstrapClusterProxy.GetClient()
 

test/e2e/e2e_suite_test.go

@@ -33,6 +33,7 @@ import (
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/klog/v2"
+	ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/bootstrap"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -119,9 +120,8 @@ var (
 
 	namespaces map[*corev1.Namespace]context.CancelFunc
 
-	// e2eIPAMKubeconfig is a kubeconfig to a cluster which provides IP address management via an in-cluster
-	// IPAM provider to claim IPs for the control plane IPs of created clusters.
-	e2eIPAMKubeconfig string
+	// e2eIPPool to be used for the e2e test.
+	e2eIPPool string
 
 	// inClusterAddressManager is used to claim and cleanup IP addresses used for kubernetes control plane API Servers.
 	inClusterAddressManager vsphereip.AddressManager
@@ -137,7 +137,7 @@ func init() {
 	flag.BoolVar(&alsoLogToFile, "e2e.also-log-to-file", true, "if true, ginkgo logs are additionally written to the `ginkgo-log.txt` file in the artifacts folder (including timestamps)")
 	flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
 	flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
-	flag.StringVar(&e2eIPAMKubeconfig, "e2e.ipam-kubeconfig", "", "path to the kubeconfig for the IPAM cluster")
+	flag.StringVar(&e2eIPPool, "e2e.ip-pool", "", "IPPool to use for the e2e test. Supports the addresses, gateway and prefix fields from the InClusterIPPool CRD https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/blob/main/api/v1alpha2/inclusterippool_types.go")
 }
 
 func TestE2E(t *testing.T) {
@@ -282,7 +282,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	switch testTarget {
 	case VCenterTestTarget:
 		// Create the in cluster address manager
-		inClusterAddressManager, err = vsphereip.InClusterAddressManager(e2eIPAMKubeconfig, ipClaimLabels, skipCleanup)
+		inClusterAddressManager, err = vsphereip.InClusterAddressManager(ctx, bootstrapClusterProxy.GetClient(), e2eIPPool, ipClaimLabels, skipCleanup)
 		Expect(err).ToNot(HaveOccurred())
 
 	case VCSimTestTarget:
@@ -332,6 +332,8 @@ var _ = SynchronizedAfterSuite(func() {
 func initScheme() *runtime.Scheme {
 	sc := runtime.NewScheme()
 	framework.TryAddDefaultSchemes(sc)
+	// TODO: should probably be added to TryAddDefaultSchemes in core CAPI.
+	_ = ipamv1.AddToScheme(sc)
 
 	if testTarget == VCSimTestTarget {
 		_ = vcsimv1.AddToScheme(sc)

test/e2e/ipam_test.go

@@ -42,6 +42,9 @@ var _ = Describe("ClusterClass Creation using Cluster API quick-start test and I
 		// Set the WithGateway option to write the gateway ip address to the variable.
 		// This variable is required for creating the InClusterIPPool for the ipam provider.
 		WithGateway("IPAM_GATEWAY"),
+		// Set the WithPrefix option to set the prefix to the variable.
+		// This variable is required for creating the InClusterIPPool for the ipam provider.
+		WithPrefix("IPAM_PREFIX"),
 		// Claim two IPs from the CI's IPAM provider to use in the InClusterIPPool of
 		// the ipam provider. The IPs then get claimed during provisioning to configure
 		// static IPs for the control-plane and worker node.

test/framework/ip/addressmanager.go

@@ -44,6 +44,7 @@ type AddressClaims []AddressClaim
 type claimOptions struct {
 	additionalIPVariableNames []string
 	gatewayIPVariableName     string
+	prefixVariableName        string
 }
 
 type ClaimOption func(*claimOptions)
@@ -63,6 +64,13 @@ func WithGateway(variableName string) ClaimOption {
 	}
 }
 
+// WithPrefix instructs Setup to store the prefix from IPAM into the provided variableName.
+func WithPrefix(variableName string) ClaimOption {
+	return func(o *claimOptions) {
+		o.prefixVariableName = variableName
+	}
+}
+
 type teardownOptions struct {
 	folderName    string
 	vSphereClient *govmomi.Client