test/e2e: use vSphere projects from Boskos

Makefile

@@ -84,7 +84,7 @@ GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 3h
 E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere.yaml)
 E2E_CONF_OVERRIDE_FILE ?= $(abspath test/e2e/config/config-overrides.yaml)
-E2E_IPAM_KUBECONFIG ?=
+E2E_VSPHERE_IP_POOL ?=
 E2E_TEMPLATE_DIR := $(abspath test/e2e/data/)
 E2E_GOVMOMI_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-govmomi
 E2E_SUPERVISOR_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-supervisor
@@ -193,6 +193,9 @@ IMPORT_BOSS_PKG := k8s.io/code-generator/cmd/import-boss
 
 CAPI_HACK_TOOLS_VER := ef04465b2ba76214eea570e27e8146c96412e32a # Note: this is the commit ID of CAPI v1.7.1
 
+BOSKOSCTL_BIN := boskosctl
+BOSKOSCTL := $(abspath $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN))
+
 CONVERSION_VERIFIER_VER := $(CAPI_HACK_TOOLS_VER)
 CONVERSION_VERIFIER_BIN := conversion-verifier
 CONVERSION_VERIFIER := $(abspath $(TOOLS_BIN_DIR)/$(CONVERSION_VERIFIER_BIN)-$(CONVERSION_VERIFIER_VER))
@@ -627,7 +630,7 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
 		--e2e.artifacts-folder="$(ARTIFACTS)" \
 		--e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \
-		--e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)"
+		--e2e.ip-pool='$(E2E_VSPHERE_IP_POOL)'
 
 ## --------------------------------------
 ## Release
@@ -958,6 +961,9 @@ $(CONVERSION_GEN_BIN): $(CONVERSION_GEN) ## Build a local copy of conversion-gen
 .PHONY: $(PROWJOB_GEN_BIN)
 $(PROWJOB_GEN_BIN): $(PROWJOB_GEN) ## Build a local copy of prowjob-gen.
 
+.PHONY: $(BOSKOSCTL_BIN)
+$(BOSKOSCTL_BIN): $(BOSKOSCTL) ## Build a local copy of boskosctl.
+
 .PHONY: $(CONVERSION_VERIFIER_BIN)
 $(CONVERSION_VERIFIER_BIN): $(CONVERSION_VERIFIER) ## Build a local copy of conversion-verifier.
 
@@ -1012,6 +1018,9 @@ $(CONTROLLER_GEN): # Build controller-gen.
 $(CONVERSION_GEN): # Build conversion-gen.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONVERSION_GEN_PKG) $(CONVERSION_GEN_BIN) $(CONVERSION_GEN_VER)
 
+$(BOSKOSCTL): # Build boskosctl from tools folder.
+	go build -o $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN) ./hack/tools/boskosctl
+
 $(CONVERSION_VERIFIER): # Build conversion-verifier.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_TOOLS_BUILD) $(CONVERSION_VERIFIER_PKG) $(CONVERSION_VERIFIER_BIN) $(CONVERSION_VERIFIER_VER)
 

hack/clean-ci.sh

@@ -61,5 +61,8 @@ wait_for_ipam_reachable
 # Set kubeconfig for IPAM cleanup
 export KUBECONFIG="${E2E_IPAM_KUBECONFIG}"
 
+# FIXME(sbueringer) before merge this should be moved to the ProwJob config in test-infra (so we can move jobs incrementally to the new environment / boskos)
+export BOSKOS_HOST=http://192.168.6.138:32222
+
 # Run e2e tests
 make clean-ci

hack/e2e.sh

@@ -36,7 +36,23 @@ fi
 # shellcheck source=./hack/ensure-kubectl.sh
 source "${REPO_ROOT}/hack/ensure-kubectl.sh"
 
+make boskosctl
+export BOSKOS_RESOURCE_OWNER=cluster-api-provider-vsphere
+if [[ "${JOB_NAME}" != "" ]]; then
+  export BOSKOS_RESOURCE_OWNER="${JOB_NAME}/${BUILD_ID}"
+fi
+export BOSKOS_RESOURCE_TYPE=vsphere-project-cluster-api-provider
+
 on_exit() {
+  # Only handle Boskos when we have to (not for vcsim)
+  if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
+    # Stop boskos heartbeat
+    [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
+
+    # If Boskos is being used then release the vsphere project.
+    [ -z "${BOSKOS_HOST:-}" ] || boskosctl release --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --resource-folder="${BOSKOS_RESOURCE_FOLDER}" --resource-pool="${BOSKOS_RESOURCE_POOL}"
+  fi
+
   # kill the VPN only when we started it (not vcsim)
   if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     docker kill vpn
@@ -90,11 +106,10 @@ export GC_KIND="false"
 # Make tests run in-parallel
 export GINKGO_NODES=5
 
-# Set the kubeconfig to the IPAM cluster so the e2e tests can claim ip addresses
-# for kube-vip.
-export E2E_IPAM_KUBECONFIG="/root/ipam-conf/capv-services.conf"
+# FIXME(sbueringer) before merge this should be moved to the ProwJob config in test-infra (so we can move jobs incrementally to the new environment / boskos)
+export BOSKOS_HOST=http://192.168.6.138:32222
 
-# Only run the vpn/check for IPAM when we need them (not vcsim)
+# Only run the vpn/check for IPAM when we need them (not for vcsim)
 if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
   # Run the vpn client in container
   docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
@@ -104,11 +119,11 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
   # Tail the vpn logs
   docker logs vpn
 
-  # Wait until the VPN connection is active and we are able to reach the ipam cluster
-  function wait_for_ipam_reachable() {
+  # Wait until the VPN connection is active.
+  function wait_for_vpn_up() {
     local n=0
     until [ $n -ge 30 ]; do
-      kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s  get inclusterippools.ipam.cluster.x-k8s.io && RET=$? || RET=$?
+      curl "https://${VSPHERE_SERVER}" --connect-timeout 2 -k -v && RET=$? || RET=$?
       if [[ "$RET" -eq 0 ]]; then
         break
       fi
@@ -117,7 +132,39 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     done
     return "$RET"
   }
-  wait_for_ipam_reachable
+  wait_for_vpn_up
+
+  # If BOSKOS_HOST is set then acquire a vsphere-project from Boskos.
+  if [ -n "${BOSKOS_HOST:-}" ]; then
+    # Check out the account from Boskos and store the produced environment
+    # variables in a temporary file.
+    account_env_var_file="$(mktemp)"
+    boskosctl acquire --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-type=${BOSKOS_RESOURCE_TYPE} 1>"${account_env_var_file}"
+    checkout_account_status="${?}"
+
+    # If the checkout process was a success then load the account's
+    # environment variables into this process.
+    # shellcheck disable=SC1090
+    [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+    export BOSKOS_RESOURCE_NAME=${BOSKOS_RESOURCE_NAME}
+    export VSPHERE_FOLDER=${BOSKOS_RESOURCE_FOLDER}
+    export VSPHERE_RESOURCE_POOL=${BOSKOS_RESOURCE_POOL}
+    export E2E_VSPHERE_IP_POOL="${BOSKOS_RESOURCE_IP_POOL}"
+
+    # Always remove the account environment variable file. It contains
+    # sensitive information.
+    rm -f "${account_env_var_file}"
+
+    if [ ! "${checkout_account_status}" = "0" ]; then
+      echo "error getting vsphere project from Boskos" 1>&2
+      exit "${checkout_account_status}"
+    fi
+
+    # Run the heartbeat to tell boskos periodically that we are still
+    # using the checked out account.
+    boskosctl heartbeat --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" >>"${ARTIFACTS}/boskos-heartbeat.log" 2>&1 &
+    HEART_BEAT_PID=$!
+  fi
 fi
 
 make envsubst