Implement Boskos

kubernetes-sigs · Jun 5, 2024 · 2574b6e · 2574b6e
1 parent 6e8e237
commit 2574b6e
Show file tree

Hide file tree

Showing 27 changed files with 2,090 additions and 104 deletions.
diff --git a/Makefile b/Makefile
@@ -84,7 +84,7 @@ GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 3h
 E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere.yaml)
 E2E_CONF_OVERRIDE_FILE ?= $(abspath test/e2e/config/config-overrides.yaml)
-E2E_IPAM_KUBECONFIG ?=
+VSPHERE_IP_POOL ?=
 E2E_TEMPLATE_DIR := $(abspath test/e2e/data/)
 E2E_GOVMOMI_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-govmomi
 E2E_SUPERVISOR_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-supervisor
@@ -193,6 +193,9 @@ IMPORT_BOSS_PKG := k8s.io/code-generator/cmd/import-boss
 
 CAPI_HACK_TOOLS_VER := ef04465b2ba76214eea570e27e8146c96412e32a # Note: this is the commit ID of CAPI v1.7.1
 
+BOSKOSCTL_BIN := boskosctl
+BOSKOSCTL := $(abspath $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN))
+
 CONVERSION_VERIFIER_VER := $(CAPI_HACK_TOOLS_VER)
 CONVERSION_VERIFIER_BIN := conversion-verifier
 CONVERSION_VERIFIER := $(abspath $(TOOLS_BIN_DIR)/$(CONVERSION_VERIFIER_BIN)-$(CONVERSION_VERIFIER_VER))
@@ -627,7 +630,7 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
 		--e2e.artifacts-folder="$(ARTIFACTS)" \
 		--e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \
-		--e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)"
+		--e2e.ip-pool="$(VSPHERE_IP_POOL)"
 
 ## --------------------------------------
 ## Release
@@ -958,6 +961,9 @@ $(CONVERSION_GEN_BIN): $(CONVERSION_GEN) ## Build a local copy of conversion-gen
 .PHONY: $(PROWJOB_GEN_BIN)
 $(PROWJOB_GEN_BIN): $(PROWJOB_GEN) ## Build a local copy of prowjob-gen.
 
+.PHONY: $(BOSKOSCTL_BIN)
+$(BOSKOSCTL_BIN): $(BOSKOSCTL) ## Build a local copy of boskosctl.
+
 .PHONY: $(CONVERSION_VERIFIER_BIN)
 $(CONVERSION_VERIFIER_BIN): $(CONVERSION_VERIFIER) ## Build a local copy of conversion-verifier.
 
@@ -1012,6 +1018,9 @@ $(CONTROLLER_GEN): # Build controller-gen.
 $(CONVERSION_GEN): # Build conversion-gen.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONVERSION_GEN_PKG) $(CONVERSION_GEN_BIN) $(CONVERSION_GEN_VER)
 
+$(BOSKOSCTL): # Build boskosctl from tools folder.
+	go build -o $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN) ./hack/tools/boskosctl
+
 $(CONVERSION_VERIFIER): # Build conversion-verifier.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_TOOLS_BUILD) $(CONVERSION_VERIFIER_PKG) $(CONVERSION_VERIFIER_BIN) $(CONVERSION_VERIFIER_VER)
 

diff --git a/go.mod b/go.mod
@@ -11,9 +11,11 @@ require (
 	github.com/go-logr/logr v1.4.2
 	github.com/google/gofuzz v1.2.0
 	github.com/google/uuid v1.6.0
+	github.com/hashicorp/go-multierror v1.0.0
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	github.com/vmware-tanzu/net-operator-api v0.0.0-20240326163340-1f32d6bf7f9d
@@ -56,6 +58,7 @@ require (
 	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

diff --git a/go.sum b/go.sum
@@ -301,10 +301,12 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rH
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=

diff --git a/hack/e2e.sh b/hack/e2e.sh
@@ -36,7 +36,20 @@ fi
 # shellcheck source=./hack/ensure-kubectl.sh
 source "${REPO_ROOT}/hack/ensure-kubectl.sh"
 
+make boskosctl
+export BOSKOS_RESOURCE_OWNER=cluster-api-provider-vsphere
+export BOSKOS_RESOURCE_TYPE=vsphere-project-cluster-api-provider
+
 on_exit() {
+
+  if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
+    # Stop boskos heartbeat
+    [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
+
+    # If Boskos is being used then release the AWS account back to Boskos.
+    [ -z "${BOSKOS_HOST:-}" ] || boskosctl release --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --resource-folder="${BOSKOS_RESOURCE_FOLDER}" --resource-pool="${BOSKOS_RESOURCE_POOL}"
+  fi
+
   # kill the VPN only when we started it (not vcsim)
   if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     docker kill vpn
@@ -90,9 +103,8 @@ export GC_KIND="false"
 # Make tests run in-parallel
 export GINKGO_NODES=5
 
-# Set the kubeconfig to the IPAM cluster so the e2e tests can claim ip addresses
-# for kube-vip.
-export E2E_IPAM_KUBECONFIG="/root/ipam-conf/capv-services.conf"
+# FIXME(sbueringer) before merge this should be moved to the ProwJob config in test-infra (so we can move jobs incrementally to the new environment / boskos)
+export BOSKOS_HOST=http://192.168.6.138:32222
 
 # Only run the vpn/check for IPAM when we need them (not vcsim)
 if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
@@ -104,11 +116,11 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
   # Tail the vpn logs
   docker logs vpn
 
-  # Wait until the VPN connection is active and we are able to reach the ipam cluster
-  function wait_for_ipam_reachable() {
+  # Wait until the VPN connection is active.
+  function wait_for_vpn_up() {
     local n=0
     until [ $n -ge 30 ]; do
-      kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s  get inclusterippools.ipam.cluster.x-k8s.io && RET=$? || RET=$?
+      curl "https://${VSPHERE_SERVER}" -k -v && RET=$? || RET=$?
       if [[ "$RET" -eq 0 ]]; then
         break
       fi
@@ -117,7 +129,39 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     done
     return "$RET"
   }
-  wait_for_ipam_reachable
+  wait_for_vpn_up
+
+  # Get vsphere-project via Boskos
+  # If BOSKOS_HOST is set then acquire a vsphere-project from Boskos.
+  if [ -n "${BOSKOS_HOST:-}" ]; then
+    # Check out the account from Boskos and store the produced environment
+    # variables in a temporary file.
+    account_env_var_file="$(mktemp)"
+    boskosctl acquire --resource-owner=${BOSKOS_RESOURCE_OWNER} --resource-type=${BOSKOS_RESOURCE_TYPE} 1>"${account_env_var_file}"
+    checkout_account_status="${?}"
+
+    # If the checkout process was a success then load the account's
+    # environment variables into this process.
+    # shellcheck disable=SC1090
+    [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+    export VSPHERE_FOLDER=${BOSKOS_RESOURCE_FOLDER}
+    export VSPHERE_RESOURCE_POOL=${BOSKOS_RESOURCE_POOL}
+    export VSPHERE_IP_POOL="${BOSKOS_RESOURCE_IP_POOL}"
+
+    # Always remove the account environment variable file. It contains
+    # sensitive information.
+    rm -f "${account_env_var_file}"
+
+    if [ ! "${checkout_account_status}" = "0" ]; then
+      echo "error getting vsphere project from boskos" 1>&2
+      exit "${checkout_account_status}"
+    fi
+
+    # run the heart beat process to tell boskos that we are still
+    # using the checked out account periodically
+    boskosctl heartbeat --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" >>"$ARTIFACTS/boskos.log" 2>&1 &
+    HEART_BEAT_PID=$!
+  fi
 fi
 
 make envsubst