Automate envtest setup from tests if not configured (#6857)

Before this, running a test that required envtest without setting the
KUBEBUILDER_ASSETS env var would fail. Now, if the env var is not set,
the envtest setup will download the necessary assets and configure the
tests to use them.

This means that `go test` (on individual tests or whole packages) now
works out of the box.

Makefile

@@ -39,13 +39,11 @@ DEV_GIT_VERSION:=v0.0.0-dev-${BRANCH_NAME}
 BUNDLE_MANIFEST_URL?=https://dev-release-assets.eks-anywhere.model-rocket.aws.dev/${BRANCH_NAME}/bundle-release.yaml
 RELEASE_MANIFEST_URL?=https://dev-release-assets.eks-anywhere.model-rocket.aws.dev/${BRANCH_NAME}/eks-a-release.yaml
 LATEST=$(BRANCH_NAME)
-$(info    Using branch-specific BUNDLE_MANIFEST_URL $(BUNDLE_MANIFEST_URL) and RELEASE_MANIFEST_URL $(RELEASE_MANIFEST_URL))
 else
 ## use the standard bundle manifest if the branch is 'main'
 DEV_GIT_VERSION:=v0.0.0-dev
 BUNDLE_MANIFEST_URL?=https://dev-release-assets.eks-anywhere.model-rocket.aws.dev/bundle-release.yaml
 RELEASE_MANIFEST_URL?=https://dev-release-assets.eks-anywhere.model-rocket.aws.dev/eks-a-release.yaml
-$(info    Using standard BUNDLE_MANIFEST_URL $(BUNDLE_MANIFEST_URL) and RELEASE_MANIFEST_URL $(RELEASE_MANIFEST_URL))
 LATEST=latest
 endif
 
@@ -142,7 +140,7 @@ LOCAL_E2E_TESTS ?= $(DOCKER_E2E_TEST)
 
 EMBED_CONFIG_FOLDER = pkg/files/config
 
-export KUBEBUILDER_ENVTEST_KUBERNETES_VERSION ?= 1.26.x
+export KUBEBUILDER_ENVTEST_KUBERNETES_VERSION ?= 1.28.x
 
 UNAME := $(shell uname -s)
 
@@ -268,6 +266,10 @@ $(GO_VULNCHECK): $(TOOLS_BIN_DIR)
 $(SETUP_ENVTEST): $(TOOLS_BIN_DIR)
 	cd $(TOOLS_BIN_DIR); $(GO) build -tags=tools -o $(SETUP_ENVTEST_BIN) sigs.k8s.io/controller-runtime/tools/setup-envtest
 
+envtest-setup: $(SETUP_ENVTEST)
+	$(eval KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch $(GO_ARCH) $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)))
+	@echo KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)
+
 .PHONY: lint
 lint: $(GOLANGCI_LINT) ## Run golangci-lint
 	$(GOLANGCI_LINT) run --new-from-rev main
@@ -428,8 +430,7 @@ test: unit-test capd-test  ## Run unit and capd tests
 
 .PHONY: unit-test
 unit-test: ## Run unit tests
-unit-test: $(SETUP_ENVTEST) 
-unit-test: KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch $(GO_ARCH) $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))
+unit-test: envtest-setup
 unit-test:
 	KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" $(GO_TEST) $(UNIT_TEST_PACKAGES) -cover -tags "$(BUILD_TAGS)" $(GO_TEST_FLAGS)
 
@@ -442,9 +443,8 @@ unit-test:
 # their tests run by this target.
 .PHONY: unit-test-patch
 unit-test-patch: ## Run unit tests for packages modified locally
-unit-test-patch: $(SETUP_ENVTEST)
+unit-test-patch: envtest-setup
 unit-test-patch: GO_PKGS ?= $(shell ./scripts/go-packages-in-patch.sh | grep -vE "$(UNIT_TEST_PACKAGE_EXCLUSION_REGEX)")
-unit-test-patch: KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch $(GO_ARCH) $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))
 unit-test-patch:
 	KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" $(GO_TEST) $(GO_PKGS) -cover -tags "$(BUILD_TAGS)" $(GO_TEST_FLAGS)
 	@echo Reminder: $@ is not a substitute for make unit-test
@@ -479,8 +479,7 @@ coverage-view: coverage-unit-test
 # their tests run by this target.
 .PHONY: coverage-view-patch
 coverage-view-patch: GO_PKGS ?= $(shell ./scripts/go-packages-in-patch.sh)
-coverage-view-patch: $(SETUP_ENVTEST)
-coverage-view-patch: KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch $(GO_ARCH) $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))
+coverage-view-patch: envtest-setup
 coverage-view-patch:
 	-$(MAKE) unit-test-patch GO_TEST_FLAGS="-coverprofile=$(COVER_PROFILE) -covermode=atomic"
 	$(GO) tool cover -html=$(COVER_PROFILE)

internal/test/envtest/environment.go

@@ -1,8 +1,11 @@
 package envtest
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	goruntime "runtime"
@@ -47,6 +50,8 @@ const (
 	tinkerbellPackage   = "github.com/tinkerbell/tink"
 	etcdProviderPackage = "github.com/aws/etcdadm-controller"
 	capcPackage         = "sigs.k8s.io/cluster-api-provider-cloudstack"
+
+	kubebuilderAssetsEnvVar = "KUBEBUILDER_ASSETS"
 )
 
 func init() {
@@ -134,6 +139,11 @@ func RunWithEnvironment(m *testing.M, opts ...EnvironmentOpt) int {
 func newEnvironment(ctx context.Context) (*Environment, error) {
 	root := getRootPath()
 	currentDir := currentDir()
+
+	if err := ensureEnvtest(ctx, root); err != nil {
+		return nil, err
+	}
+
 	crdDirectoryPaths := make([]string, 0, len(packages)+2)
 	crdDirectoryPaths = append(crdDirectoryPaths,
 		filepath.Join(root, "config", "crd", "bases"),
@@ -194,6 +204,33 @@ func newEnvironment(ctx context.Context) (*Environment, error) {
 	return env, nil
 }
 
+func ensureEnvtest(ctx context.Context, rootDir string) error {
+	// Only if the envtest config envvar is not set, try to setup assets
+	if _, ok := os.LookupEnv(kubebuilderAssetsEnvVar); ok {
+		return nil
+	}
+
+	cmd := exec.CommandContext(ctx, "make", "-s", "envtest-setup")
+	cmd.Dir = rootDir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed setting up env-test:\n%s", string(out))
+	}
+
+	// Last line of output is expected to be KUBEBUILDER_ASSETS=[path to envtest setup]
+	lines := bytes.Split(out, []byte("\n"))
+	lastLine := lines[len(lines)-2]
+	split := bytes.Split(lastLine, []byte("="))
+	if len(split) != 2 || string(split[0]) != kubebuilderAssetsEnvVar {
+		return fmt.Errorf("invalid last line of env-test setup: %s", string(lastLine))
+	}
+
+	fmt.Printf("Envtest auto-setup using installation path %s\n", string(split[1]))
+	os.Setenv(kubebuilderAssetsEnvVar, string(split[1]))
+
+	return nil
+}
+
 func (e *Environment) stop() error {
 	fmt.Println("Stopping the test environment")
 	e.cancelF() // Cancels context that will stop the manager