Merge pull request #2294 from Nordix/lentzi90/fix-conformance-suite

🌱 Fix conformance suite

.gitignore

@@ -49,6 +49,9 @@ out
 # Vscode files
 .vscode
 
+# Zed files
+.zed_server
+
 # This is where the result of the go build goes
 /output*/
 /_output*/

Makefile

@@ -215,10 +215,12 @@ test-e2e-image-prerequisites:
 
 CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH)
 CONFORMANCE_E2E_ARGS += $(E2E_ARGS)
-CONFORMANCE_GINKGO_ARGS ?= -stream
 .PHONY: test-conformance
 test-conformance: $(GINKGO) e2e-prerequisites ## Run clusterctl based conformance test on workload cluster (requires Docker).
-	time $(GINKGO) -trace -show-node-events -v -tags=e2e -focus="conformance" $(CONFORMANCE_GINKGO_ARGS) ./test/e2e/suites/conformance/... -- -config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" --data-folder="$(E2E_DATA_DIR)" $(CONFORMANCE_E2E_ARGS)
+	time $(GINKGO) -trace -show-node-events -v -tags=e2e -focus="conformance" $(CONFORMANCE_GINKGO_ARGS) \
+	   ./test/e2e/suites/conformance/... -- \
+			-config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" \
+			--data-folder="$(E2E_DATA_DIR)" $(CONFORMANCE_E2E_ARGS)
 
 test-conformance-fast: ## Run clusterctl based conformance test on workload cluster (requires Docker) using a subset of the conformance suite in parallel.
 	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_FAST_CONF_PATH) -kubetest.ginkgo-nodes=5 $(E2E_ARGS)"
@@ -378,9 +380,9 @@ staging-manifests:
 ##@ Release
 ## --------------------------------------
 
-ifneq (,$(findstring -,$(RELEASE_TAG)))                                                                                                 
-    PRE_RELEASE=true                                                                                                                    
-endif                                                                                                                                   
+ifneq (,$(findstring -,$(RELEASE_TAG)))
+    PRE_RELEASE=true
+endif
 PREVIOUS_TAG ?= $(shell git tag -l | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$$" | sort -V | grep -B1 $(RELEASE_TAG) | head -n 1 2>/dev/null)
 ## set by Prow, ref name of the base branch, e.g., main
 RELEASE_DIR := out
@@ -389,11 +391,11 @@ RELEASE_NOTES_DIR := _releasenotes
 .PHONY: $(RELEASE_DIR)
 $(RELEASE_DIR):
 	mkdir -p $(RELEASE_DIR)/
- 
+
 .PHONY: $(RELEASE_NOTES_DIR)
 $(RELEASE_NOTES_DIR):
 	mkdir -p $(RELEASE_NOTES_DIR)/
- 
+
 .PHONY: $(BUILD_DIR)
 $(BUILD_DIR):
 	@mkdir -p $(BUILD_DIR)
@@ -593,7 +595,7 @@ compile-e2e: ## Test e2e compilation
 .PHONY: FORCE
 FORCE:
 
-## --------------------------------------    
+## --------------------------------------
 ## Helpers
 ## --------------------------------------
 

test/e2e/data/e2e_conf.yaml

@@ -4,7 +4,6 @@
 
 # To run tests, run the following from the root of this repository.
 # `OPENSTACK_CLOUD=capo-e2e OPENSTACK_CLOUD_YAML_FILE=/tmp/clouds.yaml make test-conformance`
-# The -stream flag will make Ginkgo print results to the screen in real-time.
 
 managementClusterName: capo-e2e
 

test/e2e/suites/conformance/conformance_test.go

@@ -27,6 +27,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gmeasure"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -39,7 +40,6 @@ import (
 var _ = Describe("conformance tests", func() {
 	var (
 		namespace *corev1.Namespace
-		ctx       context.Context
 		specName  = "conformance"
 	)
 
@@ -53,7 +53,7 @@ var _ = Describe("conformance tests", func() {
 		shared.ApplyCoreImagesPlus(ctx, e2eCtx)
 	})
 
-	Measure(specName, func(b Benchmarker) {
+	It(specName, func(ctx context.Context) {
 		name := fmt.Sprintf("cluster-%s", namespace.Name)
 		kubernetesVersion := e2eCtx.E2EConfig.GetVariable(shared.KubernetesVersion)
 
@@ -70,7 +70,10 @@ var _ = Describe("conformance tests", func() {
 		controlPlaneMachineCount, err := strconv.ParseInt(e2eCtx.E2EConfig.GetVariable("CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT"), 10, 64)
 		Expect(err).NotTo(HaveOccurred())
 
-		b.Time("cluster creation", func() {
+		experiment := gmeasure.NewExperiment(specName)
+		AddReportEntry(experiment.Name, experiment)
+
+		experiment.MeasureDuration("cluster creation", func() {
 			result := &clusterctl.ApplyClusterTemplateAndWaitResult{}
 			clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 				ClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
@@ -93,7 +96,7 @@ var _ = Describe("conformance tests", func() {
 		})
 
 		workloadProxy := e2eCtx.Environment.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, name)
-		b.Time("conformance suite", func() {
+		experiment.MeasureDuration("conformance suite", func() {
 			err := kubetest.Run(ctx,
 				kubetest.RunInput{
 					ClusterProxy:   workloadProxy,
@@ -104,10 +107,5 @@ var _ = Describe("conformance tests", func() {
 			)
 			Expect(err).To(BeNil(), "error on kubetest execution")
 		})
-	}, 1)
-
-	AfterEach(func() {
-		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
-		shared.DumpSpecResourcesAndCleanup(ctx, specName, namespace, e2eCtx)
 	})
 })