From a847b63aa5168f3cc5f60b62859f3dee9bd6f946 Mon Sep 17 00:00:00 2001 From: Bastian Krol Date: Mon, 17 Jun 2024 16:17:58 +0200 Subject: [PATCH] test(e2e): test instrument existing workloads for all types in parallel This brings down the time for the e2e test suite from over 10 minutes to around five minutes. --- test/e2e/e2e_helpers.go | 63 ++++++++++++-------- test/e2e/e2e_test.go | 125 ++++++++++++++++++++++++---------------- 2 files changed, 115 insertions(+), 73 deletions(-) diff --git a/test/e2e/e2e_helpers.go b/test/e2e/e2e_helpers.go index 6ce186c0..6181b301 100644 --- a/test/e2e/e2e_helpers.go +++ b/test/e2e/e2e_helpers.go @@ -448,6 +448,19 @@ func UndeployDash0Resource(namespace string) { ))).To(Succeed()) } +func RebuildNodeJsApplicationContainerImage() { + By("building the dash0-operator-nodejs-20-express-test-app image") + Expect( + RunAndIgnoreOutput( + exec.Command( + "docker", + "build", + "test-resources/node.js/express", + "-t", + "dash0-operator-nodejs-20-express-test-app", + ))).To(Succeed()) +} + func InstallNodeJsCronJob(namespace string) error { return installNodeJsApplication( namespace, @@ -573,18 +586,14 @@ func RemoveAllTestApplications(namespace string) { } func installNodeJsApplication(namespace string, kind string, waitCommand *exec.Cmd) error { - imageName := "dash0-operator-nodejs-20-express-test-app" - err := RunMultipleFromStrings([][]string{ - {"docker", "build", "test-resources/node.js/express", "-t", imageName}, - { - "kubectl", - "apply", - "--namespace", - namespace, - "-f", - fmt.Sprintf("test-resources/node.js/express/%s.yaml", kind), - }, - }) + err := RunAndIgnoreOutput(exec.Command( + "kubectl", + "apply", + "--namespace", + namespace, + "-f", + fmt.Sprintf("test-resources/node.js/express/%s.yaml", kind), + )) if err != nil { return err } @@ -620,7 +629,8 @@ func VerifyThatWorkloadHasBeenInstrumented( restartPodsManually bool, instrumentationBy string, ) string { - By("waiting for the workload to get instrumented (polling its labels and events to check)") + By(fmt.Sprintf("%s: waiting for the workload to get instrumented (polling its labels and events to check)", + workloadType)) Eventually(func(g Gomega) { VerifyLabels(g, namespace, workloadType, true, instrumentationBy) verifySuccessfulInstrumentationEvent(g, namespace, workloadType, instrumentationBy) @@ -630,7 +640,7 @@ func VerifyThatWorkloadHasBeenInstrumented( restartAllPods(namespace) } - By("waiting for spans to be captured") + By(fmt.Sprintf("%s: waiting for spans to be captured", workloadType)) var testId string if isBatch { By(fmt.Sprintf("waiting for the test ID file to be written by the %s under test", workloadType)) @@ -657,7 +667,7 @@ func VerifyThatWorkloadHasBeenInstrumented( Eventually(func(g Gomega) { verifySpans(g, isBatch, workloadType, port, httpPathWithQuery) }, verifyTelemetryTimeout, verifyTelemetryPollingInterval).Should(Succeed()) - By("matchin spans have been received") + By(fmt.Sprintf("%s: matching spans have been received", workloadType)) return testId } @@ -670,7 +680,9 @@ func VerifyThatInstrumentationHasBeenReverted( testId string, instrumentationBy string, ) { - By("waiting for the instrumentation to get removed from the workload (polling its labels and events to check)") + By(fmt.Sprintf( + "%s: waiting for the instrumentation to get removed from the workload (polling its labels and events to check)", + workloadType)) Eventually(func(g Gomega) { verifyLabelsHaveBeenRemoved(g, namespace, workloadType) verifySuccessfulUninstrumentationEvent(g, namespace, workloadType, instrumentationBy) @@ -691,12 +703,16 @@ func VerifyThatInstrumentationHasBeenReverted( secondsToCheckForSpans = 80 } httpPathWithQuery := fmt.Sprintf("/dash0-k8s-operator-test?id=%s", testId) - By(fmt.Sprintf("verifying that spans are no longer being captured (checking for %d seconds)", secondsToCheckForSpans)) + By( + fmt.Sprintf("%s: verifying that spans are no longer being captured (checking for %d seconds)", + workloadType, + secondsToCheckForSpans, + )) Consistently(func(g Gomega) { - verifyNoSpans(isBatch, port, httpPathWithQuery) + verifyNoSpans(isBatch, workloadType, port, httpPathWithQuery) }, time.Duration(secondsToCheckForSpans)*time.Second, 1*time.Second).Should(Succeed()) - By("matching spans are no longer captured") + By(fmt.Sprintf("%s: matching spans are no longer captured", workloadType)) } func VerifyThatFailedInstrumentationAttemptLabelsHaveBeenRemovedRemoved(namespace string, workloadType string) { @@ -855,17 +871,18 @@ func restartAllPods(namespace string) { "--namespace", namespace, "--selector", - "app=dash0-operator-nodejs-20-express-test-app", + "app=dash0-operator-nodejs-20-express-test-replicaset-app", ))).To(Succeed()) } func verifySpans(g Gomega, isBatch bool, workloadType string, port int, httpPathWithQuery string) { spansFound := sendRequestAndFindMatchingSpans(g, isBatch, workloadType, port, httpPathWithQuery, true, nil) - g.Expect(spansFound).To(BeTrue(), "expected to find at least one matching HTTP server span") + g.Expect(spansFound).To(BeTrue(), + fmt.Sprintf("%s: expected to find at least one matching HTTP server span", workloadType)) } -func verifyNoSpans(isBatch bool, port int, httpPathWithQuery string) { +func verifyNoSpans(isBatch bool, workloadType string, port int, httpPathWithQuery string) { timestampLowerBound := time.Now() spansFound := sendRequestAndFindMatchingSpans( Default, @@ -876,7 +893,7 @@ func verifyNoSpans(isBatch bool, port int, httpPathWithQuery string) { false, ×tampLowerBound, ) - Expect(spansFound).To(BeFalse(), "expected to find no matching HTTP server span") + Expect(spansFound).To(BeFalse(), fmt.Sprintf("%s: expected to find no matching HTTP server span", workloadType)) } func sendRequestAndFindMatchingSpans( diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c4fd89ca..f52a1445 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os/exec" "strings" + "sync" "time" . "github.com/onsi/ginkgo/v2" @@ -31,6 +32,14 @@ var ( setupFinishedSuccessfully bool ) +type controllerTestWorkloadConfig struct { + workloadType string + port int + installWorkload func(string) error + isBatch bool + restartPodsManually bool +} + var _ = Describe("Dash0 Kubernetes Operator", Ordered, func() { BeforeAll(func() { @@ -47,6 +56,7 @@ var _ = Describe("Dash0 Kubernetes Operator", Ordered, func() { RecreateNamespace(applicationUnderTestNamespace) RebuildOperatorControllerImage(operatorImageRepository, operatorImageTag) RebuildDash0InstrumentationImage() + RebuildNodeJsApplicationContainerImage() setupFinishedSuccessfully = true }) @@ -85,73 +95,72 @@ var _ = Describe("Dash0 Kubernetes Operator", Ordered, func() { UndeployOperatorAndCollector(operatorNamespace) }) - type controllerTest struct { - workloadType string - port int - installWorkload func(string) error - isBatch bool - restartPodsManually bool - } - - DescribeTable( - "when instrumenting existing workloads", - func(config controllerTest) { - By(fmt.Sprintf("installing the Node.js %s", config.workloadType)) - Expect(config.installWorkload(applicationUnderTestNamespace)).To(Succeed()) - By("deploy the operator and the Dash0 custom resource") - - DeployOperatorWithCollectorAndClearExportedTelemetry(operatorNamespace, operatorImageRepository, operatorImageTag) - DeployDash0Resource(applicationUnderTestNamespace) - By(fmt.Sprintf("verifying that the Node.js %s has been instrumented by the controller", config.workloadType)) - testId := VerifyThatWorkloadHasBeenInstrumented( - applicationUnderTestNamespace, - config.workloadType, - config.port, - config.isBatch, - config.restartPodsManually, - "controller", - ) - - UndeployDash0Resource(applicationUnderTestNamespace) - - VerifyThatInstrumentationHasBeenReverted( - applicationUnderTestNamespace, - config.workloadType, - config.port, - config.isBatch, - config.restartPodsManually, - testId, - "controller", - ) - }, - Entry("should instrument and uninstrument existing cron jobs", controllerTest{ + workloadConfigs := []controllerTestWorkloadConfig{ + { workloadType: "cronjob", port: 1205, installWorkload: InstallNodeJsCronJob, isBatch: true, - }), - Entry("should instrument and uninstrument existing daemon sets", controllerTest{ + }, { workloadType: "daemonset", port: 1206, installWorkload: InstallNodeJsDaemonSet, - }), - Entry("should instrument and uninstrument existing deployments", controllerTest{ + }, { workloadType: "deployment", port: 1207, installWorkload: InstallNodeJsDeployment, - }), - Entry("should instrument and uninstrument existing replica set", controllerTest{ + }, { workloadType: "replicaset", port: 1209, installWorkload: InstallNodeJsReplicaSet, restartPodsManually: true, - }), - Entry("should instrument and uninstrument existing stateful set", controllerTest{ + }, { workloadType: "statefulset", port: 1210, installWorkload: InstallNodeJsStatefulSet, - }), - ) + }, + } + + Describe("when instrumenting existing workloads", func() { + It("should instrument and uninstrument all workload types", func() { + By("deploying all workloads") + runInParallelForAllWorkloadTypes(workloadConfigs, func(config controllerTestWorkloadConfig) { + By(fmt.Sprintf("deploying the Node.js %s", config.workloadType)) + Expect(config.installWorkload(applicationUnderTestNamespace)).To(Succeed()) + }) + + By("deploy the operator and the Dash0 custom resource") + DeployOperatorWithCollectorAndClearExportedTelemetry(operatorNamespace, operatorImageRepository, operatorImageTag) + DeployDash0Resource(applicationUnderTestNamespace) + + testIds := make(map[string]string) + runInParallelForAllWorkloadTypes(workloadConfigs, func(config controllerTestWorkloadConfig) { + By(fmt.Sprintf("verifying that the Node.js %s has been instrumented by the controller", config.workloadType)) + testIds[config.workloadType] = VerifyThatWorkloadHasBeenInstrumented( + applicationUnderTestNamespace, + config.workloadType, + config.port, + config.isBatch, + config.restartPodsManually, + "controller", + ) + }) + + UndeployDash0Resource(applicationUnderTestNamespace) + + runInParallelForAllWorkloadTypes(workloadConfigs, func(config controllerTestWorkloadConfig) { + VerifyThatInstrumentationHasBeenReverted( + applicationUnderTestNamespace, + config.workloadType, + config.port, + config.isBatch, + config.restartPodsManually, + testIds[config.workloadType], + "controller", + ) + }) + }) + }) Describe("when it detects existing immutable jobs", func() { It("should label them accordingly", func() { @@ -286,3 +295,19 @@ var _ = Describe("Dash0 Kubernetes Operator", Ordered, func() { ) }) }) + +func runInParallelForAllWorkloadTypes( + workloadConfigs []controllerTestWorkloadConfig, + testStep func(controllerTestWorkloadConfig), +) { + var wg sync.WaitGroup + for _, config := range workloadConfigs { + wg.Add(1) + go func(cfg controllerTestWorkloadConfig) { + defer GinkgoRecover() + defer wg.Done() + testStep(cfg) + }(config) + } + wg.Wait() +}