From 7202cfa3a2fd18403b80d83dae397bf9010db6d4 Mon Sep 17 00:00:00 2001 From: Bastian Krol Date: Wed, 22 May 2024 15:20:50 +0200 Subject: [PATCH] feat: add labels --- cmd/main.go | 26 ++++++++ config/manager/manager.yaml | 5 ++ internal/controller/dash0_controller.go | 8 ++- internal/controller/dash0_controller_test.go | 7 +++ internal/k8sresources/modify.go | 62 +++++++++++++++++--- internal/k8sresources/modify_test.go | 28 ++++++++- internal/webhook/dash0_webhook.go | 3 +- internal/webhook/webhook_suite_test.go | 7 +++ test/e2e/e2e_helpers.go | 42 +++++++++++-- test/e2e/e2e_test.go | 4 +- test/util/verification.go | 10 ++++ 11 files changed, 182 insertions(+), 20 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4238b7a9..f6abcd66 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -25,6 +25,7 @@ import ( operatorv1alpha1 "github.com/dash0hq/dash0-operator/api/v1alpha1" "github.com/dash0hq/dash0-operator/internal/controller" + "github.com/dash0hq/dash0-operator/internal/k8sresources" dash0webhook "github.com/dash0hq/dash0-operator/internal/webhook" //+kubebuilder:scaffold:imports ) @@ -137,11 +138,32 @@ func startOperatorManager( return fmt.Errorf("unable to create the clientset client") } + operatorVersion, isSet := os.LookupEnv("DASH0_OPERATOR_VERSION") + if !isSet { + operatorVersion = "unknown" + } + initContainerImageVersion, isSet := os.LookupEnv("DASH0_INIT_CONTAINER_IMAGE_VERSION") + if !isSet { + return fmt.Errorf("cannot start Dash0 operator, the mandatory environment variable " + + "\"DASH0_INIT_CONTAINER_IMAGE_VERSION\" is missing") + } + setupLog.Info( + "version information", + "operator version", + operatorVersion, + "init container image version", + initContainerImageVersion, + ) + if err = (&controller.Dash0Reconciler{ Client: mgr.GetClient(), ClientSet: clientSet, Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("dash0-controller"), + Versions: k8sresources.Versions{ + OperatorVersion: operatorVersion, + InitContainerImageVersion: initContainerImageVersion, + }, }).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to set up the Dash0 reconciler: %w", err) } @@ -150,6 +172,10 @@ func startOperatorManager( if os.Getenv("ENABLE_WEBHOOKS") != "false" { if err = (&dash0webhook.Handler{ Recorder: mgr.GetEventRecorderFor("dash0-webhook"), + Versions: k8sresources.Versions{ + OperatorVersion: operatorVersion, + InitContainerImageVersion: initContainerImageVersion, + }, }).SetupWebhookWithManager(mgr); err != nil { return fmt.Errorf("unable to create the Dash0 webhook: %w", err) } diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 7bc77b7c..5f1b3d0f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -47,6 +47,11 @@ spec: - --leader-elect image: dash0-operator-controller:latest name: manager + env: + - name: DASH0_OPERATOR_VERSION + value: 1.0.0 + - name: DASH0_INIT_CONTAINER_IMAGE_VERSION + value: 1.0.0 # Note: Use "imagePullPolicy: Never" when only building the image locally without pushing them anywhere. Omit # the attribute otherwise to use the default pull policy. diff --git a/internal/controller/dash0_controller.go b/internal/controller/dash0_controller.go index 0b9a4c80..a769efd8 100644 --- a/internal/controller/dash0_controller.go +++ b/internal/controller/dash0_controller.go @@ -29,6 +29,7 @@ type Dash0Reconciler struct { ClientSet *kubernetes.Clientset Scheme *runtime.Scheme Recorder record.EventRecorder + Versions k8sresources.Versions } func (r *Dash0Reconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -186,7 +187,12 @@ func (r *Dash0Reconciler) modifyExistingResources(ctx context.Context, dash0Cust }, &deployment); err != nil { return fmt.Errorf("error when fetching deployment %s/%s: %w", deployment.GetNamespace(), deployment.GetName(), err) } - hasBeenModified := k8sresources.ModifyPodSpec(&deployment.Spec.Template.Spec, deployment.GetNamespace(), logger) + hasBeenModified := k8sresources.ModifyDeployment( + &deployment, + deployment.GetNamespace(), + r.Versions, + logger, + ) if hasBeenModified { return r.Client.Update(ctx, &deployment) } else { diff --git a/internal/controller/dash0_controller_test.go b/internal/controller/dash0_controller_test.go index f2176e98..b9a2cfa8 100644 --- a/internal/controller/dash0_controller_test.go +++ b/internal/controller/dash0_controller_test.go @@ -19,11 +19,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" operatorv1alpha1 "github.com/dash0hq/dash0-operator/api/v1alpha1" + "github.com/dash0hq/dash0-operator/internal/k8sresources" ) var ( timeout = 15 * time.Second pollingInterval = 50 * time.Millisecond + + versions = k8sresources.Versions{ + OperatorVersion: "1.2.3", + InitContainerImageVersion: "4.5.6", + } ) var _ = Describe("Dash0 Controller", func() { @@ -61,6 +67,7 @@ var _ = Describe("Dash0 Controller", func() { ClientSet: clientset, Recorder: recorder, Scheme: k8sClient.Scheme(), + Versions: versions, } }) diff --git a/internal/k8sresources/modify.go b/internal/k8sresources/modify.go index bf8ef8db..5d2db217 100644 --- a/internal/k8sresources/modify.go +++ b/internal/k8sresources/modify.go @@ -9,14 +9,15 @@ import ( "slices" "github.com/go-logr/logr" - + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( - initContainerName = "dash0-instrumentation" - initContainerImage = "dash0-instrumentation:1.0.0" + initContainerName = "dash0-instrumentation" + initContainerImageTemplate = "dash0-instrumentation:%s" dash0VolumeName = "dash0-instrumentation" dash0DirectoryEnvVarName = "DASH0_INSTRUMENTATION_FOLDER_DESTINATION" @@ -28,6 +29,10 @@ const ( envVarNodeOptionsValue = "--require /opt/dash0/instrumentation/node.js/node_modules/@dash0/opentelemetry/src/index.js" envVarDash0CollectorBaseUrlName = "DASH0_OTEL_COLLECTOR_BASE_URL" envVarDash0CollectorBaseUrlNameValueTemplate = "http://dash0-opentelemetry-collector-daemonset.%s.svc.cluster.local:4318" + + instrumentedLabelKey = "dash0.instrumented" + operatorVersionLabelKey = "dash0.operator.version" + initContainerImageVersionLabelKey = "dash0.initcontainer.image.version" ) var ( @@ -38,14 +43,40 @@ var ( initContainerReadOnlyRootFilesystem = true ) -func ModifyPodSpec(podSpec *corev1.PodSpec, namespace string, logger logr.Logger) bool { +type Versions struct { + OperatorVersion string + InitContainerImageVersion string +} + +func ModifyDeployment( + deployment *appsv1.Deployment, + namespace string, + versions Versions, + logger logr.Logger, +) bool { + podTemplateSpec := &deployment.Spec.Template + hasBeenModified := modifyPodSpec( + &podTemplateSpec.Spec, + namespace, + versions.InitContainerImageVersion, + logger, + ) + if hasBeenModified { + addInstrumentationLabels(&deployment.ObjectMeta, versions) + addInstrumentationLabels(&podTemplateSpec.ObjectMeta, versions) + } + return hasBeenModified +} + +func modifyPodSpec(podSpec *corev1.PodSpec, namespace string, initContainerImageVersion string, logger logr.Logger) bool { originalSpec := podSpec.DeepCopy() addInstrumentationVolume(podSpec) - addInitContainer(podSpec) + addInitContainer(podSpec, initContainerImageVersion) for idx := range podSpec.Containers { container := &podSpec.Containers[idx] instrumentContainer(container, namespace, logger) } + return !reflect.DeepEqual(originalSpec, podSpec) } @@ -72,7 +103,7 @@ func addInstrumentationVolume(podSpec *corev1.PodSpec) { } } -func addInitContainer(podSpec *corev1.PodSpec) { +func addInitContainer(podSpec *corev1.PodSpec, initContainerImageVersion string) { // The init container has all the instrumentation packages (e.g. the Dash0 Node.js distribution etc.), stored under // /dash0/instrumentation. Its main responsibility is to copy these files to the Kubernetes volume created and mounted in // addInstrumentationVolume (mounted at /opt/dash0/instrumentation in the init container and also in the target containers). @@ -83,7 +114,7 @@ func addInitContainer(podSpec *corev1.PodSpec) { idx := slices.IndexFunc(podSpec.InitContainers, func(c corev1.Container) bool { return c.Name == initContainerName }) - initContainer := createInitContainer(podSpec) + initContainer := createInitContainer(podSpec, initContainerImageVersion) if idx < 0 { podSpec.InitContainers = append(podSpec.InitContainers, *initContainer) } else { @@ -91,7 +122,7 @@ func addInitContainer(podSpec *corev1.PodSpec) { } } -func createInitContainer(podSpec *corev1.PodSpec) *corev1.Container { +func createInitContainer(podSpec *corev1.PodSpec, initContainerImageVersion string) *corev1.Container { initContainerUser := &defaultInitContainerUser initContainerGroup := &defaultInitContainerGroup @@ -106,7 +137,7 @@ func createInitContainer(podSpec *corev1.PodSpec) *corev1.Container { return &corev1.Container{ Name: initContainerName, - Image: initContainerImage, + Image: fmt.Sprintf(initContainerImageTemplate, initContainerImageVersion), Env: []corev1.EnvVar{ { Name: dash0DirectoryEnvVarName, @@ -212,3 +243,16 @@ func addOrReplaceEnvironmentVariable(container *corev1.Container, name string, v container.Env[idx].Value = value } } + +func addInstrumentationLabels(meta *v1.ObjectMeta, labelInformation Versions) { + addLabel(meta, instrumentedLabelKey, "true") + addLabel(meta, operatorVersionLabelKey, labelInformation.OperatorVersion) + addLabel(meta, initContainerImageVersionLabelKey, labelInformation.InitContainerImageVersion) +} + +func addLabel(meta *v1.ObjectMeta, key string, value string) { + if meta.Labels == nil { + meta.Labels = make(map[string]string, 1) + } + meta.Labels[key] = value +} diff --git a/internal/k8sresources/modify_test.go b/internal/k8sresources/modify_test.go index 0505088d..d15c070d 100644 --- a/internal/k8sresources/modify_test.go +++ b/internal/k8sresources/modify_test.go @@ -18,6 +18,13 @@ import ( // intentional. However, this test should be used for more fine-grained test cases, while dash0_webhook_test.go should // be used to verify external effects (recording events etc.) that cannot be covered in this test. +var ( + versions = Versions{ + OperatorVersion: "1.2.3", + InitContainerImageVersion: "4.5.6", + } +) + var _ = Describe("Dash0 Resource Modification", func() { ctx := context.Background() @@ -25,7 +32,12 @@ var _ = Describe("Dash0 Resource Modification", func() { Context("when mutating new deployments", func() { It("should inject Dash into a new basic deployment", func() { deployment := BasicDeployment(TestNamespaceName, DeploymentName) - result := ModifyPodSpec(&deployment.Spec.Template.Spec, TestNamespaceName, log.FromContext(ctx)) + result := ModifyDeployment( + deployment, + TestNamespaceName, + versions, + log.FromContext(ctx), + ) Expect(result).To(BeTrue()) VerifyModifiedDeployment(deployment, DeploymentExpectations{ @@ -46,7 +58,12 @@ var _ = Describe("Dash0 Resource Modification", func() { It("should inject Dash into a new deployment that has multiple Containers, and already has Volumes and init Containers", func() { deployment := DeploymentWithMoreBellsAndWhistles(TestNamespaceName, DeploymentName) - result := ModifyPodSpec(&deployment.Spec.Template.Spec, TestNamespaceName, log.FromContext(ctx)) + result := ModifyDeployment( + deployment, + TestNamespaceName, + versions, + log.FromContext(ctx), + ) Expect(result).To(BeTrue()) VerifyModifiedDeployment(deployment, DeploymentExpectations{ @@ -77,7 +94,12 @@ var _ = Describe("Dash0 Resource Modification", func() { It("should update existing Dash artifacts in a new deployment", func() { deployment := DeploymentWithExistingDash0Artifacts(TestNamespaceName, DeploymentName) - result := ModifyPodSpec(&deployment.Spec.Template.Spec, TestNamespaceName, log.FromContext(ctx)) + result := ModifyDeployment( + deployment, + TestNamespaceName, + versions, + log.FromContext(ctx), + ) Expect(result).To(BeTrue()) VerifyModifiedDeployment(deployment, DeploymentExpectations{ diff --git a/internal/webhook/dash0_webhook.go b/internal/webhook/dash0_webhook.go index 4c3dc09d..25cca3d6 100644 --- a/internal/webhook/dash0_webhook.go +++ b/internal/webhook/dash0_webhook.go @@ -27,6 +27,7 @@ var ( type Handler struct { Recorder record.EventRecorder + Versions k8sresources.Versions } func (h *Handler) SetupWebhookWithManager(mgr ctrl.Manager) error { @@ -59,7 +60,7 @@ func (h *Handler) Handle(_ context.Context, request admission.Request) admission return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error while parsing the resource: %w", err)) } - hasBeenModified := k8sresources.ModifyPodSpec(&deployment.Spec.Template.Spec, request.Namespace, logger) + hasBeenModified := k8sresources.ModifyDeployment(deployment, request.Namespace, h.Versions, logger) if !hasBeenModified { return admission.Allowed("no changes") } diff --git a/internal/webhook/webhook_suite_test.go b/internal/webhook/webhook_suite_test.go index 4aeeeb1a..e9b17b88 100644 --- a/internal/webhook/webhook_suite_test.go +++ b/internal/webhook/webhook_suite_test.go @@ -19,6 +19,7 @@ import ( . "github.com/onsi/gomega" operatorv1alpha1 "github.com/dash0hq/dash0-operator/api/v1alpha1" + "github.com/dash0hq/dash0-operator/internal/k8sresources" admissionv1 "k8s.io/api/admission/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" @@ -40,6 +41,11 @@ var ( testEnv *envtest.Environment ctx context.Context cancel context.CancelFunc + + versions = k8sresources.Versions{ + OperatorVersion: "1.2.3", + InitContainerImageVersion: "4.5.6", + } ) func TestWebhook(t *testing.T) { @@ -115,6 +121,7 @@ var _ = BeforeSuite(func() { err = (&Handler{ Recorder: mgr.GetEventRecorderFor("dash0-webhook"), + Versions: versions, }).SetupWebhookWithManager(mgr) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/e2e_helpers.go b/test/e2e/e2e_helpers.go index 0f268984..b536e43a 100644 --- a/test/e2e/e2e_helpers.go +++ b/test/e2e/e2e_helpers.go @@ -290,14 +290,15 @@ func UninstallNodeJsDeployment(namespace string) error { )) } -func SendRequestsAndVerifySpansHaveBeenProduced() { +func SendRequestsAndVerifySpansHaveBeenProduced(namespace string) { timestampLowerBound := time.Now() By("verify that the resource has been instrumented and is sending telemetry", func() { Eventually(func(g Gomega) { - output, err := Run(exec.Command("curl", "http://localhost:1207/ohai"), false) + verifyLabels(g, namespace) + response, err := Run(exec.Command("curl", "http://localhost:1207/ohai"), false) g.ExpectWithOffset(1, err).NotTo(HaveOccurred()) - g.ExpectWithOffset(1, string(output)).To(ContainSubstring( + g.ExpectWithOffset(1, string(response)).To(ContainSubstring( "We make Observability easy for every developer.")) fileHandle, err := os.Open("e2e-test-received-data/traces.jsonl") g.ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -326,6 +327,30 @@ func SendRequestsAndVerifySpansHaveBeenProduced() { }) } +func verifyLabels(g Gomega, namespace string) { + instrumented := readLabel(g, namespace, "dash0.instrumented") + g.ExpectWithOffset(1, instrumented).To(Equal("true")) + operatorVersion := readLabel(g, namespace, "dash0.operator.version") + g.ExpectWithOffset(1, operatorVersion).To(MatchRegexp("\\d+\\.\\d+\\.\\d+")) + initContainerImageVersion := readLabel(g, namespace, "dash0.initcontainer.image.version") + g.ExpectWithOffset(1, initContainerImageVersion).To(MatchRegexp("\\d+\\.\\d+\\.\\d+")) +} + +func readLabel(g Gomega, namespace string, labelKey string) string { + labelValue, err := Run(exec.Command( + "kubectl", + "get", + "deployment", + "--namespace", + namespace, + "dash0-operator-nodejs-20-express-test-deployment", + "-o", + fmt.Sprintf("jsonpath={.metadata.labels['%s']}", strings.ReplaceAll(labelKey, ".", "\\.")), + ), false) + g.ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return string(labelValue) +} + func hasMatchingSpans(traces ptrace.Traces, timestampLowerBound time.Time, matchFn func(span ptrace.Span) bool) bool { for i := 0; i < traces.ResourceSpans().Len(); i++ { resourceSpan := traces.ResourceSpans().At(i) @@ -362,11 +387,17 @@ func RunAndIgnoreOutput(cmd *exec.Cmd, logCommandArgs ...bool) error { // Run executes the provided command within this context func Run(cmd *exec.Cmd, logCommandArgs ...bool) ([]byte, error) { var logCommand bool - if len(logCommandArgs) > 0 { + var alwaysLogOutput bool + if len(logCommandArgs) >= 1 { logCommand = logCommandArgs[0] } else { logCommand = true } + if len(logCommandArgs) >= 2 { + alwaysLogOutput = logCommandArgs[1] + } else { + alwaysLogOutput = false + } dir, _ := GetProjectDir() cmd.Dir = dir @@ -381,6 +412,9 @@ func Run(cmd *exec.Cmd, logCommandArgs ...bool) ([]byte, error) { fmt.Fprintf(GinkgoWriter, "running: %s\n", command) } output, err := cmd.CombinedOutput() + if alwaysLogOutput { + fmt.Fprintf(GinkgoWriter, "output: %s\n", string(output)) + } if err != nil { return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 348862ef..c0b4af13 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -140,7 +140,7 @@ var _ = Describe("Dash0 Kubernetes Operator", Ordered, func() { By("installing the Node.js deployment") Expect(InstallNodeJsDeployment(applicationUnderTestNamespace)).To(Succeed()) By("verifying that the Node.js deployment has been instrumented by the webhook") - SendRequestsAndVerifySpansHaveBeenProduced() + SendRequestsAndVerifySpansHaveBeenProduced(applicationUnderTestNamespace) }) }) @@ -156,7 +156,7 @@ var _ = Describe("Dash0 Kubernetes Operator", Ordered, func() { DeployOperator(operatorNamespace, operatorImage) DeployDash0Resource(applicationUnderTestNamespace) By("verifying that the Node.js deployment has been instrumented by the controller") - SendRequestsAndVerifySpansHaveBeenProduced() + SendRequestsAndVerifySpansHaveBeenProduced(applicationUnderTestNamespace) }) }) }) diff --git a/test/util/verification.go b/test/util/verification.go index e36c8b67..1d43f0ac 100644 --- a/test/util/verification.go +++ b/test/util/verification.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type ContainerExpectations struct { @@ -94,4 +95,13 @@ func VerifyModifiedDeployment(deployment *appsv1.Deployment, expectations Deploy } } } + + verifyLabels(deployment.Spec.Template.ObjectMeta) + verifyLabels(deployment.ObjectMeta) +} + +func verifyLabels(meta metav1.ObjectMeta) { + Expect(meta.Labels["dash0.instrumented"]).To(Equal("true")) + Expect(meta.Labels["dash0.operator.version"]).To(Equal("1.2.3")) + Expect(meta.Labels["dash0.initcontainer.image.version"]).To(Equal("4.5.6")) }