Generate spans within our validation lambdas (#326)

* feat: collect telemetry from collector validation lambdas

* feat: move to extensions instead of layers for support of container functions

* chore: revert otelbin-validation-image changes now that auto instrumentation is not an option

* feat: hook up JS instrumentation inside image

* fix: collector config deserialzation

* chore: remove wrapper script

* chore: trigger CI

* chore: avoid port conflicts

* fix: move to bundled logging exporter

* feat: set up lambda wrapper properly

* chore: enable debug logging

* chore: avoid minification for more helpful stack traces

* feat: specify the handler

* chore: remove debug logging

* improve: review

.github/workflows/ci.yaml

@@ -151,6 +151,7 @@ jobs:
           CDK_DEPLOY_REGION: 'us-east-2'
           GH_TOKEN: ${{ github.token }}
           TEST_ENVIRONMENT_NAME: ${{ steps.get_test_env_name.outputs.test_env_name }}
+          DASH0_AUTHORIZATION_TOKEN: ${{ secrets.DASH0_AUTHORIZATION_TOKEN }}
         run: |
           npx projen deploy --require-approval never --outputs-file ./cdk-outputs.json
 

packages/otelbin-validation-image/Dockerfile

@@ -26,12 +26,20 @@ RUN /usr/app/rpm_download.sh
 
 FROM public.ecr.aws/lambda/nodejs:18 AS builder
 
-WORKDIR /usr/app    
+WORKDIR /usr/app
 COPY package.json package.json
 COPY src/ src/
 RUN npm install
 RUN npm run build
 
+FROM golang:1.22 AS builder-extension
+
+RUN apt-get update && apt-get install -y zip make
+RUN wget https://github.com/open-telemetry/opentelemetry-lambda/archive/refs/tags/layer-collector/0.6.0.zip
+RUN unzip 0.6.0.zip
+WORKDIR /go/opentelemetry-lambda-layer-collector-0.6.0/collector
+RUN make package
+
 FROM public.ecr.aws/lambda/nodejs:18
 
 # Install useradd command needed by the OtelContrib RPM
@@ -40,6 +48,7 @@ RUN yum -y install python3 python3-pip shadow-utils
 
 WORKDIR /usr/otelcol
 COPY --from=downloader /download/otelcol.rpm otelcol.rpm
+COPY --from=builder-extension /go/opentelemetry-lambda-layer-collector-0.6.0/collector/build/extensions /opt/extensions
 
 # Install the collector's RPM package
 RUN rpm -i otelcol.rpm \
@@ -53,8 +62,22 @@ RUN rpm -i otelcol.rpm \
 
 # Add the event handler
 COPY --from=builder /usr/app/dist /usr/app/src
-COPY --from=builder /usr/app/package.json /usr/app/package.json
+COPY --from=builder /usr/app/package.json /usr/app/src/package.json
+
+# Copy the collector config so that telemetry data can be exported.
+# https://github.com/open-telemetry/opentelemetry-lambda/blob/layer-collector/0.6.0/collector/internal/collector/collector.go#L51-L54
+COPY collector.yaml /opt/collector-config/config.yaml
+
+# Set relevant environment variable to enable OpenTelemetry
+ENV OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4328
+ENV OTEL_PROPAGATORS="tracecontext,baggage"
+ENV OTEL_LAMBDA_DISABLE_AWS_CONTEXT_PROPAGATION=true
+# Fix "DEBUG Recording is off, propagating context in a non-recording span"
+ENV OTEL_TRACES_SAMPLER=always_on
+#ENV OTEL_LOG_LEVEL=DEBUG
+ENV NODE_OPTIONS="--require /usr/app/src/lambda-wrapper.js"
+ENV _HANDLER="index.handler"
 
 WORKDIR /usr/app/src
 
-CMD ["/usr/app/src/index.handler"]
+CMD ["/usr/app/src/index.handler"]

packages/otelbin-validation-image/collector.yaml

@@ -0,0 +1,30 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        # non-standard port to avoid port conflicts with the collectors we spawn for validation
+        endpoint: "localhost:4327"
+      http:
+        # non-standard port to avoid port conflicts with the collectors we spawn for validation
+        endpoint: "localhost:4328"
+
+exporters:
+  otlp:
+    endpoint: ingress.eu-west-1.aws.dash0.com:4317
+    headers:
+      Authorization: "Bearer ${DASH0_AUTHORIZATION_TOKEN}"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp]
+    metrics:
+      receivers: [otlp]
+      exporters: [otlp]
+    logs:
+      receivers: [otlp]
+      exporters: [otlp]
+  telemetry:
+    metrics:
+      address: localhost:8888