add example for OTLP logging via stdout and k8s (#547)

Co-authored-by: Cyrille Le Clerc <[email protected]>
Co-authored-by: Trask Stalnaker <[email protected]>

.github/renovate.json5

@@ -35,5 +35,18 @@
       "matchPackageNames": ["eclipse-temurin"],
       "enabled": false
     }
+  ],
+  "customManagers": [
+    {
+      "customType": "regex",
+      "description": "Update _VERSION variables in Dockerfiles",
+      "fileMatch": [
+        "(^|/|\\.)Dockerfile$",
+        "(^|/)Dockerfile\\.[^/]*$"
+      ],
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>[a-z-]+?)(?: depName=(?<depName>.+?))? packageName=(?<packageName>.+?)(?: versioning=(?<versioning>[a-z-]+?))?\\s(?:ENV|ARG) .+?_VERSION=(?<currentValue>.+?)\\s"
+      ]
+    }
   ]
 }

.github/scripts/run-oats-tests.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+pushd logging-k8s-stdout-otlp-json
+../gradlew assemble
+popd
+
+wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+
+cd oats/yaml
+go install github.com/onsi/ginkgo/v2/ginkgo@latest
+export TESTCASE_TIMEOUT=5m
+export TESTCASE_BASE_PATH=../..
+ginkgo -r

.github/workflows/oats-tests.yml

@@ -0,0 +1,51 @@
+name: OATS Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/workflows/oats-tests.yml
+      - 'logging-k8s-stdout-otlp-json/**'
+  workflow_dispatch:
+
+jobs:
+  acceptance-tests:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+
+      - name: Set up JDK for running Gradle
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Set up gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          cache-read-only: ${{ github.event_name == 'pull_request' }}
+
+      - name: Check out oats
+        uses: actions/checkout@v4
+        with:
+          repository: grafana/oats
+          ref: v0.1.0
+          path: oats
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: oats/go.sum
+
+      - name: Run acceptance tests
+        run: .github/scripts/run-oats-tests.sh
+
+      - name: upload log file
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: OATS logs
+          path: oats/yaml/build/**/*.log

logging-k8s-stdout-otlp-json/Dockerfile

@@ -0,0 +1,13 @@
+FROM eclipse-temurin:21-jre
+
+WORKDIR /usr/src/app/
+
+# renovate: datasource=github-releases depName=opentelemetry-java-instrumentation packageName=open-telemetry/opentelemetry-java-instrumentation
+ENV OPENTELEMETRY_JAVA_INSTRUMENTATION_VERSION=v2.10.0
+
+ADD build/libs/*SNAPSHOT.jar ./app.jar
+ADD --chmod=644 https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/$OPENTELEMETRY_JAVA_INSTRUMENTATION_VERSION/opentelemetry-javaagent.jar ./opentelemetry-javaagent.jar
+ENV JAVA_TOOL_OPTIONS=-javaagent:./opentelemetry-javaagent.jar
+
+EXPOSE 8080
+ENTRYPOINT [ "java", "-jar", "./app.jar" ]

logging-k8s-stdout-otlp-json/README.md

@@ -0,0 +1,43 @@
+# Exporting Application logs using JSON logging in Kubernetes
+
+If you want to get logs from your Java application ingested into an
+OpenTelemetry-compatible logs backend, the easiest and recommended way is using
+an OpenTelemetry protocol (OTLP) exporter,
+which is explained in the [logging](../logging) example. 
+
+However, some scenarios require logs
+to be output to files or stdout due to organizational or reliability needs.
+Refer to [Collecting OpenTelemetry-compliant Java logs from files](https://opentelemetry.io/blog/2024/collecting-otel-compliant-java-logs-from-files/) for more details.
+
+This example contains
+
+- a Java application that uses the experimental 
+  [experimental-otlp/stdout](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#in-development-exporter-selection) logs exporter
+- an OpenTelemetry collector configuration that uses the 
+  [OTLP/JSON connector](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector/otlpjsonconnector) to turn Pod logs into `OTLP`
+
+## Architecture
+
+![OTLP/JSON Architecture](otlpjson-architecture.png)
+
+The OpenTelemetry Collector pipeline:
+
+![OpenTelemetry Collector Pipeline](otel-collector-otlpjson-pipeline.png)
+
+## Getting Started
+
+The k8s directory contains the Kubernetes manifests to deploy the application and the collector.
+
+Ignore the `lgtm.yaml` file, which is only used for running locally with 
+[LGTM](https://github.com/grafana/docker-otel-lgtm/) 
+and automated testing using [OATs](https://github.com/grafana/oats).
+
+## Running locally
+
+You can run the application locally using the following steps:
+
+1. Run [k3d.sh](./k3d.sh) to start a local Kubernetes cluster with all the necessary components.
+2. Generate traffic using [generate-traffic.sh](./generate-traffic.sh)
+3. Log in to [http://localhost:3000](http://localhost:3000)
+4. Go to "Explore"
+5. Select "Loki" as data source to view the logs

logging-k8s-stdout-otlp-json/build.gradle.kts

@@ -0,0 +1,24 @@
+import org.springframework.boot.gradle.plugin.SpringBootPlugin
+import org.springframework.boot.gradle.tasks.bundling.BootJar
+
+plugins {
+    id("java")
+    id("org.springframework.boot") version "3.4.0"
+}
+
+description = "OpenTelemetry Example for Java Agent with Stdout logging"
+val moduleName by extra { "io.opentelemetry.examples.javagent.logging-k8s-stdout-otlp-json" }
+
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(17))
+    }
+}
+
+dependencies {
+    implementation(platform(SpringBootPlugin.BOM_COORDINATES))
+
+    implementation("org.springframework.boot:spring-boot-starter-web")
+    implementation("org.springframework.boot:spring-boot-starter-actuator")
+}
+

logging-k8s-stdout-otlp-json/generate-traffic.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+watch 'curl -s http://localhost:8080/rolldice'

logging-k8s-stdout-otlp-json/k3d.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+docker build -f Dockerfile -t "dice:1.1-SNAPSHOT" .
+k3d cluster create jsonlogging || k3d cluster start jsonlogging
+k3d image import -c jsonlogging dice:1.1-SNAPSHOT
+
+kubectl apply -f k8s/
+
+kubectl wait --for=condition=ready pod -l app=dice
+kubectl wait --for=condition=ready --timeout=5m pod -l app=lgtm
+
+kubectl port-forward service/dice 8080:8080 &
+kubectl port-forward service/lgtm 3000:3000 &

logging-k8s-stdout-otlp-json/k8s/collector-configmap.yaml

@@ -0,0 +1,75 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-config
+data:
+  otel-collector-config.yaml: |-
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+      prometheus/collector: # needed if you use the docker-lgtm image
+        config:
+          scrape_configs:
+            - job_name: 'opentelemetry-collector'
+              static_configs:
+                - targets: [ 'localhost:8888' ]
+      filelog/otlp-json-logs:
+        include:
+          - /var/log/pods/*/*/*.log
+        include_file_path: true
+        operators:
+          - id: container-parser
+            type: container
+    
+    processors:
+      batch:
+      resourcedetection:
+        detectors: [ "env", "system" ]
+        override: false
+
+      k8sattributes:
+      # Config details in 
+      # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor
+    connectors:
+      otlpjson:
+
+    exporters:
+      otlphttp/metrics:
+        endpoint: http://localhost:9090/api/v1/otlp
+      otlphttp/traces:
+        endpoint: http://localhost:4418
+      otlphttp/logs:
+        endpoint: http://localhost:3100/otlp
+      debug/metrics:
+        verbosity: detailed
+      debug/traces:
+        verbosity: detailed
+      debug/logs:
+        verbosity: detailed
+      nop:
+    
+    service:
+      pipelines:
+        traces:
+          receivers: [ otlp ]
+          processors: [ k8sattributes, resourcedetection, batch ]
+          exporters: [ otlphttp/traces ]
+        metrics:
+          receivers: [ otlp, prometheus/collector ]
+          processors: [ k8sattributes, resourcedetection, batch ]
+          exporters: [ otlphttp/metrics ]
+        logs/raw_otlpjson:
+          receivers: [ filelog/otlp-json-logs ]
+          # No need for processors before the otlpjson connector
+          # Declare processors in the shared "logs" pipeline below
+          processors: [ ]
+          exporters: [ otlpjson ]
+        logs/otlp:
+          receivers: [ otlp, otlpjson ]
+          processors: [ k8sattributes, resourcedetection, batch ]
+          exporters: [ otlphttp/logs ]
+          # exporters: [ otlphttp/logs, debug/logs ]  # Uncomment this line to enable debug logging

logging-k8s-stdout-otlp-json/k8s/dice.yaml

@@ -0,0 +1,48 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: dice
+spec:
+  selector:
+    app: dice
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dice
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dice
+  template:
+    metadata:
+      labels:
+        app: dice
+    spec:
+      containers:
+      - name: dice
+        image: dice:1.1-SNAPSHOT
+        imagePullPolicy: Never
+        ports:
+        - containerPort: 8080
+        env:
+          - name: OTEL_EXPORTER_OTLP_ENDPOINT
+            value: "http://lgtm:4318"
+          - name: OTEL_LOGS_EXPORTER
+            value: "experimental-otlp/stdout"
+          - name: OTEL_RESOURCE_ATTRIBUTES
+            value: service.name=dice,service.namespace=shop,service.version=1.1,deployment.environment=staging
+          - name: OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_LOG_ATTRIBUTES
+            value: "true"
+          - name: OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_CAPTURE_KEY_VALUE_PAIR_ATTRIBUTES
+            value: "true"
+          - name: OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_CAPTURE_MDC_ATTRIBUTES
+            value: "true"
+          - name: SERVICE_NAME
+            value: dice
+