diff --git a/Makefile b/Makefile index 10ffec9162d..29cb1f06106 100644 --- a/Makefile +++ b/Makefile @@ -513,6 +513,13 @@ stop-dependencies-cdc: start: start-sqlite +start-otel: + @OTEL_TRACES_EXPORTER=otlp OTEL_EXPORTER_OTLP_TRACES_INSECURE=true TEMPORAL_OTEL_DEBUG=true make start-sqlite + +start-grafana-tempo: + docker compose $(DOCKER_COMPOSE_FILES) up -d tempo + printf $(COLOR) "Grafana Tempo available at http://localhost:3000" + start-cass-es: temporal-server ./temporal-server --env development-cass-es --allow-no-auth start diff --git a/develop/docker-compose/docker-compose.darwin.yml b/develop/docker-compose/docker-compose.darwin.yml index 62e6e1c519c..5fdf95b1249 100644 --- a/develop/docker-compose/docker-compose.darwin.yml +++ b/develop/docker-compose/docker-compose.darwin.yml @@ -14,6 +14,14 @@ services: - "3000:3000" networks: - temporal-dev-network + tempo: + depends_on: + - grafana + ports: + - "3200:3200" # tempo + - "4317:4317" # otlp grpc + networks: + - temporal-dev-network temporal-ui: environment: - TEMPORAL_ADDRESS=host.docker.internal:7233 diff --git a/develop/docker-compose/docker-compose.jaeger.yml b/develop/docker-compose/docker-compose.jaeger.yml deleted file mode 100644 index 9f64337d86b..00000000000 --- a/develop/docker-compose/docker-compose.jaeger.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Include this file to run Jaeger as one more dependency: -# docker-compose -f docker-compose.yml -f docker-compose.linux.yml -f docker-compose.jaeger.yml up -version: "3.5" - -services: - jaeger: - image: jaegertracing/all-in-one:latest - container_name: temporal-dev-jaeger - ports: - - "6831:6831/udp" - - "16686:16686" - networks: - - temporal-dev-network diff --git a/develop/docker-compose/docker-compose.linux.yml b/develop/docker-compose/docker-compose.linux.yml index 9437e5945ca..6356d56a578 100644 --- a/develop/docker-compose/docker-compose.linux.yml +++ b/develop/docker-compose/docker-compose.linux.yml @@ -8,6 +8,10 @@ services: - ./prometheus-linux:/etc/prometheus grafana: network_mode: host + tempo: + depends_on: + - grafana + network_mode: host temporal-ui: environment: - TEMPORAL_ADDRESS=localhost:7233 diff --git a/develop/docker-compose/docker-compose.windows.yml b/develop/docker-compose/docker-compose.windows.yml index 4bdd31707de..14383a474c8 100644 --- a/develop/docker-compose/docker-compose.windows.yml +++ b/develop/docker-compose/docker-compose.windows.yml @@ -14,6 +14,14 @@ services: - "3000:3000" networks: - temporal-dev-network + tempo: + depends_on: + - grafana + ports: + - "3200:3200" # tempo + - "4317:4317" # otlp grpc + networks: + - temporal-dev-network temporal-ui: environment: - TEMPORAL_ADDRESS=host.docker.internal:7233 diff --git a/develop/docker-compose/docker-compose.yml b/develop/docker-compose/docker-compose.yml index 8296a4779f1..f3f3b7bb560 100644 --- a/develop/docker-compose/docker-compose.yml +++ b/develop/docker-compose/docker-compose.yml @@ -77,6 +77,13 @@ services: - prometheus volumes: - ./grafana/provisioning/:/etc/grafana/provisioning/ + tempo: + image: grafana/tempo:2.4.2 + container_name: temporal-dev-grafana-tempo + command: + - "-config.file=/etc/tempo.yaml" + volumes: + - ./grafana/provisioning/tempo/tempo.yaml:/etc/tempo.yaml temporal-ui: image: temporalio/ui:2.26.2 container_name: temporal-dev-ui diff --git a/develop/docker-compose/grafana/provisioning/datasources/prometheus.yml b/develop/docker-compose/grafana/provisioning/datasources/prometheus.yml index bf84d927105..71f7ad28ad1 100644 --- a/develop/docker-compose/grafana/provisioning/datasources/prometheus.yml +++ b/develop/docker-compose/grafana/provisioning/datasources/prometheus.yml @@ -9,4 +9,14 @@ datasources: type: prometheus url: http://prometheus:9090 access: proxy + isDefault: false +- name: Tempo-Linux + type: tempo + access: proxy + url: http://localhost:3200 + isDefault: false +- name: Tempo-Darwin + type: tempo + access: proxy + url: http://tempo:3200 isDefault: false \ No newline at end of file diff --git a/develop/docker-compose/grafana/provisioning/tempo/tempo.yaml b/develop/docker-compose/grafana/provisioning/tempo/tempo.yaml new file mode 100644 index 00000000000..fe9ce4124d4 --- /dev/null +++ b/develop/docker-compose/grafana/provisioning/tempo/tempo.yaml @@ -0,0 +1,59 @@ +stream_over_http_enabled: true +server: + http_listen_port: 3200 + log_level: info + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s + +distributor: + receivers: # this configuration will listen on all ports and protocols that tempo is capable of. + jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can + protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver + thrift_http: # + grpc: # for a production deployment you should only enable the receivers you need! + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + grpc: + opencensus: + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + traces_storage: + path: /var/tempo/generator/traces + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /var/tempo/wal # where to store the the wal locally + local: + path: /var/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator \ No newline at end of file diff --git a/develop/docker-compose/prometheus-darwin/prometheus.yml b/develop/docker-compose/prometheus-darwin/prometheus.yml index 04f5a07764b..6905f237553 100644 --- a/develop/docker-compose/prometheus-darwin/prometheus.yml +++ b/develop/docker-compose/prometheus-darwin/prometheus.yml @@ -10,3 +10,7 @@ scrape_configs: - targets: # port should match value from server config listenAddress. - host.docker.internal:8000 + - job_name: tempo + static_configs: + - targets: + - host.docker.internal:3200 \ No newline at end of file diff --git a/develop/docker-compose/prometheus-linux/prometheus.yml b/develop/docker-compose/prometheus-linux/prometheus.yml index 94e41f06239..96425b71022 100644 --- a/develop/docker-compose/prometheus-linux/prometheus.yml +++ b/develop/docker-compose/prometheus-linux/prometheus.yml @@ -10,3 +10,7 @@ scrape_configs: - targets: # port should match value from server config listenAddress. - localhost:8000 + - job_name: tempo + static_configs: + - targets: + - localhost:3200 \ No newline at end of file diff --git a/docs/development/tracing.md b/docs/development/tracing.md index e53f38adba7..15edcfe3516 100644 --- a/docs/development/tracing.md +++ b/docs/development/tracing.md @@ -2,9 +2,8 @@ The Temporal server supports ability to configure OTEL trace exporters to support emitting spans and traces for observability. More specifically, the -server uses the [Go Open Telemetry -library](https://github.com/open-telemetry/opentelemetry-go) for instrumentation -and multi-protocol multi-model telemetry exporting. This document is intended to +server uses the [Go Open Telemetry library](https://github.com/open-telemetry/opentelemetry-go) +for instrumentation and multi-protocol multi-model telemetry exporting. This document is intended to help developers understand how to configure exporters and instrument their code. A full exploration of tracing and telemetry is out of scope of this document and the reader is referred to [external reference @@ -15,16 +14,9 @@ itself](https://github.com/open-telemetry/opentelemetry-specification/blob/main/ ## Quickstart -1. Run [`jaeger-all-in-one`](https://www.jaegertracing.io/download/) (either from a binary, or using the docker image). -2. View the Jaeger UI at http://localhost:16686/search -3. Set these environment variables: - ``` - export OTEL_TRACES_EXPORTER=otlp - export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true - export TEMPORAL_OTEL_DEBUG=true - ``` -4. Start the server using `make start` -5. Refresh the Jaeger UI. You should see traces from Temporal services. +1. Run `make start-grafana-tempo` (or use a different OTEL tool like Jaeger) +2. Start the server using `make start-otel` +3. Visit http://localhost:3000/explore and select "Tempo" from the datasource dropdown. ## Configuring