From aa4702b6a7b8c9b10dc36e45a64d60c3e8e27957 Mon Sep 17 00:00:00 2001
From: Trayan Azarov <tazarov@users.noreply.github.com>
Date: Wed, 29 Nov 2023 21:04:36 +0200
Subject: [PATCH] [ENH]: Local Observability Stack with OTEL and Zipkin (#1279)

## Description of changes

*Summarize the changes made by this PR.*
 - New functionality
- Added the possibility to run the observability stack (OTEL collector +
Zipkin) alongside Chroma server in a docker-compose

## Test plan
*How are these changes tested?*

- [x] Manual testing

## Documentation Changes

Observability section on chroma docs updated -
https://github.com/chroma-core/docs/pull/149
---
 examples/observability/README.md              | 10 ++++
 .../docker-compose.local-observability.yml    | 52 +++++++++++++++++++
 .../observability/otel-collector-config.yaml  | 16 ++++++
 3 files changed, 78 insertions(+)
 create mode 100644 examples/observability/README.md
 create mode 100644 examples/observability/docker-compose.local-observability.yml
 create mode 100644 examples/observability/otel-collector-config.yaml

diff --git a/examples/observability/README.md b/examples/observability/README.md
new file mode 100644
index 00000000000..d81f8911484
--- /dev/null
+++ b/examples/observability/README.md
@@ -0,0 +1,10 @@
+# Observability
+
+## Local Observability Stack
+
+To run the Chroma with local observability stack (OpenTelemetry + Zipkin),
+run the following command from the root of the repository:
+
+```bash
+docker compose -f examples/observability/docker-compose.local-observability.yml up --build -d
+```
diff --git a/examples/observability/docker-compose.local-observability.yml b/examples/observability/docker-compose.local-observability.yml
new file mode 100644
index 00000000000..173d8875a9f
--- /dev/null
+++ b/examples/observability/docker-compose.local-observability.yml
@@ -0,0 +1,52 @@
+version: '3.9'
+networks:
+  net:
+
+services:
+  zipkin:
+    image: openzipkin/zipkin
+    ports:
+      - "9411:9411" # you can access Zipkin UI at http://localhost:9411
+    depends_on: [otel-collector]
+    networks:
+      - net
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ${PWD}/examples/observability/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317"  # OTLP
+      - "55681:55681" # Legacy
+    networks:
+      - net
+  server:
+    image: server
+    build:
+      context: ${PWD}
+      dockerfile: Dockerfile
+    volumes:
+      - ${PWD}/:/chroma
+      # Be aware that indexed data are located in "/chroma/chroma/"
+      # Default configuration for persist_directory in chromadb/config.py
+    command: uvicorn chromadb.app:app --reload --workers 1 --host 0.0.0.0 --port 8000 --log-config chromadb/log_config.yml
+    environment:
+      - IS_PERSISTENT=TRUE
+      - CHROMA_SERVER_AUTH_PROVIDER=${CHROMA_SERVER_AUTH_PROVIDER}
+      - CHROMA_SERVER_AUTH_CREDENTIALS_FILE=${CHROMA_SERVER_AUTH_CREDENTIALS_FILE}
+      - CHROMA_SERVER_AUTH_CREDENTIALS=${CHROMA_SERVER_AUTH_CREDENTIALS}
+      - CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER=${CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER}
+      - PERSIST_DIRECTORY=${PERSIST_DIRECTORY:-/chroma/chroma}
+      - CHROMA_OTEL_COLLECTION_ENDPOINT=http://otel-collector:4317/
+      - CHROMA_OTEL_EXPORTER_HEADERS=${CHROMA_OTEL_EXPORTER_HEADERS}
+      - CHROMA_OTEL_SERVICE_NAME=${CHROMA_OTEL_SERVICE_NAME:-chroma}
+      - CHROMA_OTEL_GRANULARITY=${CHROMA_OTEL_GRANULARITY:-all}
+    ports:
+      - 8000:8000
+    depends_on: [otel-collector]
+    networks:
+      - net
+
+volumes:
+  backups:
+    driver: local
diff --git a/examples/observability/otel-collector-config.yaml b/examples/observability/otel-collector-config.yaml
new file mode 100644
index 00000000000..cce02b6bf5c
--- /dev/null
+++ b/examples/observability/otel-collector-config.yaml
@@ -0,0 +1,16 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+exporters:
+  logging:
+  zipkin:
+    endpoint: "http://zipkin:9411/api/v2/spans"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [zipkin]