add OT tracing config in helm chart

Signed-off-by: Akihiko Kuroda <[email protected]>
Qiskit · Aug 9, 2023 · 807e9bd · 807e9bd
1 parent 162b31f
commit 807e9bd
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 12 deletions.
diff --git a/client/quantum_serverless/core/constants.py b/client/quantum_serverless/core/constants.py
@@ -19,6 +19,7 @@
 OT_JAEGER_PORT_KEY = "OT_JAEGER_PORT_KEY"
 OT_TRACEPARENT_ID_KEY = "OT_TRACEPARENT_ID_KEY"
 OT_INSECURE = "OT_INSECURE"
+OT_RAY_TRACER = "TO_RAY_TRACER"
 OT_SPAN_DEFAULT_NAME = "entrypoint"
 OT_ATTRIBUTE_PREFIX = "qs"
 OT_LABEL_CALL_LOCATION = "qs.location"

diff --git a/client/quantum_serverless/core/decorators.py b/client/quantum_serverless/core/decorators.py
@@ -35,6 +35,7 @@
 from typing import Optional, Dict, Any, Union, List, Callable, Sequence
 
 import ray
+from opentelemetry import trace
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 from qiskit import QuantumCircuit
 from ray.runtime_env import RuntimeEnv
@@ -44,6 +45,7 @@
     OT_JAEGER_HOST_KEY,
     OT_JAEGER_PORT_KEY,
     OT_TRACEPARENT_ID_KEY,
+    OT_RAY_TRACER,
 )
 from quantum_serverless.core.state import StateHandler
 from quantum_serverless.core.tracing import get_tracer, _trace_env_vars
@@ -195,11 +197,14 @@ def _tracible_function(
     def decorator(func: Callable):
         @functools.wraps(func)
         def wraps(*args, **kwargs):
-            tracer = get_tracer(
-                func.__module__,
-                agent_host=os.environ.get(OT_JAEGER_HOST_KEY, None),
-                agent_port=int(os.environ.get(OT_JAEGER_PORT_KEY, 6831)),
-            )
+            if bool(int(os.environ.get(OT_RAY_TRACER, "0"))):
+                tracer = trace.get_tracer(func.__module__)
+            else:
+                tracer = get_tracer(
+                    func.__module__,
+                    agent_host=os.environ.get(OT_JAEGER_HOST_KEY, None),
+                    agent_port=int(os.environ.get(OT_JAEGER_PORT_KEY, 6831)),
+                )
             ctx = TraceContextTextMapPropagator().extract(
                 {
                     TraceContextTextMapPropagator._TRACEPARENT_HEADER_NAME: trace_id  # pylint:disable=protected-access

diff --git a/client/quantum_serverless/core/tracing.py b/client/quantum_serverless/core/tracing.py
@@ -43,6 +43,7 @@
     OT_JAEGER_HOST_KEY,
     OT_JAEGER_PORT_KEY,
     OT_TRACEPARENT_ID_KEY,
+    OT_RAY_TRACER,
     OT_INSECURE,
     OT_SPAN_DEFAULT_NAME,
     OT_LABEL_CALL_LOCATION,
@@ -95,11 +96,14 @@ def _trace_env_vars(env_vars: dict, location: Optional[str] = None):
     Returns:
         dict of env variables
     """
-    tracer = get_tracer(
-        __name__,
-        agent_host=os.environ.get(OT_JAEGER_HOST_KEY, None),
-        agent_port=int(os.environ.get(OT_JAEGER_PORT_KEY, 6831)),
-    )
+    if bool(int(os.environ.get(OT_RAY_TRACER, "0"))):
+        tracer = trace.get_tracer("Quantum-Serverless")
+    else:
+        tracer = get_tracer(
+            __name__,
+            agent_host=os.environ.get(OT_JAEGER_HOST_KEY, None),
+            agent_port=int(os.environ.get(OT_JAEGER_PORT_KEY, 6831)),
+        )
     if env_vars.get(OT_TRACEPARENT_ID_KEY, None) is not None:
         env_vars[OT_TRACEPARENT_ID_KEY] = env_vars.get(OT_TRACEPARENT_ID_KEY)
     elif os.environ.get(OT_TRACEPARENT_ID_KEY) is not None:
@@ -115,8 +119,9 @@ def _trace_env_vars(env_vars: dict, location: Optional[str] = None):
         traceparent = carrier.get(
             TraceContextTextMapPropagator._TRACEPARENT_HEADER_NAME  # pylint:disable=protected-access
         )
-        env_vars[OT_TRACEPARENT_ID_KEY] = traceparent
-        os.environ[OT_TRACEPARENT_ID_KEY] = traceparent
+        if traceparent:
+            env_vars[OT_TRACEPARENT_ID_KEY] = traceparent
+            os.environ[OT_TRACEPARENT_ID_KEY] = traceparent
     return env_vars
 
 

diff --git a/infrastructure/helm/quantum-serverless/charts/gateway/templates/rayclustertemplate.yaml b/infrastructure/helm/quantum-serverless/charts/gateway/templates/rayclustertemplate.yaml
@@ -123,6 +123,18 @@ data:
                   value: "/etc/ray/tls/tls.key"
                 - name: RAY_TLS_CA_CERT
                   value: "/etc/ca/tls/ca.crt"
+{{- end }}
+{{- if .Values.application.ray.openTelemetry }}
+                - name: OT_RAY_TRACER
+                  value: "1"
+{{- end }}
+{{- if .Values.application.ray.openTelemetryCollector.local }}
+                - name: OT_JAEGER_HOST_KEY
+                  value: {{ .Values.application.ray.openTelemetryCollector.host }}
+                - name: OT_JAEGER_PORT_KEY
+                  value: {{ .Values.application.ray.openTelemetryCollector.port | quote }}
+                - name: OT_INSECURE
+                  value: {{ .Values.application.ray.openTelemetryCollector.insecure | quote }}
 {{- end }}
             - image: fluent/fluent-bit:1.9.10
               name: ray-head-logs
@@ -267,6 +279,18 @@ data:
                   value: "/etc/ray/tls/tls.key"
                 - name: RAY_TLS_CA_CERT
                   value: "/etc/ca/tls/ca.crt"
+{{- end }}
+{{- if .Values.application.ray.openTelemetry }}
+                - name: OT_RAY_TRACER
+                  value: "1"
+{{- end }}
+{{- if .Values.application.ray.openTelemetryCollector.local }}
+                - name: OT_JAEGER_HOST_KEY
+                  value: {{ .Values.application.ray.openTelemetryCollector.host }}
+                - name: OT_JAEGER_PORT_KEY
+                  value: {{ .Values.application.ray.openTelemetryCollector.port | quote }}
+                - name: OT_INSECURE
+                  value: {{ .Values.application.ray.openTelemetryCollector.insecure | quote }}
 {{- end }}
               image: {{ .Values.application.ray.nodeImage | quote}}
               imagePullPolicy: IfNotPresent

diff --git a/infrastructure/helm/quantum-serverless/charts/gateway/values.yaml b/infrastructure/helm/quantum-serverless/charts/gateway/values.yaml
@@ -28,6 +28,12 @@ application:
     kubectlImage: alpine/k8s:1.27.4
     scrapeWithPrometheus: true
     openTelemetry: false
+    openTelemetryCollector:
+      local: false
+      host: "http://otel-collector"
+      port: 4317
+      insecure: 1
+
   limits:
     maxJobsPerUser: 2
     maxComputeResources: 4