fix(sdk): manual reporting of llm spans (#1555)

packages/traceloop-sdk/tests/cassettes/test_manual/test_manual_report.yaml

@@ -0,0 +1,98 @@
+interactions:
+- request:
+    body: '{"messages": [{"role": "user", "content": "Tell me a joke about opentelemetry"}],
+      "model": "gpt-3.5-turbo"}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '107'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.35.13
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.35.13
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.9.5
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAA1RRy07DMBC85ysWn1vUB6XQCxISIB5HJF5CletsElPHa603hYD678hpaMXFh5md
+        8ezsTwagbK4WoEylxdTBDc/dw93zi71/vbotwtNN+Xy9fnl4XH9/3xdnp2qQFLT6QCN/qmNDdXAo
+        lvyONoxaMLmO55PxaD6azmcdUVOOLsnKIMPp8WwoDa9oOBpPZr2yImswqgW8ZQAAP92bMvocv9QC
+        RoM/pMYYdYlqsR8CUEwuIUrHaKNoL2pwIA15Qd/FfqpayG0OUiFQQC/osEbhFnLcoKOADCtGvYYm
+        wKeVKk1ahqBZPPIFXKLRTcQEt/CJjCCWMQcqwJDvvnYtCGtjfdmLcYPcQk0bPFJ9qu1+HUdlYFql
+        1X3j3B4vrLexWjLqSD5Fj0JhJ99mAO9dbc2/JlRgqoMshdbok+F4trNTh0MdyMlJTwqJdgd8ep71
+        +VRso2C9LKwvkQPbrsOUMttmvwAAAP//AwAebllYQgIAAA==
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8a3c07512da0135d-ATL
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Mon, 15 Jul 2024 19:06:15 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=14DjWb_t0PhGL5mOhK8gsqaD2anNOF1J7Y8Lo_SpKpw-1721070375-1.0.1.1-HZ1yyYErVn.USbzwQt76wp1v0Fpbz2MvF04IOMJMUI7ZFXPv0Np1tZ8z2AthYPyy1oxDYakl9du4ysPr.pp_jg;
+        path=/; expires=Mon, 15-Jul-24 19:36:15 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=KuBmiwwXOTWsR0nU52KjyIkpVEjiHsE8MSSzFnGTEv0-1721070375445-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - traceloop
+      openai-processing-ms:
+      - '381'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15552000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '5000'
+      x-ratelimit-limit-tokens:
+      - '160000'
+      x-ratelimit-remaining-requests:
+      - '4999'
+      x-ratelimit-remaining-tokens:
+      - '159974'
+      x-ratelimit-reset-requests:
+      - 12ms
+      x-ratelimit-reset-tokens:
+      - 9ms
+      x-request-id:
+      - req_7e9ec34ca2189a55d52eeb1828fcef25
+    status:
+      code: 200
+      message: OK
+version: 1

packages/traceloop-sdk/tests/cassettes/test_manual/test_resource_attributes.yaml

@@ -0,0 +1,98 @@
+interactions:
+- request:
+    body: '{"messages": [{"role": "user", "content": "Tell me a joke about opentelemetry"}],
+      "model": "gpt-3.5-turbo"}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '107'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.35.13
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.35.13
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.9.5
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAA1SRT2/CMAzF7/0UXi67AGphhcFlGmib2DhO2mGaUEhNG0jjKHE1EOK7Tyn/tIsP
+        7+fnPDuHBEDoQkxAqEqyqp3pjs3i/W3cX3zwc77D9fbVvZSfZj6bTeeboehEB602qPji6imqnUHW
+        ZE9YeZSMcWo26mfpKB3kjy2oqUATbaXj7qCXd7nxK+qmWT8/OyvSCoOYwHcCAHBoa8xoC9yJCaSd
+        i1JjCLJEMbk2AQhPJipChqADS8uic4OKLKNtY39Veyh0AVwhKCoQPK6bgMAEJcUagZOe908wRSUj
+        0xwt9p7hV1puW5FByaasGLQFaYEcWkaDNbLfA3vp7sT5/eM1uKHSeVrFJW1jzFVfa6tDtfQoA9kY
+        MjC5k/2YAPy0B2r+7Sycp9rxkmmLNg7M8tM4cfuSG+wPz5CJpbnpD1lyzifCPjDWy7W2JXrndXut
+        mDI5Jn8AAAD//wMA+iUaUiwCAAA=
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8a3c06ec694fada4-ATL
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Mon, 15 Jul 2024 19:05:59 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=7r92jUdUEA4wJGNCNqX1y_usNja6ZbX4SM4xdbD8r3E-1721070359-1.0.1.1-se82CXovc7ndM.lFT9BKWR72qvK2lRgsPlK5YnmE9otDNYE8e9R9v3CMBKy3SHO9cHAlhedMkC0x0GHKzILUPA;
+        path=/; expires=Mon, 15-Jul-24 19:35:59 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=DQePSM_v9bEa3hSaZw7w90aZlxFtRtbZunAmUoOiG98-1721070359012-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - traceloop
+      openai-processing-ms:
+      - '443'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=15552000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '5000'
+      x-ratelimit-limit-tokens:
+      - '160000'
+      x-ratelimit-remaining-requests:
+      - '4999'
+      x-ratelimit-remaining-tokens:
+      - '159974'
+      x-ratelimit-reset-requests:
+      - 12ms
+      x-ratelimit-reset-tokens:
+      - 9ms
+      x-request-id:
+      - req_2c640127e33a8865f70ee056e6105cf3
+    status:
+      code: 200
+      message: OK
+version: 1

packages/traceloop-sdk/tests/test_manual.py

@@ -0,0 +1,48 @@
+from opentelemetry.semconv.ai import SpanAttributes
+import pytest
+from openai import OpenAI
+from traceloop.sdk.tracing.manual import LLMMessage, track_llm_call
+
+
+@pytest.fixture
+def openai_client():
+    return OpenAI()
+
+
+@pytest.mark.vcr
+def test_manual_report(exporter, openai_client):
+    with track_llm_call(vendor="openai", type="chat") as span:
+        span.report_request(
+            model="gpt-3.5-turbo",
+            messages=[
+                LLMMessage(role="user", content="Tell me a joke about opentelemetry")
+            ],
+        )
+
+        res = openai_client.chat.completions.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {"role": "user", "content": "Tell me a joke about opentelemetry"}
+            ],
+        )
+
+        span.report_response(res.model, [text.message.content for text in res.choices])
+
+    spans = exporter.get_finished_spans()
+    open_ai_span = spans[0]
+    assert open_ai_span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "gpt-3.5-turbo"
+    assert open_ai_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.role"] == "user"
+    assert (
+        open_ai_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"]
+        == "Tell me a joke about opentelemetry"
+    )
+    assert (
+        open_ai_span.attributes[SpanAttributes.LLM_RESPONSE_MODEL]
+        == "gpt-3.5-turbo-0125"
+    )
+    assert (
+        open_ai_span.attributes[f"{SpanAttributes.LLM_COMPLETIONS}.0.content"]
+        == "Why did the opentelemetry developer break up with their partner? Because they were tired"
+        + " of constantly tracing their every move!"
+    )
+    assert open_ai_span.end_time > open_ai_span.start_time

packages/traceloop-sdk/traceloop/sdk/tracing/manual.py

@@ -0,0 +1,57 @@
+from contextlib import contextmanager
+from opentelemetry.semconv.ai import SpanAttributes
+from opentelemetry.trace import Span
+from pydantic import BaseModel
+from traceloop.sdk.tracing.context_manager import get_tracer
+
+
+class LLMMessage(BaseModel):
+    role: str
+    content: str
+
+
+class LLMUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class LLMSpan:
+    _span: Span = None
+
+    def __init__(self, span: Span):
+        self._span = span
+        pass
+
+    def report_request(self, model: str, messages: list[LLMMessage]):
+        self._span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, model)
+        for idx, message in enumerate(messages):
+            self._span.set_attribute(
+                f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", message.role
+            )
+            self._span.set_attribute(
+                f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", message.content
+            )
+
+    def report_response(self, model: str, completions: list[str]):
+        self._span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL, model)
+        for idx, completion in enumerate(completions):
+            self._span.set_attribute(
+                f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", "assistant"
+            )
+            self._span.set_attribute(
+                f"{SpanAttributes.LLM_COMPLETIONS}.{idx}", completion
+            )
+
+
+@contextmanager
+def track_llm_call(vendor: str, type: str):
+    with get_tracer() as tracer:
+        with tracer.start_as_current_span(name=f"{vendor}.{type}") as span:
+            span.set_attribute(SpanAttributes.LLM_SYSTEM, vendor)
+            span.set_attribute(SpanAttributes.LLM_REQUEST_TYPE, type)
+            llm_span = LLMSpan(span)
+            try:
+                yield llm_span
+            finally:
+                span.end()