chore(bump): next -> main (0.6.0) (#227)

.changeset/brave-needles-boil.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Allow attributes to be set on accept

.changeset/calm-ants-build.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-openai": patch
+---
+
+fix multiple function calls not firing

.changeset/clean-lies-grow.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": patch
+---
+
+fix(multimodal): crash on reconnect to same room

.changeset/eight-mugs-glow.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-openai": patch
+---
+
+fix(tts): add missing crypto import to OpenAI tts

.changeset/few-games-destroy.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": minor
+---
+
+MultimodalAgent: emit user started speaking event

.changeset/few-schools-run.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": patch
+---
+
+fix tokenizer

.changeset/hot-otters-pay.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": minor
+---
+
+support nested speech handles in pipeline agent

.changeset/khaki-ties-design.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": patch
+---
+
+fix(pipeline): add transcription for AGENT_SPEECH_COMMITTED

.changeset/moody-poems-juggle.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-openai": patch
+---
+
+groq: add support for llama 3.3 70b

.changeset/nine-planes-ring.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-elevenlabs": patch
+---
+
+use AudioByteStream for TTS

.changeset/proud-monkeys-bow.md

@@ -0,0 +1,9 @@
+---
+"@livekit/agents": minor
+"@livekit/agents-plugin-deepgram": patch
+"@livekit/agents-plugin-elevenlabs": patch
+"@livekit/agents-plugin-openai": patch
+"@livekit/agents-plugin-silero": patch
+---
+
+add metrics monitoring

.changeset/sweet-papayas-complain.md

@@ -0,0 +1,7 @@
+---
+"@livekit/agents": patch
+"@livekit/agents-plugin-elevenlabs": patch
+"@livekit/agents-plugin-openai": patch
+---
+
+add testutils, tests for oai, 11labs

agents/src/index.ts

@@ -11,6 +11,7 @@
  */
 import * as cli from './cli.js';
 import * as llm from './llm/index.js';
+import * as metrics from './metrics/index.js';
 import * as multimodal from './multimodal/index.js';
 import * as pipeline from './pipeline/index.js';
 import * as stt from './stt/index.js';
@@ -28,4 +29,4 @@ export * from './generator.js';
 export * from './audio.js';
 export * from './transcription.js';
 
-export { cli, stt, tts, llm, pipeline, multimodal, tokenize };
+export { cli, stt, tts, llm, pipeline, multimodal, tokenize, metrics };

agents/src/llm/index.ts

@@ -25,6 +25,8 @@ export {
   type CompletionUsage,
   type Choice,
   type ChatChunk,
+  type LLMCallbacks,
+  LLMEvent,
   LLM,
   LLMStream,
 } from './llm.js';

agents/src/llm/llm.ts

@@ -1,6 +1,9 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import { EventEmitter } from 'node:events';
+import type { LLMMetrics } from '../metrics/base.js';
 import { AsyncIterableQueue } from '../utils.js';
 import type { ChatContext, ChatRole } from './chat_context.js';
 import type { FunctionCallInfo, FunctionContext } from './function_context.js';
@@ -28,7 +31,15 @@ export interface ChatChunk {
   usage?: CompletionUsage;
 }
 
-export abstract class LLM {
+export enum LLMEvent {
+  METRICS_COLLECTED,
+}
+
+export type LLMCallbacks = {
+  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+};
+
+export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {
   /**
    * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
    */
@@ -48,16 +59,56 @@ export abstract class LLM {
 }
 
 export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+  protected output = new AsyncIterableQueue<ChatChunk>();
   protected queue = new AsyncIterableQueue<ChatChunk>();
   protected closed = false;
   protected _functionCalls: FunctionCallInfo[] = [];
+  abstract label: string;
 
+  #llm: LLM;
   #chatCtx: ChatContext;
   #fncCtx?: FunctionContext;
 
-  constructor(chatCtx: ChatContext, fncCtx?: FunctionContext) {
+  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {
+    this.#llm = llm;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.monitorMetrics();
+  }
+
+  protected async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    let ttft: bigint | undefined;
+    let requestId = '';
+    let usage: CompletionUsage | undefined;
+
+    for await (const ev of this.queue) {
+      this.output.put(ev);
+      requestId = ev.requestId;
+      if (!ttft) {
+        ttft = process.hrtime.bigint() - startTime;
+      }
+      if (ev.usage) {
+        usage = ev.usage;
+      }
+    }
+    this.output.close();
+
+    const duration = process.hrtime.bigint() - startTime;
+    const metrics: LLMMetrics = {
+      timestamp: Date.now(),
+      requestId,
+      ttft: Math.trunc(Number(ttft! / BigInt(1000000))),
+      duration: Math.trunc(Number(duration / BigInt(1000000))),
+      cancelled: false, // XXX(nbsp)
+      label: this.label,
+      completionTokens: usage?.completionTokens || 0,
+      promptTokens: usage?.promptTokens || 0,
+      totalTokens: usage?.totalTokens || 0,
+      tokensPerSecond:
+        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),
+    };
+    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);
   }
 
   /** List of called functions from this stream. */
@@ -88,10 +139,11 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   }
 
   next(): Promise<IteratorResult<ChatChunk>> {
-    return this.queue.next();
+    return this.output.next();
   }
 
   close() {
+    this.output.close();
     this.queue.close();
     this.closed = true;
   }

agents/src/metrics/base.ts

@@ -0,0 +1,127 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export interface LLMMetrics {
+  requestId: string;
+  timestamp: number;
+  ttft: number;
+  duration: number;
+  label: string;
+  cancelled: boolean;
+  completionTokens: number;
+  promptTokens: number;
+  totalTokens: number;
+  tokensPerSecond: number;
+  error?: Error;
+}
+
+export interface STTMetrics {
+  requestId: string;
+  timestamp: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface TTSMetrics {
+  requestId: string;
+  timestamp: number;
+  ttfb: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  cancelled: boolean;
+  charactersCount: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface VADMetrics {
+  timestamp: number;
+  idleTime: number;
+  inferenceDurationTotal: number;
+  inferenceCount: number;
+  label: string;
+}
+
+export interface PipelineEOUMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+  /** Timestamp of when the event was recorded */
+  timestamp: number;
+  /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
+  endOfUtteranceDelay: number;
+  /**
+   * Time taken to obtain the transcript after the end of the user's speech.
+   *
+   * @remarks
+   * May be 0 if the transcript was already available.
+   */
+  transcriptionDelay: number;
+}
+
+export interface PipelineLLMMetrics extends LLMMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export interface PipelineTTSMetrics extends TTSMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export type PipelineSTTMetrics = STTMetrics;
+export type PipelineVADMetrics = VADMetrics;
+
+export class MultimodalLLMError extends Error {
+  type?: string;
+  reason?: string;
+  code?: string;
+  constructor(
+    {
+      type,
+      reason,
+      code,
+      message,
+    }: { type?: string; reason?: string; code?: string; message?: string } = {},
+    options?: ErrorOptions,
+  ) {
+    super(message, options);
+    this.type = type;
+    this.reason = reason;
+    this.code = code;
+  }
+}
+
+export interface MultimodalLLMMetrics extends LLMMetrics {
+  inputTokenDetails: {
+    cachedTokens: number;
+    textTokens: number;
+    audioTokens: number;
+  };
+  outputTokenDetails: {
+    textTokens: number;
+    audioTokens: number;
+  };
+}
+
+export type AgentMetrics =
+  | STTMetrics
+  | LLMMetrics
+  | TTSMetrics
+  | VADMetrics
+  | PipelineSTTMetrics
+  | PipelineEOUMetrics
+  | PipelineLLMMetrics
+  | PipelineTTSMetrics
+  | PipelineVADMetrics
+  | MultimodalLLMMetrics;

agents/src/metrics/index.ts

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type {
+  AgentMetrics,
+  STTMetrics,
+  LLMMetrics,
+  TTSMetrics,
+  VADMetrics,
+  PipelineSTTMetrics,
+  PipelineEOUMetrics,
+  PipelineLLMMetrics,
+  PipelineTTSMetrics,
+  PipelineVADMetrics,
+  MultimodalLLMMetrics,
+} from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';

agents/src/metrics/usage_collector.ts

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AgentMetrics } from './base.js';
+import { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';
+
+export interface UsageSummary {
+  llmPromptTokens: number;
+  llmCompletionTokens: number;
+  ttsCharactersCount: number;
+  sttAudioDuration: number;
+}
+
+export class UsageCollector {
+  #summary: UsageSummary;
+
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0,
+    };
+  }
+
+  collect(metrics: AgentMetrics) {
+    if (isLLMMetrics(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (isTTSMetrics(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (isSTTMetrics(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+
+  get summary(): UsageSummary {
+    return { ...this.#summary };
+  }
+}