@hamming/hamming-sdk/dist/index.cjs.map

SDK for Hamming Evals Framework

{"version":3,"sources":["../src/index.ts","../src/fetchClient.ts","../src/httpClient.ts","../src/utils/manualResetEvent.ts","../src/logger.ts","../src/prompt-template.ts","../src/resources/anthropic.ts","../src/resources/anthropic-client.ts","../src/resources/anthropic-bedrock-client.ts","../src/resources/datasets.ts","../src/asyncStorage.ts","../src/types.ts","../src/worker.ts","../src/resources/experiments.ts","../src/resources/monitoring.ts","../src/utils/voice.ts","../src/resources/tracing.ts","../src/resources/openai-client.ts","../src/resources/prompts.ts","../src/client.ts"],"sourcesContent":["export * from \"./client\";\nexport * from \"./types\";\nexport * from \"./prompt-template\";\n","type RequestDelayFunction = (\n  attempt: number,\n  error: Error | null,\n  response: Response | null,\n  input?: string | Request,\n) => number;\n\ntype RequestRetryOnFunction = (\n  attempt: number,\n  error: Error | null,\n  response: Response | null,\n) => boolean | Promise<boolean>;\n\nexport interface RequestInitRetryParams {\n  retries?: number;\n  retryDelay?: number | RequestDelayFunction;\n  retryOn?: number[] | RequestRetryOnFunction;\n}\n\nexport type RequestInitWithRetry = RequestInit & RequestInitRetryParams;\n\nclass FetchClient {\n  private retries: number;\n  private retryDelay: number | RequestDelayFunction;\n  private retryOn: number[] | RequestRetryOnFunction;\n\n  constructor(defaults?: RequestInitRetryParams) {\n    const baseDefaults: RequestInitRetryParams = {\n      retries: 3,\n      retryDelay: 1000,\n      retryOn: [],\n    };\n\n    const finalDefaults = { ...baseDefaults, ...defaults };\n\n    this.validateDefaults(finalDefaults);\n\n    this.retries = finalDefaults.retries!;\n    this.retryDelay = finalDefaults.retryDelay!;\n    this.retryOn = finalDefaults.retryOn!;\n  }\n\n  private validateDefaults(defaults: RequestInitRetryParams): void {\n    if (\n      defaults.retries !== undefined &&\n      !this.isPositiveInteger(defaults.retries)\n    ) {\n      throw new ArgumentError(\"retries must be a positive integer\");\n    }\n\n    if (\n      defaults.retryDelay !== undefined &&\n      !this.isPositiveInteger(defaults.retryDelay) &&\n      typeof defaults.retryDelay !== \"function\"\n    ) {\n      throw new ArgumentError(\n        \"retryDelay must be a positive integer or a function returning a positive integer\",\n      );\n    }\n\n    if (\n      defaults.retryOn !== undefined &&\n      !Array.isArray(defaults.retryOn) &&\n      typeof defaults.retryOn !== \"function\"\n    ) {\n      throw new ArgumentError(\"retryOn property expects an array or function\");\n    }\n  }\n\n  private isPositiveInteger(value: any): value is number {\n    return Number.isInteger(value) && value >= 0;\n  }\n\n  public fetchRetry(\n    input: RequestInfo,\n    init?: RequestInitWithRetry,\n  ): Promise<Response> {\n    let retries = this.retries;\n    let retryDelay = this.retryDelay;\n    let retryOn = this.retryOn;\n\n    if (init) {\n      if (init.retries !== undefined && this.isPositiveInteger(init.retries)) {\n        retries = init.retries;\n      }\n\n      if (init.retryDelay !== undefined) {\n        if (\n          this.isPositiveInteger(init.retryDelay) ||\n          typeof init.retryDelay === \"function\"\n        ) {\n          retryDelay = init.retryDelay;\n        }\n      }\n\n      if (init.retryOn) {\n        if (Array.isArray(init.retryOn) || typeof init.retryOn === \"function\") {\n          retryOn = init.retryOn;\n        }\n      }\n    }\n\n    return new Promise((resolve, reject) => {\n      const wrappedFetch = (attempt: number) => {\n        const _input = input instanceof Request ? input.clone() : input;\n        fetch(_input, init)\n          .then((response) => {\n            if (Array.isArray(retryOn) && !retryOn.includes(response.status)) {\n              resolve(response);\n            } else if (typeof retryOn === \"function\") {\n              Promise.resolve(retryOn(attempt, null, response))\n                .then((retryOnResponse) => {\n                  if (retryOnResponse) {\n                    retry(attempt, null, response);\n                  } else {\n                    resolve(response);\n                  }\n                })\n                .catch(reject);\n            } else {\n              if (attempt < retries) {\n                retry(attempt, null, response);\n              } else {\n                resolve(response);\n              }\n            }\n          })\n          .catch((error) => {\n            if (typeof retryOn === \"function\") {\n              Promise.resolve(retryOn(attempt, error, null))\n                .then((retryOnResponse) => {\n                  if (retryOnResponse) {\n                    retry(attempt, error, null);\n                  } else {\n                    reject(error);\n                  }\n                })\n                .catch(reject);\n            } else if (attempt < retries) {\n              retry(attempt, error, null);\n            } else {\n              reject(error);\n            }\n          });\n      };\n\n      const retry = (\n        attempt: number,\n        error: Error | null,\n        response: Response | null,\n      ) => {\n        const delay =\n          typeof retryDelay === \"function\"\n            ? retryDelay(attempt, error, response, input)\n            : retryDelay;\n        setTimeout(() => {\n          wrappedFetch(++attempt);\n        }, delay);\n      };\n\n      wrappedFetch(0);\n    });\n  }\n}\n\nclass ArgumentError extends Error {\n  constructor(message: string) {\n    super(message);\n    this.name = \"ArgumentError\";\n  }\n}\n\nexport default FetchClient;\n","import FetchClient from \"./fetchClient\";\n\nconst TOO_MANY_REQUESTS = 429;\nconst INTERNAL_SERVER_ERROR = 500;\nconst UNAUTHORIZED = 401;\n\ninterface HttpClientOptions {\n  apiKey: string;\n  baseURL: string;\n}\n\n/**\n * The HttpClient provides methods to perform HTTP requests.\n * The `fetch` method is used to make a request to a specified endpoint.\n * It includes retry logic for transient errors, where it will retry the request\n * according to the `maxRetries` and `retryDelay` parameters.\n * For non-transient errors, it will fail fast and not retry the request.\n * @param input - The endpoint to which the request will be made.\n * @param init - The request options.\n * @param maxRetries - The maximum number of retries for the request.\n * @param retryDelay - The delay between retries.\n * @returns A promise that resolves to the response of the request, or rejects\n *          with an error if the request fails or all retries are exhausted.\n */\nexport class HttpClient {\n  apiKey: string;\n  baseURL: string;\n  fetchClient: FetchClient;\n  debug: boolean = false;\n  retries: number = 3;\n\n  constructor(opts: HttpClientOptions) {\n    this.apiKey = opts.apiKey;\n    this.baseURL = this.sanitizeBaseUrl(opts.baseURL);\n    this.fetchClient = new FetchClient();\n    this.debug = process.env.NODE_ENV === \"development\";\n  }\n\n  /**\n   * Sanitizes the base URL by trimming whitespace and removing trailing slashes.\n   * @param baseURL - The base URL to sanitize.\n   * @returns The sanitized base URL.\n   */\n  private sanitizeBaseUrl(baseURL: string): string {\n    return baseURL.trim().replace(/\\/$/, \"\");\n  }\n\n  async fetch(\n    input: string,\n    init?: RequestInit | undefined,\n  ): Promise<Response> {\n    const url = this.baseURL + input;\n\n    const requestInit = {\n      ...init,\n      headers: {\n        ...init?.headers,\n        \"Content-Type\": init?.headers?.[\"Content-Type\"] ?? \"application/json\",\n        authorization: `Bearer ${this.apiKey}`,\n      },\n    };\n\n    const isDebug = this.debug;\n\n    if (isDebug) {\n      console.debug(\n        `\\nFetching URL: ${url}` +\n          `\\nMethod: ${requestInit.method || \"GET\"}` +\n          `${requestInit.body ? `\\nBody: ${requestInit.body}` : \"\"}` +\n          `\\nHeaders: ${JSON.stringify(requestInit.headers, null, 2)}`,\n      );\n    }\n\n    const numRetries = this.retries;\n    const resp = await this.fetchClient.fetchRetry(url, {\n      ...requestInit,\n      retryOn: function (attempt, error, response) {\n        if (attempt >= numRetries) return false;\n\n        // Retry on too many requests, internal server error, or TypeError\n        const status = response?.status;\n\n        return (\n          error instanceof TypeError ||\n          status === TOO_MANY_REQUESTS ||\n          (status !== undefined && status >= INTERNAL_SERVER_ERROR)\n        );\n      },\n      retryDelay: function (attempt, error, response, input) {\n        console.warn(\n          `Fetch attempt #${attempt}: input=${input}, error=${error?.message}, response status=${response?.status}, response status text=${response?.statusText}`,\n        );\n        return Math.pow(2, attempt) * 1000;\n      },\n    });\n\n    if (resp.status === UNAUTHORIZED) {\n      throw new Error(\n        `Unauthorized. Please check that your HAMMING_API_KEY is correct by visiting: ${this.baseURL}/settings`,\n      );\n    }\n\n    if (isDebug) {\n      console.debug(`Response for ${url}: ${resp.status} ${resp.statusText}\\n`);\n    }\n\n    return resp;\n  }\n}\n","export class ManualResetEvent {\n  private isSet: boolean;\n  private waiters: Array<(...args: any) => void>;\n\n  constructor(isSet = false) {\n    this.isSet = isSet;\n    this.waiters = [];\n    if (isSet) {\n      this.resolveWaiters();\n    }\n  }\n\n  set() {\n    this.isSet = true;\n    this.resolveWaiters();\n  }\n\n  reset() {\n    this.isSet = false;\n  }\n\n  wait() {\n    if (this.isSet) {\n      return Promise.resolve();\n    }\n    return new Promise((resolve) => {\n      this.waiters.push(resolve);\n    });\n  }\n\n  resolveWaiters() {\n    this.waiters.forEach((resolve) => resolve());\n    this.waiters = [];\n  }\n}\n","import type { Hamming } from \"./index\";\nimport { LogMessage } from \"./types\";\nimport { ManualResetEvent } from \"./utils/manualResetEvent\";\n\nconst LOG_BATCH_SIZE = 512;\n\nexport class Logger {\n  private client: Hamming;\n\n  private queue: LogMessage[] = [];\n  private stopped: boolean = false;\n  private queueHasMessages = new ManualResetEvent();\n\n  constructor(client: Hamming) {\n    this.client = client;\n  }\n\n  log(message: LogMessage): void {\n    this.queue.push(message);\n    this.queueHasMessages.set();\n  }\n\n  async start(): Promise<void> {\n    console.log(\"Starting logger thread..\");\n    while (!this.stopped) {\n      await this.queueHasMessages.wait();\n      await this._processQueue();\n    }\n    await this._processQueue();\n    console.log(\"Logger thread exited!\");\n  }\n\n  stop(): void {\n    console.log(\"Waiting for logger thread to exit..\");\n    this.stopped = true;\n  }\n\n  private _drainQueue(): LogMessage[] {\n    const batchSize = Math.min(this.queue.length, LOG_BATCH_SIZE);\n    const drainedMessages = this.queue.splice(0, batchSize);\n    return drainedMessages;\n  }\n\n  private async _processQueue(): Promise<void> {\n    const messages = this._drainQueue();\n    await this._publish(messages);\n    // TODO: test and set\n    if (this.queue.length === 0) {\n      this.queueHasMessages.reset();\n    }\n  }\n\n  private async _publish(logs: LogMessage[]): Promise<void> {\n    if (logs.length === 0) {\n      return;\n    }\n    if (process.env.NODE_ENV === \"development\") {\n      console.log(`Publishing ${logs.length} message(s)..`);\n    }\n    try {\n      await this.client.fetch(\"/logs\", {\n        method: \"POST\",\n        body: JSON.stringify({ logs }),\n      });\n      if (process.env.NODE_ENV === \"development\") {\n        console.log(`Published ${logs.length} messages!`);\n      }\n    } catch (e) {\n      console.error(`Failed to publish messages: ${e}`);\n    }\n  }\n}\n","import { ChatMessage, PromptContent } from \"./types\";\n\nexport class PromptTemplate {\n  readonly prompt: PromptContent;\n  readonly vars: Array<string>;\n\n  constructor(prompt: PromptContent) {\n    this.prompt = prompt;\n    this.vars = this.extractVariables(prompt.chatMessages ?? []);\n  }\n\n  private extractVariables(messages: ChatMessage[]): string[] {\n    const content = messages.map((message) => message.content).join(\"\\n\\n\");\n    const matches = content.match(/\\{\\{([^}]+)\\}\\}/g) ?? [];\n    return matches.map((match) => match.replace(/\\{\\{([^}]+)\\}\\}/g, \"$1\"));\n  }\n\n  compile(values: Record<string, string>): PromptContent {\n    return {\n      ...this.prompt,\n      chatMessages: this.prompt.chatMessages.map((message) => {\n        return {\n          ...message,\n          content: message.content.replace(\n            /\\{\\{([^}]+)\\}\\}/g,\n            (match, pattern) => {\n              return values[pattern] || match;\n            },\n          ),\n        };\n      }),\n    };\n  }\n}\n","import type {\n  MessageCreateParams,\n  MessageCreateParamsNonStreaming,\n  MessageParam,\n  Tool,\n} from \"@anthropic-ai/sdk/resources/messages.mjs\";\nimport { ChatMessage, PromptContent, ToolChoice } from \"../types\";\n\nexport const DEFAULT_ANTHROPIC_MAX_TOKENS = 4096;\n\nexport function convertAnthropicToolChoice(\n  input: ToolChoice,\n):\n  | MessageCreateParams.ToolChoiceAuto\n  | MessageCreateParams.ToolChoiceAny\n  | MessageCreateParams.ToolChoiceTool {\n  switch (input.choice) {\n    case \"auto\":\n      return { type: \"auto\" };\n    case \"any\":\n      return { type: \"any\" };\n    case \"tool\":\n      return { type: \"tool\", name: input.functionName ?? \"\" };\n    default:\n      throw new Error(\"Invalid tool choice type\");\n  }\n}\n\nexport function convertChatMessage(message: ChatMessage): MessageParam {\n  switch (message.role) {\n    case \"user\":\n      return {\n        role: \"user\",\n        content: message.content,\n      };\n    case \"assistant\":\n      return {\n        role: \"assistant\",\n        content: message.content,\n      };\n    default:\n      throw new Error(`Unsupported role: ${message.role}`);\n  }\n}\n\nexport function createMessageParams(\n  content: PromptContent,\n): MessageCreateParamsNonStreaming {\n  const systemMessage = content.chatMessages.find(\n    (message) => message.role === \"system\",\n  );\n  const messages = content.chatMessages.filter(\n    (message) => message.role !== \"system\",\n  );\n  return {\n    model: content.languageModel,\n    system: systemMessage?.content,\n    messages: messages.map(convertChatMessage),\n    max_tokens:\n      content.promptSettings.maxTokens ?? DEFAULT_ANTHROPIC_MAX_TOKENS,\n    top_p: content.promptSettings.topP,\n    temperature: content.promptSettings.temperature,\n    tools: content.tools ? (JSON.parse(content.tools) as Tool[]) : undefined,\n    tool_choice:\n      content.promptSettings.toolChoice && content.tools\n        ? convertAnthropicToolChoice(content.promptSettings.toolChoice)\n        : undefined,\n  };\n}\n","import type { Anthropic } from \"@anthropic-ai/sdk\";\nimport type {\n  Message,\n  RawMessageStreamEvent,\n} from \"@anthropic-ai/sdk/resources/messages.mjs\";\nimport type { Stream } from \"@anthropic-ai/sdk/streaming.mjs\";\nimport { Hamming } from \"../client\";\nimport { PromptTemplate } from \"../prompt-template\";\nimport { PromptWithContent } from \"../types\";\nimport { createMessageParams } from \"./anthropic\";\n\nclass AnthropicClient {\n  private anthropic?: Anthropic;\n\n  constructor(private readonly client: Hamming) {}\n\n  async load(): Promise<Anthropic> {\n    if (this.anthropic) {\n      return this.anthropic;\n    }\n    if (!this.client.anthropicApiKey) {\n      throw new Error(\"Anthropic API key is not set\");\n    }\n    const module = await import(\"@anthropic-ai/sdk\");\n    this.anthropic = new module.Anthropic({\n      apiKey: this.client.anthropicApiKey,\n    });\n    return this.anthropic;\n  }\n\n  async createMessage(\n    prompt: PromptWithContent,\n    variables?: Record<string, string>,\n  ): Promise<Message> {\n    if (!prompt.content) {\n      throw new Error(\"Prompt content is not set\");\n    }\n    const template = new PromptTemplate(prompt.content);\n    const content = template.compile(variables || {});\n\n    const client = await this.load();\n    const params = createMessageParams(content);\n\n    return client.messages.create({\n      ...params,\n      stream: false,\n    });\n  }\n\n  async createMessageStream(\n    prompt: PromptWithContent,\n    variables?: Record<string, string>,\n  ): Promise<Stream<RawMessageStreamEvent>> {\n    if (!prompt.content) {\n      throw new Error(\"Prompt content is not set\");\n    }\n    const template = new PromptTemplate(prompt.content);\n    const content = template.compile(variables || {});\n\n    const client = await this.load();\n    const params = createMessageParams(content);\n\n    return client.messages.create({\n      ...params,\n      stream: true,\n    });\n  }\n}\n\nexport default AnthropicClient;\n","import type { AnthropicBedrock } from \"@anthropic-ai/bedrock-sdk\";\nimport type {\n  Message,\n  RawMessageStreamEvent,\n} from \"@anthropic-ai/sdk/resources/messages.mjs\";\nimport type { Stream } from \"@anthropic-ai/sdk/streaming.mjs\";\nimport { Hamming } from \"../client\";\nimport { PromptTemplate } from \"../prompt-template\";\nimport { PromptWithContent } from \"../types\";\nimport { createMessageParams } from \"./anthropic\";\n\nclass AnthropicBedrockClient {\n  private anthropic?: AnthropicBedrock;\n\n  constructor(private readonly client: Hamming) {}\n\n  async load(): Promise<AnthropicBedrock> {\n    if (this.anthropic) {\n      return this.anthropic;\n    }\n    if (!this.client.bedrock) {\n      // We're relying on ~/.aws/credentials or AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID env vars\n      console.log(\n        \"Anthropic Bedrock config is not set. Using environment credentials.\",\n      );\n    }\n    const module = await import(\"@anthropic-ai/bedrock-sdk\");\n    this.anthropic = new module.AnthropicBedrock({\n      awsSecretKey: this.client.bedrock?.awsSecretKey,\n      awsAccessKey: this.client.bedrock?.awsAccessKey,\n      awsRegion: this.client.bedrock?.awsRegion,\n      awsSessionToken: this.client.bedrock?.awsSessionToken,\n    });\n    return this.anthropic;\n  }\n\n  async createMessage(\n    prompt: PromptWithContent,\n    variables?: Record<string, string>,\n  ): Promise<Message> {\n    if (!prompt.content) {\n      throw new Error(\"Prompt content is not set\");\n    }\n    const template = new PromptTemplate(prompt.content);\n    const content = template.compile(variables || {});\n\n    const client = await this.load();\n    const params = createMessageParams(content);\n\n    return client.messages.create({\n      ...params,\n      stream: false,\n    });\n  }\n\n  async createMessageStream(\n    prompt: PromptWithContent,\n    variables?: Record<string, string>,\n  ): Promise<Stream<RawMessageStreamEvent>> {\n    if (!prompt.content) {\n      throw new Error(\"Prompt content is not set\");\n    }\n    const template = new PromptTemplate(prompt.content);\n    const content = template.compile(variables || {});\n\n    const client = await this.load();\n    const params = createMessageParams(content);\n\n    return client.messages.create({\n      ...params,\n      stream: true,\n    });\n  }\n}\n\nexport default AnthropicBedrockClient;\n","import type { Hamming } from \"../client\";\nimport type {\n  Dataset,\n  DatasetId,\n  DatasetWithItems,\n  CreateDatasetOptions,\n} from \"../types\";\n\nexport class Datasets {\n  private client: Hamming;\n\n  constructor(client: Hamming) {\n    this.client = client;\n  }\n\n  async load(id: DatasetId): Promise<DatasetWithItems> {\n    const resp = await this.client.fetch(`/datasets/${id}`, {\n      method: \"GET\",\n    });\n\n    let data: { dataset: DatasetWithItems };\n    try {\n      data = await resp.json();\n    } catch (error) {\n      throw new Error(\n        `Failed to parse dataset response as JSON for dataset ID: ${id}: ${error}`,\n      );\n    }\n    return data.dataset as DatasetWithItems;\n  }\n\n  async list(): Promise<Dataset[]> {\n    const resp = await this.client.fetch(`/datasets`);\n    const data = await resp.json();\n    return data.datasets as Dataset[];\n  }\n\n  async create(opts: CreateDatasetOptions): Promise<DatasetWithItems> {\n    const { name, description, items } = opts;\n    const resp = await this.client.fetch(\"/datasets\", {\n      method: \"POST\",\n      body: JSON.stringify({\n        name,\n        description,\n        items,\n      }),\n    });\n    const data = await resp.json();\n    return data.dataset as DatasetWithItems;\n  }\n}\n","import { AsyncLocalStorage } from \"node:async_hooks\";\n\nimport { RunContext } from \"./types\";\n\nexport const asyncRunContext = new AsyncLocalStorage<RunContext>();\n","export enum ScoreType {\n  AccuracyAI = \"accuracy_ai\",\n  FactsCompare = \"facts_compare\",\n  ContextRecall = \"context_recall\",\n  ContextPrecision = \"context_precision\",\n  Hallucination = \"hallucination\",\n  StringDiff = \"string_diff\",\n  Refusal = \"refusal\",\n  SqlAst = \"sql_ast\",\n}\n\nexport type InputType = Record<string, any>;\nexport type OutputType = Record<string, any>;\nexport type MetadataType = Record<string, any>;\n\nexport enum ExperimentStatus {\n  CREATED = \"CREATED\",\n  RUNNING = \"RUNNING\",\n  SCORING = \"SCORING\",\n  SCORING_FAILED = \"SCORING_FAILED\",\n  FINISHED = \"FINISHED\",\n  FAILED = \"FAILED\",\n}\n\nexport interface Experiment {\n  id: string;\n  name: string;\n  description?: string | null;\n  datasetId: number;\n  datasetVersionId?: number;\n  status: ExperimentStatus;\n}\n\nexport interface ExperimentItemMetrics {\n  durationMs?: number;\n}\n\nexport interface ExperimentItem {\n  id: string;\n  experimentId: string;\n  datasetItemId: string;\n  output: OutputType;\n  metrics: ExperimentItemMetrics;\n}\n\nexport interface ExperimentItemContext {\n  item: ExperimentItem;\n  startTs: number;\n}\n\nexport type DatasetId = string;\n\nexport interface DatasetItemValue {\n  input: InputType;\n  output: OutputType;\n  metadata: MetadataType;\n}\n\nexport type DatasetItem = DatasetItemValue & { id: string };\n\nexport interface Dataset {\n  id: string;\n  name: string;\n  description?: string;\n}\n\nexport type DatasetWithItems = Dataset & { items: DatasetItem[] };\n\nexport interface RunOptions {\n  dataset: DatasetId;\n  name?: string;\n  scoring?: (ScoreType | ScoringFunction)[];\n  metadata?: MetadataType;\n  parallel?: boolean | number;\n  sampling?: number;\n}\n\ninterface TracingContext {\n  experiment?: {\n    itemId?: string;\n  };\n  monitoring?: {\n    seqId?: number;\n  };\n}\n\nexport type RunContext = {\n  tracing: TracingContext;\n};\n\nexport type Runner = (input: InputType) => Promise<OutputType>;\n\nexport interface ClientOptions {\n  apiKey: string;\n  baseURL?: string;\n  openaiApiKey?: string;\n  anthropicApiKey?: string;\n  bedrock?: {\n    awsAccessKey?: string;\n    awsSecretKey?: string;\n    awsRegion?: string;\n    awsSessionToken?: string;\n  };\n}\n\nexport interface CreateDatasetOptions {\n  name: string;\n  description?: string;\n  items: DatasetItemValue[];\n}\n\nexport type TraceEvent = Record<string, unknown>;\n\nexport type LLMProvider = \"openai\" | \"anthropic\" | \"azure_openai\";\n\nexport interface GenerationMetadata {\n  provider?: LLMProvider;\n  model?: string;\n  stream?: boolean;\n  max_tokens?: number;\n  n?: number;\n  seed?: number;\n  temperature?: number;\n  usage?: {\n    completion_tokens?: number;\n    prompt_tokens?: number;\n    total_tokens?: number;\n  };\n  duration_ms?: number;\n  error?: boolean;\n  error_message?: string;\n}\n\nexport interface GenerationParams {\n  input?: string;\n  output?: string;\n  metadata?: GenerationMetadata;\n}\n\nexport interface Document {\n  pageContent: string;\n  metadata: Record<string, any>;\n}\n\nexport interface RetrievalParams {\n  query?: string;\n  results?: Document[] | string[];\n  metadata?: {\n    engine?: string;\n    [key: string]: unknown;\n  };\n}\n\nexport interface Trace {\n  id: number;\n  experimentItemId: string;\n  parentId?: number;\n  event: TraceEvent;\n}\n\nexport enum TracingMode {\n  OFF = \"off\",\n  MONITORING = \"monitoring\",\n  EXPERIMENT = \"experiment\",\n}\n\nexport interface ITracing {\n  logGeneration(params: GenerationParams): void;\n  logRetrieval(params: RetrievalParams): void;\n  log(key: string, value: unknown): void;\n  log(trace: TraceEvent): void;\n}\n\nexport interface MonitoringItem {\n  setInput(input: InputType): void;\n  setOutput(output: OutputType): void;\n  setMetadata(metadata: MetadataType): void;\n  end(error?: boolean, errorMessage?: string): void;\n  tracing: ITracing;\n}\n\nexport enum MonitoringItemType {\n  CALL = \"CALL\",\n  TEXT = \"TEXT\",\n}\n\nexport enum MonitoringItemStatus {\n  STARTED = \"STARTED\",\n  COMPLETED = \"COMPLETED\",\n  FAILED = \"FAILED\",\n}\n\nexport enum SessionEnvironment {\n  DEVELOPMENT = \"development\",\n  STAGING = \"staging\",\n  PRODUCTION = \"production\",\n}\n\nexport interface MonitoringStartOpts {\n  environment?: SessionEnvironment;\n}\n\nexport interface MonitoringSession {\n  id: string;\n  seqId: number;\n}\n\nexport interface MonitoringTraceContext {\n  session_id: string;\n  seq_id: number;\n  parent_seq_id?: number;\n}\n\nexport interface MonitoringTrace extends MonitoringTraceContext {\n  event: TraceEvent;\n}\n\nexport enum LogMessageType {\n  MONITORING = 1,\n}\n\nexport interface LogMessage {\n  type: LogMessageType;\n  payload?: MonitoringTrace;\n}\n\nexport interface Score {\n  value: number;\n  reason?: string;\n}\n\nexport enum FunctionType {\n  Numeric = \"numeric\",\n  Classification = \"classification\",\n}\n\nexport type NumericScoreConfig = {\n  type: FunctionType.Numeric;\n  aggregate: \"mean\" | \"median\";\n};\n\nexport type ClassificationScoreConfig = {\n  type: FunctionType.Classification;\n  labels: Record<number, string>;\n  colors?: Record<number, LabelColor>;\n};\n\nexport type ScoreConfig = ClassificationScoreConfig | NumericScoreConfig;\n\nexport enum ScorerExecutionType {\n  Local = \"local\",\n  Remote = \"remote\",\n}\n\ntype Scorer = LocalScorer | LLMClassifyScorer;\n\nexport interface LocalScorer {\n  type: ScorerExecutionType.Local;\n  scoreFn: (args: {\n    input: InputType;\n    output: OutputType;\n    expected: OutputType;\n  }) => Promise<Score>;\n}\n\ninterface RemoteScorer {\n  type: ScorerExecutionType.Remote;\n}\n\ninterface PromptConfig {\n  slug: string;\n  label?: string;\n}\n\nexport enum ScoreParserType {\n  XML = \"xml\",\n  JSON = \"json\",\n}\n\ninterface ScoreParserConfig {\n  type: ScoreParserType;\n}\n\nexport interface LLMClassifyScorer extends RemoteScorer {\n  method: \"classify\";\n  prompt: PromptConfig;\n  variableMappings?: Record<string, string>;\n  scoreParser: ScoreParserConfig;\n}\n\nexport interface ScoringFunction {\n  name: string;\n  version: number;\n  scoreConfig?: ScoreConfig;\n  scorer: Scorer;\n}\n\nexport interface CustomScoringConfig {\n  id: string;\n  key_name: string;\n}\n\nexport const ScoringErrorValue = -1;\n\nexport const ScoringErrorPrefix = \"<!--hamming_scoring_error-->\";\n\nexport enum LabelColor {\n  Gray = \"gray\",\n  LightGreen = \"light-green\",\n  LightBlue = \"light-blue\",\n  Amber = \"amber\",\n  Purple = \"purple\",\n  Pink = \"pink\",\n  Green = \"green\",\n  PastelGreen = \"pastel-green\",\n  Yellow = \"yellow\",\n  Blue = \"blue\",\n  Red = \"red\",\n}\n\nexport interface Prompt {\n  slug: string;\n}\n\nexport interface ToolChoice {\n  choice: string;\n  functionName: string;\n}\n\nexport interface PromptSettings {\n  temperature?: number;\n  maxTokens?: number;\n  topP?: number;\n  frequencyPenalty?: number;\n  presencePenalty?: number;\n  toolChoice?: ToolChoice;\n}\n\nexport interface ChatMessage {\n  role: string;\n  content: string;\n}\n\nexport interface PromptContent {\n  languageModel: string;\n  promptSettings: PromptSettings;\n  chatMessages: ChatMessage[];\n  tools?: string;\n}\n\nexport interface PromptWithContent extends Prompt {\n  content?: PromptContent;\n}\n\nexport enum EventKind {\n  Root = \"root\",\n  Call = \"call\",\n  CallEvent = \"call_event\",\n}\n\nexport enum CallProvider {\n  Custom = \"custom\",\n  Retell = \"retell\",\n  Vapi = \"vapi\",\n}\n\nexport enum RetellCallEventType {\n  Started = \"call_started\",\n  Ended = \"call_ended\",\n  Analyzed = \"call_analyzed\",\n}\n\nexport interface RetellCallEvent {\n  event: RetellCallEventType;\n  call: Record<string, unknown>;\n}\n\nexport enum VapiCallEventType {\n  StatusUpdate = \"status-update\",\n  EndOfCallReport = \"end-of-call-report\",\n}\n\nexport interface VapiCallEvent {\n  message: {\n    type: VapiCallEventType;\n  } & Record<string, unknown>;\n}\n\nexport type CustomCallEvent = Record<string, unknown>;\n\nexport type CallEvent = CustomCallEvent | RetellCallEvent | VapiCallEvent;\n","const MAX_WORKERS = 100;\n\nexport async function runWorkers<T>(\n  workItems: T[],\n  runFn: (workItem: T) => Promise<void>,\n  count: number = MAX_WORKERS,\n) {\n  const iterator = workItems.entries();\n  const workerCount = Math.min(count, workItems.length, MAX_WORKERS);\n  const workers = Array(workerCount)\n    .fill(iterator)\n    .map(async (iterator, idx) => {\n      for (const [index, workItem] of iterator) {\n        if (process.env.NODE_ENV === \"development\") {\n          console.log(`Worker ${idx} is processing task ${index}`);\n        }\n        await runFn(workItem);\n        if (process.env.NODE_ENV === \"development\") {\n          console.log(`Worker ${idx} has finished task ${index}`);\n        }\n      }\n    });\n  await Promise.all(workers);\n}\n","import { asyncRunContext } from \"../asyncStorage\";\nimport type { Hamming } from \"../client\";\nimport {\n  CustomScoringConfig,\n  DatasetId,\n  DatasetItem,\n  Experiment,\n  ExperimentItem,\n  ExperimentItemContext,\n  ExperimentStatus,\n  InputType,\n  LocalScorer,\n  MetadataType,\n  OutputType,\n  RunContext,\n  Runner,\n  RunOptions,\n  Score,\n  ScorerExecutionType,\n  ScoreType,\n  ScoringErrorPrefix,\n  ScoringErrorValue,\n  ScoringFunction,\n  TracingMode,\n} from \"../types\";\nimport { runWorkers } from \"../worker\";\n\nconst MAX_SAMPLES = 10;\n\nfunction newRunContext(itemId: string): RunContext {\n  return {\n    tracing: {\n      experiment: {\n        itemId,\n      },\n    },\n  };\n}\n\nconst defaultScoreTypes = [ScoreType.StringDiff];\n\ninterface RegisteredScoringFunction extends ScoringFunction {\n  registration: CustomScoringConfig;\n}\n\nclass ExperimentItems {\n  private client: Hamming;\n\n  constructor(client: Hamming) {\n    this.client = client;\n  }\n\n  async start(\n    experiment: Experiment,\n    datasetItem: DatasetItem,\n    sampleId?: number,\n  ): Promise<ExperimentItemContext> {\n    const resp = await this.client.fetch(\n      `/experiments/${experiment.id}/items`,\n      {\n        method: \"POST\",\n        body: JSON.stringify({\n          datasetItemId: datasetItem.id,\n          output: {},\n          metrics: {},\n          sampleId,\n        }),\n      },\n    );\n    const data = await resp.json();\n    const item = data.item as ExperimentItem;\n\n    const startTs = Date.now();\n    return {\n      item,\n      startTs,\n    };\n  }\n\n  async end(\n    itemContext: ExperimentItemContext,\n    output: OutputType,\n    scores: Record<string, Score> = {},\n    failed: boolean = false,\n  ) {\n    const { item, startTs } = itemContext;\n    const durationMs = Date.now() - startTs;\n    await this.client.tracing._flush(item.id);\n    // Completing the experiment item should happen after the traces are\n    // flushed, since it will automatically trigger scoring.\n    await this.client.fetch(\n      `/experiments/${item.experimentId}/items/${item.id}`,\n      {\n        method: \"PATCH\",\n        body: JSON.stringify({\n          output,\n          scores,\n          metrics: {\n            durationMs,\n          },\n          failed,\n        }),\n      },\n    );\n  }\n}\n\nexport class Experiments {\n  private client: Hamming;\n  private items: ExperimentItems;\n\n  constructor(client: Hamming) {\n    this.client = client;\n    this.items = new ExperimentItems(this.client);\n  }\n\n  async run(opts: RunOptions, run: Runner) {\n    const { dataset: datasetId } = opts;\n    const dataset = await this.client.datasets.load(datasetId);\n\n    this.client.tracing._setMode(TracingMode.EXPERIMENT);\n\n    const {\n      name = this.generateName(dataset.name),\n      scoring = defaultScoreTypes,\n      metadata = {},\n      sampling,\n    } = opts;\n\n    const sampleCount = sampling ?? 1;\n    if (sampleCount > MAX_SAMPLES) {\n      throw new Error(`The maximum number of samples is ${MAX_SAMPLES}.`);\n    }\n\n    const scoringHelper = new ScoringHelper(this.client, scoring);\n    await scoringHelper.initialize();\n\n    const experiment = await this.start(\n      name,\n      datasetId,\n      scoringHelper.getConfig(),\n      metadata,\n      sampling,\n    );\n    const baseUrl = new URL(this.client.baseURL);\n    const experimentUrl = `${baseUrl.origin}/experiments/${experiment.id}`;\n\n    try {\n      for (let sampleId = 0; sampleId < sampleCount; sampleId++) {\n        if (opts.parallel) {\n          const runFn = async (datasetItem: DatasetItem) => {\n            const itemCtx = await this.items.start(\n              experiment,\n              datasetItem,\n              sampleId,\n            );\n            try {\n              const output = await asyncRunContext.run(\n                newRunContext(itemCtx.item.id),\n                async () => run(datasetItem.input),\n              );\n              if (!output || typeof output !== \"object\") {\n                throw new Error(`Invalid output: ${output}`);\n              }\n              const scores = await scoringHelper.score(\n                datasetItem.input,\n                datasetItem.output,\n                output,\n              );\n              await this.items.end(itemCtx, output, scores);\n            } catch (err) {\n              console.error(err);\n              const msg = err instanceof Error ? err.message : \"Unknown error\";\n              const output = { error: msg };\n              await this.items.end(itemCtx, output, {}, true);\n            }\n          };\n          const workerCount =\n            typeof opts.parallel === \"number\" ? opts.parallel : undefined;\n          await runWorkers(dataset.items, runFn, workerCount);\n        } else {\n          for (const datasetItem of dataset.items) {\n            const itemCtx = await this.items.start(\n              experiment,\n              datasetItem,\n              sampleId,\n            );\n            try {\n              const output = await asyncRunContext.run(\n                newRunContext(itemCtx.item.id),\n                async () => await run(datasetItem.input),\n              );\n              if (!output || typeof output !== \"object\") {\n                throw new Error(`Invalid output: ${output}`);\n              }\n              const scores = await scoringHelper.score(\n                datasetItem.input,\n                datasetItem.output,\n                output,\n              );\n              await this.items.end(itemCtx, output, scores);\n            } catch (err) {\n              console.error(err);\n              const msg = err instanceof Error ? err.message : \"Unknown error\";\n              const output = { error: msg };\n              await this.items.end(itemCtx, output, {}, true);\n            }\n          }\n        }\n      }\n    } catch (err) {\n      await this.end(experiment, ExperimentStatus.FAILED);\n      throw err;\n    } finally {\n      await this.end(experiment);\n      console.log(\"See experiment results at:\", experimentUrl);\n    }\n    return { experimentUrl };\n  }\n\n  private async start(\n    name: string,\n    dataset: DatasetId,\n    scoring: (ScoreType | CustomScoringConfig)[],\n    metadata: MetadataType,\n    sampling?: number,\n  ): Promise<Experiment> {\n    const status = ExperimentStatus.RUNNING;\n    const resp = await this.client.fetch(`/experiments`, {\n      method: \"POST\",\n      body: JSON.stringify({\n        name,\n        dataset,\n        status,\n        scoring,\n        metadata,\n        sampling,\n      }),\n    });\n\n    const data = await resp.json();\n    return data.experiment as Experiment;\n  }\n\n  private async end(\n    experiment: Experiment,\n    status: ExperimentStatus = ExperimentStatus.FINISHED,\n  ) {\n    await this.client.fetch(`/experiments/${experiment.id}`, {\n      method: \"PATCH\",\n      body: JSON.stringify({\n        status,\n      }),\n    });\n  }\n\n  private generateName(datasetName: string): string {\n    const now = new Date();\n    return `Experiment for ${datasetName} - ${now.toLocaleString()}`;\n  }\n}\n\nclass ScoringHelper {\n  private readonly client: Hamming;\n\n  public readonly standardScoring: ScoreType[];\n  public readonly customScoring: ScoringFunction[];\n\n  private registeredFunctions: RegisteredScoringFunction[] = [];\n  private initialized = false;\n\n  constructor(client: Hamming, scoring: (ScoringFunction | ScoreType)[]) {\n    this.client = client;\n\n    this.standardScoring = scoring.filter(\n      (score): score is ScoreType => typeof score === \"string\",\n    );\n    this.customScoring = scoring.filter(\n      (score): score is ScoringFunction => typeof score !== \"string\",\n    );\n  }\n\n  async initialize() {\n    await this.registerScoringFunctions();\n    this.initialized = true;\n  }\n\n  getConfig(): (ScoreType | CustomScoringConfig)[] {\n    if (!this.initialized) {\n      throw new Error(\"ScoringHelper is not initialized\");\n    }\n    return [\n      ...this.standardScoring,\n      ...this.registeredFunctions.map((f) => f.registration),\n    ];\n  }\n\n  async score(\n    input: InputType,\n    expected: OutputType,\n    output: OutputType,\n  ): Promise<Record<string, Score>> {\n    if (!this.initialized) {\n      throw new Error(\"ScoringHelper is not initialized\");\n    }\n    const scores = {} as Record<string, Score>;\n    const promises = this.registeredFunctions\n      .filter((f) => f.scorer.type === \"local\")\n      .map(async (f) => {\n        const scorer = f.scorer as LocalScorer;\n\n        try {\n          scores[f.registration.key_name] = await scorer.scoreFn({\n            input,\n            output,\n            expected,\n          });\n        } catch (err) {\n          console.error(\n            `Failed to locally run score ${f.name.toLowerCase()}.`,\n            \"Note: This error will be displayed in the dashboard. All other scoring will be preserved and displayed accordingly.\",\n            \"Error received:\",\n            err,\n          );\n          scores[f.registration.key_name] = {\n            value: ScoringErrorValue,\n            reason: `${ScoringErrorPrefix}${err.message}`,\n          };\n        }\n      });\n    await Promise.allSettled(promises);\n    return scores;\n  }\n\n  private async registerScoringFunctions() {\n    const scoring = this.customScoring.map((scoringFunc) => ({\n      name: scoringFunc.name,\n      version: scoringFunc.version,\n      score_config: scoringFunc.scoreConfig,\n      execution_config: getExecutionConfig(scoringFunc),\n    }));\n    const resp = await this.client.fetch(`/scoring/register-functions`, {\n      method: \"POST\",\n      body: JSON.stringify({\n        scoring: scoring,\n      }),\n    });\n\n    const data = await resp.json();\n    const registrations = (data.scoring ?? []) as CustomScoringConfig[];\n    this.registeredFunctions = this.customScoring.map(\n      (scoringFunction, idx) => ({\n        ...scoringFunction,\n        registration: registrations[idx],\n      }),\n    );\n  }\n}\n\nfunction getExecutionConfig(scoringFunc: ScoringFunction): Record<string, any> {\n  if (scoringFunc.scorer.type === ScorerExecutionType.Remote) {\n    const { prompt, variableMappings, scoreParser } = scoringFunc.scorer;\n    return {\n      kind: \"remote\",\n      prompt,\n      variableMappings,\n      scoreParser,\n    };\n  }\n  return {\n    kind: \"local\",\n  };\n}\n","import { LRUCache } from \"lru-cache\";\n\nimport type {\n  CallEvent,\n  Hamming,\n  MonitoringStartOpts,\n  RunContext,\n  TraceEvent,\n  VapiCallEvent,\n} from \"../index\";\n\nimport {\n  CallProvider,\n  EventKind,\n  MonitoringItemType,\n  RetellCallEvent,\n  RetellCallEventType,\n  VapiCallEventType,\n} from \"../types\";\n\nimport { asyncRunContext } from \"../asyncStorage\";\nimport {\n  MonitoringItem as IMonitoringItem,\n  ITracing,\n  InputType,\n  MetadataType,\n  MonitoringItemStatus,\n  MonitoringSession,\n  MonitoringTrace,\n  MonitoringTraceContext,\n  OutputType,\n  TracingMode,\n} from \"../types\";\nimport { parseRetellCallId, parseVapiCallId } from \"../utils/voice\";\nimport { TracerBase } from \"./tracing\";\n\nenum MonitoringState {\n  STARTED,\n  STOPPED,\n}\n\nconst INVALID_SESSION_ID = \"INVALID_SESSION\";\n\nfunction newRunContext(seqId: number): RunContext {\n  return {\n    tracing: {\n      monitoring: {\n        seqId,\n      },\n    },\n  };\n}\n\nclass MonitoringItemTracing extends TracerBase implements ITracing {\n  client: Hamming;\n  runCtx: RunContext;\n\n  constructor(client: Hamming, seqId: number) {\n    super();\n    this.client = client;\n    this.runCtx = newRunContext(seqId);\n  }\n\n  logEvent(event: TraceEvent) {\n    const trace = this.client.monitoring._getTraceContext(this.runCtx);\n    if (!trace) return;\n    this.client.tracing._logLiveTrace({\n      event,\n      ...trace,\n    });\n  }\n}\n\nclass MonitoringItem implements IMonitoringItem {\n  client: Hamming;\n  sessionId: string;\n  seqId: number;\n  input: InputType | undefined;\n  output: OutputType | undefined;\n  metadata: MetadataType | undefined;\n  metrics: Record<string, any>;\n  status: MonitoringItemStatus;\n  errorMessage: string | undefined;\n  startTs: number;\n  itemType: MonitoringItemType;\n  tracing: ITracing;\n\n  constructor(\n    client: Hamming,\n    sessionId: string,\n    seqId: number,\n    itemType: MonitoringItemType,\n  ) {\n    this.itemType = itemType;\n    this.client = client;\n    this.sessionId = sessionId;\n    this.seqId = seqId;\n    this.metrics = {};\n    this.tracing = new MonitoringItemTracing(client, seqId);\n  }\n\n  setInput(input: InputType) {\n    this.input = input;\n  }\n\n  setOutput(output: OutputType) {\n    this.output = output;\n  }\n\n  setMetadata(metadata: MetadataType) {\n    this.metadata = metadata;\n  }\n\n  end(error: boolean = false, errorMessage?: string) {\n    this._end(error, errorMessage);\n  }\n\n  _start() {\n    this.startTs = Date.now();\n    this.status = MonitoringItemStatus.STARTED;\n  }\n\n  _end(error: boolean = false, errorMessage?: string) {\n    if (this._hasEnded()) return;\n\n    this.metrics.duration_ms = Date.now() - this.startTs;\n    this.status = error\n      ? MonitoringItemStatus.FAILED\n      : MonitoringItemStatus.COMPLETED;\n    this.errorMessage = errorMessage;\n    this.client.monitoring._endItem(this._toTrace());\n  }\n\n  _hasEnded() {\n    return [\n      MonitoringItemStatus.COMPLETED,\n      MonitoringItemStatus.FAILED,\n    ].includes(this.status);\n  }\n\n  _toTrace(): MonitoringTrace {\n    return {\n      session_id: this.sessionId,\n      seq_id: this.seqId,\n      parent_seq_id: undefined,\n      event: {\n        kind:\n          this.itemType === MonitoringItemType.CALL\n            ? EventKind.Call\n            : EventKind.Root,\n        input: this.input,\n        output: this.output,\n        metadata: this.metadata,\n        metrics: this.metrics,\n        status: this.status,\n        error_message: this.errorMessage,\n      },\n    };\n  }\n}\n\nexport class Monitoring {\n  client: Hamming;\n  private state: MonitoringState = MonitoringState.STOPPED;\n  private session: MonitoringSession | null;\n  private monitoringStartOpts: MonitoringStartOpts | undefined;\n  private callEvents: LRUCache<string, IMonitoringItem>;\n\n  constructor(client: Hamming) {\n    this.client = client;\n  }\n\n  start(opts?: MonitoringStartOpts) {\n    // Delay creating session until the first async call of runItem\n    this.monitoringStartOpts = opts;\n    this.client._logger.start();\n    this.client.tracing._setMode(TracingMode.MONITORING);\n    this.state = MonitoringState.STARTED;\n    this.callEvents = new LRUCache<string, IMonitoringItem>({\n      max: 1000,\n      ttl: 1000 * 60 * 60 * 2, // 2 hours\n    });\n    console.log(\"Monitoring started!\");\n  }\n\n  stop() {\n    this.session = null;\n    this.client.tracing._setMode(TracingMode.OFF);\n    this.client._logger.stop();\n    this.state = MonitoringState.STOPPED;\n    this.callEvents.clear();\n    console.log(\"Monitoring stopped!\");\n  }\n\n  async runItem(\n    callback: (item: IMonitoringItem) => unknown | Promise<unknown>,\n  ): Promise<unknown> {\n    await this._createSessionIfNotExist();\n    const [sessionId, seqId] = this._nextSeqId();\n\n    const item = new MonitoringItem(\n      this.client,\n      sessionId,\n      seqId,\n      MonitoringItemType.TEXT,\n    );\n    item._start();\n\n    try {\n      const response = await asyncRunContext.run(\n        newRunContext(item.seqId),\n        async () => await callback(item),\n      );\n      if (!item.output) {\n        if (\n          response &&\n          response instanceof Object &&\n          !Array.isArray(response)\n        ) {\n          item.setOutput(response);\n        } else {\n          item.setOutput({ response });\n        }\n      }\n      item._end();\n\n      return response;\n    } catch (error) {\n      item._end(true, error.message);\n      throw error;\n    }\n  }\n\n  async _startItem(itemType: MonitoringItemType): Promise<IMonitoringItem> {\n    await this._createSessionIfNotExist();\n    const [sessionId, seqId] = this._nextSeqId();\n\n    const item = new MonitoringItem(this.client, sessionId, seqId, itemType);\n    item._start();\n    return item;\n  }\n\n  async startItem(): Promise<IMonitoringItem> {\n    return this._startItem(MonitoringItemType.TEXT);\n  }\n\n  async _startCall(): Promise<IMonitoringItem> {\n    return this._startItem(MonitoringItemType.CALL);\n  }\n\n  _endItem(trace: MonitoringTrace) {\n    if (this.state === MonitoringState.STOPPED) {\n      return;\n    }\n    this.client.tracing._logLiveTrace(trace);\n  }\n\n  _getTraceContext(ctx?: RunContext): MonitoringTraceContext | null {\n    if (this.state === MonitoringState.STOPPED) {\n      return null;\n    }\n    if (!this.session) throw Error(\"Monitoring not started\");\n\n    const [sessionId, seqId] = this._nextSeqId();\n    const parentSeqId = ctx?.tracing?.monitoring?.seqId;\n\n    return {\n      session_id: sessionId,\n      seq_id: seqId,\n      parent_seq_id: parentSeqId,\n    };\n  }\n\n  async callEvent(\n    provider: CallProvider,\n    event: CallEvent,\n    metadata?: MetadataType,\n  ) {\n    switch (provider) {\n      case CallProvider.Custom:\n        throw Error(\"Custom call provider not implemented!\");\n      case CallProvider.Retell:\n        await this.handleRetellCallEvent(event as RetellCallEvent, metadata);\n        break;\n      case CallProvider.Vapi:\n        await this.handleVapiCallEvent(event as VapiCallEvent, metadata);\n        break;\n      default:\n        throw Error(`Unsupported call provider: ${provider}`);\n    }\n  }\n\n  async handleRetellCallEvent(\n    evt: RetellCallEvent,\n    metadata: MetadataType = {},\n  ) {\n    const callId = parseRetellCallId(evt);\n    if (!callId) {\n      throw Error(\"call_id is missing\");\n    }\n    let monitoringItem: IMonitoringItem | undefined;\n    if (evt.event === RetellCallEventType.Started) {\n      monitoringItem = await this._startCall();\n      monitoringItem.tracing.log({\n        kind: EventKind.CallEvent,\n        event: evt,\n      });\n      this.callEvents.set(callId, monitoringItem);\n    } else if (evt.event === RetellCallEventType.Ended) {\n      monitoringItem = this.callEvents.get(callId);\n      if (!monitoringItem) {\n        console.warn(`Missing call_started event for id: ${callId}`);\n        monitoringItem = await this._startCall();\n        this.callEvents.set(callId, monitoringItem);\n      }\n      monitoringItem.tracing.log({\n        kind: EventKind.CallEvent,\n        event: evt,\n      });\n      monitoringItem.setInput({\n        provider: CallProvider.Retell,\n      });\n      monitoringItem.setMetadata(metadata);\n      monitoringItem.setOutput(evt.call);\n      monitoringItem.end();\n    } else if (evt.event === RetellCallEventType.Analyzed) {\n      monitoringItem = this.callEvents.get(callId);\n      if (!monitoringItem) {\n        monitoringItem = await this._startCall();\n        monitoringItem.tracing.log({\n          kind: EventKind.CallEvent,\n          event: evt,\n        });\n        monitoringItem.setInput({\n          provider: CallProvider.Retell,\n        });\n        monitoringItem.setMetadata(metadata);\n        monitoringItem.setOutput(evt.call);\n        monitoringItem.end();\n      } else {\n        monitoringItem.tracing.log({\n          kind: EventKind.CallEvent,\n          event: evt,\n        });\n      }\n    }\n  }\n\n  async handleVapiCallEvent(evt: VapiCallEvent, metadata: MetadataType = {}) {\n    if (\n      evt.message.type !== VapiCallEventType.StatusUpdate &&\n      evt.message.type !== VapiCallEventType.EndOfCallReport\n    ) {\n      return;\n    }\n    const callId = parseVapiCallId(evt);\n    if (!callId) {\n      throw Error(\"call ID is missing\");\n    }\n    const crea