@dooor-ai/toolkit/src/evals/latency.ts

Guards, Evals & Observability for AI applications - works seamlessly with LangChain/LangGraph

import { Eval } from "./base";
import { EvalResult, EvalConfig } from "../core/types";

interface LatencyEvalConfig extends EvalConfig {
  /** Threshold in milliseconds (default: 3000ms = 3s) */
  thresholdMs?: number;
  /** Target latency for scoring (default: 1000ms = 1s) */
  targetMs?: number;
  /** Max latency before score is 0 (default: 10000ms = 10s) */
  maxMs?: number;
}

/**
 * Evaluates the latency of LLM responses
 * 
 * Score calculation:
 * - latency <= target: score = 1.0
 * - target < latency < max: linear scale from 1.0 to 0.0
 * - latency >= max: score = 0.0
 */
export class LatencyEval extends Eval {
  private thresholdMs: number;
  private targetMs: number;
  private maxMs: number;

  constructor(config: LatencyEvalConfig = {}) {
    super(config);
    this.thresholdMs = config.thresholdMs ?? 3000; // 3 seconds
    this.targetMs = config.targetMs ?? 1000; // 1 second
    this.maxMs = config.maxMs ?? 10000; // 10 seconds
  }

  get name(): string {
    return "LatencyEval";
  }

  evaluate(
    input: string,
    output: string,
    metadata?: Record<string, any>
  ): EvalResult {
    const latency = metadata?.latency ?? 0;

    // Calculate score
    let score: number;
    if (latency <= this.targetMs) {
      score = 1.0;
    } else if (latency >= this.maxMs) {
      score = 0.0;
    } else {
      // Linear interpolation between target and max
      const range = this.maxMs - this.targetMs;
      const delta = latency - this.targetMs;
      score = 1.0 - (delta / range);
    }

    const passed = latency <= this.thresholdMs;

    return {
      name: this.name,
      score: Math.max(0, Math.min(1, score)), // Clamp to [0, 1]
      passed,
      details: passed
        ? `Latency ${latency}ms is acceptable (threshold: ${this.thresholdMs}ms)`
        : `Latency ${latency}ms exceeds threshold of ${this.thresholdMs}ms`,
      metadata: {
        latency,
        thresholdMs: this.thresholdMs,
        targetMs: this.targetMs,
        maxMs: this.maxMs,
      },
      timestamp: new Date(),
    };
  }
}