@dooor-ai/toolkit/src/evals/base.ts

Guards, Evals & Observability for AI applications - works seamlessly with LangChain/LangGraph

import { EvalResult, EvalConfig } from "../core/types";

/**
 * Abstract base class for all evals
 */
export abstract class Eval {
  protected config: EvalConfig;

  constructor(config: EvalConfig = {}) {
    this.config = {
      threshold: 0.7,
      enabled: true,
      ...config,
    };
  }

  /**
   * Get the name of this eval
   */
  abstract get name(): string;

  /**
   * Evaluate the LLM response
   * @param input - The input that was sent to the LLM
   * @param output - The output from the LLM
   * @param metadata - Additional metadata (e.g., latency, tokens, cost)
   * @returns EvalResult with score and pass/fail
   */
  abstract evaluate(
    input: string,
    output: string,
    metadata?: Record<string, any>
  ): Promise<EvalResult> | EvalResult;

  /**
   * Check if this eval is enabled
   */
  isEnabled(): boolean {
    return this.config.enabled ?? true;
  }

  /**
   * Get the threshold
   */
  getThreshold(): number {
    return this.config.threshold ?? 0.7;
  }
}