@arizeai/phoenix-evals/dist/esm/helpers/createEvaluator.d.ts

A library for running evaluations for AI use cases

import type { EvaluatorBase } from "../core/EvaluatorBase.js";
import type { EvaluationKind, OptimizationDirection, TelemetryConfig } from "../types/index.js";
type AnyFn = (...args: any[]) => any;
/**
 * Options for creating a custom evaluator using {@link createEvaluator}.
 *
 * @public
 */
export type CreateEvaluatorOptions = {
    /**
     * The name of the evaluator / metric that it measures.
     *
     * If not provided, the function will attempt to infer the name from the function's `name` property.
     * If the function has no name, a unique name will be generated.
     *
     * @example
     * ```typescript
     * const evaluator = createEvaluator(myFunction, { name: "custom-metric" });
     * ```
     */
    name?: string;
    /**
     * The kind of the evaluation.
     *
     * - `"CODE"`: Code-based evaluator that runs custom logic
     * - `"LLM"`: LLM-based evaluator that uses a language model
     *
     * @defaultValue `"CODE"`
     *
     * @example
     * ```typescript
     * const evaluator = createEvaluator(myFunction, { kind: "CODE" });
     * ```
     */
    kind?: EvaluationKind;
    /**
     * The direction to optimize the numeric evaluation score.
     *
     * - `"MAXIMIZE"`: Higher scores are better (e.g., accuracy, F1 score)
     * - `"MINIMIZE"`: Lower scores are better (e.g., error rate, latency)
     *
     * @defaultValue `"MAXIMIZE"`
     *
     * @example
     * ```typescript
     * const evaluator = createEvaluator(myFunction, {
     *   optimizationDirection: "MAXIMIZE"
     * });
     * ```
     */
    optimizationDirection?: OptimizationDirection;
    /**
     * The telemetry configuration for the evaluator.
     *
     * When enabled, the evaluator will automatically create OpenTelemetry spans
     * for tracing and observability. This allows you to track evaluator performance
     * and debug issues in distributed systems.
     *
     * @defaultValue `{ isEnabled: true }`
     *
     * @example
     * ```typescript
     * const evaluator = createEvaluator(myFunction, {
     *   telemetry: { isEnabled: true, tracer: myTracer }
     * });
     * ```
     */
    telemetry?: TelemetryConfig;
};
/**
 * A factory function for creating a custom evaluator from any function.
 *
 * This function wraps a user-provided function into an evaluator that can be used
 * with Phoenix experiments and evaluations. The function can be synchronous or
 * asynchronous, and can return a number, an {@link EvaluationResult} object, or
 * a value that will be automatically converted to an evaluation result.
 *
 * The evaluator will automatically:
 * - Convert the function's return value to an {@link EvaluationResult}
 * - Handle both sync and async functions
 * - Wrap the function with OpenTelemetry spans if telemetry is enabled
 * - Infer the evaluator name from the function name if not provided
 *
 * @typeParam RecordType - The type of the input record that the evaluator expects.
 *   Must extend `Record<string, unknown>`.
 * @typeParam Fn - The type of the function being wrapped. Must be a function that
 *   accepts the record type and returns a value compatible with {@link EvaluationResult}.
 *
 * @param fn - The function to wrap as an evaluator. Can be synchronous or asynchronous.
 *   The function should accept a record of type `RecordType` and return either:
 *   - A number (will be converted to `{ score: number }`)
 *   - An {@link EvaluationResult} object
 *   - Any value that can be converted to an evaluation result
 *
 * @param options - Optional configuration for the evaluator. See {@link CreateEvaluatorOptions}
 *   for details on available options.
 *
 * @returns An {@link EvaluatorInterface} that can be used with Phoenix experiments
 *   and evaluation workflows.
 *
 * @example
 * Basic usage with a simple scoring function:
 * ```typescript
 * const accuracyEvaluator = createEvaluator(
 *   ({ output, expected }) => {
 *     return output === expected ? 1 : 0;
 *   },
 *   {
 *     name: "accuracy",
 *     kind: "CODE",
 *     optimizationDirection: "MAXIMIZE"
 *   }
 * );
 *
 * const result = await accuracyEvaluator.evaluate({
 *   output: "correct answer",
 *   expected: "correct answer"
 * });
 * // result: { score: 1 }
 * ```
 *
 *
 * @example
 * Returning a full EvaluationResult:
 * ```typescript
 * const qualityEvaluator = createEvaluator(
 *   ({ output }) => {
 *     const score = calculateQuality(output);
 *     return {
 *       score,
 *       label: score > 0.8 ? "high" : "low",
 *       explanation: `Quality score: ${score}`
 *     };
 *   },
 *   { name: "quality" }
 * );
 * ```
 */
export declare function createEvaluator<RecordType extends Record<string, unknown> = Record<string, unknown>, Fn extends AnyFn = AnyFn>(fn: Fn, options?: CreateEvaluatorOptions): EvaluatorBase<RecordType>;
export {};
//# sourceMappingURL=createEvaluator.d.ts.map