axiom/dist/evals.d.ts

Axiom AI SDK provides an API to wrap your AI calls with observability instrumentation.

import { S as Score, a as Scorer } from './scorer.types-D6mnSKTJ.js';
import { TestError, SerializedError } from 'vitest';
import { Reporter, TestSuite, TestModule, TestRunEndReason } from 'vitest/node.js';

/**
 * Function type for evaluation tasks that process input data and produce output.
 *
 * Used with {@link EvalParams} to define the task that will be evaluated against a dataset.
 * The task output will be scored by functions defined in {@link EvalParams.scorers}.
 *
 * @experimental This API is experimental and may change in future versions.
 *
 * @param input - The input data to process
 * @param expected - The expected output for comparison/validation
 * @returns Promise that resolves to the task output, or the output directly
 *
 * @example
 * ```typescript
 * const textGenerationTask: EvalTask<string, string> = async (input, expected) => {
 *   const result = await generateText({
 *     model: myModel,
 *     prompt: input
 *   });
 *   return result.text;
 * };
 * ```
 */
type EvalTask<TInput, TExpected> = (input: TInput, expected: TExpected) => Promise<any> | any;
/**
 * Record type of a matric of collection data
 */
type CollectionRecord = {
    /** Optional name for the record, if not set, it will default to eval name + index of the record */
    name?: string;
    input: string | Record<string, any>;
    expected: string | Record<string, any>;
};
/**
 * Configuration parameters for running an evaluation.
 *
 * Used with {@link Eval} to define how an evaluation should be executed.
 * Results are captured in {@link EvalReport} format.
 *
 * @experimental This API is experimental and may change in future versions.
 */
type EvalParams = {
    /** Function that returns the dataset with input/expected pairs for evaluation */
    data: () => Promise<CollectionRecord[]> | CollectionRecord[];
    /** The {@link EvalTask} function to execute for each data item */
    task: EvalTask<any, any>;
    /** Array of scoring functions to evaluate the task output, producing {@link Score} results */
    scorers: Array<Scorer>;
    /** Minimum score threshold for passing (0.0 to 1.0) */
    threshold: number;
    /** Optional timeout in milliseconds for task execution */
    timeout?: number;
};
/**
 * Complete report for a single evaluation case including results and metadata.
 *
 * Generated for each test case when running {@link Eval} with {@link EvalParams}.
 * Contains all {@link Score} results and execution metadata.
 *
 * @experimental This API is experimental and may change in future versions.
 */
type EvalReport = {
    /** Order/index of this case in the evaluation suite */
    index: number;
    /** Name of the evaluation */
    name: string;
    /** Input data that was provided to the {@link EvalTask} */
    input: string | Record<string, any>;
    /** Output produced by the {@link EvalTask} */
    output: string | Record<string, any>;
    /** Expected output for comparison */
    expected: string | Record<string, any>;
    /** Array of {@link Score} results from all scorers that were run */
    scores: Record<string, Score>;
    /** Any errors that occurred during evaluation */
    errors: TestError[] | null;
    /** Status of the evaluation case */
    status: 'success' | 'fail' | 'pending';
    /** Duration in milliseconds for the entire case */
    duration: number | undefined;
    /** Timestamp when the case started */
    startedAt: number | undefined;
    /** Score threshold from {@link EvalParams.threshold} that was used for pass/fail determination */
    threshold: number | undefined;
};

declare module 'vitest' {
    interface TaskMeta {
        eval?: EvalReport;
    }
}
/**
 * Creates and registers an evaluation suite with the given name and parameters.
 *
 * This function sets up a complete evaluation pipeline that will run your {@link EvalTask}
 * against a dataset, score the results, and provide detailed {@link EvalReport} reporting.
 *
 * @experimental This API is experimental and may change in future versions.
 *
 * @param name - Human-readable name for the evaluation suite
 * @param params - {@link EvalParams} configuration parameters for the evaluation
 *
 * @example
 * ```typescript
 * import { experimental_Eval as Eval } from 'axiom/ai/evals';
 *
 * Eval('Text Generation Quality', {
 *   data: async () => [
 *     { input: 'Explain photosynthesis', expected: 'Plants convert light to energy...' },
 *     { input: 'What is gravity?', expected: 'Gravity is a fundamental force...' }
 *   ],
 *   task: async (input) => {
 *     const result = await generateText({
 *       model: yourModel,
 *       prompt: input
 *     });
 *     return result.text;
 *   },
 *   scorers: [similarityScorer, factualAccuracyScorer],
 *   threshold: 0.7
 * });
 * ```
 */
declare const Eval: (name: string, params: EvalParams) => void;

/**
 * Custom Vitest reporter for Axiom AI evaluations.
 *
 * This reporter collects evaluation results and scores from tests
 * and processes them for further analysis and reporting.
 *
 * @experimental This API is experimental and may change in future versions.
 */
declare class AxiomReporter implements Reporter {
    onTestSuiteReady(_testSuite: TestSuite): void;
    onTestSuiteResult(testSuite: TestSuite): void;
    onTestRunEnd(_testModules: ReadonlyArray<TestModule>, _errors: ReadonlyArray<SerializedError>, _reason: TestRunEndReason): Promise<void>;
}

export { AxiomReporter as experimental_AxiomReporter, Eval as experimental_Eval, type EvalParams as experimental_EvalParams, type EvalReport as experimental_EvalReport, type EvalTask as experimental_EvalTask };