axiom/dist/scorers/scorers.d.ts

Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals - online evals

import { S as Score, V as ValidateName, a as ScorerOptions, b as Scorer } from '../name-validation.d-BKPGh6r3.js';
export { c as ScoreWithName, d as ScorerLike } from '../name-validation.d-BKPGh6r3.js';
import './aggregations.js';

type Simplify<T> = {
    [K in keyof T]: T[K];
} & {};
type ScorerReturnValue = number | boolean | Score;
type AwaitedValue<T> = T extends Promise<infer U> ? U : T;
type InferScorerMetadata<T> = AwaitedValue<T> extends Score<infer TMetadata> ? TMetadata : Record<string, any>;
type NormalizeScorerReturn<T, TMetadata extends Record<string, any>> = T extends Promise<any> ? Promise<Score<TMetadata>> : Score<TMetadata>;
/**
 * Creates a scorer to be used in evals.
 *
 * Scorers need to return a number or a boolean. If returning a number, it is
 * suggested that this number is between 0 and 1.
 *
 * @example
 * const scorer = createScorer('exact-match',
 *   (args: { output: string; expected: string; }) => {
 *     return args.output === args.expected ? true : false;
 *   }
 * );
 *
 * @example
 * // With aggregation for trials
 * import { PassAtK } from '@axiomhq/ai/scorers/aggregations';
 * const scorer = createScorer('tool-called',
 *   (args: { output: string }) => args.output.includes('tool') ? 1 : 0,
 *   { aggregation: PassAtK({ threshold: 0.8 }) }
 * );
 */
declare function createScorer<TArgs extends Record<string, any> = {}, TInput = [TArgs] extends [{
    input: infer I;
}] ? I : unknown, TExpected = [TArgs] extends [{
    expected: infer E;
}] ? Exclude<E, undefined> : unknown, TOutput = [TArgs] extends [{
    output: infer O;
}] ? Exclude<O, undefined> : never, TExtra extends Record<string, any> = Simplify<Omit<TArgs, 'input' | 'expected' | 'output' | 'trialIndex'>>, TReturn extends ScorerReturnValue | Promise<ScorerReturnValue> = ScorerReturnValue | Promise<ScorerReturnValue>, TName extends string = string>(
/**
 * The name of the scorer
 */
name: ValidateName<TName>, 
/**
 * The scorer function. Can be sync or async.
 */
fn: (args: TArgs) => TReturn, 
/**
 * Optional configuration for the scorer, including aggregation for trials.
 */
options?: ScorerOptions): [TOutput] extends [never] ? never : Scorer<TInput, TExpected, TOutput, TExtra, InferScorerMetadata<TReturn>, NormalizeScorerReturn<TReturn, InferScorerMetadata<TReturn>>>;

export { Score, createScorer as Scorer, ScorerOptions, Scorer as ScorerType };