@future-agi/ai-evaluation/dist/esnext/evaluator.d.ts

We help GenAI teams maintain high-accuracy for their Models in production.

import { APIKeyAuth, ResponseHandler } from '@future-agi/sdk';
import { AxiosResponse } from 'axios';
import { EvalTemplate } from './templates';
import { BatchRunResult } from './types';
/**
 * Handles responses for evaluation requests
 */
export declare class EvalResponseHandler extends ResponseHandler<BatchRunResult, any> {
    static _parseSuccess(response: AxiosResponse): BatchRunResult;
    static _handleError(response: AxiosResponse): never;
}
/**
 * Handles responses for evaluation info requests
 */
export declare class EvalInfoResponseHandler extends ResponseHandler<Record<string, any>, any> {
    static _parseSuccess(response: AxiosResponse): Record<string, any>;
    static _handleError(response: AxiosResponse): never;
}
/**
 * Client for evaluating LLM test cases
 */
export declare class Evaluator extends APIKeyAuth {
    private readonly maxWorkers;
    private evalInfoCache;
    constructor(options?: {
        fiApiKey?: string;
        fiSecretKey?: string;
        fiBaseUrl?: string;
        timeout?: number;
        maxQueue?: number;
        maxWorkers?: number;
    });
    evaluate(evalTemplates: string | EvalTemplate | (string | EvalTemplate)[], inputs: Record<string, string | string[]>, options: {
        timeout?: number;
        modelName: string;
        customEvalName?: string;
        traceEval?: boolean;
    }): Promise<BatchRunResult>;
    private _get_eval_info;
    list_evaluations(): Promise<Record<string, any>[]>;
}
/**
 * Convenience function to run a single or batch of evaluations.
 * @param evalTemplates - Evaluation name string (e.g., "Factual Accuracy") or list of templates.
 * @param inputs - Single test case or list of test cases as dictionaries.
 * @param options - Optional parameters for the evaluation.
 * @returns BatchRunResult containing evaluation results.
 */
export declare const evaluate: (evalTemplates: string | EvalTemplate | (string | EvalTemplate)[], inputs: Record<string, string | string[]>, options: {
    fiApiKey?: string;
    fiSecretKey?: string;
    fiBaseUrl?: string;
    timeout?: number;
    modelName: string;
    customEvalName?: string;
    traceEval?: boolean;
}) => Promise<BatchRunResult>;
/**
 * Convenience function to fetch information about all available evaluation templates.
 * @returns A list of evaluation template information dictionaries.
 */
export declare const list_evaluations: (options?: {
    fiApiKey?: string;
    fiSecretKey?: string;
    fiBaseUrl?: string;
}) => Promise<Record<string, any>[]>;
//# sourceMappingURL=evaluator.d.ts.map