adk-typescript/dist/evaluation/TrajectoryEvaluator.d.ts

TypeScript port of Google's Agent Development Kit (ADK)

import { EvalConstants } from './EvaluationConstants';
import { EvalEntry } from './EvaluationGenerator';
/**
 * Interface for trajectory evaluation result
 */
export interface TrajectoryEvalResult {
    query: string;
    response?: string;
    actualToolUse: Array<{
        [EvalConstants.TOOL_NAME]: string;
        [EvalConstants.TOOL_INPUT]: Record<string, any>;
    }>;
    expectedToolUse: Array<{
        [EvalConstants.TOOL_NAME]: string;
        [EvalConstants.TOOL_INPUT]: Record<string, any>;
        [EvalConstants.MOCK_TOOL_OUTPUT]?: any;
    }>;
    toolUseAccuracy: number;
    turn?: number;
    [key: string]: any;
}
/**
 * Evaluates tool use trajectories for accuracy
 */
export declare class TrajectoryEvaluator {
    /**
     * Evaluates the mean tool use accuracy of the eval dataset.
     *
     * Tool use accuracy is calculated by comparing the expected and actual tool
     * use trajectories. An exact match scores a 1, 0 otherwise. The final number
     * is an average of these individual scores.
     *
     * Value range: [0, 1], where 0 means none of the tool use entries aligned,
     * and 1 would mean all of them aligned. Higher value is good.
     *
     * @param evalDataset The dataset that will be evaluated
     * @param printDetailedResults Prints detailed results on the console (default: false)
     * @returns The mean tool use accuracy of the eval dataset
     */
    static evaluate(evalDataset: EvalEntry[][], printDetailedResults?: boolean): number;
    /**
     * Evaluate a single row from the dataset
     * @param row The evaluation entry to evaluate
     * @returns The evaluation result and any failure information
     */
    private static _evaluateRow;
    /**
     * Check if two tool use lists are equal
     * @param listA First list of tools
     * @param listB Second list of tools
     * @returns True if the lists are equal, false otherwise
     */
    static areToolsEqual(listA: Array<any>, listB: Array<any>): boolean;
    /**
     * Helper method to compare two objects for semantic equality,
     * ignoring property order differences
     * @param objA First object
     * @param objB Second object
     * @returns True if the objects have the same properties and values
     */
    private static _areObjectsEqual;
    /**
     * Removes 'mock_tool_output' from each dictionary in the list
     * @param toolUseList List of tool use entries
     * @returns Cleaned list without mock_tool_output entries
     */
    private static _removeToolOutputs;
    /**
     * Report evaluation failures to the console
     * @param failures List of evaluation failures
     */
    private static _reportFailures;
    /**
     * Print detailed evaluation results to the console
     * @param results List of evaluation results
     */
    private static _printResults;
    /**
     * Evaluates a list of agent trajectories (tool use) against expected tool use.
     * @param evalData Array of evaluation entries
     * @returns Array of trajectory evaluation results
     * @deprecated Use evaluate() instead for more comprehensive evaluation
     */
    static evaluateTrajectories(evalData: EvalEntry[]): TrajectoryEvalResult[];
}