adk-typescript/dist/evaluation/AgentEvaluator.d.ts

TypeScript port of Google's Agent Development Kit (ADK)

import { BaseAgent } from '../agents/BaseAgent';
declare const TOOL_TRAJECTORY_SCORE_KEY = "tool_trajectory_avg_score";
declare const RESPONSE_EVALUATION_SCORE_KEY = "response_evaluation_score";
declare const RESPONSE_MATCH_SCORE_KEY = "response_match_score";
/**
 * Interface for evaluation criteria
 */
export interface EvaluationCriteria {
    [TOOL_TRAJECTORY_SCORE_KEY]?: number;
    [RESPONSE_EVALUATION_SCORE_KEY]?: number;
    [RESPONSE_MATCH_SCORE_KEY]?: number;
    [key: string]: number | undefined;
}
/**
 * Interface for evaluation parameters with direct agent
 */
export interface EvaluationParams {
    agent: BaseAgent;
    evalDatasetFilePathOrDir: string;
    numRuns?: number;
    agentName?: string;
    initialSessionFile?: string;
    resetFunc?: () => void;
}
/**
 * Simple result object for backward compatibility with tests
 */
export interface EvaluationResult {
    success: boolean;
    [key: string]: any;
}
/**
 * An evaluator for Agents, mainly intended for helping with test cases
 */
export declare class AgentEvaluator {
    /**
     * Find the test_config.json file in the same folder as the test file
     * @param testFile Path to the test file
     * @returns Evaluation criteria defined in the config or defaults
     */
    static findConfigForTestFile(testFile: string): EvaluationCriteria;
    /**
     * Evaluates an Agent given eval data
     * @param params Evaluation parameters
     * @returns Array of evaluation results
     */
    static evaluate(params: EvaluationParams): Promise<EvaluationResult[]>;
    /**
     * Load evaluation dataset from file or directory
     * @param inputData Path to file or directory containing test data
     * @returns Array of evaluation datasets
     */
    private static _loadDataset;
    /**
     * Validates that the evaluation criteria align with the provided dataset
     * @param evalDataset The evaluation dataset to validate
     * @param criteria The evaluation criteria to validate against
     */
    private static _validateInput;
    /**
     * Infers evaluation criteria based on the provided dataset
     * @param evalDataset The evaluation dataset
     * @returns Inferred evaluation criteria
     */
    private static _getInferCriteria;
    /**
     * Generates evaluation responses by directly using the agent
     * @param agent The agent to evaluate
     * @param evalDataset The evaluation dataset
     * @param numRuns Number of times to run evaluation
     * @param resetFunc Optional function to reset agent state between runs
     * @param initialSession Initial session data
     * @returns Array of evaluation responses
     */
    private static _generateResponsesWithAgent;
    /**
     * Checks if response evaluation is needed
     * @param criteria The evaluation criteria
     * @param evalDataset The evaluation dataset
     * @returns True if response evaluation is required
     */
    private static _responseEvaluationRequired;
    /**
     * Checks if trajectory evaluation is needed
     * @param criteria The evaluation criteria
     * @param evalDataset The evaluation dataset
     * @returns True if trajectory evaluation is required
     */
    private static _trajectoryEvaluationRequired;
    /**
     * Evaluates response scores and raises an assertion error if they don't meet the criteria
     * @param evaluationResponse The evaluation response data
     * @param criteria The evaluation criteria
     */
    private static _evaluateResponseScores;
    /**
     * Evaluates tool trajectory scores and raises an assertion error if they don't meet the criteria
     * @param evaluationResponse The evaluation response data
     * @param criteria The evaluation criteria
     */
    private static _evaluateToolTrajectory;
    /**
     * Asserts that a metric meets the specified threshold
     * @param metrics The metrics to check
     * @param metricKey The key of the metric to check
     * @param threshold The threshold the metric must meet
     * @param description Description of the check for error messages
     */
    private static _assertScore;
}
export {};