adk-typescript/dist/evaluation/EvaluationGenerator.d.ts

TypeScript port of Google's Agent Development Kit (ADK)

import { EvalConstants } from './EvaluationConstants';
import { BaseAgent } from '../agents/BaseAgent';
import { SessionInterface as Session } from '../sessions/types';
import { InMemorySessionService } from '../sessions/InMemorySessionService';
import { InMemoryArtifactService } from '../artifacts/InMemoryArtifactService';
import { BaseTool } from '../tools/BaseTool';
import { ToolContext } from '../tools/ToolContext';
/**
 * Type for tool callback function
 */
export type BeforeToolCallback = (tool: BaseTool, args: Record<string, any>, toolContext: ToolContext, evalDataset: EvalEntry[]) => Record<string, any> | undefined;
/**
 * Interface for an evaluation data entry
 */
export interface EvalEntry {
    id?: string;
    query: string;
    response?: string;
    expected_tool_use?: Array<{
        [EvalConstants.TOOL_NAME]: string;
        [EvalConstants.TOOL_INPUT]?: Record<string, any>;
        [EvalConstants.MOCK_TOOL_OUTPUT]?: any;
    }>;
    actual_tool_use?: Array<{
        [EvalConstants.TOOL_NAME]: string;
        [EvalConstants.TOOL_INPUT]: Record<string, any>;
    }>;
    [key: string]: any;
}
/**
 * Generates evaluation data from test files
 */
export declare class EvaluationGenerator {
    /**
     * Generates evaluation responses for the given dataset and agent.
     * @param evalDataset The dataset to evaluate
     * @param agentModulePath Path to the module that contains the root agent
     * @param repeatNum Number of times to repeat the eval dataset
     * @param agentName The name of the agent to evaluate (optional)
     * @param initialSession Initial session data (optional)
     */
    static generateResponses(evalDataset: EvalEntry[], agentModulePath: string, repeatNum?: number, agentName?: string, initialSession?: Record<string, any>): Promise<EvalEntry[]>;
    /**
     * Generates evaluation responses by combining session data with evaluation data.
     * @param sessionPath Path to a JSON file that contains session data
     * @param evalDataset The evaluation dataset to combine with session data
     */
    static generateResponsesFromSession(sessionPath: string, evalDataset: EvalEntry[]): Promise<EvalEntry[]>;
    /**
     * Process a query using the agent and evaluation dataset.
     * @param data The evaluation data entry
     * @param moduleName The module name/path
     * @param agentName The agent name (optional)
     * @param initialSession Initial session data (optional)
     */
    static _processQuery(data: EvalEntry, moduleName: string, agentName?: string, initialSession?: Record<string, any>): Promise<EvalEntry>;
    /**
     * Process a query using the agent and evaluation dataset (core logic).
     * @param data The evaluation data entry
     * @param rootAgent The root agent instance
     * @param resetFunc Function to reset agent state (optional)
     * @param initialSession Initial session data (optional)
     * @param sessionId Session ID (optional)
     * @param sessionService Session service (optional)
     * @param artifactService Artifact service (optional)
     */
    static _processQueryWithRootAgent(data: EvalEntry, rootAgent: BaseAgent, resetFunc?: () => void, initialSession?: Record<string, any>, sessionId?: string, sessionService?: InMemorySessionService, artifactService?: InMemoryArtifactService): Promise<EvalEntry>;
    /**
     * Process the queries using the existing session data without invoking the runner.
     * @param sessionData The session data
     * @param data The evaluation data entry
     */
    static _processQueryWithSession(sessionData: Session, data: EvalEntry): EvalEntry;
    /**
     * Intercept specific tool calls and return predefined outputs from eval_dataset.
     * @param tool The tool being called
     * @param args The tool arguments
     * @param toolContext The tool context
     * @param evalDataset The evaluation dataset
     */
    static beforeToolCallback(tool: BaseTool, args: Record<string, any>, toolContext: ToolContext, evalDataset: EvalEntry[]): Record<string, any> | undefined;
    /**
     * Helper method to check if two argument objects are equal
     */
    private static areArgsEqual;
    /**
     * Recursively apply the before_tool_callback to the root agent and all its subagents.
     * @param agent The agent to apply the callback to
     * @param callback The callback function
     * @param allMockTools Set of tool names that need to be mocked
     * @param evalDataset The evaluation dataset
     */
    static applyBeforeToolCallback(agent: BaseAgent, callback: BeforeToolCallback, allMockTools: Set<string>, evalDataset: EvalEntry[]): void;
}