adk-typescript
Version:
TypeScript port of Google's Agent Development Kit (ADK)
455 lines (454 loc) • 21.5 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.AgentEvaluator = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const uuid_1 = require("uuid");
const EvaluationGenerator_1 = require("./EvaluationGenerator");
const ResponseEvaluator_1 = require("./ResponseEvaluator");
const TrajectoryEvaluator_1 = require("./TrajectoryEvaluator");
const LlmAgent_1 = require("../agents/LlmAgent");
const InMemorySessionService_1 = require("../sessions/InMemorySessionService");
const InMemoryArtifactService_1 = require("../artifacts/InMemoryArtifactService");
const runners_1 = require("../runners");
const EvaluationConstants_1 = require("./EvaluationConstants");
// Constants for default runs and evaluation criteria
const NUM_RUNS = 2;
const TOOL_TRAJECTORY_SCORE_KEY = "tool_trajectory_avg_score";
// This evaluation is not very stable.
// This is always optional unless explicitly specified.
const RESPONSE_EVALUATION_SCORE_KEY = "response_evaluation_score";
const RESPONSE_MATCH_SCORE_KEY = "response_match_score";
const ALLOWED_CRITERIA = [
TOOL_TRAJECTORY_SCORE_KEY,
RESPONSE_EVALUATION_SCORE_KEY,
RESPONSE_MATCH_SCORE_KEY,
];
const QUERY_COLUMN = "query";
const REFERENCE_COLUMN = "reference";
const EXPECTED_TOOL_USE_COLUMN = "expected_tool_use";
const DEFAULT_CRITERIA = {
[TOOL_TRAJECTORY_SCORE_KEY]: 1.0, // 1-point scale; 1.0 is perfect.
[RESPONSE_MATCH_SCORE_KEY]: 0.8, // Rouge-1 text match; 0.8 is default.
};
/**
* Load JSON data from a file
* @param filePath Path to the JSON file
* @returns Parsed JSON content
*/
function loadJson(filePath) {
const data = fs.readFileSync(filePath, 'utf8');
return JSON.parse(data);
}
/**
* An evaluator for Agents, mainly intended for helping with test cases
*/
class AgentEvaluator {
/**
* Find the test_config.json file in the same folder as the test file
* @param testFile Path to the test file
* @returns Evaluation criteria defined in the config or defaults
*/
static findConfigForTestFile(testFile) {
const testFolder = path.dirname(testFile);
const configPath = path.join(testFolder, "test_config.json");
if (fs.existsSync(configPath)) {
const configData = loadJson(configPath);
if ("criteria" in configData && typeof configData.criteria === "object") {
return configData.criteria;
}
else {
throw new Error(`Invalid format for test_config.json at ${configPath}. Expected a 'criteria' dictionary.`);
}
}
return DEFAULT_CRITERIA;
}
/**
* Evaluates an Agent given eval data
* @param params Evaluation parameters
* @returns Array of evaluation results
*/
static async evaluate(params) {
const { agent, evalDatasetFilePathOrDir, numRuns = NUM_RUNS, agentName, initialSessionFile, resetFunc } = params;
let testFiles = [];
// Determine if we're dealing with a directory or a single file
if (fs.existsSync(evalDatasetFilePathOrDir) &&
fs.lstatSync(evalDatasetFilePathOrDir).isDirectory()) {
// Walk directory recursively to find .test.json files
const walkDir = (dir) => {
let results = [];
const list = fs.readdirSync(dir);
for (const file of list) {
const filePath = path.join(dir, file);
const stat = fs.lstatSync(filePath);
if (stat.isDirectory()) {
// Recursively walk subdirectories
results = results.concat(walkDir(filePath));
}
else if (file.endsWith('.test.json')) {
results.push(filePath);
}
}
return results;
};
testFiles = walkDir(evalDatasetFilePathOrDir);
}
else {
testFiles = [evalDatasetFilePathOrDir];
}
// Load initial session state if provided
let initialSessionState = {};
if (initialSessionFile) {
const fileContent = fs.readFileSync(initialSessionFile, 'utf8');
initialSessionState = JSON.parse(fileContent).state || {};
}
// Process each test file
for (const testFile of testFiles) {
const dataset = AgentEvaluator._loadDataset(testFile)[0];
const criteria = AgentEvaluator.findConfigForTestFile(testFile);
AgentEvaluator._validateInput([dataset], criteria);
// Use the provided agent directly
let agentToEvaluate = agent;
// If a specific sub-agent is requested by name
if (agentName) {
// First try to find it using the standard property
if (agent.subAgents && Array.isArray(agent.subAgents)) {
// Check sub-agents by their name property
const subAgent = agent.subAgents.find(a => a.name === agentName);
if (subAgent) {
agentToEvaluate = subAgent;
}
else {
console.log(`Couldn't find sub-agent with name '${agentName}' in subAgents array. Will try findAgent method if available.`);
}
}
// If we still haven't found it, try the findAgent method if available
if (agentToEvaluate === agent && agent.findAgent) {
try {
const foundAgent = agent.findAgent(agentName);
if (foundAgent) {
agentToEvaluate = foundAgent;
}
}
catch (error) {
console.log(`Error while finding agent '${agentName}': ${error}`);
}
}
if (agentToEvaluate === agent) {
console.log(`Could not find sub-agent '${agentName}'. Using the provided agent.`);
}
else {
console.log(`Found and using sub-agent '${agentName}'.`);
}
}
console.log("running agent with tools ", agentToEvaluate);
const evaluationResponse = await AgentEvaluator._generateResponsesWithAgent(agentToEvaluate, [dataset], numRuns, resetFunc, { state: initialSessionState });
if (AgentEvaluator._responseEvaluationRequired(criteria, [dataset])) {
await AgentEvaluator._evaluateResponseScores(evaluationResponse, criteria);
}
if (AgentEvaluator._trajectoryEvaluationRequired(criteria, [dataset])) {
await AgentEvaluator._evaluateToolTrajectory(evaluationResponse, criteria);
}
}
// For backward compatibility with tests
return Array(numRuns).fill({ success: true });
}
/**
* Load evaluation dataset from file or directory
* @param inputData Path to file or directory containing test data
* @returns Array of evaluation datasets
*/
static _loadDataset(inputData) {
const loadJsonFile = (filePath) => {
const data = loadJson(filePath);
if (!Array.isArray(data) || !data.every(d => typeof d === 'object')) {
throw new Error(`${filePath} must contain a list of dictionaries.`);
}
return data;
};
if (typeof inputData === 'string') {
if (fs.existsSync(inputData) && fs.lstatSync(inputData).isDirectory()) {
const testFiles = [];
const walkDir = (dir) => {
let results = [];
const list = fs.readdirSync(dir);
for (const file of list) {
const filePath = path.join(dir, file);
const stat = fs.lstatSync(filePath);
if (stat.isDirectory()) {
results = results.concat(walkDir(filePath));
}
else if (file.endsWith('.test.json')) {
results.push(filePath);
}
}
return results;
};
const files = walkDir(inputData);
return files.map(f => loadJsonFile(f));
}
else if (fs.existsSync(inputData) && fs.lstatSync(inputData).isFile()) {
return [loadJsonFile(inputData)];
}
else {
throw new Error(`Input path ${inputData} is invalid.`);
}
}
else if (Array.isArray(inputData)) {
if (inputData.every(i => typeof i === 'string' && fs.existsSync(i))) {
return inputData.map(f => loadJsonFile(f));
}
throw new TypeError("Input list must contain valid file paths.");
}
throw new TypeError("Invalid input type for dataset loading.");
}
/**
* Validates that the evaluation criteria align with the provided dataset
* @param evalDataset The evaluation dataset to validate
* @param criteria The evaluation criteria to validate against
*/
static _validateInput(evalDataset, criteria) {
if (!evalDataset || evalDataset.length === 0) {
throw new Error("The evaluation dataset is None or empty.");
}
for (const key in criteria) {
if (!ALLOWED_CRITERIA.includes(key)) {
throw new Error(`Invalid criteria key: ${key}. Expected one of ${ALLOWED_CRITERIA.join(', ')}.`);
}
}
if (!evalDataset) {
throw new Error("The evaluation dataset is empty.");
}
const sample = evalDataset[0];
const firstQuery = sample[0];
if (!Array.isArray(sample) || typeof firstQuery !== 'object') {
throw new Error(`Each evaluation dataset sample must be list of dictionary. But it's ${JSON.stringify(evalDataset)}`);
}
if (TOOL_TRAJECTORY_SCORE_KEY in criteria) {
if (!(QUERY_COLUMN in firstQuery) ||
!(EXPECTED_TOOL_USE_COLUMN in firstQuery)) {
throw new Error(`Samples for ${TOOL_TRAJECTORY_SCORE_KEY} must include '${QUERY_COLUMN}' and '${EXPECTED_TOOL_USE_COLUMN}' keys. The sample is ${JSON.stringify(sample)}.`);
}
}
if (RESPONSE_EVALUATION_SCORE_KEY in criteria) {
if (!(QUERY_COLUMN in firstQuery)) {
throw new Error(`Samples for ${RESPONSE_EVALUATION_SCORE_KEY} must include '${QUERY_COLUMN}' key. The sample is ${JSON.stringify(sample)}.`);
}
}
if (RESPONSE_MATCH_SCORE_KEY in criteria) {
if (!(QUERY_COLUMN in firstQuery) ||
!(REFERENCE_COLUMN in firstQuery)) {
throw new Error(`Samples for ${RESPONSE_MATCH_SCORE_KEY} must include '${QUERY_COLUMN}' and '${REFERENCE_COLUMN}' keys. The sample is ${JSON.stringify(sample)}.`);
}
}
}
/**
* Infers evaluation criteria based on the provided dataset
* @param evalDataset The evaluation dataset
* @returns Inferred evaluation criteria
*/
static _getInferCriteria(evalDataset) {
const inferredCriteria = {};
const sample = evalDataset[0][0];
if (QUERY_COLUMN in sample && EXPECTED_TOOL_USE_COLUMN in sample) {
inferredCriteria[TOOL_TRAJECTORY_SCORE_KEY] = DEFAULT_CRITERIA[TOOL_TRAJECTORY_SCORE_KEY];
}
if (QUERY_COLUMN in sample && REFERENCE_COLUMN in sample) {
inferredCriteria[RESPONSE_MATCH_SCORE_KEY] = DEFAULT_CRITERIA[RESPONSE_MATCH_SCORE_KEY];
}
return inferredCriteria;
}
/**
* Generates evaluation responses by directly using the agent
* @param agent The agent to evaluate
* @param evalDataset The evaluation dataset
* @param numRuns Number of times to run evaluation
* @param resetFunc Optional function to reset agent state between runs
* @param initialSession Initial session data
* @returns Array of evaluation responses
*/
static async _generateResponsesWithAgent(agent, evalDataset, numRuns, resetFunc, initialSession = {}) {
const results = [];
for (let i = 0; i < numRuns; i++) {
const runResults = [];
for (const dataGroup of evalDataset) {
// Initialize services once per conversation
const sessionService = new InMemorySessionService_1.InMemorySessionService();
const artifactService = new InMemoryArtifactService_1.InMemoryArtifactService();
// Setup the session
const appName = initialSession.appName || "EvaluationGenerator";
const userId = initialSession.userId || "test_user_id";
const sessionId = (0, uuid_1.v4)();
// Create a session for this conversation
const session = sessionService.createSession({
appName,
userId,
sessionId,
state: initialSession.state || {}
});
// Reset agent state if reset function is provided
if (resetFunc && typeof resetFunc === 'function') {
resetFunc();
}
// Process each turn in the conversation using the same session
for (const data of dataGroup) {
// Extract tool names that need to be mocked
const allMockTools = new Set();
const expectedToolUse = data.expected_tool_use || [];
for (const expected of expectedToolUse) {
if (expected[EvaluationConstants_1.EvalConstants.MOCK_TOOL_OUTPUT] !== undefined) {
allMockTools.add(expected[EvaluationConstants_1.EvalConstants.TOOL_NAME]);
}
}
// Create a copy of the evaluation data to use in callbacks
const evalDataCopy = { ...data };
// Apply the tool callback to mock tool outputs if agent is an LlmAgent
if (agent instanceof LlmAgent_1.LlmAgent) {
EvaluationGenerator_1.EvaluationGenerator.applyBeforeToolCallback(agent, (tool, args, toolContext, evalDataset) => EvaluationGenerator_1.EvaluationGenerator.beforeToolCallback(tool, args, toolContext, evalDataset), allMockTools, [evalDataCopy]);
}
// Create a runner for the agent
const runner = new runners_1.Runner({
appName,
agent,
artifactService,
sessionService
});
// Process the response
const response = { ...data };
const query = data.query;
// Create a content object from the query
const content = {
role: 'user',
parts: [{ text: query }]
};
const turnActualToolUses = [];
// Run the agent and collect responses
for await (const event of runner.run({
userId,
sessionId,
newMessage: content
})) {
if (event.isFinalResponse() && event.content && event.content.parts.length > 0) {
const textPart = event.content.parts.find((part) => part.text !== undefined);
if (textPart) {
response.response = textPart.text;
}
}
// Check for direct function calls
if (event.getFunctionCalls && event.getFunctionCalls().length > 0) {
for (const call of event.getFunctionCalls()) {
turnActualToolUses.push({
tool_name: call.name,
tool_input: call.args
});
}
}
else {
console.log('did not get function calls', event?.content?.parts);
}
}
// Update the response with collected tool uses
response.actual_tool_use = turnActualToolUses;
runResults.push(response);
}
}
results.push(runResults);
}
return results;
}
/**
* Checks if response evaluation is needed
* @param criteria The evaluation criteria
* @param evalDataset The evaluation dataset
* @returns True if response evaluation is required
*/
static _responseEvaluationRequired(criteria, evalDataset) {
return REFERENCE_COLUMN in evalDataset[0][0] && (RESPONSE_EVALUATION_SCORE_KEY in criteria ||
RESPONSE_MATCH_SCORE_KEY in criteria);
}
/**
* Checks if trajectory evaluation is needed
* @param criteria The evaluation criteria
* @param evalDataset The evaluation dataset
* @returns True if trajectory evaluation is required
*/
static _trajectoryEvaluationRequired(criteria, evalDataset) {
return EXPECTED_TOOL_USE_COLUMN in evalDataset[0][0] &&
TOOL_TRAJECTORY_SCORE_KEY in criteria;
}
/**
* Evaluates response scores and raises an assertion error if they don't meet the criteria
* @param evaluationResponse The evaluation response data
* @param criteria The evaluation criteria
*/
static async _evaluateResponseScores(evaluationResponse, criteria) {
const metrics = ResponseEvaluator_1.ResponseEvaluator.evaluateResponses(evaluationResponse.flat());
const meanScore = metrics.reduce((sum, result) => sum + result.score, 0) / metrics.length;
const metricsMap = {
"coherence/mean": meanScore,
"rouge_1/mean": meanScore
};
AgentEvaluator._assertScore(metricsMap, "coherence/mean", criteria[RESPONSE_EVALUATION_SCORE_KEY], "Average response evaluation score");
AgentEvaluator._assertScore(metricsMap, "rouge_1/mean", criteria[RESPONSE_MATCH_SCORE_KEY], "Average response match score");
}
/**
* Evaluates tool trajectory scores and raises an assertion error if they don't meet the criteria
* @param evaluationResponse The evaluation response data
* @param criteria The evaluation criteria
*/
static async _evaluateToolTrajectory(evaluationResponse, criteria) {
const score = TrajectoryEvaluator_1.TrajectoryEvaluator.evaluate(evaluationResponse, true // print_detailed_results
);
AgentEvaluator._assertScore({ [TOOL_TRAJECTORY_SCORE_KEY]: score }, TOOL_TRAJECTORY_SCORE_KEY, criteria[TOOL_TRAJECTORY_SCORE_KEY], "Average tool trajectory evaluation score");
}
/**
* Asserts that a metric meets the specified threshold
* @param metrics The metrics to check
* @param metricKey The key of the metric to check
* @param threshold The threshold the metric must meet
* @param description Description of the check for error messages
*/
static _assertScore(metrics, metricKey, threshold, description) {
if (metricKey in metrics && threshold !== undefined) {
const actualScore = metrics[metricKey];
if (actualScore < threshold) {
throw new Error(`${description} is lower than expected. ` +
`Expected >= ${threshold}, but got ${actualScore}.`);
}
}
}
}
exports.AgentEvaluator = AgentEvaluator;