adk-typescript
Version:
TypeScript port of Google's Agent Development Kit (ADK)
214 lines (213 loc) • 10.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.DEFAULT_CRITERIA = exports.EVAL_SESSION_ID_PREFIX = exports.RESPONSE_EVALUATION_SCORE_KEY = exports.RESPONSE_MATCH_SCORE_KEY = exports.TOOL_TRAJECTORY_SCORE_KEY = exports.MISSING_EVAL_DEPENDENCIES_MESSAGE = exports.EvalStatus = void 0;
exports.getEvaluationCriteriaOrDefault = getEvaluationCriteriaOrDefault;
exports.getRootAgent = getRootAgent;
exports.tryGetResetFunc = tryGetResetFunc;
exports.parseAndGetEvalsToRun = parseAndGetEvalsToRun;
exports.runEvals = runEvals;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const uuid_1 = require("uuid");
// Import evaluation modules (assume these exist in src/evaluation/)
const EvaluationGenerator_1 = require("../evaluation/EvaluationGenerator");
const ResponseEvaluator_1 = require("../evaluation/ResponseEvaluator");
const TrajectoryEvaluator_1 = require("../evaluation/TrajectoryEvaluator");
// --- Enums & Data Models ---
var EvalStatus;
(function (EvalStatus) {
EvalStatus[EvalStatus["PASSED"] = 1] = "PASSED";
EvalStatus[EvalStatus["FAILED"] = 2] = "FAILED";
EvalStatus[EvalStatus["NOT_EVALUATED"] = 3] = "NOT_EVALUATED";
})(EvalStatus || (exports.EvalStatus = EvalStatus = {}));
// --- Constants ---
exports.MISSING_EVAL_DEPENDENCIES_MESSAGE = 'Eval module is not installed, please install via `npm install @google-cloud/aiplatform` or the appropriate package.';
exports.TOOL_TRAJECTORY_SCORE_KEY = 'tool_trajectory_avg_score';
exports.RESPONSE_MATCH_SCORE_KEY = 'response_match_score';
exports.RESPONSE_EVALUATION_SCORE_KEY = 'response_evaluation_score';
exports.EVAL_SESSION_ID_PREFIX = '___eval___session___';
exports.DEFAULT_CRITERIA = {
[exports.TOOL_TRAJECTORY_SCORE_KEY]: 1.0,
[exports.RESPONSE_MATCH_SCORE_KEY]: 0.8,
};
// --- Helper Functions ---
function importFromPath(moduleName, filePath) {
// Dynamic import using require
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mod = require(filePath);
return mod;
}
function getAgentModule(agentModuleFilePath) {
const filePath = path.join(agentModuleFilePath, '__init__.js');
return importFromPath('agent', filePath);
}
function getEvaluationCriteriaOrDefault(evalConfigFilePath) {
if (evalConfigFilePath) {
const configData = JSON.parse(fs.readFileSync(evalConfigFilePath, 'utf-8'));
if ('criteria' in configData && typeof configData['criteria'] === 'object') {
return configData['criteria'];
}
else {
throw new Error(`Invalid format for test_config.json at ${evalConfigFilePath}. Expected a 'criteria' dictionary.`);
}
}
else {
console.info('No config file supplied. Using default criteria.');
return exports.DEFAULT_CRITERIA;
}
}
function getRootAgent(agentModuleFilePath) {
const agentModule = getAgentModule(agentModuleFilePath);
return agentModule.agent.rootAgent;
}
function tryGetResetFunc(agentModuleFilePath) {
const agentModule = getAgentModule(agentModuleFilePath);
return agentModule.agent.resetData || undefined;
}
function parseAndGetEvalsToRun(evalSetFilePaths) {
const evalSetToEvals = {};
for (const inputEvalSet of evalSetFilePaths) {
let evalSetFile;
let evals = [];
if (inputEvalSet.includes(':')) {
const [file, evalsStr] = inputEvalSet.split(':');
evalSetFile = file;
evals = evalsStr.split(',');
}
else {
evalSetFile = inputEvalSet;
}
if (!(evalSetFile in evalSetToEvals)) {
evalSetToEvals[evalSetFile] = [];
}
evalSetToEvals[evalSetFile].push(...evals);
}
return evalSetToEvals;
}
// --- Main Evaluation Logic ---
async function* runEvals({ evalSetToEvals, rootAgent, resetFunc, evalMetrics, sessionService, artifactService, printDetailedResults = false, }) {
for (const [evalSetFile, evalsToRun] of Object.entries(evalSetToEvals)) {
const evalItems = JSON.parse(fs.readFileSync(evalSetFile, 'utf-8'));
if (!evalItems || evalItems.length === 0) {
throw new Error(`No eval data found in eval set file: ${evalSetFile}`);
}
for (const evalItem of evalItems) {
const evalName = evalItem['name'];
const evalData = evalItem['data'];
const initialSession = evalItem['initial_session'] || {};
if (evalsToRun.length > 0 && !evalsToRun.includes(evalName)) {
continue;
}
try {
console.log(`Running Eval: ${evalSetFile}:${evalName}`);
const sessionId = `${exports.EVAL_SESSION_ID_PREFIX}${(0, uuid_1.v4)()}`;
const scrapeResult = await EvaluationGenerator_1.EvaluationGenerator._processQueryWithRootAgent(evalData, rootAgent, resetFunc, initialSession, sessionId, sessionService, artifactService);
const evalMetricResults = [];
for (const evalMetric of evalMetrics) {
let evalMetricResult = undefined;
if (evalMetric.metricName === exports.TOOL_TRAJECTORY_SCORE_KEY) {
const trajectoryResults = TrajectoryEvaluator_1.TrajectoryEvaluator.evaluateTrajectories([scrapeResult]);
const score = trajectoryResults[0][exports.TOOL_TRAJECTORY_SCORE_KEY];
evalMetricResult = getEvalMetricResult(evalMetric, score);
}
else if (evalMetric.metricName === exports.RESPONSE_MATCH_SCORE_KEY) {
const responseResults = ResponseEvaluator_1.ResponseEvaluator.evaluateResponses([scrapeResult]);
const score = responseResults[0][exports.RESPONSE_MATCH_SCORE_KEY];
evalMetricResult = getEvalMetricResult(evalMetric, score['rouge_1/mean']);
}
else if (evalMetric.metricName === exports.RESPONSE_EVALUATION_SCORE_KEY) {
const responseResults = ResponseEvaluator_1.ResponseEvaluator.evaluateResponses([scrapeResult]);
const score = responseResults[0][exports.RESPONSE_EVALUATION_SCORE_KEY];
evalMetricResult = getEvalMetricResult(evalMetric, score['coherence/mean']);
}
else {
console.warn(`[33m${evalMetric.metricName} is not supported.[0m`);
evalMetricResults.push([
evalMetric,
{ score: undefined, evalStatus: EvalStatus.NOT_EVALUATED },
]);
continue;
}
evalMetricResults.push([
evalMetric,
evalMetricResult,
]);
printEvalMetricResult(evalMetric, evalMetricResult);
}
// Determine final eval status
let finalEvalStatus = EvalStatus.NOT_EVALUATED;
for (const [, result] of evalMetricResults) {
if (result.evalStatus === EvalStatus.PASSED) {
finalEvalStatus = EvalStatus.PASSED;
}
else if (result.evalStatus === EvalStatus.NOT_EVALUATED) {
continue;
}
else if (result.evalStatus === EvalStatus.FAILED) {
finalEvalStatus = EvalStatus.FAILED;
break;
}
else {
throw new Error('Unknown eval status.');
}
}
yield {
evalSetFile,
evalId: evalName,
finalEvalStatus,
evalMetricResults,
sessionId,
};
if (finalEvalStatus === EvalStatus.PASSED) {
console.log('Result: ✅ Passed\n');
}
else {
console.log('Result: ❌ Failed\n');
}
}
catch (e) {
console.error(`Error: ${e}`);
}
}
}
}
function getEvalMetricResult(evalMetric, score) {
const evalStatus = score >= evalMetric.threshold ? EvalStatus.PASSED : EvalStatus.FAILED;
return { score, evalStatus };
}
function printEvalMetricResult(evalMetric, evalMetricResult) {
console.log(`Metric: ${evalMetric.metricName}\tStatus: ${EvalStatus[evalMetricResult.evalStatus]}\tScore: ${evalMetricResult.score}\tThreshold: ${evalMetric.threshold}`);
}