adk-typescript
Version:
TypeScript port of Google's Agent Development Kit (ADK)
176 lines (175 loc) • 7.34 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ResponseEvaluator = void 0;
const EvaluationConstants_1 = require("./EvaluationConstants");
/**
* Mock implementation of MetricPromptTemplateExamples
* In a real implementation, this would be imported from a Vertex AI package
*/
const MetricPromptTemplateExamples = {
Pointwise: {
COHERENCE: 'coherence'
}
};
/**
* Runs response evaluation for agents
*/
class ResponseEvaluator {
/**
* Evaluates a list of agent responses against references.
* @param evalData Array of evaluation entries
* @returns Array of evaluation results
*/
static evaluateResponses(evalData) {
return evalData.map(entry => {
const query = entry[EvaluationConstants_1.EvalConstants.QUERY] ?? '';
const response = entry[EvaluationConstants_1.EvalConstants.RESPONSE] ?? '';
const reference = entry[EvaluationConstants_1.EvalConstants.REFERENCE];
// Simple exact match scoring - in a real implementation,
// this would use more sophisticated metrics like ROUGE
const score = response && reference &&
ResponseEvaluator.calculateSimilarity(response, reference) || 0;
const reason = score > 0.8 ? 'High similarity' :
score > 0.5 ? 'Moderate similarity' : 'Low similarity';
return { query, response, reference, score, reason };
});
}
/**
* Calculate similarity between two text strings
* @param text1 First text
* @param text2 Second text
* @returns Similarity score between 0 and 1
*/
static calculateSimilarity(text1, text2) {
// Simple implementation - in a real context, you would use
// a sophisticated text similarity algorithm like ROUGE-1
// This is just a placeholder that checks for rough similarity
const normalize = (text) => {
return text.toLowerCase()
.replace(/[^\w\s]/g, '')
.split(/\s+/)
.filter(word => word.length > 0);
};
const words1 = normalize(text1);
const words2 = normalize(text2);
if (words1.length === 0 || words2.length === 0)
return 0;
const commonWords = words1.filter(word => words2.includes(word));
const unionLength = new Set([...words1, ...words2]).size;
// Calculate Jaccard similarity
return commonWords.length / unionLength;
}
/**
* Returns the value of requested evaluation metrics.
* @param rawEvalDataset The dataset that will be evaluated
* @param evaluationCriteria The evaluation criteria to use
* @param printDetailedResults Whether to print detailed results
* @returns Summary metrics
*/
static async evaluate(rawEvalDataset, evaluationCriteria, printDetailedResults = false) {
if (!rawEvalDataset || rawEvalDataset.length === 0) {
throw new Error("The evaluation dataset is empty.");
}
const metrics = ResponseEvaluator._getMetrics(rawEvalDataset, Object.keys(evaluationCriteria));
// Flatten dataset for processing
const flattenedQueries = rawEvalDataset.flat();
// Convert to the format expected by the evaluation system
const evalDataset = flattenedQueries.map(item => ({
prompt: item.query,
response: item.response || '',
reference: item.reference,
reference_trajectory: item.expected_tool_use
}));
// Perform evaluation
const evalResult = await ResponseEvaluator._performEval(evalDataset, metrics);
if (printDetailedResults) {
ResponseEvaluator._printResults(evalResult);
}
return evalResult.summary_metrics;
}
/**
* Determines which metrics to use based on dataset and criteria
* @param rawEvalDataset The evaluation dataset
* @param criteria The criteria to evaluate
* @returns List of metrics to use
*/
static _getMetrics(rawEvalDataset, criteria) {
const metrics = [];
const firstEntry = rawEvalDataset[0][0];
if (criteria.includes("response_evaluation_score") &&
"query" in firstEntry &&
"expected_tool_use" in firstEntry) {
metrics.push(MetricPromptTemplateExamples.Pointwise.COHERENCE);
}
if (criteria.includes("response_match_score") &&
"reference" in firstEntry) {
metrics.push("rouge_1");
}
return metrics;
}
/**
* Performs evaluation on the dataset using specified metrics
* @param dataset The evaluation dataset
* @param metrics The metrics to evaluate
* @returns Evaluation results
*/
static async _performEval(dataset, metrics) {
// This is a simplified mock implementation
// In a real implementation, this would call an external service
// like Vertex AI Evaluation
const summaryMetrics = {};
const metricsTable = [];
// Calculate values for each metric
if (metrics.includes('coherence')) {
// Simulate coherence scoring
const coherenceScores = dataset.map((entry, index) => {
// Simple coherence score based on response length and structure
// In a real implementation, this would use an LLM or more sophisticated metrics
const response = entry.response || '';
const score = Math.min(0.5 + response.length / 200, 1.0);
return {
index,
score,
metric: 'coherence'
};
});
// Calculate mean score
const meanCoherence = coherenceScores.reduce((sum, entry) => sum + entry.score, 0) /
(coherenceScores.length || 1);
summaryMetrics['coherence/mean'] = meanCoherence;
metricsTable.push(...coherenceScores);
}
if (metrics.includes('rouge_1')) {
// Calculate ROUGE-1 scores for each entry
const rougeScores = dataset.map((entry, index) => {
const response = entry.response || '';
const reference = entry.reference || '';
const score = response && reference ?
ResponseEvaluator.calculateSimilarity(response, reference) : 0;
return {
index,
score,
metric: 'rouge_1'
};
});
// Calculate mean score
const meanRouge = rougeScores.reduce((sum, entry) => sum + entry.score, 0) /
(rougeScores.length || 1);
summaryMetrics['rouge_1/mean'] = meanRouge;
metricsTable.push(...rougeScores);
}
return {
summary_metrics: summaryMetrics,
metrics_table: metricsTable
};
}
/**
* Prints evaluation results to the console
* @param evalResult Evaluation results
*/
static _printResults(evalResult) {
console.log("Evaluation Summary Metrics:", evalResult.summary_metrics);
console.table(evalResult.metrics_table);
}
}
exports.ResponseEvaluator = ResponseEvaluator;