@adonis-agora/agent
Version:
Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.
76 lines • 2.82 kB
JavaScript
/**
* Score a batch of stored runs and persist the results — the offline path, and the one to reach for.
*
* A scorer that throws is recorded as a {@link ScorerFailure} and the batch carries on. One
* malformed run, one judge that returned prose instead of a verdict, one adapter row a scorer did
* not expect: none of those should cost you the other 499 runs of a backfill, and a failure that
* silently became a score of 0 would be worse still — it would read as the agent's fault.
*/
export async function runEvaluation(options) {
const now = options.now ?? (() => new Date());
const runs = await options.source.listRuns(options.query);
const runIds = runs.map((run) => run.runId);
const alreadyScored = new Map();
if (options.rescore !== true) {
for (const scorer of options.scorers) {
alreadyScored.set(scorer.name, new Set(await options.store.scoredRunIds(scorer.name, runIds)));
}
}
const scores = [];
const failures = [];
const scoredRuns = new Set();
let pairsSkipped = 0;
for (const run of runs) {
for (const scorer of options.scorers) {
if (alreadyScored.get(scorer.name)?.has(run.runId) === true) {
pairsSkipped += 1;
continue;
}
let result;
try {
result = await scorer.score(run);
}
catch (error) {
failures.push({
runId: run.runId,
scorer: scorer.name,
message: error instanceof Error ? error.message : String(error),
});
continue;
}
if (result === null) {
continue;
}
scores.push(toRunScore({ run, scorer, result, scoredAt: now() }));
scoredRuns.add(run.runId);
}
}
if (scores.length > 0) {
await options.store.recordScores(scores);
}
return {
runsRead: runs.length,
runsScored: scoredRuns.size,
pairsSkipped,
scoresRecorded: scores.length,
failures,
};
}
/** Assemble the persisted row: the scorer's verdict plus the provenance a summary groups by. */
export function toRunScore(args) {
const { run, scorer, result, scoredAt } = args;
return {
runId: run.runId,
threadId: run.threadId,
agentName: run.agentName,
scorer: scorer.name,
kind: scorer.kind,
score: result.score,
reason: result.reason,
...(result.metadata !== undefined ? { metadata: result.metadata } : {}),
// The RUN's day, not the batch's — see RunScore.day.
day: run.startedAt.slice(0, 10),
scoredAt: scoredAt.toISOString(),
};
}
//# sourceMappingURL=evaluate.js.map