@adonis-agora/agent
Version:
Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.
411 lines • 15.6 kB
JavaScript
import { percentile, summarizeLatency } from './run-metrics.js';
/** Which kind of mistake a UI tool's error message is (what the genui tools and the registry throw). */
export function classifyGenuiError(message) {
const code = /^invalid .+? code: (.*)$/s.exec(message);
if (code !== null) {
const first = code[1] ?? '';
if (/the bridge (is|has one method)/.test(first))
return 'bridge';
if (/: no `App`/.test(first))
return 'missing-app';
if (/is not a kit component|does not exist here/.test(first))
return 'unknown-component';
return 'syntax';
}
if (/is not available here|not available (on this channel|in this conversation)/i.test(message))
return 'unavailable';
if (/^invalid (UI tree|.+ props|.+ call): /.test(message))
return 'shape';
if (/^Invalid input for tool "/.test(message))
return 'shape';
return 'other';
}
/** An error message without the values it quotes (a schema's "received 'Jane'"), at most 300 chars. */
export function scrubGenuiError(message) {
const scrubbed = message
.replace(/"(?:[^"\\\n]|\\.)*"/g, '"…"')
.replace(/'(?:[^'\\\n]|\\.)*'/g, "'…'");
return scrubbed.length > 300 ? `${scrubbed.slice(0, 299)}…` : scrubbed;
}
// ─── Aggregation (every store's `genuiStats`) ───────────────────────────────────────────────────
/** Rows created within the (inclusive, UTC) day range. */
export function genuiRowsInRange(rows, range) {
const start = new Date(`${range.fromDay}T00:00:00.000Z`);
const end = new Date(`${range.toDay}T23:59:59.999Z`);
return rows.filter((row) => row.createdAt >= start && row.createdAt <= end);
}
function rate(successes, attempts) {
return attempts === 0 ? null : successes / attempts;
}
function addCost(total, cost) {
if (cost === undefined || cost === null)
return total;
return (total ?? 0) + cost;
}
/** {@link GenuiStats} from the rows of a range — the same numbers whichever store holds them. */
export function summarizeGenuiCalls(rows) {
const byRun = new Map();
for (const row of rows) {
const list = byRun.get(row.runId) ?? [];
list.push(row);
byRun.set(row.runId, list);
}
const toolModel = new Map();
const keyOf = (row) => `${row.toolName}\u0000${row.modelId}`;
const groupOf = (row) => {
let group = toolModel.get(keyOf(row));
if (group === undefined) {
group = {
toolName: row.toolName,
modelId: row.modelId,
mode: row.mode,
calls: 0,
firstAttempts: 0,
firstTrySuccesses: 0,
firstTryRate: null,
retried: 0,
failed: 0,
costUsd: null,
};
toolModel.set(keyOf(row), group);
}
return group;
};
const totals = {
calls: rows.length,
turns: byRun.size,
firstAttempts: 0,
firstTrySuccesses: 0,
firstTryRate: null,
retried: 0,
failed: 0,
inputTokens: 0,
outputTokens: 0,
costUsd: null,
};
const errors = new Map();
const days = new Map();
for (const row of rows) {
const group = groupOf(row);
group.calls += 1;
group.costUsd = addCost(group.costUsd, row.costUsd);
totals.inputTokens += row.inputTokens;
totals.outputTokens += row.outputTokens;
totals.costUsd = addCost(totals.costUsd, row.costUsd);
const first = row.attempt === 1;
if (first) {
group.firstAttempts += 1;
totals.firstAttempts += 1;
}
if (row.validity === 'first_try') {
group.firstTrySuccesses += 1;
totals.firstTrySuccesses += 1;
}
else if (row.validity === 'retried') {
group.retried += 1;
totals.retried += 1;
}
else {
group.failed += 1;
totals.failed += 1;
const category = row.errorCategory ?? 'other';
const message = row.errorMessage ?? '';
const key = `${category}\u0000${row.toolName}\u0000${message}`;
const at = row.createdAt.toISOString();
const entry = errors.get(key) ?? {
category,
message,
toolName: row.toolName,
count: 0,
lastAt: at,
};
entry.count += 1;
if (at > entry.lastAt)
entry.lastAt = at;
errors.set(key, entry);
}
const day = row.createdAt.toISOString().slice(0, 10);
const bucket = days.get(day) ?? {
day,
calls: 0,
turns: 0,
inputTokens: 0,
outputTokens: 0,
costUsd: null,
firstTryRate: null,
runs: new Set(),
attempts: 0,
successes: 0,
};
bucket.calls += 1;
bucket.runs.add(row.runId);
bucket.inputTokens += row.inputTokens;
bucket.outputTokens += row.outputTokens;
bucket.costUsd = addCost(bucket.costUsd, row.costUsd);
if (first)
bucket.attempts += 1;
if (row.validity === 'first_try')
bucket.successes += 1;
days.set(day, bucket);
}
totals.firstTryRate = rate(totals.firstTrySuccesses, totals.firstAttempts);
for (const group of toolModel.values()) {
group.firstTryRate = rate(group.firstTrySuccesses, group.firstAttempts);
}
const turnCosts = [];
const firstFrames = [];
const completes = [];
for (const list of byRun.values()) {
let cost = null;
let firstFrame;
let complete;
for (const row of list) {
cost = addCost(cost, row.costUsd);
if (row.firstFrameMs !== undefined && row.firstFrameMs !== null) {
firstFrame = Math.min(firstFrame ?? row.firstFrameMs, row.firstFrameMs);
}
if (row.outcome === 'ok' &&
row.completeFrameMs !== undefined &&
row.completeFrameMs !== null) {
complete = Math.min(complete ?? row.completeFrameMs, row.completeFrameMs);
}
}
if (cost !== null)
turnCosts.push(cost);
if (firstFrame !== undefined)
firstFrames.push(firstFrame);
if (complete !== undefined)
completes.push(complete);
}
const costs = [...turnCosts].sort((a, b) => a - b);
return {
totals,
byToolModel: [...toolModel.values()].sort((a, b) => b.calls - a.calls || a.toolName.localeCompare(b.toolName)),
errors: [...errors.values()]
.sort((a, b) => b.count - a.count || b.lastAt.localeCompare(a.lastAt))
.slice(0, 20),
byDay: [...days.values()]
.sort((a, b) => a.day.localeCompare(b.day))
.map(({ runs, attempts, successes, ...day }) => ({
...day,
turns: runs.size,
firstTryRate: rate(successes, attempts),
})),
costPerTurn: {
turns: byRun.size,
avgUsd: costs.length === 0 ? null : costs.reduce((a, b) => a + b, 0) / costs.length,
p50Usd: percentile(costs, 0.5),
p95Usd: percentile(costs, 0.95),
},
latency: {
firstFrame: summarizeLatency(firstFrames),
complete: summarizeLatency(completes),
},
};
}
/**
* Wraps the turn's stream writer to note when each tool call's UI frames (`component`) are written —
* what the user sees, so the time to first frame counts a streaming preview, and a frame a gate
* held back counts when it was released.
*/
export function observeGenuiFrames(writer, now = Date.now) {
const times = new Map();
const note = (frame) => {
let toolCallId;
let partial;
if (frame.t === 'component') {
toolCallId = frame.toolCallId;
partial = frame.partial;
}
else if (frame.t === 'event' && frame.event.kind === 'ui') {
toolCallId = frame.event.toolCallId;
partial = frame.event.partial;
}
else {
return;
}
if (typeof toolCallId !== 'string')
return;
const at = now();
const entry = times.get(toolCallId) ?? {};
entry.first ??= at;
if (partial !== true)
entry.complete ??= at;
times.set(toolCallId, entry);
};
const wrapped = Object.create(writer);
wrapped.write = (frame) => {
try {
note(frame);
}
catch {
// Telemetry never stands in the stream's way.
}
return writer.write(frame);
};
wrapped.end = () => writer.end();
if (typeof writer.flush === 'function') {
wrapped.flush = () => writer.flush.call(writer);
}
return { writer: wrapped, times };
}
/**
* Tracks the turn's UI attempts across steps: a refused UI call followed by another is a retry,
* whichever UI tool the second one uses.
*/
export class GenuiTurnTelemetry {
refusedSinceAccepted = 0;
rows(facts) {
const sizes = facts.calls.map((call) => jsonSize(call.input));
const written = facts.text.length + sizes.reduce((a, b) => a + b, 0);
const rows = [];
facts.calls.forEach((call, index) => {
const marker = facts.markerOf(call.name);
if (marker === undefined || marker.telemetry === false)
return;
const result = facts.results.find((each) => each.id === call.id);
if (result === undefined)
return;
const share = written === 0 ? 1 / facts.calls.length : (sizes[index] ?? 0) / written;
const failed = result.error !== undefined;
const attempt = this.refusedSinceAccepted + 1;
if (failed)
this.refusedSinceAccepted += 1;
else
this.refusedSinceAccepted = 0;
const sample = marker.telemetry !== undefined && marker.telemetry.samplePayloads === true;
const times = facts.frames.get(call.id);
const pushed = !failed && hasUi(result.output);
const complete = times?.complete ?? (pushed ? facts.now : undefined);
rows.push({
toolCallId: call.id,
runId: facts.runId,
threadId: facts.threadId,
actorRef: facts.actorRef,
...(facts.agentName !== undefined ? { agentName: facts.agentName } : {}),
step: facts.step,
toolName: call.name,
channel: facts.channel,
mode: marker.mode,
modelId: facts.modelId,
attempt,
outcome: failed ? 'error' : 'ok',
validity: failed ? 'failed' : attempt === 1 ? 'first_try' : 'retried',
...(failed
? {
errorCategory: classifyGenuiError(result.error ?? ''),
errorMessage: sample
? (result.error ?? '').slice(0, 300)
: scrubGenuiError(result.error ?? ''),
}
: {}),
inputTokens: Math.round(facts.usage.inputTokens * share),
outputTokens: Math.round(facts.usage.outputTokens * share),
...(facts.costUsd !== null ? { costUsd: facts.costUsd * share } : {}),
...(facts.costUsd !== null && facts.costSource !== undefined
? { costSource: facts.costSource }
: {}),
...latency(facts.startedAt, times?.first ?? complete, failed ? undefined : complete),
payloadBytes: sizes[index] ?? 0,
...(sample ? { payload: JSON.stringify(call.input ?? null).slice(0, 200_000) } : {}),
});
});
return rows;
}
}
/** The two latencies from the turn's start, when it is known. */
function latency(startedAt, first, complete) {
if (startedAt === undefined)
return {};
return {
...(first !== undefined ? { firstFrameMs: Math.max(0, first - startedAt) } : {}),
...(complete !== undefined ? { completeFrameMs: Math.max(0, complete - startedAt) } : {}),
};
}
/**
* Genui telemetry for an engine whose tools are called OUTSIDE the agent loop (OpenCode's MCP
* endpoint, which its code mode calls): one row per UI call, attempts counted per run. There the
* model's step is not split per call (tokens 0, no cost) and the frames are not timed (no latency).
* Recording is best-effort and never changes what the call returns or throws.
*/
export class GenuiRunRecorder {
maxRuns;
runs = new Map();
constructor(maxRuns = 1000) {
this.maxRuns = maxRuns;
}
async track(input, invoke) {
const { marker, store } = input;
if (marker === undefined ||
marker.telemetry === false ||
store.recordGenuiCalls === undefined) {
return invoke();
}
let output;
let error;
try {
output = await invoke();
return output;
}
catch (thrown) {
error = thrown instanceof Error ? thrown.message : String(thrown);
throw thrown;
}
finally {
try {
const rows = this.turn(input.runId).rows({
runId: input.runId,
threadId: input.threadId,
actorRef: input.actorRef,
...(input.agentName !== undefined ? { agentName: input.agentName } : {}),
step: 0,
channel: input.channel,
modelId: input.modelId,
usage: { inputTokens: 0, outputTokens: 0 },
costUsd: null,
text: '',
calls: [input.call],
results: [
error !== undefined ? { id: input.call.id, error } : { id: input.call.id, output },
],
markerOf: () => marker,
frames: new Map(),
now: Date.now(),
});
if (rows.length > 0)
await store.recordGenuiCalls(rows);
}
catch {
// Telemetry is best-effort.
}
}
}
turn(runId) {
let turn = this.runs.get(runId);
if (turn === undefined) {
turn = new GenuiTurnTelemetry();
this.runs.set(runId, turn);
if (this.runs.size > this.maxRuns) {
const oldest = this.runs.keys().next().value;
if (oldest !== undefined)
this.runs.delete(oldest);
}
}
return turn;
}
}
/** A genui tool's output (`{ shown, id }`) says it pushed a frame. */
function hasUi(output) {
return (typeof output === 'object' &&
output !== null &&
typeof output.id === 'string');
}
function jsonSize(value) {
try {
return JSON.stringify(value ?? null).length;
}
catch {
return 0;
}
}
//# sourceMappingURL=genui-telemetry.js.map