agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
616 lines • 27 kB
JavaScript
/**
* `accountForAnswer` — one answer's recording (+ what the app declared) in, a
* typed account out. A pure read-time Fold over the Trace: no clock, network,
* model or randomness; the same inputs give the same bytes. It never writes
* back into the run.
*
* It throws ONLY on a caller error (a recording that is not an object,
* declarations that break their rules). Content it cannot read is passed over
* and counted (`unread`); a line whose fill fails becomes `unreadable.line@1`.
*/
import { assumedLinesFor } from '../core/agent/arguments/serve.js';
import { assessAnswer } from '../core/agent/assessment/assess.js';
import { assessmentDataFrom, assumedValuesSourceOf, rewrittenBehind, sourcesByReason, standingLineOf, standingNamesFrom, } from '../core/agent/assessment/compose.js';
import { ASSUMED_BLOCK_HEADING, COVERAGE_BLOCK_HEADING } from '../semantics/coverage/answer.js';
import { scriptRowsOfTurn } from '../core/agent/scripts/rows.js';
import { rebuildBoundary, recordedScripts, scriptsBlock, } from '../core/agent/scripts/attachment.js';
import { scriptRowIsWellFormed } from '../core/agent/scripts/rows.js';
import { chip, clipFact, MAX_VAR_CHARS, saidBy, sentence } from './render.js';
import { ANSWER_ACCOUNT_TEMPLATE_SET_VERSION } from './templates.js';
import { validateDeclarations } from './declarations.js';
import { fitAccount } from './size.js';
import { isRecord, keptOut, num, str, recordingView, } from './view.js';
import { at, countedFields, derived, FACT_TEXT_CHARS, makeSay, stateAt, } from './facts/common.js';
import { readAsked } from './facts/asked.js';
import { readUnderstood } from './facts/understood.js';
import { MAX_CALLS, readCalls } from './facts/calls.js';
import { answeringIteration, readInView } from './facts/inView.js';
import { readCheckedRows } from './facts/checked.js';
import { readPausedLeg } from './facts/pausedLeg.js';
import { beforePausePointers, readFoundRow } from './facts/found.js';
import { readHowSure } from './facts/howSure.js';
import { runChecks, summaryOf, wrongLines } from './signals.js';
/** The answer text the account carries (the PDF folds at 1,200); the rest is in the record. */
export const MAX_ANSWER_CHARS = 4000;
const HEADINGS = {
asked: 'row.asked',
understood: 'row.understood',
checked: 'row.checked',
'not-checked': 'row.notChecked',
found: 'row.found',
'how-sure': 'row.howSure',
'anything-wrong': 'row.wrong',
};
/** recorded if any line is; not-applicable if every line is; else not-recorded. */
function rowStatus(lines) {
if (lines.some((l) => l.status === 'recorded'))
return 'recorded';
if (lines.length > 0 && lines.every((l) => l.status === 'not-applicable'))
return 'not-applicable';
return 'not-recorded';
}
function saidByChips(lines) {
const sources = [];
for (const line of lines) {
if (line.status === 'recorded' && !sources.includes(line.source))
sources.push(line.source);
}
return sources.map(saidBy);
}
function row(id, lines, extra = {}) {
const status = rowStatus(lines);
return {
id,
heading: sentence(HEADINGS[id], { status: 'not-applicable' }),
lines,
...(extra.more !== undefined && { more: extra.more }),
chips: [...(extra.chips ?? []), ...saidByChips(lines)],
status,
};
}
/** The separator the framework's one composer puts between the answer and what it appends. */
const APPENDED_SEPARATOR = '\n\n---\n\n';
/**
* The headings of the two BLOCKS the framework appends after an answer
* (`coverage/answer.ts` · `composeAnswerWithCoverage`, the one composer), each
* read from its owner: the limits block's and the "Assumed" block's — long,
* fixed sentences only the framework writes.
*/
const BLOCK_HEADINGS = [
`${COVERAGE_BLOCK_HEADING} — declared by the tools that produced it, not by the model:`,
ASSUMED_BLOCK_HEADING,
];
/**
* The standing line THIS run appended, rebuilt from its record — or
* `undefined` when the record says none could be there.
*
* The line opens with the owner's plain words ("Not sure — ", "Known — "),
* which a model can write too, so an opening proves nothing. The record does:
* `turn_end.answerAssessment` exists only on a run whose answer layer folded
* the standing, and the line is ONE function of that projection
* (`assessment/compose.ts` · `standingLineOf`) — plus, when a value was
* assumed, of the committed rows it names, read from the source the layer's
* stage read (`assumedValuesSourceOf`) and the fold's rewrite witness. Only
* that exact line, where the composer puts it, is the library's. Without the
* committed state an assumed value cannot be rebuilt, and the line is not
* claimed.
*
* A line that named calls in the app's words (a tool's `displayName`) is
* rebuilt from the words the record kept beside it (`turn_end.standingLineNames`)
* and the calls the fold reads off the committed state (each reason's
* `source`) — the same two inputs the layer's stage composed it from. Words
* the record does not hold in full, or no committed state, and the line is
* not claimed.
*/
function appendedStandingLine(end, state) {
const data = assessmentDataFrom(end.payload.answerAssessment);
if (data === undefined)
return undefined;
const kept = end.payload.standingLineNames;
const displayNames = kept === undefined ? undefined : standingNamesFrom(kept);
if (kept !== undefined && displayNames === undefined)
return undefined;
if (!data.reasons.includes('argument-assumed') && displayNames === undefined) {
return standingLineOf(data, [], false);
}
if (state === undefined)
return undefined;
const source = assumedValuesSourceOf(data.reasons, state);
const assumed = source === undefined ? [] : assumedLinesFor(source.ledger, source.turn, source.readDecisions);
const assessed = assessAnswer({ snapshot: { sharedState: state } });
return standingLineOf(data, assumed, rewrittenBehind(assessed), displayNames === undefined ? undefined : { sources: sourcesByReason(assessed), displayNames });
}
/**
* The opening of the scripts block THIS run attached
* (`.scriptTravelsWithTheAnswer()`), rebuilt from its record — never matched
* by words. The block is one function of `turn_end.scripts` (the shown
* scripts and the omitted count) and, for a program over the bound, of that
* call's committed `script` row (`core/agent/scripts/attachment.ts` ·
* `scriptsBlock`), behind the boundary the record states (`closedFence`: the
* block opens with the fence it closed; `withheldFromText`: each script is
* its label line alone). With a script shown, the block's text up to its
* first label line opens it; with none shown, the omission lines do —
* rebuilt for the last call that reached the tool and for every one (the
* record does not keep `which`). Anything the record does not hold in full,
* and the block is not claimed.
*/
function appendedScriptsOpenings(end, state, within) {
// The one reader of the shape (`attachment.ts` · `recordedScripts`): the
// text is rebuilt only from a record read back whole — a value no writer
// of this library files rebuilds nothing (the How-sure lines count it).
const read = recordedScripts(end.payload.scripts, within);
if (read === undefined || read.unread > 0)
return [];
const { boundary: recorded, omitted } = read;
if (recorded === undefined || omitted === undefined)
return [];
const boundary = rebuildBoundary(recorded, within);
if (boundary === undefined)
return [];
const first = read.shown[0];
if (first !== undefined) {
const only = { shown: [first.script], omitted: 0 };
// Up to and including the first label line: the closed fence, if any, then it.
const lines = scriptsBlock(only, boundary).split('\n');
return [lines.slice(0, boundary.kind === 'closed' ? 3 : 1).join('\n')];
}
if (omitted === 0 || state === undefined)
return [];
const turn = num(state.turnNumber);
if (turn === undefined)
return [];
const ledger = Array.isArray(state.findingsLedger) ? state.findingsLedger : [];
const current = scriptRowsOfTurn(ledger, turn).current;
// The rows come from saved state: rebuild only from rows this library writes.
if (!current.every((row) => scriptRowIsWellFormed(row))) {
return [];
}
const reached = current.filter((row) => row.outcome !== 'submitted');
const overSizeOf = (rows) => rows.filter((row) => row.text === undefined);
return [overSizeOf(reached.slice(-1)), overSizeOf(reached)].map((overSize) => scriptsBlock({ shown: [], omitted, overSize }, boundary));
}
/** Whether `text` opens the framework's appended section: the run's own line, a block's heading, or its scripts. */
function opensAppended(text, line, scripts = []) {
if (line !== undefined && (text === line || text.startsWith(`${line}\n\n`)))
return true;
if (scripts.some((opening) => opening !== '' && text.startsWith(opening)))
return true;
return BLOCK_HEADINGS.some((heading) => text.startsWith(heading));
}
/**
* Split the framework's appended section off an answer: the LAST separator
* whose next line opens it — the run's own standing line, exactly, or a
* block's heading. A `---` the model wrote itself opens neither, and the
* composer joins its own blocks with blank lines, never a second separator.
* An answer with no text of its own is the section alone (the composer adds
* no separator then).
*/
function splitAppended(content, line, scripts = []) {
let at = content.lastIndexOf(APPENDED_SEPARATOR);
while (at >= 0) {
const rest = content.slice(at + APPENDED_SEPARATOR.length);
if (opensAppended(rest, line, scripts))
return { model: content.slice(0, at), appended: rest };
at = at === 0 ? -1 : content.lastIndexOf(APPENDED_SEPARATOR, at - 1);
}
return opensAppended(content, line, scripts)
? { model: '', appended: content }
: { model: content };
}
/** How many items one of the data's lists holds — `0` for a list the record does not carry. */
const listLength = (value) => (Array.isArray(value) ? value.length : 0);
/**
* A TYPED answer's limits, read off `turn_end.answerCoverage` — the data
* `.limitsTravelWithTheAnswer()` files beside an answer that must stay JSON.
* Counts, with a pointer to every item's `what` (an assumed value's
* `toolName` — never its `value`, an argument's value, which "show me" never
* shows), capped at the account's call cap. `undefined` when the event
* carries none.
*/
function readLimitsData(end) {
// Limits the record keeps out (a redaction policy selects `answerCoverage`):
// they travelled as data, and their words and counts are not in this record.
if (keptOut(end, 'answerCoverage')) {
return {
value: null,
source: 'library',
status: 'not-recorded',
missing: 'redacted',
pointers: [at(end, 'answerCoverage')],
};
}
const data = end.payload.answerCoverage;
if (!isRecord(data))
return undefined;
const pointers = [];
for (const list of ['checked', 'notChecked', 'cannotCover']) {
const items = data[list];
if (!Array.isArray(items))
continue;
items.forEach((item, i) => {
if (isRecord(item) && typeof item.what === 'string') {
pointers.push(at(end, 'answerCoverage', list, i, 'what'));
}
});
}
const assumed = data.assumed;
if (Array.isArray(assumed)) {
assumed.forEach((item, i) => {
if (isRecord(item) && typeof item.toolName === 'string') {
pointers.push(at(end, 'answerCoverage', 'assumed', i, 'toolName'));
}
});
}
if (pointers.length === 0)
return undefined;
return {
value: {
checked: listLength(data.checked),
notChecked: listLength(data.notChecked),
cannotCover: listLength(data.cannotCover),
assumed: listLength(assumed),
},
source: 'library',
status: 'recorded',
pointers: pointers.slice(0, MAX_CALLS),
};
}
/**
* The answer, and what the framework carried with it: the appended section of
* a PROSE answer split off it (the limits block, the "Assumed" block, the
* standing line — `limitsBlock`), or a TYPED answer's limits as data
* (`limitsData`). A typed answer's limits never read `not-applicable`: the
* block reads `not-recorded` with `missing: 'as-data'`, and the data is
* `recorded`. An answer the record keeps out (a redaction policy's
* placeholder on `finalContent`) reads `not-recorded` with `missing:
* 'redacted'` — the run still answered, so the account never reads it as a
* run that did not finish.
*/
function readAnswer(view) {
const end = view.last('agent.turn_end');
const content = str(end?.payload.finalContent);
const none = {
value: null,
source: 'library',
status: 'not-recorded',
pointers: [],
missing: 'no-event',
};
const noData = {
value: null,
source: 'library',
status: 'not-applicable',
pointers: [],
};
if (end !== undefined && content === undefined && keptOut(end, 'finalContent')) {
const kept = {
value: null,
source: 'library',
status: 'not-recorded',
missing: 'redacted',
};
return {
answer: { ...kept, pointers: [at(end, 'finalContent')] },
limits: { ...kept, pointers: [] },
limitsData: readLimitsData(end) ?? noData,
};
}
if (end === undefined || content === undefined) {
return {
answer: none,
limits: { value: null, source: 'library', status: 'not-applicable', pointers: [] },
limitsData: noData,
};
}
const pointer = at(end, 'finalContent');
const data = readLimitsData(end);
const { model, appended } = splitAppended(content, appendedStandingLine(end, view.state), appendedScriptsOpenings(end, view.state, content.length));
const answer = {
...clipFact(model, MAX_ANSWER_CHARS),
source: 'model',
status: 'recorded',
pointers: [pointer],
};
if (appended !== undefined) {
return {
answer,
limits: {
...clipFact(appended, MAX_ANSWER_CHARS),
source: 'library',
status: 'recorded',
pointers: [pointer],
},
limitsData: data ?? noData,
};
}
return {
answer,
limits: data !== undefined
? {
value: null,
source: 'library',
status: 'not-recorded',
pointers: [],
missing: 'as-data',
}
: { value: null, source: 'library', status: 'not-applicable', pointers: [] },
limitsData: data ?? noData,
};
}
function readRun(view, resumedLeg) {
const configured = view.first('agent.run_configured');
const model = str(configured?.payload.llm?.model);
const turnNumber = num(view.state?.turnNumber);
if (view.runId === undefined) {
return {
value: null,
source: 'library',
status: 'not-recorded',
pointers: [],
missing: 'no-event',
};
}
const anchor = configured ?? view.events[0];
return {
value: {
runId: view.runId,
...(turnNumber !== undefined && { turnNumber }),
...(model !== undefined && { model }),
resumedLeg,
},
source: 'library',
status: 'recorded',
pointers: [
...(anchor !== undefined ? [derived(anchor, '#meta/runId')] : []),
...(configured !== undefined && model !== undefined ? [at(configured, 'llm', 'model')] : []),
...(turnNumber !== undefined ? [stateAt('turnNumber')] : []),
],
};
}
/**
* The account of a run this record cannot tell about — said once per row, and
* nothing is claimed about the run (no "did not run any tools", no "did not
* finish"); the hosting op may map it to `summary.notAvailable@1`. Three causes:
*
* - `no-own-events` — the run owns NO event of this record (an
* options.runId that is not this recording's, or a recording with no
* readable event). The record's silence is not evidence.
* - `kept-out` — a reader touched a value the record keeps out (a
* redaction policy's placeholder) without asking about it first
* (`view.ts` · `keptOutRead`), so a fact read from it would be a fact
* about a placeholder. The kept-out values the account DOES handle — the
* question, the answer, a tool result, a call's outcome fields, the
* history — never get here: each is said to be kept out where it is read.
* - `too-large` — the account would pass its size limit even with nothing
* listed (`size.ts` · `fitAccount`): only a record built to be, every
* text at its cut in characters JSON writes six bytes wide. Counted unread.
*/
function notToldAccount(view, say, unread, cause) {
const missing = {
status: 'not-recorded',
missing: cause === 'kept-out'
? 'redacted'
: cause === 'too-large'
? 'unreadable'
: 'no-event',
};
const scopeLine = cause === 'kept-out'
? 'scope.keptOut'
: cause === 'too-large'
? 'scope.tooLarge'
: 'scope.noOwnEvents';
const rowLine = cause === 'kept-out'
? 'row.keptOut'
: cause === 'too-large'
? 'row.tooLarge'
: 'row.notInRecord';
const none = () => ({
value: null,
source: 'library',
pointers: [],
...missing,
});
const rows = Object.keys(HEADINGS).map((id) => row(id, [
id === 'anything-wrong'
? say(scopeLine, {
...missing,
chips: [chip('not-recorded', 'chip.notRecorded')],
})
: say(rowLine, missing),
]));
return {
kind: 'agentfootprint/answer-account',
shape: 1,
templates: { set: 'answer-account', version: ANSWER_ACCOUNT_TEMPLATE_SET_VERSION },
run: none(),
question: none(),
answer: none(),
facts: {
routing: {
configured: none(),
verdict: none(),
scores: none(),
confidence: none(),
appDecision: none(),
delivered: none(),
refusals: [],
},
calls: [],
beforePause: [],
inView: [],
evidence: none(),
standing: none(),
limitsBlock: none(),
limitsData: none(),
errors: { failed: 0, refused: 0, declined: 0, notDispatched: 0, withheld: 0 },
checks: { reachable: [], unreachable: [], notApplicable: [] },
},
rows,
signals: [],
unreachable: [],
summary: {
sentence: say(scopeLine, missing),
tone: 'unknown',
},
unread: unread(),
foreign: view.foreign,
scope: view.scope,
};
}
/** One call answered before the pause, as the facts list it (read from the committed state). */
function beforePauseFact(c) {
const reading = c.reading;
const coverage = c.coverage;
return {
toolName: c.toolName.slice(0, FACT_TEXT_CHARS),
toolCallId: c.toolCallId.slice(0, FACT_TEXT_CHARS),
emptiness: reading.emptiness,
...countedFields(reading),
...(coverage !== undefined && {
coverage: {
kind: coverage.kind,
...(coverage.lookedFor !== undefined && {
lookedFor: coverage.lookedFor.text.slice(0, FACT_TEXT_CHARS),
}),
checked: coverage.items.filter((i) => i.section === 'checked').length,
notChecked: coverage.items.filter((i) => i.section === 'notChecked').length,
cannotCover: coverage.items.filter((i) => i.section === 'cannotCover').length,
kinds: coverage.items.filter((i) => i.kind !== undefined).length,
},
}),
pointers: beforePausePointers(c),
};
}
/** @internal — the account with test hooks (`failTemplate`). The public door is `accountForAnswer`. */
export function buildAccount(recording, declarations, options, internals) {
if (typeof recording !== 'object' || recording === null || Array.isArray(recording)) {
throw new TypeError('accountForAnswer: the recording must be an object ({ snapshot, events, structure })');
}
const decl = validateDeclarations(declarations);
const view = recordingView(recording, options);
let unread = view.unread;
const say = makeSay(internals, () => {
unread += 1;
});
if (view.events.length === 0)
return notToldAccount(view, say, () => unread, 'no-own-events');
const resumedLeg = view.ofType('pause.resume').length > 0 ||
(view.ofType('agent.turn_end').length > 0 && view.ofType('agent.turn_start').length === 0);
const iteration = answeringIteration(view);
const ctx = {
view,
declarations: decl,
resumedLeg,
...(iteration !== undefined && { answeringIteration: iteration }),
say,
noteUnread: () => {
unread += 1;
},
budget: { itemChars: 0, items: 0 },
};
const asked = readAsked(ctx);
const understood = readUnderstood(ctx);
const calls = readCalls(ctx);
const inView = readInView(ctx, calls.ids);
const pausedLeg = readPausedLeg(ctx, calls);
const beforePause = pausedLeg.calls;
const checkedRows = readCheckedRows(ctx, calls, pausedLeg);
const found = readFoundRow(ctx, calls, inView, pausedLeg);
const howSure = readHowSure(ctx, calls);
const checks = runChecks(ctx, understood, calls, inView, pausedLeg);
const wrong = wrongLines(ctx, checks, calls, pausedLeg);
const { answer, limits, limitsData } = readAnswer(view);
// A kept-out answer is still an answer the run gave (`readAnswer`).
const answered = answer.status === 'recorded' || answer.missing === 'redacted';
const summary = summaryOf(ctx, checks, answered, pausedLeg);
// Every reader has asked the view by now: one that touched a kept-out value
// without asking about it read a placeholder, not the run.
if (view.keptOutRead().length > 0)
return notToldAccount(view, say, () => unread, 'kept-out');
const firstSignal = checks.signals[0];
const wrongChips = firstSignal !== undefined
? [
chip('signals', 'chip.signals', firstSignal.tone, {
// Every signal the checks raised, not only the listed ones.
n: { value: checks.signals.length + checks.omittedSignals, source: 'library' },
}),
]
: [];
const rows = [
row('asked', asked.lines),
row('understood', understood.lines),
row('checked', checkedRows.checked, checkedRows.checkedMore ? { more: checkedRows.checkedMore } : {}),
row('not-checked', checkedRows.notChecked, checkedRows.notCheckedMore ? { more: checkedRows.notCheckedMore } : {}),
row('found', found),
row('how-sure', howSure.lines),
row('anything-wrong', wrong, { chips: wrongChips }),
];
// Counts are judgements: over EVERY call. Only the listing (`facts.calls`) is capped.
const all = calls.all.map((c) => c.fact);
const count = (o) => all.filter((c) => c.outcome === o).length;
const told = {
kind: 'agentfootprint/answer-account',
shape: 1,
templates: { set: 'answer-account', version: ANSWER_ACCOUNT_TEMPLATE_SET_VERSION },
run: readRun(view, resumedLeg),
question: asked.question.value !== null && asked.question.value.length > MAX_VAR_CHARS
? { ...asked.question, ...clipFact(asked.question.value, MAX_VAR_CHARS) }
: asked.question,
answer,
facts: {
routing: understood.routing,
calls: calls.calls.map((c) => c.fact),
...(calls.omitted > 0 && { callsOmitted: calls.omitted }),
beforePause: beforePause.slice(0, MAX_CALLS).map((c) => beforePauseFact(c)),
...(beforePause.length > MAX_CALLS && { beforePauseOmitted: beforePause.length - MAX_CALLS }),
inView: inView.all.slice(0, MAX_CALLS).map((r) => r.fact),
...(inView.all.length > MAX_CALLS && { inViewOmitted: inView.all.length - MAX_CALLS }),
evidence: howSure.evidence,
standing: howSure.standing,
limitsBlock: limits,
limitsData,
errors: {
failed: count('failed'),
refused: count('refused'),
declined: count('declined'),
notDispatched: count('not-dispatched'),
withheld: all.filter((c) => c.withheldBy !== undefined).length,
},
checks: {
reachable: checks.reachable,
unreachable: checks.unreachableChecks,
notApplicable: checks.notApplicable,
},
},
rows,
signals: checks.signals,
unreachable: checks.unreachable,
summary,
unread,
foreign: view.foreign,
scope: view.scope,
};
// README law 8, measured in its own unit — serialized UTF-8 bytes — by the one
// authority: the listings fitted and counted, never the judgements.
return fitAccount(told, () => notToldAccount(view, say, () => unread + 1, 'too-large'));
}
/**
* Explain one answer from its recording.
*
* @param recording the answer's recording — `{ snapshot, events, structure }` (`recordRun`).
* @param declarations what the APP declares when the report is made (skill labels, `rowsAt`);
* everything filled from it is vouched `app`.
* @param options `runId` — the stored artifact's `meta.origin.runId`, when known.
*
* @example
* ```ts
* import { accountForAnswer } from 'agentfootprint/observe';
*
* const account = accountForAnswer(recording, { tools: { lookup_volumes: { rowsAt: 'volumes' } } });
* account.summary.sentence.text; // "Nothing this report looks for turned up in the record."
* ```
*/
export function accountForAnswer(recording, declarations, options) {
return buildAccount(recording, declarations, options);
}
//# sourceMappingURL=account.js.map