UNPKG

agentfootprint

Version:

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

621 lines • 28 kB
"use strict"; /** * `accountForAnswer` — one answer's recording (+ what the app declared) in, a * typed account out. A pure read-time Fold over the Trace: no clock, network, * model or randomness; the same inputs give the same bytes. It never writes * back into the run. * * It throws ONLY on a caller error (a recording that is not an object, * declarations that break their rules). Content it cannot read is passed over * and counted (`unread`); a line whose fill fails becomes `unreadable.line@1`. */ Object.defineProperty(exports, "__esModule", { value: true }); exports.accountForAnswer = exports.buildAccount = exports.MAX_ANSWER_CHARS = void 0; const serve_js_1 = require("../core/agent/arguments/serve.js"); const assess_js_1 = require("../core/agent/assessment/assess.js"); const compose_js_1 = require("../core/agent/assessment/compose.js"); const answer_js_1 = require("../semantics/coverage/answer.js"); const rows_js_1 = require("../core/agent/scripts/rows.js"); const attachment_js_1 = require("../core/agent/scripts/attachment.js"); const rows_js_2 = require("../core/agent/scripts/rows.js"); const render_js_1 = require("./render.js"); const templates_js_1 = require("./templates.js"); const declarations_js_1 = require("./declarations.js"); const size_js_1 = require("./size.js"); const view_js_1 = require("./view.js"); const common_js_1 = require("./facts/common.js"); const asked_js_1 = require("./facts/asked.js"); const understood_js_1 = require("./facts/understood.js"); const calls_js_1 = require("./facts/calls.js"); const inView_js_1 = require("./facts/inView.js"); const checked_js_1 = require("./facts/checked.js"); const pausedLeg_js_1 = require("./facts/pausedLeg.js"); const found_js_1 = require("./facts/found.js"); const howSure_js_1 = require("./facts/howSure.js"); const signals_js_1 = require("./signals.js"); /** The answer text the account carries (the PDF folds at 1,200); the rest is in the record. */ exports.MAX_ANSWER_CHARS = 4000; const HEADINGS = { asked: 'row.asked', understood: 'row.understood', checked: 'row.checked', 'not-checked': 'row.notChecked', found: 'row.found', 'how-sure': 'row.howSure', 'anything-wrong': 'row.wrong', }; /** recorded if any line is; not-applicable if every line is; else not-recorded. */ function rowStatus(lines) { if (lines.some((l) => l.status === 'recorded')) return 'recorded'; if (lines.length > 0 && lines.every((l) => l.status === 'not-applicable')) return 'not-applicable'; return 'not-recorded'; } function saidByChips(lines) { const sources = []; for (const line of lines) { if (line.status === 'recorded' && !sources.includes(line.source)) sources.push(line.source); } return sources.map(render_js_1.saidBy); } function row(id, lines, extra = {}) { const status = rowStatus(lines); return { id, heading: (0, render_js_1.sentence)(HEADINGS[id], { status: 'not-applicable' }), lines, ...(extra.more !== undefined && { more: extra.more }), chips: [...(extra.chips ?? []), ...saidByChips(lines)], status, }; } /** The separator the framework's one composer puts between the answer and what it appends. */ const APPENDED_SEPARATOR = '\n\n---\n\n'; /** * The headings of the two BLOCKS the framework appends after an answer * (`coverage/answer.ts` · `composeAnswerWithCoverage`, the one composer), each * read from its owner: the limits block's and the "Assumed" block's — long, * fixed sentences only the framework writes. */ const BLOCK_HEADINGS = [ `${answer_js_1.COVERAGE_BLOCK_HEADING} — declared by the tools that produced it, not by the model:`, answer_js_1.ASSUMED_BLOCK_HEADING, ]; /** * The standing line THIS run appended, rebuilt from its record — or * `undefined` when the record says none could be there. * * The line opens with the owner's plain words ("Not sure — ", "Known — "), * which a model can write too, so an opening proves nothing. The record does: * `turn_end.answerAssessment` exists only on a run whose answer layer folded * the standing, and the line is ONE function of that projection * (`assessment/compose.ts` · `standingLineOf`) — plus, when a value was * assumed, of the committed rows it names, read from the source the layer's * stage read (`assumedValuesSourceOf`) and the fold's rewrite witness. Only * that exact line, where the composer puts it, is the library's. Without the * committed state an assumed value cannot be rebuilt, and the line is not * claimed. * * A line that named calls in the app's words (a tool's `displayName`) is * rebuilt from the words the record kept beside it (`turn_end.standingLineNames`) * and the calls the fold reads off the committed state (each reason's * `source`) — the same two inputs the layer's stage composed it from. Words * the record does not hold in full, or no committed state, and the line is * not claimed. */ function appendedStandingLine(end, state) { const data = (0, compose_js_1.assessmentDataFrom)(end.payload.answerAssessment); if (data === undefined) return undefined; const kept = end.payload.standingLineNames; const displayNames = kept === undefined ? undefined : (0, compose_js_1.standingNamesFrom)(kept); if (kept !== undefined && displayNames === undefined) return undefined; if (!data.reasons.includes('argument-assumed') && displayNames === undefined) { return (0, compose_js_1.standingLineOf)(data, [], false); } if (state === undefined) return undefined; const source = (0, compose_js_1.assumedValuesSourceOf)(data.reasons, state); const assumed = source === undefined ? [] : (0, serve_js_1.assumedLinesFor)(source.ledger, source.turn, source.readDecisions); const assessed = (0, assess_js_1.assessAnswer)({ snapshot: { sharedState: state } }); return (0, compose_js_1.standingLineOf)(data, assumed, (0, compose_js_1.rewrittenBehind)(assessed), displayNames === undefined ? undefined : { sources: (0, compose_js_1.sourcesByReason)(assessed), displayNames }); } /** * The opening of the scripts block THIS run attached * (`.scriptTravelsWithTheAnswer()`), rebuilt from its record — never matched * by words. The block is one function of `turn_end.scripts` (the shown * scripts and the omitted count) and, for a program over the bound, of that * call's committed `script` row (`core/agent/scripts/attachment.ts` · * `scriptsBlock`), behind the boundary the record states (`closedFence`: the * block opens with the fence it closed; `withheldFromText`: each script is * its label line alone). With a script shown, the block's text up to its * first label line opens it; with none shown, the omission lines do — * rebuilt for the last call that reached the tool and for every one (the * record does not keep `which`). Anything the record does not hold in full, * and the block is not claimed. */ function appendedScriptsOpenings(end, state, within) { // The one reader of the shape (`attachment.ts` · `recordedScripts`): the // text is rebuilt only from a record read back whole — a value no writer // of this library files rebuilds nothing (the How-sure lines count it). const read = (0, attachment_js_1.recordedScripts)(end.payload.scripts, within); if (read === undefined || read.unread > 0) return []; const { boundary: recorded, omitted } = read; if (recorded === undefined || omitted === undefined) return []; const boundary = (0, attachment_js_1.rebuildBoundary)(recorded, within); if (boundary === undefined) return []; const first = read.shown[0]; if (first !== undefined) { const only = { shown: [first.script], omitted: 0 }; // Up to and including the first label line: the closed fence, if any, then it. const lines = (0, attachment_js_1.scriptsBlock)(only, boundary).split('\n'); return [lines.slice(0, boundary.kind === 'closed' ? 3 : 1).join('\n')]; } if (omitted === 0 || state === undefined) return []; const turn = (0, view_js_1.num)(state.turnNumber); if (turn === undefined) return []; const ledger = Array.isArray(state.findingsLedger) ? state.findingsLedger : []; const current = (0, rows_js_1.scriptRowsOfTurn)(ledger, turn).current; // The rows come from saved state: rebuild only from rows this library writes. if (!current.every((row) => (0, rows_js_2.scriptRowIsWellFormed)(row))) { return []; } const reached = current.filter((row) => row.outcome !== 'submitted'); const overSizeOf = (rows) => rows.filter((row) => row.text === undefined); return [overSizeOf(reached.slice(-1)), overSizeOf(reached)].map((overSize) => (0, attachment_js_1.scriptsBlock)({ shown: [], omitted, overSize }, boundary)); } /** Whether `text` opens the framework's appended section: the run's own line, a block's heading, or its scripts. */ function opensAppended(text, line, scripts = []) { if (line !== undefined && (text === line || text.startsWith(`${line}\n\n`))) return true; if (scripts.some((opening) => opening !== '' && text.startsWith(opening))) return true; return BLOCK_HEADINGS.some((heading) => text.startsWith(heading)); } /** * Split the framework's appended section off an answer: the LAST separator * whose next line opens it — the run's own standing line, exactly, or a * block's heading. A `---` the model wrote itself opens neither, and the * composer joins its own blocks with blank lines, never a second separator. * An answer with no text of its own is the section alone (the composer adds * no separator then). */ function splitAppended(content, line, scripts = []) { let at = content.lastIndexOf(APPENDED_SEPARATOR); while (at >= 0) { const rest = content.slice(at + APPENDED_SEPARATOR.length); if (opensAppended(rest, line, scripts)) return { model: content.slice(0, at), appended: rest }; at = at === 0 ? -1 : content.lastIndexOf(APPENDED_SEPARATOR, at - 1); } return opensAppended(content, line, scripts) ? { model: '', appended: content } : { model: content }; } /** How many items one of the data's lists holds — `0` for a list the record does not carry. */ const listLength = (value) => (Array.isArray(value) ? value.length : 0); /** * A TYPED answer's limits, read off `turn_end.answerCoverage` — the data * `.limitsTravelWithTheAnswer()` files beside an answer that must stay JSON. * Counts, with a pointer to every item's `what` (an assumed value's * `toolName` — never its `value`, an argument's value, which "show me" never * shows), capped at the account's call cap. `undefined` when the event * carries none. */ function readLimitsData(end) { // Limits the record keeps out (a redaction policy selects `answerCoverage`): // they travelled as data, and their words and counts are not in this record. if ((0, view_js_1.keptOut)(end, 'answerCoverage')) { return { value: null, source: 'library', status: 'not-recorded', missing: 'redacted', pointers: [(0, common_js_1.at)(end, 'answerCoverage')], }; } const data = end.payload.answerCoverage; if (!(0, view_js_1.isRecord)(data)) return undefined; const pointers = []; for (const list of ['checked', 'notChecked', 'cannotCover']) { const items = data[list]; if (!Array.isArray(items)) continue; items.forEach((item, i) => { if ((0, view_js_1.isRecord)(item) && typeof item.what === 'string') { pointers.push((0, common_js_1.at)(end, 'answerCoverage', list, i, 'what')); } }); } const assumed = data.assumed; if (Array.isArray(assumed)) { assumed.forEach((item, i) => { if ((0, view_js_1.isRecord)(item) && typeof item.toolName === 'string') { pointers.push((0, common_js_1.at)(end, 'answerCoverage', 'assumed', i, 'toolName')); } }); } if (pointers.length === 0) return undefined; return { value: { checked: listLength(data.checked), notChecked: listLength(data.notChecked), cannotCover: listLength(data.cannotCover), assumed: listLength(assumed), }, source: 'library', status: 'recorded', pointers: pointers.slice(0, calls_js_1.MAX_CALLS), }; } /** * The answer, and what the framework carried with it: the appended section of * a PROSE answer split off it (the limits block, the "Assumed" block, the * standing line — `limitsBlock`), or a TYPED answer's limits as data * (`limitsData`). A typed answer's limits never read `not-applicable`: the * block reads `not-recorded` with `missing: 'as-data'`, and the data is * `recorded`. An answer the record keeps out (a redaction policy's * placeholder on `finalContent`) reads `not-recorded` with `missing: * 'redacted'` — the run still answered, so the account never reads it as a * run that did not finish. */ function readAnswer(view) { const end = view.last('agent.turn_end'); const content = (0, view_js_1.str)(end?.payload.finalContent); const none = { value: null, source: 'library', status: 'not-recorded', pointers: [], missing: 'no-event', }; const noData = { value: null, source: 'library', status: 'not-applicable', pointers: [], }; if (end !== undefined && content === undefined && (0, view_js_1.keptOut)(end, 'finalContent')) { const kept = { value: null, source: 'library', status: 'not-recorded', missing: 'redacted', }; return { answer: { ...kept, pointers: [(0, common_js_1.at)(end, 'finalContent')] }, limits: { ...kept, pointers: [] }, limitsData: readLimitsData(end) ?? noData, }; } if (end === undefined || content === undefined) { return { answer: none, limits: { value: null, source: 'library', status: 'not-applicable', pointers: [] }, limitsData: noData, }; } const pointer = (0, common_js_1.at)(end, 'finalContent'); const data = readLimitsData(end); const { model, appended } = splitAppended(content, appendedStandingLine(end, view.state), appendedScriptsOpenings(end, view.state, content.length)); const answer = { ...(0, render_js_1.clipFact)(model, exports.MAX_ANSWER_CHARS), source: 'model', status: 'recorded', pointers: [pointer], }; if (appended !== undefined) { return { answer, limits: { ...(0, render_js_1.clipFact)(appended, exports.MAX_ANSWER_CHARS), source: 'library', status: 'recorded', pointers: [pointer], }, limitsData: data ?? noData, }; } return { answer, limits: data !== undefined ? { value: null, source: 'library', status: 'not-recorded', pointers: [], missing: 'as-data', } : { value: null, source: 'library', status: 'not-applicable', pointers: [] }, limitsData: data ?? noData, }; } function readRun(view, resumedLeg) { const configured = view.first('agent.run_configured'); const model = (0, view_js_1.str)(configured?.payload.llm?.model); const turnNumber = (0, view_js_1.num)(view.state?.turnNumber); if (view.runId === undefined) { return { value: null, source: 'library', status: 'not-recorded', pointers: [], missing: 'no-event', }; } const anchor = configured ?? view.events[0]; return { value: { runId: view.runId, ...(turnNumber !== undefined && { turnNumber }), ...(model !== undefined && { model }), resumedLeg, }, source: 'library', status: 'recorded', pointers: [ ...(anchor !== undefined ? [(0, common_js_1.derived)(anchor, '#meta/runId')] : []), ...(configured !== undefined && model !== undefined ? [(0, common_js_1.at)(configured, 'llm', 'model')] : []), ...(turnNumber !== undefined ? [(0, common_js_1.stateAt)('turnNumber')] : []), ], }; } /** * The account of a run this record cannot tell about — said once per row, and * nothing is claimed about the run (no "did not run any tools", no "did not * finish"); the hosting op may map it to `summary.notAvailable@1`. Three causes: * * - `no-own-events` — the run owns NO event of this record (an * options.runId that is not this recording's, or a recording with no * readable event). The record's silence is not evidence. * - `kept-out` — a reader touched a value the record keeps out (a * redaction policy's placeholder) without asking about it first * (`view.ts` · `keptOutRead`), so a fact read from it would be a fact * about a placeholder. The kept-out values the account DOES handle — the * question, the answer, a tool result, a call's outcome fields, the * history — never get here: each is said to be kept out where it is read. * - `too-large` — the account would pass its size limit even with nothing * listed (`size.ts` · `fitAccount`): only a record built to be, every * text at its cut in characters JSON writes six bytes wide. Counted unread. */ function notToldAccount(view, say, unread, cause) { const missing = { status: 'not-recorded', missing: cause === 'kept-out' ? 'redacted' : cause === 'too-large' ? 'unreadable' : 'no-event', }; const scopeLine = cause === 'kept-out' ? 'scope.keptOut' : cause === 'too-large' ? 'scope.tooLarge' : 'scope.noOwnEvents'; const rowLine = cause === 'kept-out' ? 'row.keptOut' : cause === 'too-large' ? 'row.tooLarge' : 'row.notInRecord'; const none = () => ({ value: null, source: 'library', pointers: [], ...missing, }); const rows = Object.keys(HEADINGS).map((id) => row(id, [ id === 'anything-wrong' ? say(scopeLine, { ...missing, chips: [(0, render_js_1.chip)('not-recorded', 'chip.notRecorded')], }) : say(rowLine, missing), ])); return { kind: 'agentfootprint/answer-account', shape: 1, templates: { set: 'answer-account', version: templates_js_1.ANSWER_ACCOUNT_TEMPLATE_SET_VERSION }, run: none(), question: none(), answer: none(), facts: { routing: { configured: none(), verdict: none(), scores: none(), confidence: none(), appDecision: none(), delivered: none(), refusals: [], }, calls: [], beforePause: [], inView: [], evidence: none(), standing: none(), limitsBlock: none(), limitsData: none(), errors: { failed: 0, refused: 0, declined: 0, notDispatched: 0, withheld: 0 }, checks: { reachable: [], unreachable: [], notApplicable: [] }, }, rows, signals: [], unreachable: [], summary: { sentence: say(scopeLine, missing), tone: 'unknown', }, unread: unread(), foreign: view.foreign, scope: view.scope, }; } /** One call answered before the pause, as the facts list it (read from the committed state). */ function beforePauseFact(c) { const reading = c.reading; const coverage = c.coverage; return { toolName: c.toolName.slice(0, common_js_1.FACT_TEXT_CHARS), toolCallId: c.toolCallId.slice(0, common_js_1.FACT_TEXT_CHARS), emptiness: reading.emptiness, ...(0, common_js_1.countedFields)(reading), ...(coverage !== undefined && { coverage: { kind: coverage.kind, ...(coverage.lookedFor !== undefined && { lookedFor: coverage.lookedFor.text.slice(0, common_js_1.FACT_TEXT_CHARS), }), checked: coverage.items.filter((i) => i.section === 'checked').length, notChecked: coverage.items.filter((i) => i.section === 'notChecked').length, cannotCover: coverage.items.filter((i) => i.section === 'cannotCover').length, kinds: coverage.items.filter((i) => i.kind !== undefined).length, }, }), pointers: (0, found_js_1.beforePausePointers)(c), }; } /** @internal — the account with test hooks (`failTemplate`). The public door is `accountForAnswer`. */ function buildAccount(recording, declarations, options, internals) { if (typeof recording !== 'object' || recording === null || Array.isArray(recording)) { throw new TypeError('accountForAnswer: the recording must be an object ({ snapshot, events, structure })'); } const decl = (0, declarations_js_1.validateDeclarations)(declarations); const view = (0, view_js_1.recordingView)(recording, options); let unread = view.unread; const say = (0, common_js_1.makeSay)(internals, () => { unread += 1; }); if (view.events.length === 0) return notToldAccount(view, say, () => unread, 'no-own-events'); const resumedLeg = view.ofType('pause.resume').length > 0 || (view.ofType('agent.turn_end').length > 0 && view.ofType('agent.turn_start').length === 0); const iteration = (0, inView_js_1.answeringIteration)(view); const ctx = { view, declarations: decl, resumedLeg, ...(iteration !== undefined && { answeringIteration: iteration }), say, noteUnread: () => { unread += 1; }, budget: { itemChars: 0, items: 0 }, }; const asked = (0, asked_js_1.readAsked)(ctx); const understood = (0, understood_js_1.readUnderstood)(ctx); const calls = (0, calls_js_1.readCalls)(ctx); const inView = (0, inView_js_1.readInView)(ctx, calls.ids); const pausedLeg = (0, pausedLeg_js_1.readPausedLeg)(ctx, calls); const beforePause = pausedLeg.calls; const checkedRows = (0, checked_js_1.readCheckedRows)(ctx, calls, pausedLeg); const found = (0, found_js_1.readFoundRow)(ctx, calls, inView, pausedLeg); const howSure = (0, howSure_js_1.readHowSure)(ctx, calls); const checks = (0, signals_js_1.runChecks)(ctx, understood, calls, inView, pausedLeg); const wrong = (0, signals_js_1.wrongLines)(ctx, checks, calls, pausedLeg); const { answer, limits, limitsData } = readAnswer(view); // A kept-out answer is still an answer the run gave (`readAnswer`). const answered = answer.status === 'recorded' || answer.missing === 'redacted'; const summary = (0, signals_js_1.summaryOf)(ctx, checks, answered, pausedLeg); // Every reader has asked the view by now: one that touched a kept-out value // without asking about it read a placeholder, not the run. if (view.keptOutRead().length > 0) return notToldAccount(view, say, () => unread, 'kept-out'); const firstSignal = checks.signals[0]; const wrongChips = firstSignal !== undefined ? [ (0, render_js_1.chip)('signals', 'chip.signals', firstSignal.tone, { // Every signal the checks raised, not only the listed ones. n: { value: checks.signals.length + checks.omittedSignals, source: 'library' }, }), ] : []; const rows = [ row('asked', asked.lines), row('understood', understood.lines), row('checked', checkedRows.checked, checkedRows.checkedMore ? { more: checkedRows.checkedMore } : {}), row('not-checked', checkedRows.notChecked, checkedRows.notCheckedMore ? { more: checkedRows.notCheckedMore } : {}), row('found', found), row('how-sure', howSure.lines), row('anything-wrong', wrong, { chips: wrongChips }), ]; // Counts are judgements: over EVERY call. Only the listing (`facts.calls`) is capped. const all = calls.all.map((c) => c.fact); const count = (o) => all.filter((c) => c.outcome === o).length; const told = { kind: 'agentfootprint/answer-account', shape: 1, templates: { set: 'answer-account', version: templates_js_1.ANSWER_ACCOUNT_TEMPLATE_SET_VERSION }, run: readRun(view, resumedLeg), question: asked.question.value !== null && asked.question.value.length > render_js_1.MAX_VAR_CHARS ? { ...asked.question, ...(0, render_js_1.clipFact)(asked.question.value, render_js_1.MAX_VAR_CHARS) } : asked.question, answer, facts: { routing: understood.routing, calls: calls.calls.map((c) => c.fact), ...(calls.omitted > 0 && { callsOmitted: calls.omitted }), beforePause: beforePause.slice(0, calls_js_1.MAX_CALLS).map((c) => beforePauseFact(c)), ...(beforePause.length > calls_js_1.MAX_CALLS && { beforePauseOmitted: beforePause.length - calls_js_1.MAX_CALLS }), inView: inView.all.slice(0, calls_js_1.MAX_CALLS).map((r) => r.fact), ...(inView.all.length > calls_js_1.MAX_CALLS && { inViewOmitted: inView.all.length - calls_js_1.MAX_CALLS }), evidence: howSure.evidence, standing: howSure.standing, limitsBlock: limits, limitsData, errors: { failed: count('failed'), refused: count('refused'), declined: count('declined'), notDispatched: count('not-dispatched'), withheld: all.filter((c) => c.withheldBy !== undefined).length, }, checks: { reachable: checks.reachable, unreachable: checks.unreachableChecks, notApplicable: checks.notApplicable, }, }, rows, signals: checks.signals, unreachable: checks.unreachable, summary, unread, foreign: view.foreign, scope: view.scope, }; // README law 8, measured in its own unit — serialized UTF-8 bytes — by the one // authority: the listings fitted and counted, never the judgements. return (0, size_js_1.fitAccount)(told, () => notToldAccount(view, say, () => unread + 1, 'too-large')); } exports.buildAccount = buildAccount; /** * Explain one answer from its recording. * * @param recording the answer's recording — `{ snapshot, events, structure }` (`recordRun`). * @param declarations what the APP declares when the report is made (skill labels, `rowsAt`); * everything filled from it is vouched `app`. * @param options `runId` — the stored artifact's `meta.origin.runId`, when known. * * @example * ```ts * import { accountForAnswer } from 'agentfootprint/observe'; * * const account = accountForAnswer(recording, { tools: { lookup_volumes: { rowsAt: 'volumes' } } }); * account.summary.sentence.text; // "Nothing this report looks for turned up in the record." * ``` */ function accountForAnswer(recording, declarations, options) { return buildAccount(recording, declarations, options); } exports.accountForAnswer = accountForAnswer; //# sourceMappingURL=account.js.map