agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
460 lines • 21 kB
JavaScript
/**
* The three checks, their signals, what could not be run, the "Anything wrong"
* row and the one-liner.
*
* Signals, not causes: each signal is a record fact under a fixed rule; the
* account never assigns fault. PRIORITY (the order signals are listed and the
* one-liner quotes them):
* 1. decided-delivered — `decided-not-delivered` (bad)
* 2. existence — `existence-not-checked`, from a `notChecked` OR a
* `cannotCover` item with `kind: 'existence'` (warn)
* 3. empty-results — `undeclared-empty-used` (bad), then
* `undeclared-empty-in-view` (warn)
* The disposition law: a check that could not run is NOT "nothing wrong" — it
* is listed as unreachable, and a check that does not apply to this answer is
* left out of the count (`reachable + unreachable + notApplicable = 3`).
*/
import { chip, dedupePointers, joinSentences, n, v } from './render.js';
import { at, declarationAt, emptinessSource, FACT_TEXT_CHARS, } from './facts/common.js';
import { itemAt } from './facts/calls.js';
import { heldChip } from './facts/checked.js';
import { beforePausePointers, inViewPointers, inViewVars } from './facts/found.js';
/** Signal lines per check, and unreachable lines per check, before the rest are counted. */
export const MAX_LINES_PER_CHECK = 3;
export function runChecks(ctx, understood, calls, inView,
/** The part before a pause, read from the committed state (`facts/pausedLeg.ts`) — judged like this leg's calls. */
pausedLeg = { stateHeld: false, calls: [] }) {
const signals = [];
const unreachable = [];
let omitted = 0;
let omittedSignals = 0;
const omittedPointers = [];
const omit = (sentence) => {
omitted += 1;
if (omittedPointers.length < MAX_LINES_PER_CHECK)
omittedPointers.push(...sentence.pointers.slice(0, 1));
};
const perCheck = (list, check) => list.filter((x) => x.check === check).length;
const addSignal = (signal) => {
if (perCheck(signals, signal.check) >= MAX_LINES_PER_CHECK) {
omit(signal.sentence);
omittedSignals += 1;
}
else
signals.push(signal);
};
const addUnreachable = (u) => {
if (perCheck(unreachable, u.check) >= MAX_LINES_PER_CHECK)
omit(u.sentence);
else
unreachable.push(u);
};
const states = {
'decided-delivered': understood.check,
existence: 'not-applicable',
'empty-results': 'not-applicable',
};
const checkPointers = [];
// 1 — decided-delivered (read by the understood row, which owns the proof).
if (understood.signal !== undefined) {
addSignal({
id: 'decided-not-delivered',
check: 'decided-delivered',
sentence: understood.signal,
tone: 'bad',
});
}
if (understood.unreachable !== undefined) {
addUnreachable({
check: 'decided-delivered',
missing: understood.unreachable.missing ?? 'no-event',
sentence: understood.unreachable,
});
}
if (understood.check === 'reachable')
checkPointers.push(...understood.checkPointers);
// A call no event names cannot be put in a sentence about its tool: every check it would
// feed is said to be unreachable for it, by its call id — never silently skipped.
// Which checks each unnamed call blocks — ONE line per call, written after both loops.
const unnamedBlocks = new Map();
const unnamed = (check, call) => {
states[check] = 'unreachable';
const blocks = unnamedBlocks.get(call) ?? new Set();
blocks.add(check);
unnamedBlocks.set(call, blocks);
};
// 2 — existence: EVERY call of this run (the listing cap never limits a check) whose
// declaration has not-checked / cannot-cover items — every item, not only the printed ones —
// and every call before a pause whose declaration the committed state holds.
const declaredCalls = [
...calls.all.map((call) => ({
...(call.coverage !== undefined && { coverage: call.coverage }),
tool: call.tool,
toolName: call.fact.toolName,
...(call.unnamed === true && { unnamedCall: call }),
})),
...pausedLeg.calls.map((call) => ({
...(call.coverage !== undefined && { coverage: call.coverage }),
tool: call.tool,
toolName: call.toolName.slice(0, FACT_TEXT_CHARS),
})),
];
for (const call of declaredCalls) {
const items = call.coverage?.items.filter((i) => i.section !== 'checked') ?? [];
if (items.length === 0)
continue;
if (call.unnamedCall !== undefined) {
unnamed('existence', call.unnamedCall);
continue;
}
const tool = call.tool;
const withKind = items.filter((i) => i.kind !== undefined);
if (withKind.length === 0) {
states.existence = 'unreachable';
addUnreachable({
check: 'existence',
missing: 'not-declared',
sentence: ctx.say('unreachable.existence', {
vars: { tool },
status: 'not-recorded',
missing: 'not-declared',
pointers: items.slice(0, 1).map((i) => itemAt(i, 'what')),
}),
});
continue;
}
if (states.existence === 'not-applicable')
states.existence = 'reachable';
checkPointers.push(...withKind.map((i) => itemAt(i, 'kind')));
for (const item of withKind.filter((i) => i.kind === 'existence')) {
const source = `tool:${call.toolName}`;
const never = item.section === 'cannotCover';
const sentence = item.keptOut
? // Its words are kept out of the record: the limit is said, its words never printed.
ctx.say(never ? 'signal.existenceCannotCover.keptOut' : 'signal.existenceNotChecked.keptOut', { vars: { tool }, pointers: [itemAt(item, 'kind'), itemAt(item, 'what')] })
: item.short !== undefined
? ctx.say(never ? 'signal.existenceCannotCover' : 'signal.existenceNotChecked', {
vars: { tool, short: v(item.short, source, itemAt(item, 'short')) },
pointers: [itemAt(item, 'kind')],
})
: ctx.say(never ? 'signal.existenceCannotCover.full' : 'signal.existenceNotChecked.full', {
vars: { tool, what: v(item.what, source, itemAt(item, 'what')) },
pointers: [itemAt(item, 'kind')],
});
addSignal({ id: 'existence-not-checked', check: 'existence', sentence, tone: 'warn' });
}
}
// 3 — empty results: this run's results (judged ones) and the earlier ones in view.
// Used before in-view: this run's results are read first, so their signals land first.
// A call the record shows PRESENTING an artifact (`artifacts.presented`, filed by the library's
// own `present`) returned a receipt for the screen, not data the answer was built on: whether
// it was empty is not a question it can be asked, so it is left out of the check — never read
// as a shape nobody declared (take 3: the receipt alone made the check unreachable).
const presented = new Set(ctx.view
.ofType('artifacts.presented')
.map((e) => e.payload.toolCallId)
.filter((id) => typeof id === 'string'));
const judged = calls.all.filter((c) => c.fact.outcome === 'ran' &&
c.fact.withheldBy === undefined &&
c.end !== undefined &&
!presented.has(c.fact.toolCallId));
if (judged.length > 0 ||
inView.all.length > 0 ||
inView.keptOut > 0 ||
pausedLeg.calls.length > 0)
states['empty-results'] = 'reachable';
// Whether a result was empty cannot be told — and WHY, from the one reader: no shape declared;
// the app's rows key holding no list; or a ticket to the artifact store that counts no rows
// (`emptiness.ts` · "Rows that travel by reference"). A declared key is never said undeclared.
const cannotTell = (reading, toolName, tool, pointer) => {
if (!reading.undeclaredShape && reading.rowsUnread === undefined)
return false;
states['empty-results'] = 'unreachable';
const declared = reading.rowsAt !== undefined;
// The value does not carry the rows (or their count) where the app declared them — or, with
// no key read, the library's placement ticket, which records bytes and no row count. A
// result the record keeps out (a redaction policy's placeholder) is its own reason: never
// "not declared" of a shape nobody can see.
const missing = reading.rowsUnread === 'redacted'
? 'redacted'
: reading.rowsUnread === 'uncounted-ticket' && !declared
? 'not-built'
: 'not-declared';
const id = reading.rowsUnread === 'redacted'
? 'unreachable.empty.redacted'
: reading.rowsUnread === 'no-list'
? 'unreachable.empty.noList'
: reading.rowsUnread === 'uncounted-ticket'
? 'unreachable.empty.uncountedTicket'
: 'unreachable.empty';
addUnreachable({
check: 'empty-results',
missing,
sentence: ctx.say(id, {
vars: { tool },
status: 'not-recorded',
missing,
// The app's rows key decides it when one was read: it is the basis, and "show me" names it.
...(declared && { basis: ['app'] }),
pointers: declared
? [pointer, declarationAt(ctx.declarations, `tools.${toolName}.rowsAt`)]
: [pointer],
}),
});
return true;
};
for (const call of judged) {
const end = call.end;
if (end === undefined)
continue; // `judged` keeps only calls with an end
if (call.unnamed) {
unnamed('empty-results', call);
continue;
}
if (cannotTell(call.emptiness, call.toolName, call.tool, at(end, 'toolCallId')))
continue;
checkPointers.push(...call.fact.pointers.slice(-1));
if (call.emptiness.emptiness !== 'undeclared-empty')
continue;
addSignal({
id: 'undeclared-empty-used',
check: 'empty-results',
tone: 'bad',
sentence: ctx.say('signal.undeclaredEmptyUsed', {
vars: { tool: call.tool },
basis: [emptinessSource(call.emptiness)],
pointers: call.fact.pointers,
}),
});
}
// The calls answered before a pause: their bytes are in the committed history, their door in
// the committed `coverageDeclared` rows — the same result the model used for THIS answer.
for (const call of pausedLeg.calls) {
const pointers = beforePausePointers(call);
if (cannotTell(call.reading, call.toolName, call.tool, pointers[0]))
continue;
checkPointers.push(...pointers.slice(-1));
if (call.reading.emptiness !== 'undeclared-empty')
continue;
addSignal({
id: 'undeclared-empty-used',
check: 'empty-results',
tone: 'bad',
sentence: ctx.say('signal.undeclaredEmptyUsed.beforePause', {
vars: { tool: call.tool },
basis: [emptinessSource(call.reading)],
pointers,
chips: [heldChip()],
}),
});
}
// Earlier results the model could see, in a history the record keeps out: unread, said so.
if (inView.keptOut > 0) {
states['empty-results'] = 'unreachable';
addUnreachable({
check: 'empty-results',
missing: 'redacted',
sentence: ctx.say('unreachable.inView.redacted', {
vars: { n: { value: inView.keptOut, source: 'library' } },
status: 'not-recorded',
missing: 'redacted',
}),
});
}
for (const read of inView.all) {
if (cannotTell(read.reading, read.toolName, read.tool, at(read.witness, 'sourceId')))
continue;
checkPointers.push(at(read.witness, 'sourceId'));
if (read.reading.emptiness !== 'undeclared-empty')
continue;
addSignal({
id: 'undeclared-empty-in-view',
check: 'empty-results',
tone: 'warn',
sentence: ctx.say('signal.undeclaredEmptyInView', {
vars: inViewVars(read),
basis: [emptinessSource(read.reading)],
pointers: inViewPointers(ctx, read),
}),
});
}
for (const [call, blocks] of unnamedBlocks) {
const both = blocks.has('existence') && blocks.has('empty-results');
const id = both
? 'unreachable.unnamed.both'
: blocks.has('existence')
? 'unreachable.unnamed.existence'
: 'unreachable.unnamed';
addUnreachable({
check: blocks.has('empty-results') ? 'empty-results' : 'existence',
missing: 'unreadable',
sentence: ctx.say(id, {
vars: { id: call.tool },
status: 'not-recorded',
missing: 'unreadable',
pointers: call.fact.pointers,
}),
});
}
const ids = ['decided-delivered', 'existence', 'empty-results'];
return {
signals,
// Pointers stand for the checks that ran — a sample is enough, and the account stays bounded.
omitted,
omittedSignals,
omittedPointers,
unreachable,
reachable: ids.filter((id) => states[id] === 'reachable'),
unreachableChecks: ids.filter((id) => states[id] === 'unreachable'),
notApplicable: ids.filter((id) => states[id] === 'not-applicable'),
checkPointers: dedupePointers(checkPointers).slice(0, MAX_LINES_PER_CHECK * 2),
routingLost: understood.beforePause === 'lost',
};
}
/** The "Anything wrong" row's lines, in order. */
export function wrongLines(ctx, checks, calls, pausedLeg = { stateHeld: false, calls: [] }) {
const lines = checks.signals.map((s) => s.sentence);
// Every call, not only the listed ones: a count is a judgement.
const all = calls.all;
const count = (o) => all.filter((c) => c.fact.outcome === o).length;
const failed = count('failed');
const refused = count('refused');
const declined = count('declined');
const notDispatched = count('not-dispatched');
const troubled = all.filter((c) => ['failed', 'refused', 'declined', 'not-dispatched'].includes(c.fact.outcome));
if (troubled.length > 0) {
lines.push(ctx.say('wrong.errors', {
vars: {
n: n(troubled.length),
failed: n(failed),
refused: n(refused),
declined: n(declined),
notDispatched: n(notDispatched),
},
pointers: troubled.flatMap((c) => c.fact.pointers.slice(-1)),
}));
}
else if (all.length > 0) {
lines.push(ctx.say('wrong.noErrors', { pointers: all.flatMap((c) => c.fact.pointers.slice(-1)) }));
}
const withheld = all.filter((c) => c.fact.withheldBy !== undefined && c.ruleEvent !== undefined);
if (withheld.length > 0) {
lines.push(ctx.say('wrong.withheld', {
vars: { n: n(withheld.length) },
pointers: withheld.flatMap((c) => (c.ruleEvent ? [at(c.ruleEvent, 'outcome')] : [])),
}));
}
const applicable = checks.reachable.length + checks.unreachableChecks.length;
if (checks.signals.length === 0 && checks.reachable.length > 0) {
lines.push(ctx.say('wrong.none', {
vars: { reachable: n(checks.reachable.length) },
pointers: checks.checkPointers,
chips: [chip('none-found', 'chip.noneFound', 'ok')],
}));
}
if (checks.unreachableChecks.length > 0) {
lines.push(checks.unreachableChecks.length === applicable
? ctx.say('wrong.unreachable.all', {
status: 'not-recorded',
missing: 'not-declared',
chips: [chip('not-recorded', 'chip.notRecorded')],
})
: ctx.say('wrong.unreachable', {
vars: { unreachable: n(checks.unreachableChecks.length), applicable: n(applicable) },
status: 'not-recorded',
missing: 'not-declared',
chips: [chip('not-recorded', 'chip.notRecorded')],
}), ...checks.unreachable.map((u) => ({ ...u.sentence, item: true })));
}
if (checks.omitted > 0) {
lines.push(ctx.say('wrong.more', { vars: { n: n(checks.omitted) }, pointers: checks.omittedPointers }));
}
if (applicable === 0 && checks.signals.length === 0) {
lines.push(ctx.say('wrong.notApplicable', { status: 'not-applicable' }));
}
if (ctx.resumedLeg) {
lines.push(pausedLeg.stateHeld
? // The routing is named as lost only when it is: a verdict the paused state holds is
// read (`facts/understood.ts` · `routingVerdictOf`), and an agent that routes nothing
// lost none.
ctx.say(checks.routingLost ? 'wrong.beforePause.held' : 'wrong.beforePause.held.events', {
status: 'not-recorded',
missing: 'before-pause',
chips: [heldChip()],
})
: ctx.say('wrong.beforePause', {
status: 'not-recorded',
missing: 'before-pause',
chips: [chip('before-pause', 'chip.beforePause')],
}));
}
if (ctx.view.scope === 'unfiltered') {
lines.push(ctx.say('scope.unfiltered', { status: 'not-recorded', missing: 'no-event' }));
}
return lines;
}
/**
* The one-liner. On a resumed leg whose record holds the committed state, the
* checks above read the part before the pause from it, so the one-liner is
* the one any answer gets, followed by what this record does NOT hold of the
* part before the pause (its events). Only a record with no committed history
* says the part before the pause is not in it.
*/
export function summaryOf(ctx, checks, finished, pausedLeg = { stateHeld: false, calls: [] }) {
const read = summaryOfChecks(ctx, checks, finished, pausedLeg.stateHeld);
if (!finished || !ctx.resumedLeg || !pausedLeg.stateHeld)
return read;
return {
sentence: joinSentences(read.sentence, ctx.say('summary.resumed.held', { status: 'not-recorded', missing: 'before-pause' })),
tone: read.tone,
};
}
function summaryOfChecks(ctx, checks, finished, stateHeld) {
if (!finished) {
return {
sentence: ctx.say('summary.unfinished', { status: 'not-recorded', missing: 'no-event' }),
tone: 'unknown',
};
}
const [first, second] = checks.signals;
if (first !== undefined) {
let sentence = joinSentences(first.sentence, second?.sentence);
if (ctx.resumedLeg && !stateHeld) {
sentence = joinSentences(sentence, ctx.say('summary.resumed.tail', { status: 'not-recorded', missing: 'before-pause' }));
}
return { sentence, tone: first.tone };
}
if (ctx.resumedLeg && !stateHeld) {
return {
sentence: ctx.say('summary.resumed', { status: 'not-recorded', missing: 'before-pause' }),
tone: 'unknown',
};
}
const applicable = checks.reachable.length + checks.unreachableChecks.length;
if (checks.unreachableChecks.length > 0) {
return checks.unreachableChecks.length === applicable
? {
sentence: ctx.say('summary.none.nothingRun', {
status: 'not-recorded',
missing: 'not-declared',
}),
tone: 'unknown',
}
: {
sentence: ctx.say('summary.none.partial', {
vars: { unreachable: n(checks.unreachableChecks.length), applicable: n(applicable) },
status: 'not-recorded',
missing: 'not-declared',
}),
tone: 'unknown',
};
}
if (applicable === 0) {
return {
sentence: ctx.say('summary.none.notApplicable', { status: 'not-applicable' }),
tone: 'unknown',
};
}
return { sentence: ctx.say('summary.none', { pointers: checks.checkPointers }), tone: 'ok' };
}
//# sourceMappingURL=signals.js.map