agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
240 lines • 13.9 kB
JavaScript
;
/**
* cacheRecorder() — the cache layer's meter.
*
* Subscribes to:
* - `FlowRecorder.onDecision` — CacheGate routing decisions
* (apply-markers / no-markers + the rule that fired + evidence from
* `decide()`).
* - `agentfootprint.stream.llm_end` — reads that event's `usage` (the PORT
* shape: `{ input, output, cacheRead?, cacheWrite? }`) and asks the
* agent's `CacheStrategy.extractMetrics` what is known about it.
*
* Produces: a per-turn report via `recorder.report()` — token tallies, hit
* rate and dollar estimates, each carried as a `Claim` so an UNMEASURED turn
* can never render as a zero one.
*
* ── Why every number here is a Claim (9.59.0) ─────────────────────────
* The shipped 9.58.0 meter reported `hitRate: 0` for a 20-call turn that hit
* cache on every call. Two faults, and the second is why the first went
* unnoticed for so long: (1) the strategies parsed RAW WIRE field names off a
* value that carries the PORT shape, so every field read `undefined`; (2) the
* report typed its aggregates as bare `number`, so "nobody measured" and
* "measured, and it was zero" rendered identically. Fixing (1) alone would
* have left a meter that still cannot say "unmeasured" — hence `Claim<T>`,
* plus `measuredIterations` / `unmeasuredIterations` so a rate computed from
* 3 of 20 calls states its own denominator.
*
* Read the number through `isKnown(report.hitRate)` (or `describeClaim` for a
* one-line render). There is deliberately no door that hands you a bare
* number without your having branched.
*
* ── And an unknown carries ITS OWN reason (9.59.1) ────────────────────
* 9.59.0 enforced that law per call and then broke it in `report()`, which
* hardcoded "the provider reported no cache fields" for the whole turn no
* matter what the rows said. With no strategy passed and a provider that DID
* report cache fields, the row said "nothing read the usage" and the summary
* blamed the provider — pointing the reader away from their actual mistake.
* The summary now carries the rows' own reasons, and says so plainly when the
* rows disagree rather than picking one.
*
* ── What this recorder does NOT do ────────────────────────────────────
* It emits no events. (Earlier prose here promised per-iteration
* `agentfootprint.cache.applied` / `agentfootprint.cache.metrics` events;
* neither name has ever existed in the event registry and no `typedEmit` has
* ever been in this file. The prose was the bug.)
*
* It does not write `scope.recentHitRate` back into agent state either, so
* CacheGate's hit-rate-floor rule never fires on its own — the loop is
* severed at both ends (the key is seeded `undefined` and written by
* nothing). Recorders do not write to chart scope, so closing that loop needs
* an agent-side accessor convention; it is separable work from measuring the
* number, which is what this file now does.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.cacheRecorder = void 0;
const foottrace_1 = require("foottrace");
const conventions_js_1 = require("../lib/conventions.js");
const claim_js_1 = require("../lib/claim/claim.js");
/**
* How many distinct reasons a disagreeing summary spells out before it
* switches to counting the rest. A summary sentence must stay readable.
*/
const MAX_LISTED_REASONS = 3;
/** The metrics of a row already filtered by `isKnown`. Narrowing helper only. */
function valueOfMetrics(entry) {
return (0, claim_js_1.isKnown)(entry.metrics)
? entry.metrics.value
: { cacheReadTokens: 0, cacheWriteTokens: 0, freshInputTokens: 0 };
}
function cacheRecorder(options = {}) {
const perIter = [];
let lastDecision;
let iterationCounter = 0;
function dollars(tokens, kind) {
if (!options.pricing)
return 0;
const model = options.model ?? 'unknown';
return tokens * options.pricing.pricePerToken(model, kind);
}
const handle = {
id: 'cache-recorder',
onDecision(event) {
// Only care about CacheGate decisions, matched by the decider's LOCAL
// stage id. Both `event.decider` (the node NAME) and the prefixed
// `traversalContext.stageId` become `sf-cache/…` now that CacheGate is
// nested in sf-cache, so we strip the subflow path with splitStageId and
// compare the local id. This is id-stable (survives a display-name
// rename) and nesting-safe (works top-level or inside sf-cache).
// (The old `event.decider !== 'cache-gate'` was a no-op: event.decider is
// the NAME 'CacheGate', never the id 'cache-gate'.)
const stageId = event.traversalContext?.stageId;
if (!stageId || (0, foottrace_1.splitStageId)(stageId).localStageId !== conventions_js_1.STAGE_IDS.CACHE_GATE)
return;
const matched = event.evidence?.rules.find((r) => r.matched);
lastDecision = {
branch: event.chosen,
...(matched?.label !== undefined && { rule: matched.label }),
};
},
onEmit(event) {
// .watch() receives the engine envelope (`name`); explicit typed-stream
// forwarding uses `type`. Both carry the same already-normalized usage.
const type = 'type' in event ? event.type : event.name;
if (type !== 'agentfootprint.stream.llm_end')
return;
iterationCounter++;
// The PORT shape, which is the only shape this event has ever carried.
const usage = event.payload.usage;
const metrics = options.strategy === undefined
? (0, claim_js_1.unknown)('no CacheStrategy was given to cacheRecorder(), so nothing read the usage')
: options.strategy.extractMetrics(usage);
const branch = lastDecision?.branch ?? 'apply-markers';
// Compute dollar math:
// spent = freshInput * inputPrice
// + cacheRead * cacheReadPrice
// + cacheWrite * cacheWritePrice
// no-cache cost = (freshInput + cacheRead + cacheWrite) * inputPrice
// saved = no-cache cost - spent
let dollarsSpent;
let savedVsNoCache;
if ((0, claim_js_1.isKnown)(metrics)) {
const m = metrics.value;
const spent = dollars(m.freshInputTokens, 'input') +
dollars(m.cacheReadTokens, 'cacheRead') +
dollars(m.cacheWriteTokens, 'cacheWrite');
const noCacheCost = dollars(m.freshInputTokens + m.cacheReadTokens + m.cacheWriteTokens, 'input');
const ev = options.pricing === undefined
? 'measured tokens, but no PricingTable was given — every price is 0'
: 'measured tokens priced with the given PricingTable';
dollarsSpent = (0, claim_js_1.known)(spent, ev);
savedVsNoCache = (0, claim_js_1.known)(noCacheCost - spent, ev);
}
else {
// No tokens measured ⇒ no dollars. Carrying the metrics claim's own
// reason forward is what keeps a $0 estimate from reading as "free".
const why = metrics.kind === 'unknown'
? metrics.reason
: `no cache tokens are measurable here — ${metrics.evidence}`;
dollarsSpent = (0, claim_js_1.unknown)(why);
savedVsNoCache = (0, claim_js_1.unknown)(why);
}
const entry = {
iteration: iterationCounter,
branch,
...(lastDecision?.rule !== undefined && { rule: lastDecision.rule }),
metrics,
dollarsSpent,
dollarsSavedVsNoCache: savedVsNoCache,
};
perIter.push(entry);
lastDecision = undefined;
},
report() {
const apply = perIter.filter((p) => p.branch === 'apply-markers').length;
const skip = perIter.filter((p) => p.branch === 'no-markers').length;
const measured = perIter.filter((p) => (0, claim_js_1.isKnown)(p.metrics));
const n = measured.length;
const total = perIter.length;
// ONE reason, stated once, reused by every claim below — so a reader
// who prints any single field learns why the whole report is empty.
//
// ── The summary CARRIES the rows' reasons; it never invents one (9.59.1)
// 9.59.0 hardcoded 'the provider reported no cache fields' here. That
// sentence is a fabrication whenever the rows said something else — most
// painfully when no CacheStrategy was passed and the provider DID report
// cacheRead/cacheWrite: the rows said "nothing read the usage", and the
// summary blamed the provider for the caller's own omission. An unknown
// must carry ITS OWN reason. That is the law this file exists to
// enforce, and this is it applied to the summary too.
const unmeasured = perIter.filter((p) => !(0, claim_js_1.isKnown)(p.metrics));
const reasons = [];
for (const row of unmeasured) {
const m = row.metrics;
// `unmeasured` already excluded 'known'; this narrows the type.
if ((0, claim_js_1.isKnown)(m))
continue;
const stated = m.kind === 'not-applicable' ? `this provider cannot report it (${m.evidence})` : m.reason;
if (!reasons.includes(stated))
reasons.push(stated);
}
// Rows can disagree — a turn may mix an adapter that cannot report with
// calls that carried no usage at all. Picking one silently would state a
// cause for calls that stated a different one, so the summary says there
// was more than one and lists them. Bounded by construction: at most
// MAX_LISTED_REASONS are spelled out and the remainder is COUNTED, so a
// strategy whose reason varies per call cannot grow this into a wall.
const listed = reasons.slice(0, MAX_LISTED_REASONS);
const remaining = reasons.length - listed.length;
const [onlyReason] = reasons;
const carried = reasons.length === 1 && onlyReason !== undefined
? onlyReason
: `the calls did not agree on why — ${reasons.length} different reasons were ` +
`given: ${listed.map((r, i) => `(${i + 1}) ${r}`).join('; ')}` +
(remaining > 0 ? `; and ${remaining} more not listed here` : '');
const why = total === 0
? 'no LLM call was observed, so there is nothing to measure'
: reasons.length === 0
? // Unreachable where `why` is USED (it is read only when nothing
// was measured, and an unmeasured row always states a reason).
`all ${total} observed call(s) reported cache usage`
: `none of the ${total} observed call(s) reported cache usage — ${carried}`;
const evidence = `summed over the ${n} of ${total} call(s) whose usage was measured`;
const claimOf = (value) => n === 0 ? (0, claim_js_1.unknown)(why) : (0, claim_js_1.known)(value, evidence);
const cacheRead = measured.reduce((s2, p) => s2 + valueOfMetrics(p).cacheReadTokens, 0);
const cacheWrite = measured.reduce((s2, p) => s2 + valueOfMetrics(p).cacheWriteTokens, 0);
const fresh = measured.reduce((s2, p) => s2 + valueOfMetrics(p).freshInputTokens, 0);
const requestTotal = cacheRead + cacheWrite + fresh;
const dollarsSpent = measured.reduce((s2, p) => s2 + ((0, claim_js_1.isKnown)(p.dollarsSpent) ? p.dollarsSpent.value : 0), 0);
const dollarsSaved = measured.reduce((s2, p) => s2 + ((0, claim_js_1.isKnown)(p.dollarsSavedVsNoCache) ? p.dollarsSavedVsNoCache.value : 0), 0);
return Object.freeze({
totalIterations: total,
applyMarkersIterations: apply,
noMarkersIterations: skip,
measuredIterations: n,
unmeasuredIterations: total - n,
cacheReadTokensTotal: claimOf(cacheRead),
cacheWriteTokensTotal: claimOf(cacheWrite),
freshInputTokensTotal: claimOf(fresh),
// A measured turn with zero prompt tokens is not a thing; guard it
// anyway rather than divide by zero and hand back NaN as a rate.
hitRate: n === 0
? (0, claim_js_1.unknown)(why)
: requestTotal === 0
? (0, claim_js_1.unknown)(`the ${n} measured call(s) reported 0 prompt tokens in total, so a hit rate has no denominator`)
: (0, claim_js_1.known)(cacheRead / requestTotal, evidence),
estimatedDollarsSpent: claimOf(dollarsSpent),
estimatedDollarsSavedVsNoCache: claimOf(dollarsSaved),
perIter: Object.freeze([...perIter]),
});
},
reset() {
perIter.length = 0;
lastDecision = undefined;
iterationCounter = 0;
},
};
return handle;
}
exports.cacheRecorder = cacheRecorder;
//# sourceMappingURL=cacheRecorder.js.map