agentfootprint/dist/esm/recorders/observability/ToolChoiceRecorder.d.ts

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

/**
 * toolChoiceRecorder — runtime tool-choice margins (RFC-002 tier 2,
 * blocks C4–C6).
 *
 * Per LLM call that OFFERED tools, this recorder captures the menu the
 * model saw (`stream.llm_start.tools`), what it actually invoked
 * (`stream.tool_start`), and the choice context — then, LAZILY on first
 * read, ranks the offered candidates against that context via
 * influence-core's `scoreMargin` (C4):
 *
 *   margin = score(best chosen) − score(best non-chosen)
 *
 * Small margin (`narrow`, < `marginThreshold`, default 0.05) = the
 * choice was a close call under the proxy. Top-scored candidate not
 * among the chosen (`proxyDisagreement`) is ALWAYS flagged — either a
 * proxy miss or a genuinely surprising model choice; both are exactly
 * what a debugger wants surfaced.
 *
 * ## The choice context (C4 — what is embedded, precisely)
 *
 * `buildChoiceContext` assembles the SAME two slots the model's
 * tool-selection reasoning ran on:
 *
 *   INCLUDED
 *   1. the user message of the current turn (`agent.turn_start.userPrompt`)
 *      — the task the model is choosing a tool FOR (first
 *      `maxSlotChars` chars: the head states the task);
 *   2. the latest assistant reasoning text — the most recent
 *      `stream.llm_end.content` of this turn, when present (last
 *      `maxSlotChars` chars: the tail is where "what next" lives).
 *      Iteration 1 has no assistant text; the slot is omitted.
 *
 *   EXCLUDED (deliberately)
 *   - the system prompt: constant across every call of the run — zero
 *     per-call discrimination, it only dilutes the embedding;
 *   - older history turns: recency dominates tool choice, and the full
 *     transcript grows the embedding cost linearly with run length;
 *   - raw tool results: the model reads them, but their distilled
 *     effect on the NEXT choice is the assistant's own reasoning text,
 *     which IS included; raw payloads skew the embedding toward data
 *     vocabulary (the honest-proxy discipline: mirror the
 *     decision-relevant text, not every visible byte);
 *   - tool schemas: those are the CANDIDATES being ranked, not context.
 *
 * Candidate text per offered tool = `confusabilityText` (tokenized name
 * + description) — the SAME construction the catalog lint (C1) embeds,
 * so build-time confusability and runtime margins measure one geometry.
 *
 * ## Laziness (C5)
 *
 * Event hooks only RECORD (string copies into a KeyedStore). The
 * embedder runs on the first `getCalls()` / `getFlagged()` /
 * `getSummary()` — embedding I/O NEVER rides the hot path, even when
 * the recorder is attached inline. Scores memoize per entry. Attach
 * with `{ delivery: 'deferred' }` (footprintjs RFC-001) to move the
 * bookkeeping off the hot path too — it is a normal CombinedRecorder.
 *
 * Pattern: CombinedRecorder (Convention 1 — single purpose: tool-choice
 *          margin evidence). Owns a `KeyedStore<ToolChoiceEntry>` keyed
 *          by the LLM call's `runtimeStageId`. Convention 4: resets on a
 *          new `runId` via `FlowRecorder.onRunStart`.
 * Role:    Tier-3 /observe recorder. Attach via `Agent.create(...)
 *          .watch(handle)` or `executor.attachCombinedRecorder`.
 *
 * Honest claim (RFC-002 §2): margins are embedding geometry between the
 * context and tool descriptions — a deterministic PROXY for the model's
 * selection function, never "the model chose because". Tier 3
 * (choice-entropy sampling) validates the proxy.
 */
import type { EmitEvent } from 'footprintjs';
import { type Embedder, type MarginResult } from '../../influence-core/index.js';
/** Minimal structural slice of footprintjs's run-boundary events — the
 *  `FlowRunEvent` (`onRunStart`/`onRunEnd`/`onRunFailed`). runId is all we
 *  read (Convention 4). */
interface RunBoundaryEvent {
    readonly traversalContext?: {
        readonly runId?: string;
    };
}
/** Minimal structural slice of footprintjs's `onResume` union — on
 *  `CombinedRecorder`, `onResume` is one of the SharedLifecycleOverlap
 *  methods (declared on both `ScopeRecorder` and `FlowRecorder` with
 *  DIFFERENT payloads), so its type is `ResumeEvent | FlowResumeEvent`, not
 *  a single `FlowRunEvent`-shaped type like the three siblings above. Only
 *  the control-flow `FlowResumeEvent` carries `traversalContext`; the
 *  scope-channel `ResumeEvent` (this recorder also receives it, via
 *  `onEmit`) has none — `traversalContext?.runId` is `undefined` there and
 *  the handler ignores it.
 *
 *  `hasInput` is listed here too, even though the handler never reads it.
 *  Both union members declare `hasInput: boolean`, and TypeScript's
 *  weak-type check rejects an assignment to an object type whose
 *  properties are ALL optional when the source has NO property name in
 *  common with it at all. A slice with only `traversalContext` shares zero
 *  property names with the scope-channel `ResumeEvent` (it has no
 *  `traversalContext` field), so TypeScript flagged the whole handle as not
 *  assignable to `CombinedRecorder` (the 7.3.0 regression: f0a87de added
 *  `onResume` typed with exactly that too-narrow slice). Declaring
 *  `hasInput` gives this slice at least one property name in common with
 *  BOTH union members, which is all the weak-type check requires — the
 *  field carries no behavior. */
interface ResumeBoundaryEvent {
    readonly hasInput?: boolean;
    readonly traversalContext?: {
        readonly runId?: string;
    };
}
/** One offered tool, as the model saw it on `llm_start`. */
export interface OfferedTool {
    readonly name: string;
    readonly description?: string;
}
/** Why an entry has no margin. */
export type ToolChoiceSkipReason = 
/** The model answered without invoking any tool — no choice to score. */
'nothing-chosen'
/** A chosen tool name was not in the offered catalog (wiring anomaly —
 *  surfaced, not silently massaged). */
 | 'chosen-not-offered';
/** One LLM call that offered tools — the recorder's public row shape. */
export interface ToolChoiceCall {
    /** runtimeStageId of the LLM-call stage (the KeyedStore key). */
    readonly runtimeStageId: string;
    readonly iteration: number;
    /** The catalog the model saw, in request order. */
    readonly offered: readonly OfferedTool[];
    /** Unique tool names actually invoked, in first-call order. */
    readonly chosen: readonly string[];
    /** Every invocation (parallel calls + repeat calls of one tool visible). */
    readonly toolCallIds: readonly string[];
    /** The choice context that was (or will be) embedded. */
    readonly contextText: string;
    /** Ranked scores + margin + flags. Undefined until scored, or when
     *  `skipped` says why it never will be. */
    readonly margin?: MarginResult;
    readonly skipped?: ToolChoiceSkipReason;
}
/** Run-summary counts (C6). */
export interface ToolChoiceSummary {
    /** LLM calls that offered tools (= recorded entries). */
    readonly llmCallsWithTools: number;
    /** Entries where the model invoked at least one tool. */
    readonly choices: number;
    /** Entries with a computed `margin`. */
    readonly scored: number;
    /** narrow OR proxy-disagreement. */
    readonly flagged: number;
    readonly narrow: number;
    readonly proxyDisagreement: number;
    /** Entries that will never score (`skipped` set). */
    readonly skipped: number;
}
export interface ToolChoiceRecorderOptions {
    /**
     * Injected embedder — runs ONLY on first read (lazy). Wrap in
     * `embeddingCache(...)` (agentfootprint/observe) so repeated tool
     * descriptions embed once across calls and runs.
     */
    readonly embedder: Embedder;
    /** Margins below this flag `narrow`. Default 0.05 (RFC-002 §4). */
    readonly marginThreshold?: number;
    /** Per-slot cap for the choice context. Default 2000 chars. */
    readonly maxSlotChars?: number;
    /** Recorder id (default 'tool-choice'). */
    readonly id?: string;
}
export interface ToolChoiceRecorderHandle {
    readonly id: string;
    /** All recorded LLM calls (scored on demand — first call runs the
     *  embedder; results memoize). Entries still open mid-run stay
     *  unscored until they close. */
    getCalls(): Promise<readonly ToolChoiceCall[]>;
    /** Calls whose choice was fragile: `narrow` margin OR
     *  `proxyDisagreement` (always flagged) — C6. */
    getFlagged(): Promise<readonly ToolChoiceCall[]>;
    /** Run-summary counts (C6). Scores pending entries first. */
    getSummary(): Promise<ToolChoiceSummary>;
    clear(): void;
    onEmit(event: EmitEvent): void;
    onRunStart(event: RunBoundaryEvent): void;
    onResume(event: ResumeBoundaryEvent): void;
    onRunEnd(event: RunBoundaryEvent): void;
    onRunFailed(event: RunBoundaryEvent): void;
}
/** C4: the precise choice-context construction (see module JSDoc for the
 *  include/exclude rationale). Exported so consumers can reproduce it. */
export declare function buildChoiceContext(args: {
    readonly userPrompt: string;
    readonly latestAssistantText?: string;
    readonly maxSlotChars?: number;
}): string;
/** Build the tool-choice margin recorder (C5). */
export declare function toolChoiceRecorder(options: ToolChoiceRecorderOptions): ToolChoiceRecorderHandle;
export {};