agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
133 lines • 6.58 kB
JavaScript
;
/**
* bisectCulprits — multi-culprit bisection over the ranked suspect set
* (RFC-003 Part B, block D9). The "git bisect" of the localizer.
*
* When single-suspect ablations don't flip the outcome — redundant causes
* (two facts that EACH justify the wrong answer), or interacting ones —
* the culprit is a SET. This harness finds a minimal culprit set by
* recursive halving over the ranked suspects (delta-debugging style,
* Zeller's ddmin specialized to two-way splits), then keeps searching the
* remainder for INDEPENDENT culprits until the remainder stops flipping.
*
* Probe semantics (the D9 discipline):
* - every probe = N seeded reruns of the consumer's `AblationRunner`
* with the probe's combined specs; "flipped" = MAJORITY of runs
* changed outcome; similarity mean ± spread is always reported —
* never single-run verdicts;
* - probe 0 is the BASELINE (no ablation): if it flips, the scenario
* itself is unstable and the result is honestly `'inconclusive'`;
* - probes are cached by spec-set, and budgeted (`maxProbes`) — running
* out of budget yields `'inconclusive'`, never a partial claim
* dressed up as a finding.
*
* §B2: the returned `verdict`/`culprits` are CAUSAL claims — they rest
* exclusively on counterfactual reruns. The input ranking only chooses
* the SEARCH ORDER (better ranking = fewer probes), it never decides the
* outcome.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.bisectCulprits = void 0;
const ablation_js_1 = require("./ablation.js");
const types_js_1 = require("./types.js");
const localize_js_1 = require("./localize.js");
// ─── The harness ─────────────────────────────────────────────────────
class ProbeBudgetExceeded extends Error {
constructor() {
super('probe budget exceeded');
}
}
/**
* Find minimal culprit set(s) by seeded counterfactual bisection. See
* module docs for semantics and the §B2 claim tier.
*/
async function bisectCulprits(options) {
const candidates = options.suspects.filter((suspect) => suspect.ablation !== undefined && suspect.ablation.kind !== 'arg');
const maxProbes = options.maxProbes ?? types_js_1.CONTEXT_BISECT_DEFAULTS.maxProbes;
const maxCulprits = options.maxCulprits ?? types_js_1.CONTEXT_BISECT_DEFAULTS.maxCulprits;
const config = { rerun: options.rerun, embedder: options.embedder };
const probes = [];
const cache = new Map();
let runsUsed = 0;
const keyOf = (set) => set
.map((suspect) => (0, localize_js_1.suspectLabel)(suspect))
.sort()
.join('|');
async function probe(set) {
const key = keyOf(set);
const cached = cache.get(key);
if (cached !== undefined)
return cached;
if (probes.length >= maxProbes)
throw new ProbeBudgetExceeded();
const stats = await (0, ablation_js_1.runAblationProbe)(config, set.flatMap((suspect) => (suspect.ablation !== undefined ? [suspect.ablation] : [])));
runsUsed += stats.samples;
const flipped = (0, ablation_js_1.probeFlipped)(stats);
probes.push({ ablated: set.map((suspect) => (0, localize_js_1.suspectLabel)(suspect)), stats, flipped });
cache.set(key, flipped);
return flipped;
}
/**
* Minimal subset of `candidates` that — together with `context` — flips
* the outcome. Precondition: probe(candidates ∪ context) flipped.
* Two-way ddmin: try each half; on interference (neither half alone
* suffices) minimize each half with the other as context.
*/
async function minimize(set, context) {
if (set.length <= 1)
return [...set];
const mid = Math.ceil(set.length / 2);
const top = set.slice(0, mid); // ranked order: the likelier half first
const rest = set.slice(mid);
if (await probe([...top, ...context]))
return minimize(top, context);
if (await probe([...rest, ...context]))
return minimize(rest, context);
// Interference: parts of BOTH halves are needed jointly.
const fromTop = await minimize(top, [...rest, ...context]);
const fromRest = await minimize(rest, [...fromTop, ...context]);
return [...fromTop, ...fromRest];
}
try {
// Baseline: an unstable scenario invalidates everything downstream.
// ZERO-TOLERANCE (review Finding 1): a single un-ablated flip marks the
// scenario unstable — the majority-rule probeFlipped() gate would let a
// 1-in-3-flaky scenario through to a 'confirmed' CAUSAL verdict, which
// violates the §B2 honest-claims discipline. Same gate localize.ts uses.
{
const baselineStats = await (0, ablation_js_1.runAblationProbe)(config, []);
runsUsed += baselineStats.samples;
const unstable = baselineStats.flips > 0;
probes.push({ ablated: [], stats: baselineStats, flipped: unstable });
cache.set(keyOf([]), (0, ablation_js_1.probeFlipped)(baselineStats));
if (unstable) {
return { verdict: 'inconclusive', culprits: [], probes, runsUsed };
}
}
// Reproduction gate: the full ranked set must flip at all.
if (candidates.length === 0 || !(await probe(candidates))) {
return { verdict: 'not-reproducible', culprits: [], probes, runsUsed };
}
// Find minimal sets; then keep searching the remainder for
// INDEPENDENT culprits until it stops flipping.
const culprits = [];
let remaining = candidates;
for (let round = 0; round < maxCulprits; round++) {
const found = await minimize(remaining, []);
culprits.push(found);
const foundKeys = new Set(found.map((suspect) => (0, localize_js_1.suspectLabel)(suspect)));
remaining = remaining.filter((suspect) => !foundKeys.has((0, localize_js_1.suspectLabel)(suspect)));
if (remaining.length === 0 || !(await probe(remaining)))
break;
}
return { verdict: 'confirmed', culprits, probes, runsUsed };
}
catch (error) {
if (error instanceof ProbeBudgetExceeded) {
return { verdict: 'inconclusive', culprits: [], probes, runsUsed };
}
throw error;
}
}
exports.bisectCulprits = bisectCulprits;
//# sourceMappingURL=bisect.js.map