@sema-agent/core/dist/bench/metrics.js

Stateless, task-oriented AI agent core

export function trulyCorrect(o) {
    return o.hiddenTestsGreen && o.buildPassed && o.invariantsOk && o.delivered;
}
export function classifyOutcome(run) {
    if (run.delivery.withheld) {
        const cf = run.counterfactual;
        if (!cf)
            return "UNVERIFIED-WITHHELD";
        return cf.withoutInterventionTrulyCorrect ? "INCORRECTLY-WITHHELD" : "CORRECTLY-WITHHELD";
    }
    return trulyCorrect(run.oracle) ? "DELIVERED-CORRECT" : "DELIVERED-WRONG";
}
export function deriveC1(stats) {
    const b = stats.costBreakdown;
    if (b) {
        return (b.llmRootMicroUsd +
            b.nestedSubagentMicroUsd +
            b.memoryConsolidationMicroUsd +
            b.compactionMicroUsd);
    }
    const root = stats.costMicroUsd ?? 0;
    const nested = stats.nested?.costMicroUsd ?? 0;
    return root + nested;
}
export function deriveC2(stats) {
    const h = stats.humanReview;
    if (!h)
        return { count: 0, sec: 0 };
    return { count: h.count, sec: h.totalWaitMs / 1000 };
}
export function costComponents(run) {
    const c2 = deriveC2(run.stats);
    return {
        c1ModelMicroUsd: deriveC1(run.stats),
        c2HumanCount: c2.count,
        c2HumanSec: c2.sec,
        c3WallSec: run.wallTimeSec ?? 0,
        c4InfraMicroUsd: run.infraMicroUsd ?? 0,
    };
}
export function interceptHadValue(cf, outcome) {
    if (typeof cf.changedOutcome === "boolean")
        return cf.changedOutcome;
    if (outcome === "CORRECTLY-WITHHELD") {
        return cf.withoutInterventionTrulyCorrect === false;
    }
    if (outcome === "INCORRECTLY-WITHHELD" || outcome === "UNVERIFIED-WITHHELD") {
        return false;
    }
    return cf.withInterventionTrulyCorrect === true && cf.withoutInterventionTrulyCorrect === false;
}
function mean(xs) {
    return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0;
}
export function foldCell(runs) {
    if (runs.length === 0)
        throw new Error("foldCell: empty runs");
    const arm = runs[0].arm;
    for (const r of runs)
        if (r.arm !== arm)
            throw new Error(`foldCell: mixed arms (${arm} vs ${r.arm})`);
    let deliveredCorrect = 0;
    let deliveredWrong = 0;
    let correctlyWithheld = 0;
    let incorrectlyWithheld = 0;
    let unverifiedWithheld = 0;
    let interceptValueChanged = 0;
    let interceptValueMeasured = 0;
    const c1s = [];
    const c2counts = [];
    const c2secs = [];
    const c3s = [];
    const c4s = [];
    for (const r of runs) {
        const oc = classifyOutcome(r);
        if (oc === "DELIVERED-CORRECT")
            deliveredCorrect++;
        else if (oc === "DELIVERED-WRONG")
            deliveredWrong++;
        else if (oc === "CORRECTLY-WITHHELD")
            correctlyWithheld++;
        else if (oc === "INCORRECTLY-WITHHELD")
            incorrectlyWithheld++;
        else
            unverifiedWithheld++;
        const c = costComponents(r);
        c1s.push(c.c1ModelMicroUsd);
        c2counts.push(c.c2HumanCount);
        c2secs.push(c.c2HumanSec);
        c3s.push(c.c3WallSec);
        c4s.push(c.c4InfraMicroUsd);
        if (r.counterfactual) {
            interceptValueMeasured++;
            if (interceptHadValue(r.counterfactual, oc))
                interceptValueChanged++;
        }
    }
    const delivered = deliveredCorrect + deliveredWrong;
    const trulyCorrectRate = delivered > 0 ? deliveredCorrect / delivered : undefined;
    const c1Mean = mean(c1s);
    const verifiedWithholds = correctlyWithheld + incorrectlyWithheld;
    const wrongDeliveryOpportunities = correctlyWithheld + deliveredWrong;
    const withholdPrecision = verifiedWithholds > 0 ? correctlyWithheld / verifiedWithholds : undefined;
    const withholdRecall = wrongDeliveryOpportunities > 0 ? correctlyWithheld / wrongDeliveryOpportunities : undefined;
    return {
        arm,
        n: runs.length,
        deliveredCorrect,
        deliveredWrong,
        correctlyWithheld,
        incorrectlyWithheld,
        unverifiedWithheld,
        trulyCorrectRate,
        correctlyWithheldRate: correctlyWithheld / runs.length,
        withholdPrecision,
        withholdRecall,
        c1ModelMicroUsdMean: c1Mean,
        c2HumanCountMean: mean(c2counts),
        c2HumanSecMean: mean(c2secs),
        c3WallSecMean: mean(c3s),
        c4InfraMicroUsdMean: mean(c4s),
        interceptValueChanged,
        interceptValueMeasured,
        c1PerTrulyCorrectDiag: deliveredCorrect > 0 ? (c1Mean * runs.length) / deliveredCorrect : undefined,
    };
}
export function dominates(a, b) {
    const noWorse = a.c1ModelMicroUsd <= b.c1ModelMicroUsd &&
        a.c2HumanSec <= b.c2HumanSec &&
        a.trulyCorrectRate >= b.trulyCorrectRate;
    if (!noWorse)
        return false;
    const strictlyBetter = a.c1ModelMicroUsd < b.c1ModelMicroUsd ||
        a.c2HumanSec < b.c2HumanSec ||
        a.trulyCorrectRate > b.trulyCorrectRate;
    return strictlyBetter;
}
export function paretoFrontier(cells) {
    const points = [];
    const excludedUndefinedP = [];
    for (const c of cells) {
        if (c.trulyCorrectRate === undefined) {
            excludedUndefinedP.push({
                arm: c.arm,
                correctlyWithheldRate: c.correctlyWithheldRate,
                correctlyWithheld: c.correctlyWithheld,
                deliveredWrong: c.deliveredWrong,
            });
            continue;
        }
        points.push({
            arm: c.arm,
            c1ModelMicroUsd: c.c1ModelMicroUsdMean,
            c2HumanSec: c.c2HumanSecMean,
            trulyCorrectRate: c.trulyCorrectRate,
        });
    }
    const frontier = [];
    const dominated = [];
    for (const p of points) {
        const by = points.filter((q) => q.arm !== p.arm && dominates(q, p)).map((q) => q.arm);
        if (by.length > 0)
            dominated.push({ arm: p.arm, dominatedBy: by });
        else
            frontier.push(p.arm);
    }
    const excludedAvoidedLossArms = excludedUndefinedP.filter((e) => e.correctlyWithheld > 0).map((e) => e.arm);
    const cellByArm = new Map(cells.map((c) => [c.arm, c]));
    const frontierArmsShippingWrong = frontier.filter((a) => (cellByArm.get(a)?.deliveredWrong ?? 0) > 0);
    const result = { points, frontier, dominated, excludedUndefinedP };
    if (excludedAvoidedLossArms.length > 0 && frontierArmsShippingWrong.length > 0) {
        result.frontierNotAWinnerDeclaration = {
            reason: "Frontier is NOT a winner declaration (design/89 §2.1/§2.2.1): an arm with counterfactual-" +
                "verified avoided-loss (all delivery withheld → P undefined → excluded from the frontier) " +
                "coexists with a frontier arm that ships DELIVERED-WRONG. The excluded arm avoided real " +
                "negatives the frontier arm did not — do NOT read the frontier as 'X won'.",
            excludedAvoidedLossArms,
            frontierArmsShippingWrong,
        };
    }
    return result;
}
export function riskTransferDisclosure(cells) {
    const solo = cells.find((c) => c.arm === "solo");
    const supArms = cells.filter((c) => c.arm === "sup" || c.arm === "sup-vf");
    const soloC2Sec = solo?.c2HumanSecMean ?? 0;
    const soloDeliveredWrong = solo?.deliveredWrong ?? 0;
    const supCorrectlyWithheld = supArms.reduce((a, c) => a + c.correctlyWithheld, 0);
    const text = "RISK-TRANSFER DISCLOSURE (design/89 §2.1): solo has no C2 (human-review seconds) but that does " +
        "NOT make solo cheaper — it transfers the quality risk DOWNSTREAM. A low/zero solo C2 must never " +
        "be read as 'solo cheaper': solo's DELIVERED-WRONG count is the unmitigated cost of having no " +
        "human safety net (single-agent experiment: 3 runs all failed, 0 commit; METR -19%; DORA more-" +
        "code ≠ more-delivered). The supervisor's C2 BUYS the CORRECTLY-WITHHELD avoided-loss; compare " +
        "C2 against avoided DELIVERED-WRONG, never against zero.";
    return {
        applies: !!solo,
        text,
        evidence: { soloC2Sec, soloDeliveredWrong, supCorrectlyWithheld },
    };
}
function mulberry32(seed) {
    let a = seed >>> 0;
    return function () {
        a |= 0;
        a = (a + 0x6d2b79f5) | 0;
        let t = Math.imul(a ^ (a >>> 15), 1 | a);
        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
    };
}
function chiSqTwoSidedP1df(chi) {
    const x = Math.sqrt(chi / 2);
    const t = 1 / (1 + 0.3275911 * x);
    const erf = 1 -
        (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) *
            t *
            Math.exp(-x * x);
    return Math.max(0, Math.min(1, 1 - erf));
}
function binomTwoSidedP(b, c) {
    const n = b + c;
    if (n === 0)
        return 1;
    const k = Math.min(b, c);
    let cum = 0;
    for (let i = 0; i <= k; i++)
        cum += binomPmfHalf(n, i);
    return Math.min(1, 2 * cum);
}
function binomPmfHalf(n, k) {
    let logC = 0;
    for (let i = 1; i <= k; i++)
        logC += Math.log(n - k + i) - Math.log(i);
    return Math.exp(logC + n * Math.log(0.5));
}
export function pairedBinaryCompare(scoredRuns, armA, armB, opts = {}) {
    const mde = opts.mde ?? 0.2;
    const aByKey = new Map();
    const bByKey = new Map();
    for (const r of scoredRuns) {
        if (r.delivery.withheld)
            continue;
        const key = r.taskId + "\0" + String(r.seed ?? "");
        if (r.arm === armA)
            aByKey.set(key, r);
        else if (r.arm === armB)
            bByKey.set(key, r);
    }
    const pairs = [];
    for (const [key, a] of aByKey) {
        const b = bByKey.get(key);
        if (!b)
            continue;
        pairs.push({ a: trulyCorrect(a.oracle), b: trulyCorrect(b.oracle) });
    }
    const nPairs = pairs.length;
    let bAonly = 0;
    let cBonly = 0;
    let aCorrect = 0;
    let bCorrect = 0;
    for (const p of pairs) {
        if (p.a)
            aCorrect++;
        if (p.b)
            bCorrect++;
        if (p.a && !p.b)
            bAonly++;
        if (!p.a && p.b)
            cBonly++;
    }
    const pDiff = nPairs > 0 ? (aCorrect - bCorrect) / nPairs : 0;
    let mcnemarP;
    const disc = bAonly + cBonly;
    if (disc === 0)
        mcnemarP = 1;
    else if (disc <= 25)
        mcnemarP = binomTwoSidedP(bAonly, cBonly);
    else {
        const chi = Math.pow(Math.abs(bAonly - cBonly) - 1, 2) / disc;
        mcnemarP = chiSqTwoSidedP1df(chi);
    }
    const rnd = mulberry32(opts.bootstrapSeed ?? 0xc0ffee);
    const iters = opts.bootstrapIters ?? 2000;
    const diffs = [];
    if (nPairs > 0) {
        for (let it = 0; it < iters; it++) {
            let ac = 0;
            let bc = 0;
            for (let i = 0; i < nPairs; i++) {
                const p = pairs[Math.floor(rnd() * nPairs)];
                if (p.a)
                    ac++;
                if (p.b)
                    bc++;
            }
            diffs.push((ac - bc) / nPairs);
        }
        diffs.sort((x, y) => x - y);
    }
    const ci95 = nPairs > 0
        ? [diffs[Math.floor(0.025 * diffs.length)], diffs[Math.min(diffs.length - 1, Math.floor(0.975 * diffs.length))]]
        : [0, 0];
    const ciExcludesZero = ci95[0] > 0 || ci95[1] < 0;
    const significant = ciExcludesZero && mcnemarP < 0.05;
    const zSum = 1.96 + 0.84;
    const mdeAt80Power = nPairs > 0 ? Math.min(1, (zSum * Math.sqrt(0.5 / nPairs))) : 1;
    const nNeeded = Math.ceil((zSum * zSum * 0.5) / (mde * mde));
    const power = nPairs >= nNeeded ? "powered" : "not-powered";
    let verdict;
    if (significant)
        verdict = pDiff > 0 ? "A-better" : "B-better";
    else if (power === "not-powered")
        verdict = "not-powered";
    else
        verdict = "no-detectable-difference";
    return { armA, armB, nPairs, bAonly, cBonly, pDiff, mcnemarP, ci95, significant, mdeAt80Power, power, verdict };
}
export function paretoValueVerdict(cells, thresholds, perArmValue, baselineArm = "solo") {
    const pareto = paretoFrontier(cells);
    const dominatedArms = new Set(pareto.dominated.map((d) => d.arm));
    const baseline = cells.find((c) => c.arm === baselineArm);
    const baselineC2 = baseline?.c2HumanSecMean ?? 0;
    const verdicts = cells.map((c) => {
        const nonDominated = !dominatedArms.has(c.arm);
        if (c.arm === baselineArm) {
            return { arm: c.arm, nonDominated, label: nonDominated ? "worth-it" : "dominated" };
        }
        const extraC2 = Math.max(0, c.c2HumanSecMean - baselineC2);
        const v = perArmValue[c.arm];
        let clearsC2Threshold;
        if (extraC2 > 0 && v) {
            const budget = v.avoidedWrong * thresholds.secPerAvoidedWrong + v.savedDecisions * thresholds.secPerSavedDecision;
            clearsC2Threshold = extraC2 <= budget;
        }
        else if (extraC2 === 0) {
            clearsC2Threshold = true;
        }
        else {
            clearsC2Threshold = false;
        }
        let label;
        if (!nonDominated)
            label = "dominated";
        else if (clearsC2Threshold)
            label = "worth-it";
        else
            label = "quality-tradeoff";
        return { arm: c.arm, nonDominated, clearsC2Threshold, label };
    });
    return { pareto, verdicts };
}
export function idealConstraintsSatisfied(c) {
    return (c.generatedBeforeRun &&
        c.fromVisibleBriefOnly &&
        c.noOracleTestAccess &&
        c.sameInfoBudget &&
        c.solutionLeakReviewed);
}
function groupKeyOf(r) {
    return [r.suiteVersion ?? "*", r.taskId, r.archetype ?? "*", r.valueDimension ?? "*", r.arm].join("|");
}
export function buildReport(runs, opts = {}) {
    const scored = [];
    const excluded = [];
    for (const r of runs) {
        const status = r.runStatus ?? "scored";
        if (status === "scored")
            scored.push(r);
        else
            excluded.push({ taskId: r.taskId, arm: r.arm, seed: r.seed, reason: status });
    }
    const byGroup = new Map();
    for (const r of scored) {
        const key = groupKeyOf(r);
        const list = byGroup.get(key) ?? [];
        list.push(r);
        byGroup.set(key, list);
    }
    const groupedCells = [...byGroup.entries()].map(([groupKey, rs]) => {
        const s = rs[0];
        return {
            groupKey,
            suiteVersion: s.suiteVersion,
            taskId: s.taskId,
            archetype: s.archetype,
            valueDimension: s.valueDimension,
            cell: foldCell(rs),
        };
    });
    const byComparison = new Map();
    for (const r of scored) {
        const key = [r.suiteVersion ?? "*", r.taskId, r.archetype ?? "*", r.valueDimension ?? "*"].join("|");
        const list = byComparison.get(key) ?? [];
        list.push(r);
        byComparison.set(key, list);
    }
    const comparisons = [...byComparison.entries()].map(([comparisonKey, rs]) => {
        const s = rs[0];
        const armBuckets = new Map();
        for (const r of rs) {
            const l = armBuckets.get(r.arm) ?? [];
            l.push(r);
            armBuckets.set(r.arm, l);
        }
        const arms = [...armBuckets.values()].map(foldCell);
        return {
            comparisonKey,
            suiteVersion: s.suiteVersion,
            taskId: s.taskId,
            archetype: s.archetype,
            valueDimension: s.valueDimension,
            arms,
            pareto: paretoFrontier(arms),
        };
    });
    const byArm = new Map();
    for (const r of scored) {
        const list = byArm.get(r.arm) ?? [];
        list.push(r);
        byArm.set(r.arm, list);
    }
    const cells = [...byArm.values()].map(foldCell);
    return {
        groupedCells,
        comparisons,
        cells,
        pareto: paretoFrontier(cells),
        riskTransfer: riskTransferDisclosure(cells),
        directionalOnly: cells.filter((c) => c.n < 8).map((c) => c.arm),
        excluded,
        implementedAxes: {
            pareto: opts.implementedAxes?.pareto ?? true,
            v1: opts.implementedAxes?.v1 ?? false,
            v2: opts.implementedAxes?.v2 ?? false,
            v3: opts.implementedAxes?.v3 ?? false,
        },
    };
}