@sema-agent/core/dist/bench/metrics.d.ts

Stateless, task-oriented AI agent core

/**
 * design/89 §2 — VALUE-JUDGMENT METRICS (the cost-per-truly-correct measurement tool).
 *
 * Pure, deterministic measurement logic for the value-judgment benchmark (design/89, milestone-gap
 * ❓-未判项: "supervisor / multi-agent vs solo 到底值不值?"). The service three-arm harness feeds REAL
 * runs in; this module computes truly-correct / three-class outcomes / cost four-components / Pareto.
 *
 * IRON LAWS (design/89 §2, blood-bought red lines — DO NOT relax):
 *   1. The numerator counts ONLY objective oracles (exec-gate exit-code, build, property-harness, and
 *      actual-delivery). LLM-judge is NEVER counted into truly-correct (it is a runtime GATE, not a
 *      measurement; §2.2). It burns cost (C1/C2) but never the numerator.
 *   2. NEVER synthesize a single scalar. Cost is FOUR separate components (§2.3); C1 (token µUSD) and
 *      C2 (human seconds) are apples + oranges (§2.1) and are NEVER added. The verdict is a 3-D Pareto
 *      frontier over (C1, C2.sec, P). A µUSD/tc ratio may be reported ONLY as a per-axis diagnostic.
 *   3. THREE outcome classes (§2.2.1): DELIVERED-CORRECT / DELIVERED-WRONG / CORRECTLY-WITHHELD. A
 *      correctly-withheld task (human/gate stopped a would-be DELIVERED-WRONG) is the supervisor's
 *      avoided-loss value — it is NOT a failure and NOT in the numerator.
 *   4. Deterministic + testable: this file has zero model/network dependency. Synthetic-run fixtures
 *      validate the measurement logic without a real model (see metrics.test.ts).
 *
 * This is a SHADOW-LAB experiment tool (not the core engine). Run via `npx tsx`.
 */
/**
 * The orchestration arms.
 *   - "solo"   — no supervision profile (the cost floor / quality-risk baseline).
 *   - "sup"    — SUP-per-step (S1a): the scripted human gate fires UNCONDITIONALLY at every plan_review +
 *                risky tool ask (C2 paid every time). S1a proved this is `dominated` on coding tasks (same P
 *                as solo, but pays C2 for nothing the model couldn't get right alone).
 *   - "sup-vf" — SUP verifier-first (S1b): the SAME supervision profile, but the scripted human gate is only
 *                ESCALATED (C2 paid) when ① a deterministic verifier (the mechanical hidden oracle / RepairOracle
 *                / shellGate hazard list) is RED or ② the action hits the danger list (destructive-fs / secret /
 *                pipe-to-shell). Otherwise the gate AUTO-ALLOWS with C2=0 (steals reflection/actor-critic + CRITIC:
 *                an external deterministic verifier is the critic; the human is the escalation tier only). Hypothesis:
 *                on coding tasks where the verifier goes green, SUP-vf collapses C2→~0 and stops being dominated by
 *                solo, WHILE keeping the trap-safety CORRECTLY-WITHHELD (the danger list still escalates `rm -rf data`).
 */
export type Arm = "solo" | "sup" | "sup-vf";
/**
 * The OBJECTIVE oracle verdicts for one run (design/89 §2.2). These are the four AND criteria.
 * Every field is produced by an external, deterministic oracle run AFTER the agent finishes — never
 * by an LLM-judge. Each is computed by the service harness (it owns exec-gate / build / git) and
 * recorded verbatim; this module only ANDs them.
 */
export interface OracleVerdicts {
    /** §2.2 criterion ① — hidden objective test suite all green: `runExecGate` exit==0 over a NON-EMPTY
     *  suite. Encode "empty suite / null exit" as `false` (exec-gate never passes those). */
    hiddenTestsGreen: boolean;
    /** §2.2 criterion ② — build/compile passed (APK assembleOfflineDebug / tsc / gradle build). Guards
     *  the "tests green but compile broke" false-positive. */
    buildPassed: boolean;
    /** §2.2 criterion ③ — mechanical invariants un-violated (property-harness). `true` when no violation
     *  OR property-harness is not applicable to this task (the AND must not penalize an N/A task). The
     *  harness sets this to `propertyHarnessApplicable ? noViolations : true`. */
    invariantsOk: boolean;
    /** §2.2 criterion ④ — ACTUALLY DELIVERED: a git commit exists AND the merge does not roll back.
     *  "ran but didn't deliver" (run 019ec5d1 触顶蒸发: 104 lines all lost) MUST count 0, else solo is
     *  systematically over-estimated. */
    delivered: boolean;
}
/**
 * Whether the human/gate WITHHELD this run's delivery (design/89 §2.2.1). When `true`, the agent
 * (or a human at an approval gate) correctly DID NOT deliver — the §2.2.1 supervisor-fairness fix:
 * a withheld run that WOULD have been DELIVERED-WRONG is avoided-loss value, not a failure, and is
 * single-listed (not in the numerator, not a failure). Only the SUP/TEAM arms can withhold; for SOLO
 * this is always `false` (no human/gate).
 */
export interface DeliveryDecision {
    /** The human/gate stopped/aborted/refused-to-deliver this run. */
    withheld: boolean;
}
/**
 * design/95 §3.2 — the lifecycle status of a run, so INFRA noise never poisons the mode signal
 * (design/89 §3.3 Beatsep red line). The Beatsep task#2b 9% pass-rate was infra
 * death (OOM / K8S passthrough / nested-root), NOT a mode signal — feeding such runs into the
 * numerator turns them into spurious DELIVERED-WRONG / zero datapoints and reproduces the very noise
 * the design claims to have isolated. `buildReport` SCORES only `scored`; `infra-failed` / `excluded`
 * runs are dropped from every cell/Pareto computation and surfaced as an exclusion ledger.
 *
 *   - "scored"        — a clean run whose oracle verdict is a real mode signal. The ONLY status scored.
 *   - "infra-failed"  — OOM / sandbox / passthrough / nested-root death; the agent never got a fair
 *                       attempt. Excluded; counted in the exclusion ledger with reason "infra-failed".
 *   - "excluded"      — manually quarantined (e.g. corpusHash drift, approver-policy change mid-batch,
 *                       a known-bad fixture). Excluded; counted with reason "excluded".
 *
 * ABSENT ⇒ treated as "scored" (back-compat: older RunRecord arrays predate this field).
 */
export type RunStatus = "scored" | "infra-failed" | "excluded";
/** A single run fed in by the service three-arm harness. */
export interface RunRecord {
    arm: Arm;
    /** Task id (for paired-seed grouping / cell folding). */
    taskId: string;
    /** Repeat/seed index within the cell (design/89 §3.4 N-repeat). */
    seed?: number | string;
    /**
     * design/95 §3.2 — lifecycle status. Only "scored" runs reach the numerator / cost /
     * Pareto. Absent ⇒ "scored" (back-compat). See {@link RunStatus}.
     */
    runStatus?: RunStatus;
    /**
     * design/95 §2 (B1 / Simpson's-paradox guard) — the heterogeneity coordinates a run belongs to. The
     * suite is heterogeneous (different archetypes / value-dimensions / difficulties); folding everything
     * into an arm mean can flip a per-cell conclusion (Simpson's paradox), making J1-J5 / JV1-JV4
     * UN-evaluable. `buildReport` groups by `(suiteVersion, taskId, archetype, valueDimension, arm)` and
     * keeps the seed-paired rows; the suite-level rollup is a WEIGHTED summary of scored cells only and is
     * NOT a decision surface. All fields optional for back-compat (a run with none falls into a single
     * "ungrouped" cell, preserving the old single-arm-mean behavior for legacy fixtures).
     */
    suiteVersion?: string;
    archetype?: string;
    valueDimension?: string;
    /** The four objective oracle verdicts (§2.2). */
    oracle: OracleVerdicts;
    /** Whether delivery was correctly withheld (§2.2.1). */
    delivery: DeliveryDecision;
    /**
     * The engine-produced `TaskResult.stats` (or the relevant subset). C1 is derived from
     * `stats.costBreakdown` (design/80 D-E-core); C2 from `stats.humanReview` (design/91, 1.110.0 ship).
     * Shape mirrors @sema-agent/core `TaskResult["stats"]` — see {@link CoreStatsSubset}.
     */
    stats: CoreStatsSubset;
    /** Wall-clock seconds for the whole run (task start→done). The C3 source. */
    wallTimeSec?: number;
    /** Infra cost in µUSD (E2B vCPU-hr / egress / k8s) — owned & filled by the SERVICE (core never has
     *  this). The C4 source. Absent for solo-local / when the service did not measure it. */
    infraMicroUsd?: number;
    /**
     * §2.4#2 intercept-value counterfactual: for each human intervention on a SUP run, the harness
     * re-runs the same brief + fixed seed with that intervention DISABLED and records whether the final
     * truly-correct flipped. Absent for SOLO (no intervention) and when the counterfactual was not run.
     *
     * CRITICAL (design/89 §2.2.1 line 96): for a WITHHELD run this is REQUIRED to decide whether the
     * withhold was CORRECT. A withhold is avoided-loss ONLY if disabling the intervention would have
     * produced a DELIVERED-WRONG (`withoutInterventionTrulyCorrect===false`). When this is absent on a
     * withheld run, the withhold is UNVERIFIED — it is NOT credited as avoided-loss.
     */
    counterfactual?: InterceptCounterfactual;
    /**
     * R2 (MINOR) — the HONESTY provenance of this run's judgment, stamped at the RECORD level (not only in the
     * report/stamp layer). A SUP/TEAM run's value rides on a SCRIPTED approver (NOT a real human review), so it is
     * an INITIAL judgment, never a firm verdict (design/95 §6.1.bis / §6.1). Carrying it on the record means a
     * record-level filter / re-aggregation / persisted run cannot silently lose the "scripted-human, not firm"
     * semantics (before this, filtering RunRecords dropped the label that lived only on `BatchStamp`). SOLO has no
     * human/approver ⇒ undefined (a pure objective-oracle run — no scripted-human judgment to caveat).
     */
    judgeMode?: "scripted-human-initjudge";
}
/**
 * The subset of @sema-agent/core `TaskResult["stats"]` this tool reads. Kept structurally
 * compatible (all optional, same field names/shapes) so a real `TaskResult.stats` is assignable here
 * with no adaptation — verified against src/core/types.ts (1.110.0).
 */
export interface CoreStatsSubset {
    costMicroUsd?: number;
    /** design/80 D-E-core finance taxonomy. C1 is the SUM of its LLM-derived lines. */
    costBreakdown?: {
        llmRootMicroUsd: number;
        nestedSubagentMicroUsd: number;
        memoryConsolidationMicroUsd: number;
        compactionMicroUsd: number;
        sideCallMicroUsd?: number;
    };
    nested?: {
        costMicroUsd?: number;
    };
    /** design/91 human-review burden (the C2 source). Absent ⇒ no approval gate was hit. */
    humanReview?: {
        count: number;
        totalWaitMs: number;
        gates: Array<{
            kind: string;
            waitMs: number;
            decision?: string;
        }>;
    };
}
/**
 * §2.2 — `truly-correct(run)` = AND of the four objective criteria (any false ⇒ 0). The numerator of
 * P. By construction this takes ONLY {@link OracleVerdicts} (objective oracles); there is no parameter
 * by which an LLM-judge could enter — that is the type-level enforcement of iron law #1.
 */
export declare function trulyCorrect(o: OracleVerdicts): boolean;
export type Outcome = "DELIVERED-CORRECT" | "DELIVERED-WRONG" | "CORRECTLY-WITHHELD" | "INCORRECTLY-WITHHELD" | "UNVERIFIED-WITHHELD";
/**
 * §2.2.1 — classify a run into one of the outcome classes. The withheld bucket is SPLIT by the §2.1
 * intercept-value counterfactual (design/89 §2.2.1 line 96 — a withhold is only avoided-loss if it
 * stopped a would-be DELIVERED-WRONG):
 *   - DELIVERED-CORRECT: not withheld AND truly-correct → the numerator.
 *   - DELIVERED-WRONG: not withheld AND not truly-correct → the true negative (solo hard-committing a
 *     buggy artifact, OR 触顶蒸发 with delivered=false). §2.2 criterion ④ folds "ran-but-didn't-
 *     deliver" into DELIVERED-WRONG, not into a withheld bucket — withheld requires an EXPLICIT
 *     human/gate decision, vaporized-on-overflow is a failure.
 *   - CORRECTLY-WITHHELD: withheld AND the counterfactual proves the task WOULD have been DELIVERED-
 *     WRONG (`withoutInterventionTrulyCorrect===false`). The supervisor's avoided-loss value — NOT in
 *     the numerator, NOT a failure.
 *   - INCORRECTLY-WITHHELD: withheld BUT the counterfactual shows the task would have been DELIVERED-
 *     CORRECT (`withoutInterventionTrulyCorrect===true`). A value-DESTROYING over-cautious intervention
 *     (a false-positive withhold). This is a COST/mistake, NEVER laundered into avoided-loss (the
 *     focus-item-② failure mode the review caught).
 *   - UNVERIFIED-WITHHELD: withheld but NO counterfactual was recorded → correctness unknown. Excluded
 *     from the avoided-loss credit (NOT defaulted to CORRECTLY-WITHHELD) and flagged (design/89 §2.2.1
 *     line 96: withheld MUST be counterfactual-verified).
 */
export declare function classifyOutcome(run: {
    oracle: OracleVerdicts;
    delivery: DeliveryDecision;
    counterfactual?: InterceptCounterfactual;
}): Outcome;
/**
 * §2.3 — the four cost components for one run. EACH single-listed; C1 and C2 are different units
 * (µUSD vs seconds) and are NEVER added (that is the §2.1 apples+oranges trap). There is deliberately
 * NO field that combines them.
 */
export interface CostComponents {
    /** C1 — model (LLM-token) cost in µUSD. Derived from `stats.costBreakdown` (design/80 D-E-core):
     *  the sum of the LLM-derived lines. */
    c1ModelMicroUsd: number;
    /** C2 — human-review burden: (count, seconds). From `stats.humanReview` (design/91). Seconds, not
     *  µUSD — never folded into C1. */
    c2HumanCount: number;
    c2HumanSec: number;
    /** C3 — wall-clock seconds (the speed axis; fan-out's true value場之一). */
    c3WallSec: number;
    /** C4 — infra µUSD (E2B vCPU-hr / egress / k8s), service-owned. 0 when not measured. */
    c4InfraMicroUsd: number;
}
/**
 * §2.3 — derive C1 from `stats.costBreakdown` (design/80 D-E-core). C1 is the LLM-token-derived total:
 * the sum of the four breakdown lines. Falls back to `costMicroUsd` (+ nested) when `costBreakdown`
 * is absent (older runs), then 0. This is the ONLY composition done — and it is WITHIN C1 (all µUSD,
 * same unit); it never reaches across to C2.
 */
export declare function deriveC1(stats: CoreStatsSubset): number;
/** §2.4 — derive C2 (count, seconds) from `stats.humanReview` (design/91). Absent ⇒ no gate ⇒ (0,0). */
export declare function deriveC2(stats: CoreStatsSubset): {
    count: number;
    sec: number;
};
/** §2.3 — assemble all four cost components for a run. They stay separate by construction. */
export declare function costComponents(run: RunRecord): CostComponents;
/**
 * §2.4#2 — the result of the intercept-value counterfactual for ONE human intervention: re-run the
 * same brief + fixed seed with that intervention DISABLED and observe how the final OUTCOME CLASS
 * changed. The intervention has intercept value when the class IMPROVED — including the avoided-loss
 * case (CORRECTLY-WITHHELD vs would-be DELIVERED-WRONG), which truly-correct alone CANNOT see
 * (a withheld run's `withInterventionTrulyCorrect` is trivially false — nothing was delivered).
 * This is what gives J3's "human-review is quality not only cost" a measurement.
 */
export interface InterceptCounterfactual {
    /** truly-correct WITH the intervention (the actual SUP run). For a WITHHELD run this is trivially
     *  false (nothing delivered) — do NOT infer intercept value from it; use the class comparison. */
    withInterventionTrulyCorrect: boolean;
    /** truly-correct WITHOUT it (the re-run with the intervention disabled). For a withhold this is the
     *  load-bearing field: `false` ⇒ the withhold prevented a DELIVERED-WRONG (avoided-loss). */
    withoutInterventionTrulyCorrect: boolean;
    /** Optional explicit override. When set, {@link interceptHadValue} uses it verbatim. Prefer leaving
     *  it unset so the predicate derives value from the outcome classes (which handles withholds). */
    changedOutcome?: boolean;
}
/**
 * §2.4#2 — did a human intervention have intercept VALUE? Outcome-CLASS-aware (NOT truly-correct-binary
 * -aware), so it correctly credits the supervisor's #1 value — an avoided-loss withhold — which a
 * truly-correct XOR cannot see (design/89 §2.2.1: a withheld run's truly-correct is always false).
 *
 * Value exists when disabling the intervention would have WORSENED the outcome class:
 *   - DELIVER-correcting intervention: WITH delivers correct, WITHOUT delivers wrong
 *     (`withInterventionTrulyCorrect===true && withoutInterventionTrulyCorrect===false`).
 *   - AVOIDED-LOSS withhold: the run was correctly withheld (CORRECTLY-WITHHELD) and WITHOUT the
 *     intervention it would have been DELIVERED-WRONG (`withoutInterventionTrulyCorrect===false`).
 *
 * An INCORRECTLY-WITHHELD intervention (would-be correct, killed by an over-cautious human) has
 * NEGATIVE value and returns `false` here — it is never counted as intercept value.
 *
 * Pass the run's outcome class so the predicate knows whether this was a withhold; the explicit
 * `changedOutcome` override still wins when the harness sets it.
 */
export declare function interceptHadValue(cf: InterceptCounterfactual, outcome: Outcome): boolean;
/**
 * §2.4 — the intercept-value seam the service three-arm harness implements: given a SUP run, clone the
 * brief + fix the seed, disable ONE human intervention, re-run, and return its truly-correct. This is
 * a SEAM (not implemented here — it needs the real Runner/model). The metrics tool consumes its result
 * via `RunRecord.counterfactual`; this type pins the contract so service implements it correctly.
 */
export type InterceptValueCounterfactualSeam = (args: {
    /** The original SUP run whose intervention is being ablated. */
    run: RunRecord;
    /** Which intervention to disable (index into `stats.humanReview.gates`). */
    interventionIndex: number;
}) => Promise<InterceptCounterfactual>;
/**
 * §2.3 output-form — one arm's per-cell vector: `(C1, C2.sec, C3, C4, P=truly-correct rate, intercept
 * value, CORRECTLY-WITHHELD count)` + the raw outcome tallies. Reported WITH n (and a CI when n large
 * enough). NO single µUSD/tc headline (iron law #2).
 */
export interface ArmCell {
    arm: Arm;
    n: number;
    /** Outcome tallies. The withheld bucket is SPLIT (§2.2.1 line 96): only counterfactual-verified
     *  avoided-loss is `correctlyWithheld`; an over-cautious would-be-correct kill is
     *  `incorrectlyWithheld` (a cost); a withhold with no counterfactual is `unverifiedWithheld`. */
    deliveredCorrect: number;
    deliveredWrong: number;
    correctlyWithheld: number;
    incorrectlyWithheld: number;
    unverifiedWithheld: number;
    /**
     * P = truly-correct RATE. design/89 §2.2.1: CORRECTLY-WITHHELD is neither numerator nor failure, so
     * it is EXCLUDED from the denominator (it is not an attempt that produced a delivered artifact). P =
     * DELIVERED-CORRECT / (DELIVERED-CORRECT + DELIVERED-WRONG). When all runs were withheld the rate is
     * undefined (no delivery attempt to score).
     */
    trulyCorrectRate: number | undefined;
    /** Avoided-loss rate = CORRECTLY-WITHHELD / n. The supervisor's §2.2.1 value, single-listed. */
    correctlyWithheldRate: number;
    /**
     * design/95 M10 — withhold treated as a binary detector of would-be-wrong delivery
     * (positive = withheld; ground truth = would-be-wrong, established by the §2.4 counterfactual). Reported
     * as precision/recall — NOT just the raw correctly-withheld count — so an arm cannot look good by
     * withholding indiscriminately (high count, low precision) or by rarely withholding (high precision,
     * low recall). Both undefined when their denominator is 0.
     *   withholdPrecision = correctlyWithheld / (correctlyWithheld + incorrectlyWithheld)
     *     — of the VERIFIED withhold decisions, the fraction that avoided a real wrong delivery. UNVERIFIED
     *       withholds are EXCLUDED (no counterfactual ⇒ TP/FP unknown; the honest, anti-over-claim choice).
     *   withholdRecall = correctlyWithheld / (correctlyWithheld + deliveredWrong)
     *     — of the wrong-delivery opportunities, the fraction the withhold caught (deliveredWrong = the FN:
     *       a wrong artifact was delivered that a withhold would have avoided).
     */
    withholdPrecision: number | undefined;
    withholdRecall: number | undefined;
    /** Mean C1 (µUSD), C2 (count, sec), C3 (sec), C4 (µUSD) across the cell. Each single-listed. */
    c1ModelMicroUsdMean: number;
    c2HumanCountMean: number;
    c2HumanSecMean: number;
    c3WallSecMean: number;
    c4InfraMicroUsdMean: number;
    /** §2.4 intercept value: how many interventions changed the outcome / how many were measured. */
    interceptValueChanged: number;
    interceptValueMeasured: number;
    /**
     * Per-axis DIAGNOSTIC only (iron law #2): mean-C1 µUSD per DELIVERED-CORRECT. Undefined when 0
     * delivered-correct. NEVER combines C2 — it is C1-only, labeled a diagnostic, and never the verdict.
     */
    c1PerTrulyCorrectDiag: number | undefined;
}
/** Fold a cell's runs (all same arm) into the per-arm vector. */
export declare function foldCell(runs: RunRecord[]): ArmCell;
/**
 * §2.1 — the three Pareto axes for one arm. Lower is better for cost (C1, C2.sec); higher is better
 * for correctness (P). NOTE: an arm with undefined P (all-withheld) cannot be placed on the frontier
 * — it is excluded from domination judgment and flagged.
 */
export interface ParetoPoint {
    arm: Arm;
    c1ModelMicroUsd: number;
    c2HumanSec: number;
    trulyCorrectRate: number;
}
/**
 * §2.1 — does point `a` Pareto-DOMINATE point `b`? `a` dominates `b` iff `a` is no worse on ALL three
 * axes (≤ C1, ≤ C2.sec, ≥ P) AND strictly better on at least one. Domination means b is provably
 * inferior. NOT dominated ⇒ both are on the frontier (a real trade-off — exactly the §2.1 outcome we
 * refuse to collapse into one scalar).
 */
export declare function dominates(a: ParetoPoint, b: ParetoPoint): boolean;
export interface ParetoResult {
    /** The input points that could be placed (P defined). */
    points: ParetoPoint[];
    /** Arms on the Pareto frontier (NOT dominated by any other) — the real trade-offs. */
    frontier: Arm[];
    /** Arms dominated by ≥1 other arm, each with WHO dominates it. A dominated arm is provably inferior
     *  (the only kind of verdict §2.1 permits — no scalar ranking of frontier members). */
    dominated: Array<{
        arm: Arm;
        dominatedBy: Arm[];
    }>;
    /**
     * Arms excluded from the frontier because P was undefined (all-withheld) — flagged, not silently
     * dropped, and CARRYING their avoided-loss so a consumer that reads `frontier` cannot miss the
     * withheld arm's value (design/89 §2.2.1: an all-withheld supervisor must not vanish, leaving a
     * wrong-shipping solo as the sole "winner").
     */
    excludedUndefinedP: Array<{
        arm: Arm;
        correctlyWithheldRate: number;
        correctlyWithheld: number;
        deliveredWrong: number;
    }>;
    /**
     * HARD COUNTERWEIGHT (design/89 §2.1/§2.2.1): set when the frontier is NOT a clean winner declaration
     * — an arm with avoided-loss (correctlyWithheld>0) was excluded for undefined P WHILE a frontier arm
     * ships real negatives (deliveredWrong>0). A downstream consumer keying on `frontier` MUST read this
     * before concluding "X won": the excluded arm avoided loss the frontier arm did not.
     */
    frontierNotAWinnerDeclaration?: {
        reason: string;
        excludedAvoidedLossArms: Arm[];
        frontierArmsShippingWrong: Arm[];
    };
}
/**
 * §2.1 — compute the 3-D Pareto frontier over the per-arm cells. Each arm contributes ONE point
 * `(C1 mean, C2.sec mean, P)`. Arms with undefined P (all-withheld) are flagged and excluded from
 * domination. The output is the frontier + dominated-by map — NEVER a single ranked scalar.
 */
export declare function paretoFrontier(cells: ArmCell[]): ParetoResult;
/**
 * §2.1 — the risk-transfer disclosure: solo having no C2 does NOT mean solo is cheaper — it transfers
 * quality risk downstream (single-agent 3-runs-all-failed 0-commit = the cost of no human safety net;
 * METR -19% / DORA more-code≠more-delivered / 31% PR zero-review). The report MUST emit this verbatim
 * whenever a solo arm is present so a low solo C2 is never read as "solo cheaper". Returns the text +
 * the supporting numbers from this run so it is not boilerplate but grounded.
 */
export declare function riskTransferDisclosure(cells: ArmCell[]): {
    applies: boolean;
    text: string;
    evidence: {
        soloC2Sec: number;
        soloDeliveredWrong: number;
        supCorrectlyWithheld: number;
    };
};
/**
 * a paired-binary comparison of two arms on the SAME tasks/seeds. The two arms' truly-
 * correct flags are paired row-by-row (paired-seed, design/89 §3.4). For binary paired data the
 * RIGHT tests are McNemar (discordant pairs) and a paired-bootstrap difference interval — NOT two
 * independent proportions with "CI non-overlap" (that ignores the pairing and is under-powered).
 */
export interface PairedBinaryComparison {
    /** Arm A (e.g. "sup") vs arm B (e.g. "solo"). pDiff = P(A) − P(B). */
    armA: Arm;
    armB: Arm;
    /** Number of PAIRED rows actually compared (only rows where both arms produced a delivery attempt). */
    nPairs: number;
    /** Discordant pair counts: b = A-correct & B-wrong; c = A-wrong & B-correct (McNemar's b,c). */
    bAonly: number;
    cBonly: number;
    /** Point estimate of the paired proportion difference P(A) − P(B). */
    pDiff: number;
    /** McNemar exact-ish two-sided p-value over discordant pairs (binomial, continuity-corrected χ² when
     *  b+c large). */
    mcnemarP: number;
    /** Bootstrap percentile 95% CI for the paired difference (lo, hi) — a percentile CI on the seeded
     *  paired-bootstrap distribution, NOT Newcombe's analytic interval. Crosses 0 ⇒ not significant. */
    ci95: [number, number];
    /** TRUE iff the difference is statistically significant at α=0.05 (CI excludes 0 AND McNemar p<0.05). */
    significant: boolean;
    /**
     * pre-registered Minimum Detectable Effect at the observed nPairs (the difference this
     * comparison COULD have detected at 80% power). When |pDiff| is below this AND not significant, the
     * verdict is "not powered" — NOT "no difference". This is the field that stops "N≥15 is an assertion".
     */
    mdeAt80Power: number;
    /** "powered" iff nPairs ≥ the n needed to detect the pre-registered MDE; else "not-powered". */
    power: "powered" | "not-powered";
    /** Human-readable verdict that NEVER over-claims a null result from an under-powered comparison. */
    verdict: "A-better" | "B-better" | "no-detectable-difference" | "not-powered";
}
/**
 * compare two arms' delivered truly-correct as PAIRED binary. Pairs runs by the COMPOSITE
 * `(taskId, seed)` key (a bare `seed` collides across tasks — the same repeat index
 * recurs per task — so cross-task input would overwrite pairs and contaminate the McNemar sample);
 * only rows where BOTH arms delivered (not withheld, both scored) form a pair — a
 * withheld run has no delivered binary to pair (it is scored in the withhold/avoided-loss axis, not
 * here). `mde` is the pre-registered minimum detectable effect (default 0.20 absolute = a 20pp swing).
 */
export declare function pairedBinaryCompare(scoredRuns: RunRecord[], armA: Arm, armB: Arm, opts?: {
    mde?: number;
    bootstrapSeed?: number;
    bootstrapIters?: number;
}): PairedBinaryComparison;
/**
 * the pre-registered C2 exchange rate(s): how many human-review SECONDS we are willing to
 * pay to buy one unit of supervisor value. Without these, "higher P / more withholds ⇒ worth it" is
 * unfalsifiable (the "helpful but too expensive" counter-thesis cannot be observed). Declared BEFORE
 * the run (design/95 §9), not fit after.
 */
export interface C2Thresholds {
    /** Max human-review seconds we will pay to avoid ONE DELIVERED-WRONG (per CORRECTLY-WITHHELD). */
    secPerAvoidedWrong: number;
    /** Max human-review seconds we will pay to save ONE repeated decision (V1). */
    secPerSavedDecision: number;
}
export interface ParetoVerdict {
    arm: Arm;
    /** Non-dominated on (C1, C2.sec, P) — necessary condition to be considered at all. */
    nonDominated: boolean;
    /** The pre-registered exchange-rate check: does the arm's extra C2 buy enough avoided-loss / saved
     *  decisions to clear the declared threshold? Undefined when the arm has no extra C2 over baseline. */
    clearsC2Threshold?: boolean;
    /**
     * the label. An arm may ONLY be "worth-it" when it is non-dominated AND clears the C2
     * threshold. A non-dominated but threshold-failing arm is "quality-tradeoff" (higher quality, but the
     * cost is not bought back) — NEVER "worth-it". A dominated arm is "dominated".
     */
    label: "worth-it" | "quality-tradeoff" | "dominated";
}
/**
 * classify each arm against the SUP/TEAM-vs-baseline value question with a pre-registered
 * exchange rate. `baselineArm` is the cost floor to compare extra C2 against (default "solo").
 * `avoidedWrong` / `savedDecisions` per arm come from the cells / campaign report.
 *
 * 🔴 ADVERSARIAL-REVIEW DISCIPLINE (B1 partial false-close, design/95 §2.3): `cells` MUST be a
 * HOMOGENEOUS cross-arm set — feed `report.comparisons[*].arms` (one `(suiteVersion,taskId,archetype,
 * valueDimension)` comparison at a time), NEVER `report.cells` (the suite-wide ARM MEAN). The arm mean
 * folds heterogeneous tasks → the value LABEL itself flips (Simpson): a cell that is "quality-tradeoff"
 * (extra human seconds NOT bought back) gets averaged with a "worth-it" cell into a single misleading
 * "worth-it", hiding the "helpful but too expensive" anti-thesis. The frontier folds too (§11 / B1).
 * Regression: metrics.test §13b proves per-cell {worth-it, quality-tradeoff} ≠ arm-mean {worth-it}.
 */
export declare function paretoValueVerdict(cells: ArmCell[], thresholds: C2Thresholds, perArmValue: Partial<Record<Arm, {
    avoidedWrong: number;
    savedDecisions: number;
}>>, baselineArm?: Arm): {
    pareto: ParetoResult;
    verdicts: ParetoVerdict[];
};
/**
 * design/95 §7.1 — V1 (免重复劳动): campaign-level human-decision delta. The ONLY authoritative V1
 * mechanism (design/95 reconcile, MAJOR#2): cross-task reuse of a decided strategy. Semantics:
 *   - `soloHumanDecisions` is the V1 BASELINE — the operator's actual up-front decision count when
 *     running solo across the campaign (NOT 0, NOT synthetic — measured from real operator
 *     prep across runs). `0` is only valid when the campaign genuinely needed no human decision.
 *   - `repeatedDecisionsSaved = soloHumanDecisions − supHumanDecisions` (may be negative = SUP cost
 *     MORE human decisions; reported honestly, not floored).
 */
export interface CampaignV1Saved {
    decisionKind: string;
    /** V1 baseline: operator's measured up-front decisions when running SOLO. Provenance MUST
     *  be recorded in the stamp (real prep, not 0/synthetic). */
    soloHumanDecisions: number;
    /** SUP arm's actual human-review decision count for this decision kind. */
    supHumanDecisions: number;
    /** = soloHumanDecisions − supHumanDecisions. Positive ⇒ SUP saved repeated labour (V1 evidence). */
    repeatedDecisionsSaved: number;
    /** Whether the baseline is a real measurement vs absent (honesty gate). */
    baselineProvenance: "measured-operator-baseline" | "absent-not-claimable";
}
/**
 * design/95 §7.2 — V2 (蓝图清晰度) THREE-arm ablation. All three are
 * paired on the SAME seeds (paired===true ⇒ the three pX arrays/values are over the same n seeds).
 * INVARIANT: `n` is the paired-seed count common to all three sub-arms; the three deltas are computed
 * over those n pairs.
 *   - pSoloRaw            — only the original brief (no blueprint baseline).
 *   - pSoloSupGenerated   — the SUP-arm-GENERATED blueprint (attributable to supervisor; gen cost charged).
 *   - pSoloExperimenterIdeal — the experimenter "ideal" blueprint. this is a CONSTRAINED upper
 *     bound on "better-prompt help", NOT "supervisor's mechanistic ceiling", UNLESS the ideal blueprint
 *     was generated under the {@link IdealBlueprintConstraints} (run-time, visible-brief-only, no oracle
 *     access, same info budget, leak-reviewed). `idealConstraintsSatisfied` records which it is.
 */
export interface BlueprintAblationTriple {
    subgoalId: string;
    paired: boolean;
    /** Paired-seed count common to all three sub-arms (array/Δ invariant base). */
    n: number;
    pSoloRaw: number;
    pSoloSupGenerated: number;
    pSoloExperimenterIdeal: number;
    /** = pSoloSupGenerated − pSoloRaw (supervisor blueprint's real net value). */
    deltaSupVsRaw: number;
    /** = pSoloExperimenterIdeal − pSoloRaw. LABELLED per `idealConstraintsSatisfied`. */
    deltaIdealVsRaw: number;
    /** when false, deltaIdealVsRaw is "manual-prompt upper bound", NOT "supervisor potential". */
    idealConstraintsSatisfied: boolean;
    /** blueprint generation cost charged to the sup-generated arm (budget-match red line). */
    blueprintGenC1MicroUsd: number;
    blueprintGenC2Count: number;
    blueprintGenC2Sec: number;
}
/** the constraints under which an "ideal" blueprint may be read as a supervisor-mechanism
 *  upper bound rather than a generic "better prompt helps" result. Recorded per ablation. */
export interface IdealBlueprintConstraints {
    generatedBeforeRun: boolean;
    fromVisibleBriefOnly: boolean;
    noOracleTestAccess: boolean;
    sameInfoBudget: boolean;
    solutionLeakReviewed: boolean;
}
/** True iff ALL ideal-blueprint constraints hold. */
export declare function idealConstraintsSatisfied(c: IdealBlueprintConstraints): boolean;
/**
 * design/95 §7.3 — V3 (无关性隔离) contamination probe (canary, mechanical, zero LLM-judge).
 * the isolated-vs-shared P delta CONFOUNDS execution order / context size / worker count / budget; the
 * PRIMARY signal must be per-subgoal oracle failure + explicit wrong-use of a sibling artifact (the
 * canary leak), with the P delta kept as a DIAGNOSTIC only.
 * INVARIANT: `isolatedSubgoalP.length === sharedSubgoalP.length === factorCount`.
 */
export interface IsolationContamination {
    factorCount: number;
    /** Per-factor P under isolation. length === factorCount. */
    isolatedSubgoalP: number[];
    /** Per-factor P under shared context. length === factorCount. */
    sharedSubgoalP: number[];
    /** PRIMARY mechanical signal: cross-factor canary leak rate (explicit token bleed). */
    canaryLeakRate: number;
    /** PRIMARY mechanical signal: count of factors that failed their oracle AND demonstrably
     *  used a sibling factor's artifact (the controlled-ablation harm signal). */
    wrongSiblingArtifactUses: number;
    /** DIAGNOSTIC ONLY (confound): mean(isolated P) − mean(shared P). Not the primary signal. */
    contaminationRateDiag: number;
    /** Extra C1 the isolation cost (multi-worker / repeated context load). */
    isolationOverheadC1: number;
}
/**
 * design/95 §7 — the CAMPAIGN-level wrapper (V1 is campaign-scoped, design/95 reconcile MAJOR#2). Built
 * ON TOP of per-task {@link ValueJudgmentReport}s; does NOT re-derive their numerator/Pareto red lines.
 * V2/V3 attach optionally per the §7.2/§7.3 structures. All §7 fields are populated by the harness
 * (the producer), not synthesized in `buildReport` — `implementedAxes` declares which were filled.
 */
export interface CampaignReport {
    perTask: ValueJudgmentReport[];
    campaignV1?: CampaignV1Saved;
    blueprintAblation?: BlueprintAblationTriple;
    contaminationProbe?: IsolationContamination;
    implementedAxes: ImplementedAxes;
}
/**
 * design/95 §2 — one heterogeneity CELL: a `(suiteVersion, taskId, archetype, valueDimension,
 * arm)` group with its folded {@link ArmCell}. The decision surface is the per-cell vector — NOT the
 * arm mean (the Simpson's-paradox guard). `groupKey` is the stable join key.
 */
export interface GroupedCell {
    groupKey: string;
    suiteVersion?: string;
    taskId: string;
    archetype?: string;
    valueDimension?: string;
    cell: ArmCell;
}
/**
 * 🔴 ADVERSARIAL-REVIEW FIX (B1 partial false-close): the cross-arm decision surface. A {@link
 * GroupedCell} keys on `arm`, so each is SINGLE-ARM — you cannot run a cross-arm Pareto / value verdict
 * (J2/J3/B3) from one. The prior `buildReport` exposed only `report.pareto`, computed on the suite-wide
 * `cells` ARM MEAN — the very Simpson's-prone surface B1 demotes to "NOT a decision surface". A consumer
 * reading the obvious `report.pareto` got the folded-across-heterogeneity frontier = exactly what B1
 * claims to prevent. This is the CORRECT decision surface: one entry per `(suiteVersion, taskId,
 * archetype, valueDimension)` (NO arm), carrying all arms' cells + their per-cell Pareto. J2/J3/B3 MUST
 * read these, never `report.pareto`.
 */
export interface ComparisonCell {
    comparisonKey: string;
    suiteVersion?: string;
    taskId: string;
    archetype?: string;
    valueDimension?: string;
    /** All arms' folded cells WITHIN this homogeneous comparison (same task/archetype/dimension). */
    arms: ArmCell[];
    /** The per-comparison Pareto over `arms` — the decision surface, NOT the suite arm mean. */
    pareto: ParetoResult;
}
/**
 * council Q1 — capability metadata: which of the four §7/Pareto value axes are actually
 * IMPLEMENTED in this report. Consumers must read this before claiming a V1/V2/V3 result; an unset axis
 * is NOT a null result, it is "not measured here". (The T1 structures above are types + pure helpers;
 * the report-level V1/V2/V3 fields are populated by the harness, not synthesized in `buildReport`.)
 */
export interface ImplementedAxes {
    pareto: boolean;
    v1: boolean;
    v2: boolean;
    v3: boolean;
}
export interface ValueJudgmentReport {
    /** per-arm cells grouped by full heterogeneity key (incl. arm). Single-arm; use for cell
     *  inspection / `pairedBinaryCompare` inputs. NOT directly a cross-arm verdict surface (each is one arm). */
    groupedCells: GroupedCell[];
    /**
     * 🔴 ADVERSARIAL-REVIEW FIX (B1): the CROSS-ARM decision surface — one per `(suiteVersion, taskId,
     * archetype, valueDimension)` (NO arm), each carrying all arms + a per-comparison Pareto over a
     * HOMOGENEOUS set of runs. J2/J3/B3 (`paretoValueVerdict`) MUST read `comparisons[*].pareto` /
     * `comparisons[*].arms`, NEVER the suite-wide `pareto`/`cells` (which fold heterogeneous tasks → Simpson).
     */
    comparisons: ComparisonCell[];
    /**
     * suite-level rollup: per-arm cells folded over ALL scored runs. A WEIGHTED summary for
     * dashboards ONLY — NOT a decision surface (folding heterogeneous tasks can flip a verdict, Simpson).
     * Consumers MUST decide on `comparisons`; this is convenience aggregation.
     */
    cells: ArmCell[];
    /**
     * 🔴 DASHBOARD ONLY — the Pareto over the suite-wide ARM MEAN (`cells`). This folds heterogeneous
     * tasks into one frontier (Simpson's paradox) and is NOT a verdict surface. The verdict surface is
     * `comparisons[*].pareto`. Kept only for a single-number dashboard glance; flagged so a consumer
     * cannot mistake it for the decision (the B1 false-close: prior code wired the verdict to this).
     */
    pareto: ParetoResult;
    riskTransfer: ReturnType<typeof riskTransferDisclosure>;
    /** design/89 §3.4: cells with n<8 are directional-only, not verdict-grade. Flagged, not dropped. */
    directionalOnly: Arm[];
    /**
     * the exclusion ledger: runs dropped from scoring (infra-failed / excluded) with reasons.
     * A regression-grade invariant: these never reach `cells` / `pareto` (no infra noise in the verdict).
     */
    excluded: Array<{
        taskId: string;
        arm: Arm;
        seed?: number | string;
        reason: RunStatus;
    }>;
    /** which value axes this report actually measured. */
    implementedAxes: ImplementedAxes;
}
/**
 * Top-level: SCORE only `scored` runs (B4 — drop infra-failed/excluded with reasons),
 * group by `(suiteVersion, taskId, archetype, valueDimension, arm)` (B1 — the per-cell decision surface,
 * Simpson's-paradox guard), fold each group, AND provide a suite-level per-arm rollup for dashboards
 * (explicitly NOT a decision surface). Deterministic; no model.
 *
 * `opts.implementedAxes` lets the harness declare which §7 axes it populated; default
 * = only Pareto (the V1/V2/V3 structures are T1 net-new, not synthesized here).
 */
export declare function buildReport(runs: RunRecord[], opts?: {
    implementedAxes?: Partial<ImplementedAxes>;
}): ValueJudgmentReport;