@arizeai/phoenix-client/src/testing/reporter-format.ts

A client for the Phoenix API

import { formatAcceptanceResult } from "./acceptance";
import type { TestResult } from "./state";
import type {
  AcceptanceResult,
  Annotation,
  OptimizationDirection,
} from "./types";

/**
 * The serializable summary of one suite that the reporter renders. This is the
 * payload written to and read from the artifact files in `report-artifacts.ts`,
 * so it holds only plain data — no clients, tracers, or live state.
 */
export interface SuiteSummary {
  /** Suite name (also the dataset / experiment name in Phoenix). */
  name: string;
  /** True when the suite did not sync to Phoenix (dry run, disabled, or error). */
  trackingDisabled?: boolean;
  /** Human-readable reason tracking was disabled, when known. */
  trackingDisabledReason?: string;
  /** Setup failure that disabled tracking, reduced to its message for printing. */
  setupError?: { message: string };
  /** Number of best-effort uploads (runs + annotations) that failed. */
  uploadFailureCount?: number;
  /** Per-test outcomes shown in the summary. */
  results: TestResult[];
  /** Aggregate acceptance results shown in the summary. */
  acceptanceResults?: AcceptanceResult[];
  /** Phoenix UI links (dataset / experiment) printed at the end of the block. */
  links: Array<{ label: string; url: string }>;
}

/**
 * Options that control how the reporter renders. Resolved once per run from the
 * environment and the output stream (see {@link resolveRenderOptions}) and then
 * threaded through every formatting function so jest and vitest behave
 * identically without either reporter class needing options of its own.
 */
export interface RenderOptions {
  /** Show every test row plus the legacy per-test `output:` detail block. */
  verbose: boolean;
  /** Emit ANSI color escapes. */
  color: boolean;
  /** Max test rows shown per suite in compact mode (failures are never hidden). */
  maxRows: number;
  /** Terminal width budget used to size tables. */
  maxWidth: number;
}

// ---------------------------------------------------------------------------
// Option / environment resolution
// ---------------------------------------------------------------------------

function isTruthyFlag(value: string | undefined): boolean {
  const v = (value ?? "").toLowerCase();
  return v === "true" || v === "1" || v === "on" || v === "yes";
}

function isFalsyFlag(value: string | undefined): boolean {
  const v = (value ?? "").toLowerCase();
  return v === "false" || v === "0" || v === "off" || v === "no";
}

/**
 * Resolve {@link RenderOptions} from the environment and output stream.
 *
 * Verbosity: `PHOENIX_TEST_REPORTER=verbose` (or the `PHOENIX_TEST_VERBOSE=1`
 * alias) restores the full per-test dump; the default is the compact view.
 * `PHOENIX_TEST_REPORTER_MAX_ROWS` caps the per-suite rows (default 10).
 *
 * Color follows the common ecosystem rules: off when `NO_COLOR` is set, in CI,
 * on a non-TTY, or a "dumb" terminal; `PHOENIX_TEST_COLOR` / `FORCE_COLOR`
 * force it on or off.
 */
export function resolveRenderOptions(
  env: NodeJS.ProcessEnv = process.env,
  stream: { isTTY?: boolean; columns?: number } = process.stdout
): RenderOptions {
  const verbose =
    (env.PHOENIX_TEST_REPORTER ?? "").toLowerCase() === "verbose" ||
    isTruthyFlag(env.PHOENIX_TEST_VERBOSE);

  const parsedRows = Number.parseInt(
    env.PHOENIX_TEST_REPORTER_MAX_ROWS ?? "",
    10
  );
  const maxRows =
    Number.isFinite(parsedRows) && parsedRows > 0 ? parsedRows : 10;

  const color = resolveColor(env, stream);

  // When piped (no TTY width) assume a roomy-but-safe 100 columns so the
  // overview table doesn't over-truncate suite names in CI logs.
  const columns =
    typeof stream.columns === "number" && stream.columns > 0
      ? stream.columns
      : 100;
  const maxWidth = Math.min(columns, 120);

  return { verbose, color, maxRows, maxWidth };
}

function resolveColor(
  env: NodeJS.ProcessEnv,
  stream: { isTTY?: boolean }
): boolean {
  if (isTruthyFlag(env.PHOENIX_TEST_COLOR)) return true;
  if (isFalsyFlag(env.PHOENIX_TEST_COLOR)) return false;
  if (
    env.FORCE_COLOR != null &&
    env.FORCE_COLOR !== "" &&
    env.FORCE_COLOR !== "0"
  )
    return true;
  if (env.NO_COLOR != null && env.NO_COLOR !== "") return false;
  if (env.CI != null && env.CI !== "") return false;
  if (env.TERM === "dumb") return false;
  return stream.isTTY === true;
}

// ---------------------------------------------------------------------------
// Zero-dependency ASCII / ANSI toolkit
// ---------------------------------------------------------------------------

const ANSI = {
  green: "\x1b[32m",
  red: "\x1b[31m",
  yellow: "\x1b[33m",
  dim: "\x1b[2m",
  bold: "\x1b[1m",
  reset: "\x1b[0m",
} as const;

type AnsiColor = keyof Omit<typeof ANSI, "reset">;

/** Wrap a string in an ANSI color, or return it unchanged when color is off. */
function colorize(s: string, code: AnsiColor, o: RenderOptions): string {
  return o.color ? `${ANSI[code]}${s}${ANSI.reset}` : s;
}

// eslint-disable-next-line no-control-regex -- matching the ESC control byte is the point
const ANSI_PATTERN = /\x1b\[[0-9;]*m/g;

/** Visible length of a string, ignoring any ANSI escape codes. */
function visibleLen(s: string): string["length"] {
  return s.replace(ANSI_PATTERN, "").length;
}

/**
 * Truncate `s` to `max` visible characters, keeping the head and tail with an
 * ellipsis in the middle. Strings containing ANSI codes are returned unchanged
 * to avoid slicing through an escape sequence (colored cells are always short
 * enough to fit, so they never need truncating).
 */
function truncateMiddle(s: string, max: number): string {
  if (max <= 1 || ANSI_PATTERN.test(s)) return s;
  if (s.length <= max) return s;
  const head = Math.ceil((max - 1) / 2);
  const tail = Math.floor((max - 1) / 2);
  return `${s.slice(0, head)}…${tail > 0 ? s.slice(s.length - tail) : ""}`;
}

/** Left-pad-end a cell to `width`, measuring with {@link visibleLen}. */
function padCell(s: string, width: number): string {
  const pad = width - visibleLen(s);
  return pad > 0 ? s + " ".repeat(pad) : s;
}

interface TableSpec {
  /** Per-column max width caps; columns without a cap auto-size. */
  caps?: number[];
  /** A summary row rendered as the final line (e.g. an AGGREGATE row). */
  footer?: string[];
}

/**
 * Render an aligned ASCII table (no table dependency). Column widths auto-size
 * to content, clamped by `spec.caps`, and the first column is shrunk toward a
 * floor when the table would exceed `o.maxWidth`. Cells longer than their final
 * width are middle-truncated; widths are computed with {@link visibleLen} so
 * ANSI color never breaks alignment.
 */
function renderTable(
  headers: string[],
  rows: string[][],
  o: RenderOptions,
  spec: TableSpec = {}
): string[] {
  const colCount = headers.length;
  const gutter = "  ";
  const allRows = [headers, ...rows, ...(spec.footer ? [spec.footer] : [])];

  const widths = headers.map((_, i) => {
    const natural = Math.max(...allRows.map((r) => visibleLen(r[i] ?? "")));
    const cap = spec.caps?.[i];
    return cap ? Math.min(natural, cap) : natural;
  });

  const FIRST_COL_FLOOR = 16;
  const totalWidth = () =>
    widths.reduce((a, b) => a + b, 0) + gutter.length * (colCount - 1);
  while (totalWidth() > o.maxWidth && widths[0]! > FIRST_COL_FLOOR) {
    widths[0]!--;
  }

  const lastCol = colCount - 1;
  const formatRow = (cells: string[]) =>
    cells
      .map((cell, i) => {
        const text = truncateMiddle(cell ?? "", widths[i]!);
        // The last column is never padded — trailing spaces are wasted tokens.
        return i === lastCol ? text : padCell(text, widths[i]!);
      })
      .join(gutter)
      // Drop the dangling gutter when the final cell is empty (e.g. a clean
      // suite's blank Result column).
      .trimEnd();

  const out = [formatRow(headers), ...rows.map(formatRow)];
  if (spec.footer) out.push(formatRow(spec.footer));
  return out;
}

// ---------------------------------------------------------------------------
// Annotation aggregation
// ---------------------------------------------------------------------------

/** Structured aggregate of one annotation across a suite's results. */
interface AnnotationStat {
  name: string;
  kind: "number" | "boolean";
  /** Mean score for numeric annotations. */
  avg?: number;
  /** Count of `true` scores for boolean annotations. */
  trueCount?: number;
  /** Number of scored samples. */
  count: number;
}

/**
 * Aggregate every (non-`pass`) annotation across results, preserving the order
 * in which annotation names are first seen. Numeric and boolean annotations are
 * tracked separately; if a name appears as both, the first kind seen wins.
 */
function computeAnnotationStats(
  results: readonly TestResult[]
): AnnotationStat[] {
  const order: string[] = [];
  const numeric = new Map<string, number[]>();
  const boolean = new Map<string, { t: number; total: number }>();
  for (const result of results) {
    for (const ann of result.annotations) {
      if (ann.name === "pass") continue;
      if (typeof ann.score === "number" && Number.isFinite(ann.score)) {
        if (!numeric.has(ann.name) && !boolean.has(ann.name))
          order.push(ann.name);
        const arr = numeric.get(ann.name);
        if (arr) arr.push(ann.score);
        else if (!boolean.has(ann.name)) numeric.set(ann.name, [ann.score]);
      } else if (typeof ann.score === "boolean") {
        if (!numeric.has(ann.name) && !boolean.has(ann.name))
          order.push(ann.name);
        const cur = boolean.get(ann.name);
        if (cur) {
          cur.total++;
          if (ann.score) cur.t++;
        } else if (!numeric.has(ann.name)) {
          boolean.set(ann.name, { t: ann.score ? 1 : 0, total: 1 });
        }
      }
    }
  }
  return order.map((name) => {
    const nums = numeric.get(name);
    if (nums) {
      return {
        name,
        kind: "number",
        avg: nums.reduce((a, b) => a + b, 0) / nums.length,
        count: nums.length,
      };
    }
    const bools = boolean.get(name)!;
    return { name, kind: "boolean", trueCount: bools.t, count: bools.total };
  });
}

/** Format an annotation stat as the aggregate-line string used historically. */
function formatStat(stat: AnnotationStat): string {
  const samples = `${stat.count} sample${stat.count === 1 ? "" : "s"}`;
  return stat.kind === "number"
    ? `avg ${stat.avg!.toFixed(3)} (${samples})`
    : `${stat.trueCount}/${stat.count} true`;
}

/**
 * Aggregate annotations into the legacy `name -> summary` string map (kept for
 * the verbose view and any external callers).
 */
function aggregateAnnotations(
  results: readonly TestResult[]
): Record<string, string> {
  const out: Record<string, string> = {};
  for (const stat of computeAnnotationStats(results)) {
    out[stat.name] = formatStat(stat);
  }
  return out;
}

// ---------------------------------------------------------------------------
// Miss detection + row selection
// ---------------------------------------------------------------------------

interface AcceptanceBar {
  bar: number;
  direction: OptimizationDirection;
}

/**
 * Per-annotation score bar a single run must clear to avoid counting as a
 * "miss". Only `average` criteria contribute one: the aggregate `threshold`
 * reused as a per-run heuristic (the suite-level acceptance block still reports
 * the true aggregate verdict). `passRate` criteria decide passing with an
 * arbitrary `passFn` predicate — there is no static numeric bar to highlight
 * against — so their rows fall back to the default miss heuristic.
 */
function buildAcceptanceBars(suite: SuiteSummary): Map<string, AcceptanceBar> {
  const bars = new Map<string, AcceptanceBar>();
  for (const result of suite.acceptanceResults ?? []) {
    if (result.metric !== "average") continue;
    bars.set(result.annotationName, {
      bar: result.threshold,
      direction: result.direction ?? "maximize",
    });
  }
  return bars;
}

/**
 * Whether a passing test's evaluator scores fall short. When maximizing, a
 * boolean `false` or a numeric score below its bar is a miss; when minimizing,
 * a boolean `true` or a score above its bar is a miss. With no criterion for an
 * annotation, only a non-positive score counts (keeps zero-config suites quiet).
 */
function isMiss(result: TestResult, bars: Map<string, AcceptanceBar>): boolean {
  for (const ann of result.annotations) {
    if (ann.name === "pass") continue;
    const acceptanceBar = bars.get(ann.name);
    const minimizing = acceptanceBar?.direction === "minimize";
    if (typeof ann.score === "boolean") {
      if (minimizing ? ann.score : !ann.score) return true;
    } else if (typeof ann.score === "number" && Number.isFinite(ann.score)) {
      if (acceptanceBar === undefined) {
        if (ann.score <= 0) return true;
      } else if (
        minimizing
          ? ann.score > acceptanceBar.bar
          : ann.score < acceptanceBar.bar
      ) {
        return true;
      }
    }
  }
  return false;
}

/**
 * Annotation columns for a suite's table: the acceptance-gated metrics first
 * (the ones a user cares about), else the union of annotation names by
 * first-seen order. Capped at three to keep the table narrow.
 */
function selectAnnotationColumns(suite: SuiteSummary): string[] {
  const MAX_COLUMNS = 3;
  const gated = (suite.acceptanceResults ?? []).map((r) => r.annotationName);
  const ordered =
    gated.length > 0
      ? gated
      : computeAnnotationStats(suite.results).map((s) => s.name);
  return [...new Set(ordered)].slice(0, MAX_COLUMNS);
}

/** The aggregate cell for an annotation column. */
function aggregateCell(stats: AnnotationStat[], name: string): string {
  const stat = stats.find((s) => s.name === name);
  if (!stat) return "—";
  return stat.kind === "number"
    ? `avg ${stat.avg!.toFixed(2)}`
    : `${stat.trueCount}/${stat.count}`;
}

// ---------------------------------------------------------------------------
// Suite rendering
// ---------------------------------------------------------------------------

type SuiteStatus = "pass" | "miss" | "fail";

/** A suite's headline numbers, computed once and shared by every renderer. */
interface SuiteVitals {
  total: number;
  passed: number;
  failed: number;
  missCount: number;
  status: SuiteStatus;
  meanLatencyMs: number;
  /** Failing tests first, then below-bar misses, each worst-score first. */
  problems: TestResult[];
  acceptanceFailed: boolean;
}

function computeSuiteVitals(suite: SuiteSummary): SuiteVitals {
  const bars = buildAcceptanceBars(suite);
  const total = suite.results.length;
  const passed = suite.results.filter((r) => r.status === "passed").length;
  const failed = suite.results.filter((r) => r.status === "failed").length;
  const misses = suite.results
    .filter((r) => r.status === "passed" && isMiss(r, bars))
    .sort((a, b) => worstScore(a) - worstScore(b));
  const acceptanceFailed = (suite.acceptanceResults ?? []).some(
    (r) => !r.passed
  );
  const status: SuiteStatus =
    failed > 0 || acceptanceFailed
      ? "fail"
      : misses.length > 0
        ? "miss"
        : "pass";
  const meanLatencyMs =
    total > 0 ? suite.results.reduce((a, r) => a + r.durationMs, 0) / total : 0;
  return {
    total,
    passed,
    failed,
    missCount: misses.length,
    status,
    meanLatencyMs,
    problems: [
      ...suite.results.filter((r) => r.status === "failed"),
      ...misses,
    ],
    acceptanceFailed,
  };
}

/** The worst (lowest) evaluator score on a row, for ordering most-broken first. */
function worstScore(result: TestResult): number {
  let worst = Number.POSITIVE_INFINITY;
  for (const ann of result.annotations) {
    if (ann.name === "pass") continue;
    worst = Math.min(worst, annotationScoreValue(ann));
  }
  return worst;
}

/** ANSI color matching a status (green pass / yellow miss / red fail). */
function statusColor(status: SuiteStatus): AnsiColor {
  return status === "fail" ? "red" : status === "miss" ? "yellow" : "green";
}

/** `2/2 passed · 1 failed · 3 misses`, dropping any zero clause. */
function countsLabel(v: SuiteVitals): string {
  const parts = [`${v.passed}/${v.total} passed`];
  if (v.failed > 0) parts.push(`${v.failed} failed`);
  if (v.missCount > 0)
    parts.push(`${v.missCount} miss${v.missCount === 1 ? "" : "es"}`);
  return parts.join(" · ");
}

/** Setup / upload problems worth surfacing regardless of test status. */
function warningLines(suite: SuiteSummary, o: RenderOptions): string[] {
  const out: string[] = [];
  if (suite.setupError?.message) {
    out.push(
      `  ${colorize("setup error:", "red", o)} ${suite.setupError.message}`
    );
  }
  const n = suite.uploadFailureCount ?? 0;
  if (n > 0) {
    out.push(
      `  ${colorize("warning:", "yellow", o)} ${n} upload${n === 1 ? "" : "s"} failed (auth or network?)`
    );
  }
  return out;
}

/**
 * Render a single suite. A clean suite collapses to one line; a suite with
 * failures or misses expands into a per-row diagnosis (scores, rationale,
 * output, and the Phoenix ids needed to pull the trace). Pass a verbose
 * {@link RenderOptions} to restore the full per-test dump.
 */
export function formatSuiteSummary(
  suite: SuiteSummary,
  o: RenderOptions = resolveRenderOptions()
): string {
  return o.verbose
    ? formatVerboseSuite(suite)
    : formatSuiteDetail(suite, computeSuiteVitals(suite), o);
}

/** Verbatim acceptance-criteria block (kept stable for downstream parsers). */
function acceptanceLines(suite: SuiteSummary): string[] {
  if (!suite.acceptanceResults || suite.acceptanceResults.length === 0)
    return [];
  return [
    "  Acceptance Criteria:",
    ...suite.acceptanceResults.map((r) => `    ${formatAcceptanceResult(r)}`),
  ];
}

function linkLines(suite: SuiteSummary): string[] {
  return suite.links.map((link) => `  ${link.label}: ${link.url}`);
}

/** Dim one-line roll-up of every metric average plus mean latency. */
function vitalsInline(
  suite: SuiteSummary,
  v: SuiteVitals,
  o: RenderOptions
): string {
  const parts = computeAnnotationStats(suite.results).map((s) =>
    s.kind === "number"
      ? `${s.name} ${s.avg!.toFixed(2)}`
      : `${s.name} ${s.trueCount}/${s.count}`
  );
  parts.push(`avg ${formatDuration(v.meanLatencyMs)}`);
  return colorize(parts.join("   "), "dim", o);
}

function formatSuiteDetail(
  suite: SuiteSummary,
  v: SuiteVitals,
  o: RenderOptions
): string {
  const title = `${colorize(suite.name, statusColor(v.status), o)}  ${colorize(
    countsLabel(v),
    "dim",
    o
  )}`;
  const warnings = warningLines(suite, o);

  // Clean suite with nothing to warn about: one line is the whole story.
  if (v.status === "pass" && warnings.length === 0) {
    return `${title}   ${vitalsInline(suite, v, o)}`;
  }

  const lines = [title, `  ${vitalsInline(suite, v, o)}`, ...warnings];
  // Never hide a hard failure; cap the number of below-bar misses shown.
  const failures = v.problems.filter((r) => r.status === "failed");
  const misses = v.problems.filter((r) => r.status !== "failed");
  const shownMisses = misses.slice(0, Math.max(o.maxRows - failures.length, 0));
  for (const r of [...failures, ...shownMisses]) {
    lines.push(...problemEntry(r, o));
  }
  const hidden = misses.length - shownMisses.length;
  if (hidden > 0) {
    lines.push(
      colorize(`  … ${hidden} more miss${hidden === 1 ? "" : "es"}`, "dim", o)
    );
  }
  for (const a of suite.acceptanceResults ?? []) {
    if (!a.passed) {
      lines.push(
        `  ${colorize("✗ acceptance", "red", o)} ${formatAcceptanceResult(
          a
        ).replace(/^FAIL /, "")}`
      );
    }
  }
  for (const link of suite.links) {
    lines.push(`  ${link.label}: ${link.url}`);
  }
  return lines.join("\n");
}

/**
 * One failing / missing row: a weighted title with its scores, then the dim
 * detail an agent needs to fix it — rationale, model output, and trace ids.
 */
function problemEntry(result: TestResult, o: RenderOptions): string[] {
  const mark = colorize("✗", result.status === "failed" ? "red" : "yellow", o);
  const name = truncateEnd(humanizeLabel(result.testName), 72);
  const dry = result.dryRun ? colorize(" (dry run)", "dim", o) : "";
  const lines = [`  ${mark} ${colorize(name, "bold", o)}${dry}`];
  const indent = "      ";

  const err = compactError(result.error);
  if (err) lines.push(`${indent}${colorize(err, "red", o)}`);

  // The sub-perfect evaluators that dragged the row down, worst score first —
  // a clean `1.0` metric isn't what broke it, so it stays out of the way.
  const rationales = [...result.annotations]
    .filter((a) => a.name !== "pass" && annotationScoreValue(a) < 1)
    .sort((a, b) => annotationScoreValue(a) - annotationScoreValue(b))
    .slice(0, 3);
  for (const ann of rationales) {
    const reason = ann.explanation ?? ann.label;
    const tail = reason
      ? ` ${colorize("·", "dim", o)} ${truncateSummary(reason, 160)}`
      : "";
    lines.push(
      `${indent}${colorize(ann.name, "dim", o)} ${formatScore(ann)}${tail}`
    );
  }

  const output = summarizeValue(result.output, 160);
  if (output !== null) {
    lines.push(`${indent}${colorize("output", "dim", o)} ${output}`);
  }

  const ids = formatResultIds(result);
  if (ids) lines.push(`${indent}${colorize(ids, "dim", o)}`);

  return lines;
}

/** The legacy verbose view: every test as a delimited block including output. */
function formatVerboseSuite(suite: SuiteSummary): string {
  const lines: string[] = [];
  lines.push("");
  lines.push(suite.name);
  if (suite.trackingDisabled) {
    lines.push(`  (tracking disabled — ${friendlyTrackingReason(suite)})`);
  }
  const total = suite.results.length;
  const passed = suite.results.filter((r) => r.status === "passed").length;
  const failed = suite.results.filter((r) => r.status === "failed").length;
  lines.push(
    `  ${passed}/${total} passed${failed ? `, ${failed} failed` : ""}`
  );
  if (suite.uploadFailureCount && suite.uploadFailureCount > 0) {
    lines.push(
      `  warning: ${suite.uploadFailureCount} upload${suite.uploadFailureCount === 1 ? "" : "s"} failed (auth or network?)`
    );
  }

  const aggregated = aggregateAnnotations(suite.results);
  for (const [name, summary] of Object.entries(aggregated)) {
    lines.push(`    ${name}: ${summary}`);
  }
  lines.push(...acceptanceLines(suite));

  for (const result of suite.results) {
    const status =
      result.status === "passed"
        ? "PASS"
        : result.status === "failed"
          ? "FAIL"
          : "SKIP";
    const tag = result.dryRun ? " (dry run — not uploaded)" : "";
    const annotations = formatAnnotationsInline(result.annotations);
    const annotationSuffix = annotations ? `  →  ${annotations}` : "";
    lines.push("");
    lines.push(
      `    [${status}] ${humanizeLabel(result.testName)} (${formatDuration(result.durationMs)})${tag}${annotationSuffix}`
    );
    if (result.error) {
      lines.push(`      error: ${result.error}`);
    }
    if (result.output !== undefined) {
      lines.push(`      output: ${stringifyForLog(result.output)}`);
    }
    for (const ann of result.annotations) {
      if (ann.name !== "pass" && ann.explanation) {
        lines.push(`      why (${ann.name}): ${ann.explanation}`);
      }
    }
    const ids = formatResultIds(result);
    if (ids) {
      lines.push(`      ids: ${ids}`);
    }
  }

  lines.push(...linkLines(suite));
  return lines.join("\n");
}

// ---------------------------------------------------------------------------
// Cross-suite overview
// ---------------------------------------------------------------------------

/** Friendly, env-var-free reason a suite ran locally. */
function friendlyTrackingReason(suite: SuiteSummary): string {
  return suite.setupError?.message ?? "local only";
}

/** A single tracking note when every suite ran locally for the same reason. */
function sharedTrackingNote(
  suites: readonly SuiteSummary[]
): string | undefined {
  return suites.length > 0 &&
    suites.every((s) => s.trackingDisabled && !s.setupError)
    ? "tracking disabled (local only)"
    : undefined;
}

/** The primary metric stat for a suite's overview row, or `null`. */
function primaryStat(suite: SuiteSummary): AnnotationStat | null {
  const stats = computeAnnotationStats(suite.results);
  const primary = selectAnnotationColumns(suite)[0];
  return stats.find((s) => s.name === primary) ?? null;
}

/**
 * Render the run header (totals + tracking note) and, for multi-suite runs, an
 * aligned overview table: one row per suite with its pass count, primary metric,
 * acceptance verdict, mean latency, and a miss/fail note. This is the index;
 * only suites with problems are expanded into a detail block below it.
 */
export function formatScoreboard(
  suites: readonly SuiteSummary[],
  o: RenderOptions = resolveRenderOptions()
): string {
  if (suites.length === 0) return "";

  const vitals = suites.map(computeSuiteVitals);
  const passed = vitals.reduce((a, v) => a + v.passed, 0);
  const total = vitals.reduce((a, v) => a + v.total, 0);
  const failedTests = vitals.reduce((a, v) => a + v.failed, 0);
  const misses = vitals.reduce((a, v) => a + v.missCount, 0);
  const acceptFails = vitals.filter((v) => v.acceptanceFailed).length;
  const uploadFails = suites.reduce(
    (a, s) => a + (s.uploadFailureCount ?? 0),
    0
  );

  // Totals are test/row-level so the clauses stay in one unit; the per-suite
  // breakdown lives in the table below.
  const header = [
    "Eval Results",
    `${suites.length} suite${suites.length === 1 ? "" : "s"}`,
    `${passed}/${total} passed`,
  ];
  if (failedTests > 0) header.push(`${failedTests} failed`);
  if (misses > 0) header.push(`${misses} miss${misses === 1 ? "" : "es"}`);
  if (acceptFails > 0) {
    header.push(
      `${acceptFails} acceptance failure${acceptFails === 1 ? "" : "s"}`
    );
  }
  if (uploadFails > 0) header.push(`${uploadFails} uploads failed`);
  const note = sharedTrackingNote(suites);
  if (note) header.push(note);
  const headerLine = colorize(header.join(" · "), "bold", o);

  // A single suite's own detail block is the overview; just print the header.
  if (suites.length === 1) return headerLine;

  // Columns appear only when at least one suite has something to put in them.
  const anyScore = suites.some((s) => primaryStat(s) !== null);
  const anyAccept = suites.some((s) => (s.acceptanceResults ?? []).length > 0);
  const anyLink = suites.some((s) => s.links.length > 0);
  const primaries = suites.map((s) => selectAnnotationColumns(s)[0]);
  const shared =
    primaries.every((p) => p && p === primaries[0]) && primaries[0]
      ? primaries[0]
      : undefined;

  const headers = ["Suite", "Tests"];
  const caps = [34, 7];
  if (anyScore) {
    headers.push(shared ?? "Score");
    caps.push(22);
  }
  if (anyAccept) {
    headers.push("Accept");
    caps.push(7);
  }
  headers.push("Latency", "Result");
  caps.push(8, 12);
  if (anyLink) {
    headers.push("Link");
    caps.push(48);
  }

  const rows = suites.map((suite, i) => {
    const v = vitals[i]!;
    const stats = computeAnnotationStats(suite.results);
    const stat = primaryStat(suite);
    const row = [suite.name, `${v.passed}/${v.total}`];
    if (anyScore) {
      row.push(
        !stat
          ? "—"
          : shared
            ? aggregateCell(stats, stat.name)
            : `${stat.name} ${aggregateCell(stats, stat.name).replace(/^avg /, "")}`
      );
    }
    if (anyAccept) {
      row.push(
        (suite.acceptanceResults ?? []).length === 0
          ? "—"
          : v.acceptanceFailed
            ? colorize("FAIL", "red", o)
            : colorize("PASS", "green", o)
      );
    }
    row.push(formatDuration(v.meanLatencyMs), resultNote(v, o));
    if (anyLink) row.push(suite.links[0]?.url ?? "—");
    return row;
  });

  return [headerLine, "", ...renderTable(headers, rows, o, { caps })].join(
    "\n"
  );
}

/** The overview "Result" cell: what went wrong, colored, or blank when clean. */
function resultNote(v: SuiteVitals, o: RenderOptions): string {
  if (v.failed > 0) return colorize(`${v.failed} failed`, "red", o);
  if (v.acceptanceFailed) return colorize("accept ✗", "red", o);
  if (v.missCount > 0) {
    return colorize(
      `${v.missCount} miss${v.missCount === 1 ? "" : "es"}`,
      "yellow",
      o
    );
  }
  return "";
}

// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------

/**
 * Print the run summary: the overview header (and, for multi-suite runs, the
 * index table), then an expanded detail block for every suite that failed or
 * had misses. Clean suites are fully described by their overview row. A
 * single-suite run always prints its block; verbose prints every block.
 */
export function printSuiteSummaries(suites: readonly SuiteSummary[]): void {
  const o = resolveRenderOptions();
  const overview = formatScoreboard(suites, o);
  // eslint-disable-next-line no-console
  if (overview) console.log(overview);

  const expand = o.verbose
    ? suites
    : suites.length === 1
      ? suites
      : suites.filter((s) => computeSuiteVitals(s).status !== "pass");
  for (const suite of expand) {
    // eslint-disable-next-line no-console
    console.log(`\n${formatSuiteSummary(suite, o)}`);
  }
}

// ---------------------------------------------------------------------------
// Shared formatters
// ---------------------------------------------------------------------------

/**
 * Render a test's annotations as a compact, single-line `name=score` list for
 * the verbose per-test header. The implicit `pass` annotation is omitted.
 */
function formatAnnotationsInline(annotations: readonly Annotation[]): string {
  return annotations
    .filter((ann) => ann.name !== "pass")
    .map((ann) => `${ann.name}=${formatScore(ann)}`)
    .join(", ");
}

function formatScore(ann: Annotation): string {
  if (typeof ann.score === "number") return ann.score.toString();
  if (typeof ann.score === "boolean") return ann.score ? "true" : "false";
  if (ann.label) return ann.label;
  return "(no score)";
}

function formatDuration(ms: number): string {
  if (ms < 1000) return `${Math.round(ms)}ms`;
  return `${(ms / 1000).toFixed(2)}s`;
}

function stringifyForLog(value: unknown): string {
  try {
    const json = JSON.stringify(value);
    return json && json.length > 200
      ? `${json.slice(0, 197)}...`
      : (json ?? "");
  } catch {
    return String(value);
  }
}

// ---------------------------------------------------------------------------
// Per-failure detail helpers
//
// A problem row's block answers the two questions an agent needs to fix it:
// *why* (the judge's rationale and the model output) and *where* (the Phoenix
// trace / run / example ids it can pull for the full picture).
// ---------------------------------------------------------------------------

/** A run's numeric score for sorting (booleans as 1/0, missing as +∞). */
function annotationScoreValue(ann: Annotation): number {
  if (typeof ann.score === "number") return ann.score;
  if (typeof ann.score === "boolean") return ann.score ? 1 : 0;
  return Number.POSITIVE_INFINITY;
}

/** First non-empty line of a multi-line error, truncated for one-line display. */
function compactError(error: string | undefined): string | null {
  if (!error) return null;
  const firstLine = error
    .split("\n")
    .map((line) => line.trim())
    .find((line) => line.length > 0);
  return firstLine ? truncateSummary(firstLine, 160) : null;
}

/** Phoenix ids for a run as a single `trace=… run=… example=…` string. */
function formatResultIds(result: TestResult): string | null {
  const parts: string[] = [];
  if (result.traceId) parts.push(`trace=${result.traceId}`);
  if (result.runId) parts.push(`run=${result.runId}`);
  if (result.exampleId) parts.push(`example=${result.exampleId}`);
  return parts.length > 0 ? parts.join("  ") : null;
}

// ---------------------------------------------------------------------------
// Label humanization
// ---------------------------------------------------------------------------

/**
 * Turn a machine test name into a readable title. A `test.each` row is named by
 * stringifying its input (`{"userQuery":"Show active users"}`); we surface the
 * value of a single-field object directly (`Show active users`) and fold a
 * multi-field object to `key=value` pairs. Non-JSON names pass through.
 */
function humanizeLabel(name: string): string {
  const trimmed = name.trim();
  if (!(trimmed.startsWith("{") || trimmed.startsWith("["))) return name;
  let parsed: unknown;
  try {
    parsed = JSON.parse(trimmed);
  } catch {
    return name;
  }
  if (Array.isArray(parsed)) return summarizeValue(parsed) ?? name;
  if (!parsed || typeof parsed !== "object") return name;
  const entries = Object.entries(parsed as Record<string, unknown>).filter(
    ([, v]) => v != null && v !== ""
  );
  if (entries.length === 0) return name;
  if (entries.length === 1 && typeof entries[0]![1] === "string") {
    return entries[0]![1] as string;
  }
  return entries
    .map(([k, v]) => `${k}=${summaryPrimitive(v) ?? ""}`)
    .join("  ");
}

/** Truncate keeping the head (most identifying for a title), trailing ellipsis. */
function truncateEnd(s: string, max: number): string {
  return s.length <= max ? s : `${s.slice(0, max - 1)}…`;
}

// ---------------------------------------------------------------------------
// Token-efficient value summarization (adapted from the vitest-evals reporter)
//
// Replaces a blind JSON truncation with a key-preferring summary: the salient
// keys of an eval output (`score`, `output`, `error`, …) come first, primitives
// are rendered compactly, and JSON-encoded strings are parsed so the same
// summary applies. The result is far denser and more legible per token.
// ---------------------------------------------------------------------------

/** Keys surfaced first when summarizing a record, in priority order. */
const PREFERRED_SUMMARY_KEYS = [
  "score",
  "label",
  "pass",
  "passed",
  "output",
  "result",
  "answer",
  "response",
  "reason",
  "rationale",
  "explanation",
  "error",
  "message",
  "name",
  "id",
  "status",
];

function truncateSummary(value: string, maxLength = 96): string {
  return value.length <= maxLength
    ? value
    : `${value.slice(0, maxLength - 1)}…`;
}

/** Render a single value as a short token: scalars inline, containers as counts. */
function summaryPrimitive(value: unknown): string | null {
  if (value === undefined) return null;
  if (value === null) return "null";
  if (typeof value === "string") {
    const truncated = truncateSummary(value, 48);
    // Bare-word strings stay unquoted; anything with spaces/punctuation is
    // quoted so the key=value pairs remain unambiguous.
    return /^[\w.:/@-]+$/.test(truncated)
      ? truncated
      : JSON.stringify(truncated);
  }
  if (typeof value === "number" || typeof value === "boolean") {
    return String(value);
  }
  if (Array.isArray(value)) return `array(${value.length})`;
  if (typeof value === "object") {
    return `object(${Object.keys(value as Record<string, unknown>).length})`;
  }
  return String(value);
}

function summarizeRecord(
  record: Record<string, unknown>,
  maxLength: number
): string | null {
  const keys = Object.keys(record);
  if (keys.length === 0) return "object(0)";
  const ordered = [
    ...PREFERRED_SUMMARY_KEYS.filter((key) => keys.includes(key)),
    ...keys.filter((key) => !PREFERRED_SUMMARY_KEYS.includes(key)),
  ].slice(0, 4);
  const parts = ordered
    .map((key) => {
      const formatted = summaryPrimitive(record[key]);
      return formatted === null ? null : `${key}=${formatted}`;
    })
    .filter((part): part is string => part !== null);
  if (parts.length === 0) return null;
  const suffix = keys.length > ordered.length ? " …" : "";
  return truncateSummary(`${parts.join(" ")}${suffix}`, maxLength);
}

/**
 * Summarize an arbitrary value to a compact, single-line string, or `null` when
 * there's nothing to show (`undefined`). JSON-encoded strings are parsed first
 * so the key-preferring record summary still applies.
 */
function summarizeValue(value: unknown, maxLength = 96): string | null {
  if (value === undefined) return null;
  if (value === null) return "null";
  if (typeof value === "string") {
    const trimmed = value.trim();
    if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
      try {
        return summarizeValue(JSON.parse(trimmed), maxLength);
      } catch {
        // Not valid JSON — fall through to plain-string handling.
      }
    }
    return truncateSummary(value, maxLength);
  }
  if (typeof value === "number" || typeof value === "boolean") {
    return String(value);
  }
  if (Array.isArray(value)) {
    if (value.length === 0) return "array(0)";
    const first = summaryPrimitive(value[0]);
    const suffix = value.length > 1 ? " …" : "";
    return truncateSummary(
      `array(${value.length}) ${first ?? ""}${suffix}`.trim(),
      maxLength
    );
  }
  if (typeof value === "object") {
    return summarizeRecord(value as Record<string, unknown>, maxLength);
  }
  return truncateSummary(String(value), maxLength);
}