@arizeai/phoenix-evals
Version:
A library for running evaluations for AI use cases
269 lines • 10.4 kB
JavaScript
function validateBeta(beta) {
if (!(beta > 0)) {
throw new Error("beta must be > 0");
}
}
function validateAverage(average) {
if (average !== "macro" && average !== "micro" && average !== "weighted") {
throw new Error('average must be one of "macro", "micro", or "weighted"');
}
}
function validateLabelSequences(expected, output) {
if (!Array.isArray(expected) || !Array.isArray(output)) {
throw new Error("expected and output must be arrays of labels");
}
if (expected.length !== output.length) {
throw new Error(`expected and output must have the same length. Got ${expected.length} and ${output.length}`);
}
if (expected.length === 0) {
throw new Error("expected and output must be non-empty");
}
}
function safeDivide(numerator, denominator, zeroDivision) {
if (denominator === 0) {
return zeroDivision;
}
return numerator / denominator;
}
function computeFScore(precision, recall, beta, zeroDivision) {
if (precision === 0 && recall === 0) {
return 0;
}
const betaSquared = beta * beta;
const numerator = (1 + betaSquared) * precision * recall;
const denominator = betaSquared * precision + recall;
return safeDivide(numerator, denominator, zeroDivision);
}
/** Collects all unique labels, ordered by first appearance in `expected` then `output`. */
function collectLabels(expected, output) {
const seen = new Set();
const ordered = [];
for (const label of [...expected, ...output]) {
if (!seen.has(label)) {
seen.add(label);
ordered.push(label);
}
}
return ordered;
}
/**
* Compares two labels the same way `Map`/`Set` key lookups do (SameValueZero),
* so a `NaN` label matches another `NaN` label instead of always mismatching
* under strict `===`.
*/
function labelsAreEqual(a, b) {
return (a === b ||
(typeof a === "number" &&
typeof b === "number" &&
Number.isNaN(a) &&
Number.isNaN(b)));
}
function computeClassCounts(expected, output, labels) {
const countsByLabel = new Map();
for (const label of labels) {
countsByLabel.set(label, {
truePositive: 0,
falsePositive: 0,
falseNegative: 0,
});
}
for (const [index, expectedLabel] of expected.entries()) {
// `output` is validated to have the same length as `expected`.
const outputLabel = output[index];
// `labels` is derived from these same `expected`/`output` arrays, so both
// labels are always pre-populated keys in `countsByLabel`.
if (labelsAreEqual(expectedLabel, outputLabel)) {
const counts = countsByLabel.get(expectedLabel);
counts.truePositive += 1;
}
else {
const predictedCounts = countsByLabel.get(outputLabel);
predictedCounts.falsePositive += 1;
const expectedCounts = countsByLabel.get(expectedLabel);
expectedCounts.falseNegative += 1;
}
}
return countsByLabel;
}
/**
* Resolves the one-vs-rest positive label, if any. Auto-detection of a
* numeric `{0, 1}` label set only applies under the default `"macro"`
* average, so an explicitly configured `"micro"`/`"weighted"` average is
* never silently replaced by binary one-vs-rest scoring.
*/
function resolvePositiveLabel(configuredPositiveLabel, labels, average) {
if (configuredPositiveLabel !== undefined) {
return configuredPositiveLabel;
}
if (average !== "macro") {
return null;
}
const uniqueLabels = new Set(labels);
if (uniqueLabels.size === 2 && uniqueLabels.has(0) && uniqueLabels.has(1)) {
return 1;
}
return null;
}
/**
* Aggregates per-class precision/recall/F-score across all classes.
*
* Mirrors scikit-learn's `average` semantics: for `"macro"`/`"weighted"` the
* F-score is computed per class and then averaged (not derived from the
* aggregated precision/recall). For `"micro"` the per-class confusion counts
* are pooled first, so the F-score is computed from the pooled
* precision/recall.
*/
function aggregatePrecisionRecall(countsByLabel, average, beta, zeroDivision) {
if (average === "micro") {
let truePositive = 0;
let falsePositive = 0;
let falseNegative = 0;
for (const counts of countsByLabel.values()) {
truePositive += counts.truePositive;
falsePositive += counts.falsePositive;
falseNegative += counts.falseNegative;
}
const precision = safeDivide(truePositive, truePositive + falsePositive, zeroDivision);
const recall = safeDivide(truePositive, truePositive + falseNegative, zeroDivision);
return {
precision,
recall,
fScore: computeFScore(precision, recall, beta, zeroDivision),
};
}
const perClassMetrics = Array.from(countsByLabel.values()).map((counts) => {
const precision = safeDivide(counts.truePositive, counts.truePositive + counts.falsePositive, zeroDivision);
const recall = safeDivide(counts.truePositive, counts.truePositive + counts.falseNegative, zeroDivision);
return {
precision,
recall,
fScore: computeFScore(precision, recall, beta, zeroDivision),
support: counts.truePositive + counts.falseNegative,
};
});
if (average === "macro") {
return {
precision: mean(perClassMetrics.map((m) => m.precision)),
recall: mean(perClassMetrics.map((m) => m.recall)),
fScore: mean(perClassMetrics.map((m) => m.fScore)),
};
}
// weighted
const totalSupport = perClassMetrics.reduce((sum, m) => sum + m.support, 0);
if (totalSupport === 0) {
return {
precision: zeroDivision,
recall: zeroDivision,
fScore: zeroDivision,
};
}
return {
precision: weightedMean(perClassMetrics, "precision", totalSupport),
recall: weightedMean(perClassMetrics, "recall", totalSupport),
fScore: weightedMean(perClassMetrics, "fScore", totalSupport),
};
}
function mean(values) {
return values.reduce((sum, value) => sum + value, 0) / values.length;
}
function weightedMean(perClassMetrics, field, totalSupport) {
const weightedSum = perClassMetrics.reduce((sum, metrics) => sum + metrics[field] * metrics.support, 0);
return weightedSum / totalSupport;
}
/**
* Computes precision, recall, and F-beta score for a batch of expected vs.
* predicted labels.
*
* `expected`/`output` are the full sequence of labels across an entire
* dataset, not a single row — this and the evaluators built on it are
* dataset-level, unlike the package's per-row LLM evaluators. Call it once
* over every row's collected labels rather than wiring it into a per-row
* pipeline (e.g. `runExperiment`'s per-row evaluators).
*
* Supports both binary classification (via `positiveLabel`, or
* auto-detected when `average` is at its default `"macro"` and the labels
* are the numeric set `{0, 1}`) and multi-class classification (via the
* `average` strategy).
*
* @example Multi-class (macro average)
* ```typescript
* computePrecisionRecallFScore({
* expected: ["cat", "dog", "cat", "bird"],
* output: ["cat", "cat", "cat", "bird"],
* });
* // { precision: 5/9, recall: 2/3, fScore: 0.6, beta: 1, average: "macro", ... }
* ```
*
* @example Binary with an explicit positive label
* ```typescript
* computePrecisionRecallFScore(
* { expected: ["spam", "ham", "spam"], output: ["spam", "spam", "ham"] },
* { beta: 0.5, positiveLabel: "spam" }
* );
* ```
*/
export function computePrecisionRecallFScore({ expected, output }, options = {}) {
const { beta = 1, average = "macro", zeroDivision = 0, positiveLabel, } = options;
validateBeta(beta);
validateAverage(average);
validateLabelSequences(expected, output);
const labels = collectLabels(expected, output);
const countsByLabel = computeClassCounts(expected, output, labels);
const resolvedPositiveLabel = resolvePositiveLabel(positiveLabel, labels, average);
if (resolvedPositiveLabel !== null) {
const classCounts = countsByLabel.get(resolvedPositiveLabel);
if (!classCounts) {
throw new Error(`positiveLabel ${JSON.stringify(resolvedPositiveLabel)} not present in labels ${JSON.stringify(labels)}`);
}
const precision = safeDivide(classCounts.truePositive, classCounts.truePositive + classCounts.falsePositive, zeroDivision);
const recall = safeDivide(classCounts.truePositive, classCounts.truePositive + classCounts.falseNegative, zeroDivision);
return {
precision,
recall,
fScore: computeFScore(precision, recall, beta, zeroDivision),
beta,
average,
labels,
positiveLabel: resolvedPositiveLabel,
};
}
const { precision, recall, fScore } = aggregatePrecisionRecall(countsByLabel, average, beta, zeroDivision);
return {
precision,
recall,
fScore,
beta,
average,
labels,
positiveLabel: null,
};
}
/**
* Formats a beta value for use in metric names, e.g. `1` -> `"f1"`,
* `0.5` -> `"f0_5"`.
*/
export function formatBetaForMetricName(beta) {
if (beta <= 0) {
return "f";
}
if (Number.isInteger(beta)) {
return `f${beta}`;
}
return `f${beta.toString().replace(".", "_")}`;
}
/**
* The suffix appended to a metric name to reflect the aggregation strategy,
* e.g. `"precision"` vs. `"precision_micro"`. No suffix is used when
* `positiveLabel` is explicitly configured, since `average` is not
* applicable in that one-vs-rest binary mode. This mirrors
* `resolvePositiveLabel`'s auto-detection rule (only under the default
* `"macro"` average) so a constructed evaluator's static name always
* matches what `computePrecisionRecallFScore` actually computes.
*/
export function getAverageMetricNameSuffix({ average = "macro", positiveLabel, }) {
if (positiveLabel !== undefined) {
return "";
}
return average === "macro" ? "" : `_${average}`;
}
//# sourceMappingURL=classificationMetrics.js.map