@arizeai/phoenix-client
Version:
A client for the Phoenix API
182 lines • 7.56 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.logOutput = logOutput;
exports.logAnnotation = logAnnotation;
exports.evaluate = evaluate;
exports.traceEvaluator = traceEvaluator;
exports.flushAnnotations = flushAnnotations;
const phoenix_test_tracking_1 = require("./phoenix-test-tracking");
const state_1 = require("./state");
/**
* Log the output produced by the test for the current run.
*
* Calling this multiple times overwrites the previously recorded value.
* The argument can be any JSON-serializable value — typically an object
* matching the shape of the example's `expected` field.
*/
function logOutput(output) {
const run = (0, state_1.currentRun)();
if (!run) {
throw new Error("logOutput() must be called inside a Phoenix eval test body");
}
run.output = output;
run.outputSet = true;
(0, phoenix_test_tracking_1.endTaskSpanForRun)(run);
}
/**
* Record an annotation on the current run.
*
* Annotations are collected during the test and posted to Phoenix as
* experiment evaluations after the test completes. The `name` is the
* Phoenix evaluation name; `score`, `label`, and `explanation` map to
* the standard Phoenix `EvaluationResult` fields.
*
* The annotation name `"pass"` is reserved — Phoenix eval tests always write
* a `pass` annotation derived from the test's assertion outcome, so a
* user-supplied annotation with that name would race / overwrite the
* built-in one. Such calls are silently ignored.
*/
function logAnnotation(annotation) {
const run = (0, state_1.currentRun)();
if (!run) {
throw new Error("logAnnotation() must be called inside a Phoenix eval test body");
}
if (annotation.name === "pass")
return;
run.annotations.push(annotation);
}
/**
* Run an evaluator object against the current test run and record the result.
*
* The evaluator may come from `@arizeai/phoenix-evals.createEvaluator`,
* `asExperimentEvaluator`, or any plain object with `{ name, evaluate }`.
* When `params` is omitted, the current test's `input`, recorded `output`,
* `expected`, `metadata`, and task `traceId` are supplied.
*/
async function evaluate(evaluator, params) {
var _a;
const run = (0, state_1.currentRun)();
if (!run) {
return await evaluator.evaluate((params !== null && params !== void 0 ? params : {}));
}
if (!run.outputSet && !(params && "output" in params)) {
warnEvaluateBeforeOutput(run.suite, evaluator.name, run.testName);
}
const evaluatorParams = Object.assign({ input: run.params.input,
// `run.output` is only ever set together with `outputSet`, so it is already
// `undefined` until a value is recorded.
output: run.output, expected: run.params.expected, metadata: run.params.metadata, traceId: (_a = run.traceId) !== null && _a !== void 0 ? _a : null }, (params !== null && params !== void 0 ? params : {}));
const { result, traceId } = await (0, phoenix_test_tracking_1.runEvaluatorWithTracing)(run.suite, evaluator.name, evaluatorParams, (paramsToEvaluate) => evaluator.evaluate(paramsToEvaluate));
logAnnotation(toAnnotation({
name: evaluator.name,
kind: evaluator.kind,
result,
traceId,
}));
return result;
}
/**
* Trace an evaluator function so its execution shows up as a separate
* `EVALUATOR` span in Phoenix and any `{ name, score }`-shaped return
* value is automatically captured as an annotation on the current run.
*
* The annotation name defaults to the traced function's name, falling
* back to `"evaluator"`.
*/
function traceEvaluator(fn, options) {
var _a;
const evaluatorName = (_a = options === null || options === void 0 ? void 0 : options.name) !== null && _a !== void 0 ? _a : (fn.name && fn.name !== "" ? fn.name : "evaluator");
return async (params) => {
const run = (0, state_1.currentRun)();
if (!run) {
// outside a test context, just call the function plainly
return await fn(params);
}
const { result, traceId } = await (0, phoenix_test_tracking_1.runEvaluatorWithTracing)(run.suite, evaluatorName, params, fn);
if (isAnnotationShaped(result)) {
logAnnotation(Object.assign(Object.assign({}, result), { traceId }));
}
return result;
};
}
/**
* Warn (at most once per suite) when an evaluator runs before any output was
* recorded and none was passed explicitly. Such an evaluator receives
* `output: undefined`, which silently scores against nothing — almost always a
* forgotten `logOutput()`. Harmless for evaluators that only read `input`.
*/
const warnedOutputSuites = new WeakSet();
function warnEvaluateBeforeOutput(suite, evaluatorName, testName) {
if (warnedOutputSuites.has(suite))
return;
warnedOutputSuites.add(suite);
// eslint-disable-next-line no-console
console.warn(`[@arizeai/phoenix-client] evaluate("${evaluatorName}") ran before ` +
`logOutput() on test "${testName}", so the evaluator received ` +
`output=undefined. Call logOutput(...) first, or pass { output } ` +
`explicitly. (Ignore if this evaluator only needs input.)`);
}
/**
* Normalize an evaluator's return value into an {@link Annotation}. The value
* is already typed as an {@link EvaluationResult}, so we only dispatch on its
* runtime shape: a string becomes a `label`, a number/boolean/null becomes a
* `score`, and an object contributes its `score`/`label`/`explanation`/
* `metadata` directly.
*/
function toAnnotation({ name, kind, result, traceId, }) {
const annotatorKind = kind !== null && kind !== void 0 ? kind : "CODE";
if (typeof result === "string") {
return { name, label: result, annotatorKind, traceId };
}
if (typeof result === "number" ||
typeof result === "boolean" ||
result === null) {
return { name, score: result, annotatorKind, traceId };
}
if (typeof result === "object" && !Array.isArray(result)) {
const { score, label, explanation, metadata } = result;
return {
name,
score,
label,
explanation,
metadata,
annotatorKind,
traceId,
};
}
return { name, annotatorKind, traceId };
}
function isAnnotationShaped(value) {
if (!value || typeof value !== "object")
return false;
const v = value;
if (typeof v.name !== "string")
return false;
if (v.score !== undefined &&
typeof v.score !== "number" &&
typeof v.score !== "boolean" &&
v.score !== null) {
return false;
}
return true;
}
/**
* Internal: persist all collected annotations for the run.
*
* Phoenix's `experiment_evaluations` endpoint is keyed by
* `(experiment_run_id, name)` so two annotations with the same name on
* the same run race each other. We collapse duplicates by name (last
* wins) up front, which makes the final state deterministic; the
* remaining writes target distinct names, so they post in parallel.
*/
async function flushAnnotations(runId, annotations, suite) {
if (!annotations.length)
return;
const byName = new Map();
for (const annotation of annotations) {
byName.set(annotation.name, annotation);
}
await Promise.all(Array.from(byName.values(), (annotation) => (0, phoenix_test_tracking_1.postAnnotation)(suite, runId, annotation)));
}
//# sourceMappingURL=helpers.js.map