@arizeai/phoenix-client
Version:
A client for the Phoenix API
258 lines • 9.93 kB
JavaScript
;
var __rest = (this && this.__rest) || function (s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.declareDescribe = declareDescribe;
exports.declareTest = declareTest;
exports.getAllSuites = getAllSuites;
exports.clearAllSuites = clearAllSuites;
const acceptance_1 = require("./acceptance");
const helpers_1 = require("./helpers");
const phoenix_test_tracking_1 = require("./phoenix-test-tracking");
const report_artifacts_1 = require("./report-artifacts");
const state_1 = require("./state");
const types_1 = require("./types");
/** Globally accessible registry of all suites we created in this process. */
const allSuites = [];
/** Drive a `describe(name, fn, config?)` invocation. */
function declareDescribe(hooks, name, fn, config = {}, variant = "default") {
var _a, _b;
const describeFn = variant === "only"
? ((_a = hooks.describeOnly) !== null && _a !== void 0 ? _a : hooks.describe)
: variant === "skip"
? ((_b = hooks.describeSkip) !== null && _b !== void 0 ? _b : hooks.describe)
: hooks.describe;
describeFn(name, () => {
const suite = {
name,
config,
registeredExamples: new Map(),
exampleIdsByTest: new Map(),
trackingDisabled: false,
results: [],
links: [],
};
(0, state_1.pushSuite)(suite);
try {
fn();
}
finally {
(0, state_1.popSuite)();
}
allSuites.push(suite);
hooks.beforeAll(async () => {
await (0, phoenix_test_tracking_1.initializeSuite)(suite);
});
hooks.afterAll(async () => {
let teardownError;
let acceptanceError;
try {
await (0, phoenix_test_tracking_1.teardownSuite)(suite);
}
catch (err) {
teardownError = err;
}
try {
suite.acceptanceResults = (0, acceptance_1.evaluateAcceptanceCriteria)({
criteria: suite.config.acceptanceCriteria,
results: suite.results,
});
acceptanceError = (0, acceptance_1.createAcceptanceFailureError)(suite.acceptanceResults);
}
finally {
(0, report_artifacts_1.writeSuiteSummaryArtifact)(suite);
}
if (teardownError) {
throw teardownError;
}
if (acceptanceError) {
throw acceptanceError;
}
});
});
}
/** Drive a `test(name, params, fn)` invocation. */
function declareTest(hooks, name, params, fn, variant = "default", timeout) {
var _a, _b, _c;
const suite = (0, state_1.currentSuite)();
if (!suite) {
throw new Error("Phoenix eval test() must be declared inside a describe() block");
}
// Collapse the `expected` / `reference` / `output` aliases into the single
// canonical `expected` slot up front, so every downstream consumer (dataset
// upload, evaluator params, test args) reads one field.
params = normalizeReferenceOutput(params);
// A per-test `dryRun` opts the case out of Phoenix entirely: no dataset
// example, no experiment run, no annotations — it runs as an ordinary
// local test. Suite-level dryRun (or PHOENIX_TEST_TRACKING=false) is
// handled in `initializeSuite`, which never uploads anything anyway.
const isDryRun = !!params.dryRun;
// Avoid silently overwriting an example registration when the same test name
// is declared twice in the same suite.
const uniqueName = ensureUniqueName(suite, name);
// Skipped cases never run, and registering them would still upload their
// example to the tracked dataset via `initializeSuite()` — contradicting the
// skip contract and letting unfinished/flaky cases mutate the dataset. Dry-run
// cases opt out of Phoenix entirely. Neither should register an example.
if (!isDryRun && variant !== "skip") {
suite.registeredExamples.set(uniqueName, {
testName: uniqueName,
params: params,
});
}
const repetitions = isDryRun
? 1
: (0, phoenix_test_tracking_1.resolveRepetitions)(params.repetitions, suite);
suite.maxRepetitions = Math.max((_a = suite.maxRepetitions) !== null && _a !== void 0 ? _a : 1, repetitions);
const testFn = variant === "only"
? ((_b = hooks.testOnly) !== null && _b !== void 0 ? _b : hooks.test)
: variant === "skip"
? ((_c = hooks.testSkip) !== null && _c !== void 0 ? _c : hooks.test)
: hooks.test;
for (let rep = 1; rep <= repetitions; rep++) {
const runnerName = repetitions > 1
? `${uniqueName} [rep ${rep}/${repetitions}]`
: uniqueName;
testFn(runnerName, async () => {
await executeRun({
suite,
runnerName,
logicalName: uniqueName,
repetitionNumber: rep,
repetitions,
dryRun: isDryRun,
params: params,
fn: fn,
});
}, timeout);
}
}
/** Execute a single (possibly-repeated) run of a declared test. */
async function executeRun(opts) {
var _a;
const { suite, runnerName, logicalName, repetitionNumber, repetitions } = opts;
const run = {
suite,
testName: runnerName,
logicalName,
repetitionNumber,
dryRun: opts.dryRun,
params: opts.params,
output: undefined,
outputSet: false,
annotations: [],
startTime: new Date(),
runMetadata: {},
};
const start = Date.now();
let status = "passed";
let thrown;
let testError;
await state_1.runStorage.run(run, async () => {
const taskOutcome = await (0, phoenix_test_tracking_1.runTaskWithTracing)(suite, runnerName, async () => {
const args = {
input: opts.params.input,
expected: opts.params.expected,
metadata: opts.params.metadata,
};
const result = await opts.fn(args);
// If the user returned a value and didn't call logOutput(),
// adopt the return value as the run's output.
if (result !== undefined &&
!run.outputSet &&
isPlainObjectOrPrimitive(result)) {
run.output = result;
run.outputSet = true;
}
return result;
});
run.traceId = taskOutcome.traceId;
if ("error" in taskOutcome && taskOutcome.error) {
status = "failed";
testError = taskOutcome.error.message;
if (taskOutcome.isTaskError) {
run.error = taskOutcome.error.message;
}
thrown = taskOutcome.error;
}
run.endTime = new Date();
// The pass/fail annotation is recorded regardless of tracking mode
// so it shows up consistently in the reporter summary.
run.annotations.unshift({
name: "pass",
score: status === "passed",
annotatorKind: "CODE",
});
const runId = await (0, phoenix_test_tracking_1.postExperimentRun)(suite, run);
run.runId = runId;
await (0, helpers_1.flushAnnotations)(runId, run.annotations, suite);
});
const result = {
suiteName: suite.name,
testName: runnerName,
status,
output: run.outputSet ? run.output : undefined,
annotations: run.annotations,
error: testError,
durationMs: Date.now() - start,
repetitionNumber: repetitions > 1 ? repetitionNumber : undefined,
repetitions: repetitions > 1 ? repetitions : undefined,
dryRun: opts.dryRun || undefined,
traceId: run.traceId,
runId: run.runId,
exampleId: (_a = suite.exampleIdsByTest.get(logicalName)) === null || _a === void 0 ? void 0 : _a.exampleId,
};
suite.results.push(result);
if (thrown) {
throw thrown;
}
}
/**
* Return a copy of `params` with the reference output collapsed onto the
* canonical `expected` key and the `reference` / `output` aliases dropped.
*/
function normalizeReferenceOutput(params) {
const expected = (0, types_1.resolveReference)(params);
const { reference: _reference, output: _output } = params, rest = __rest(params, ["reference", "output"]);
return Object.assign(Object.assign({}, rest), { expected });
}
function isPlainObjectOrPrimitive(value) {
if (value === null)
return true;
const type = typeof value;
return (type === "string" ||
type === "number" ||
type === "boolean" ||
type === "object");
}
function ensureUniqueName(suite, name) {
if (!suite.registeredExamples.has(name)) {
return name;
}
let i = 2;
while (suite.registeredExamples.has(`${name} (${i})`)) {
i++;
}
return `${name} (${i})`;
}
/** Suite registry exposed for reporters and the public summary helper. */
function getAllSuites() {
return allSuites;
}
/**
* Reset the suite registry. Reporters call this at the start of every test
* run so module-cached state from previous watch invocations is released.
*/
function clearAllSuites() {
allSuites.length = 0;
}
//# sourceMappingURL=runner.js.map