@arizeai/phoenix-client
Version:
A client for the Phoenix API
579 lines • 24.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.isFalsyFlag = isFalsyFlag;
exports.__resetTrackingLatchForTests = __resetTrackingLatchForTests;
exports.isTrackingEnabled = isTrackingEnabled;
exports.resolveRepetitions = resolveRepetitions;
exports.initializeSuite = initializeSuite;
exports.runTaskWithTracing = runTaskWithTracing;
exports.endTaskSpanForRun = endTaskSpanForRun;
exports.postExperimentRun = postExperimentRun;
exports.postAnnotation = postAnnotation;
exports.runEvaluatorWithTracing = runEvaluatorWithTracing;
exports.teardownSuite = teardownSuite;
const phoenix_otel_1 = require("@arizeai/phoenix-otel");
const datasets_1 = require("../datasets");
const tracing_1 = require("../experiments/tracing");
const index_1 = require("../index");
const ensureString_1 = require("../utils/ensureString");
const toObjectHeaders_1 = require("../utils/toObjectHeaders");
const urlUtils_1 = require("../utils/urlUtils");
const state_1 = require("./state");
function isFalsyFlag(value) {
// Tolerate surrounding whitespace and matching quotes that survive some
// shells and `.env` loaders (e.g. `PHOENIX_TEST_TRACKING="false"` or a value
// with a trailing newline). Without this, such a value reads as truthy and
// silently re-enables recording even though the user asked to disable it.
const v = (value !== null && value !== void 0 ? value : "")
.trim()
.replace(/^(['"])(.*)\1$/, "$2")
.trim()
.toLowerCase();
return v === "false" || v === "0" || v === "off" || v === "no";
}
/**
* Snapshot of `PHOENIX_TEST_TRACKING` as it was when this module first loaded.
*
* When the flag is exported on the command line (the documented `eval:offline`
* workflow), this captures the user's intent at process start and is immune to
* any later in-process mutation of `process.env` by a sibling suite or setup
* file. That mutation is what made tracking leak across suites in #13930: one
* suite flipped the env var and re-enabled recording for the others.
*/
const trackingDisabledAtLoad = isFalsyFlag(process.env.PHOENIX_TEST_TRACKING);
/**
* Latches to `true` the first time tracking is observed disabled in this
* process. Recording is opt-out and shared process-wide, so a single falsy
* reading turns the whole run off and keeps it off — later suites cannot
* re-enable recording regardless of declaration or execution order.
*/
let trackingLatchedOff = trackingDisabledAtLoad;
/**
* Reset the process-level tracking latch. Test-only seam so unit tests can
* exercise the enable/disable transitions in isolation; not part of the
* public API.
*
* @internal
*/
function __resetTrackingLatchForTests() {
trackingLatchedOff = isFalsyFlag(process.env.PHOENIX_TEST_TRACKING);
}
/**
* Decide whether tests should sync to Phoenix.
*
* Tracing is enabled by default. It can be disabled globally by setting
* `PHOENIX_TEST_TRACKING=false`, or per suite via `SuiteConfig.dryRun`.
*
* The global disable is sticky for the lifetime of the process: once the flag
* is seen falsy — at load time or on any later call — tracking stays off for
* every suite. This keeps offline mode deterministic regardless of which
* suites are included in a run, or the order they execute in.
*/
function isTrackingEnabled(suite) {
if (trackingLatchedOff || isFalsyFlag(process.env.PHOENIX_TEST_TRACKING)) {
trackingLatchedOff = true;
return { enabled: false, reason: "PHOENIX_TEST_TRACKING is disabled" };
}
if (suite === null || suite === void 0 ? void 0 : suite.config.dryRun) {
return { enabled: false, reason: "suite configured dryRun" };
}
return { enabled: true };
}
/**
* Resolve the repetition count for a test: per-test value, else suite-level,
* else the `PHOENIX_TEST_REPETITIONS` env var, else `1`. Non-positive or
* non-finite values fall back to `1`.
*/
function resolveRepetitions(perTest, suite) {
const envValue = Number(process.env.PHOENIX_TEST_REPETITIONS);
const candidates = [
perTest,
suite.config.repetitions,
Number.isFinite(envValue) ? envValue : undefined,
];
for (const c of candidates) {
if (typeof c === "number" && Number.isFinite(c) && c >= 1) {
return Math.floor(c);
}
}
return 1;
}
/**
* Deterministic key for matching dataset examples by content. Uses sorted
* keys so two inputs that differ only in property order hash the same.
*/
function stableKey(value) {
return stableStringify(value);
}
function stableStringify(value) {
if (value === null || typeof value !== "object") {
return JSON.stringify(value);
}
if (Array.isArray(value)) {
return "[" + value.map(stableStringify).join(",") + "]";
}
const keys = Object.keys(value).sort();
return ("{" +
keys
.map((k) => JSON.stringify(k) +
":" +
stableStringify(value[k]))
.join(",") +
"}");
}
/** TEXT for raw strings, JSON for everything else. */
function mimeTypeFor(value) {
return typeof value === "string" ? phoenix_otel_1.MimeType.TEXT : phoenix_otel_1.MimeType.JSON;
}
const taskSpansByRun = new WeakMap();
/**
* Warn once when `PHOENIX_HOST` is plain `http:` while an `Authorization`
* header is being forwarded to the OTLP exporter — that combination
* exfiltrates the bearer token in cleartext.
*/
let warnedAboutHttpScheme = false;
function maybeWarnHttpScheme(baseUrl, headers) {
if (warnedAboutHttpScheme || !baseUrl)
return;
let parsed;
try {
parsed = new URL(baseUrl);
}
catch (_a) {
return;
}
if (parsed.protocol !== "http:")
return;
if (parsed.hostname === "localhost" || parsed.hostname === "127.0.0.1")
return;
if (!headers)
return;
const picked = (0, toObjectHeaders_1.toObjectHeaders)(headers);
const hasAuth = Object.keys(picked).some((h) => h.toLowerCase() === "authorization");
if (!hasAuth)
return;
warnedAboutHttpScheme = true;
// eslint-disable-next-line no-console
console.warn(`[@arizeai/phoenix-client] PHOENIX_HOST="${baseUrl}" uses http:// with ` +
`an Authorization header set; the bearer token will travel in cleartext. ` +
`Use https:// for non-localhost Phoenix endpoints.`);
}
function buildLinks(client, datasetId, experimentId) {
const baseUrl = client.config.baseUrl;
if (!baseUrl)
return [];
return [
{ label: "Dataset", url: (0, urlUtils_1.getDatasetUrl)({ baseUrl, datasetId }) },
{
label: "Experiments",
url: (0, urlUtils_1.getDatasetExperimentsUrl)({ baseUrl, datasetId }),
},
{
label: "Experiment",
url: (0, urlUtils_1.getExperimentUrl)({ baseUrl, datasetId, experimentId }),
},
];
}
/**
* Initialize the suite: upload the dataset, create the experiment, and
* register the OpenInference tracer.
*
* If tracing is disabled (no Phoenix env vars, or PHOENIX_TEST_TRACKING=false),
* this populates a no-op tracer and exits without making any network calls.
*/
async function initializeSuite(suite) {
var _a;
var _b, _c, _d, _e, _f, _g, _h, _j, _k;
const tracking = isTrackingEnabled(suite);
if (!tracking.enabled) {
suite.trackingDisabled = true;
suite.trackingDisabledReason = tracking.reason;
suite.tracer = (0, phoenix_otel_1.createNoOpProvider)().getTracer("no-op");
suite.evaluatorTracer = suite.tracer;
return;
}
const client = (_b = suite.config.client) !== null && _b !== void 0 ? _b : (0, index_1.createClient)();
suite.client = client;
const datasetName = (_c = suite.config.datasetName) !== null && _c !== void 0 ? _c : suite.name;
const description = (_d = suite.config.description) !== null && _d !== void 0 ? _d : `Phoenix test dataset auto-generated from ${suite.name}`;
const examples = Array.from(suite.registeredExamples.values()).map((registered) => { var _a, _b, _c; return ({
id: (_a = registered.params.id) !== null && _a !== void 0 ? _a : null,
input: registered.params.input,
output: (_b = registered.params.expected) !== null && _b !== void 0 ? _b : {},
metadata: (_c = registered.params.metadata) !== null && _c !== void 0 ? _c : {},
splits: registered.params.splits,
}); });
let datasetId;
try {
const created = await (0, datasets_1.createDataset)({
client,
name: datasetName,
description,
examples,
});
datasetId = created.datasetId;
}
catch (err) {
suite.trackingDisabled = true;
suite.setupError = err instanceof Error ? err : new Error(String(err));
suite.tracer = (0, phoenix_otel_1.createNoOpProvider)().getTracer("no-op");
suite.evaluatorTracer = suite.tracer;
return;
}
suite.datasetId = datasetId;
// Map test names to server-side example ids by re-fetching the dataset.
// The server doesn't promise that the GET response order matches the
// upload order, so we match by user-supplied `id` first, then by
// `JSON.stringify(input)` deep-equality with FIFO-on-collision.
try {
const { data: response } = await client.GET("/v1/datasets/{id}/examples", {
params: { path: { id: datasetId } },
});
const fetched = (_e = (_a = response === null || response === void 0 ? void 0 : response.data) === null || _a === void 0 ? void 0 : _a.examples) !== null && _e !== void 0 ? _e : [];
const idToTestName = new Map();
const inputKeyToTestNames = new Map();
for (const [testName, registered] of suite.registeredExamples.entries()) {
if (registered.params.id) {
idToTestName.set(registered.params.id, testName);
continue;
}
const key = stableKey(registered.params.input);
const arr = (_f = inputKeyToTestNames.get(key)) !== null && _f !== void 0 ? _f : [];
arr.push(testName);
inputKeyToTestNames.set(key, arr);
}
for (const ex of fetched) {
const byId = idToTestName.get(ex.id);
if (byId) {
suite.exampleIdsByTest.set(byId, {
exampleId: ex.id,
nodeId: ex.node_id,
});
idToTestName.delete(ex.id);
continue;
}
const queue = inputKeyToTestNames.get(stableKey(ex.input));
if (queue && queue.length) {
const testName = queue.shift();
suite.exampleIdsByTest.set(testName, {
exampleId: ex.id,
nodeId: ex.node_id,
});
}
}
}
catch (_l) {
// If we cannot resolve example ids, runs will be logged without one.
}
const projectName = `${datasetName}-${new Date().toISOString()}`;
suite.projectName = projectName;
try {
const experimentResponse = await client
.POST("/v1/datasets/{dataset_id}/experiments", {
params: { path: { dataset_id: datasetId } },
body: {
name: (_g = suite.config.datasetName) !== null && _g !== void 0 ? _g : suite.name,
description,
metadata: Object.assign(Object.assign({}, ((_h = suite.config.metadata) !== null && _h !== void 0 ? _h : {})), envMetadata()),
project_name: projectName,
repetitions: Math.max(1, (_j = suite.maxRepetitions) !== null && _j !== void 0 ? _j : 1),
},
})
.then((res) => { var _a; return (_a = res.data) === null || _a === void 0 ? void 0 : _a.data; });
if (!experimentResponse) {
throw new Error("Failed to create experiment");
}
suite.experimentId = experimentResponse.id;
suite.projectName = (_k = experimentResponse.project_name) !== null && _k !== void 0 ? _k : projectName;
}
catch (err) {
suite.trackingDisabled = true;
suite.setupError = err instanceof Error ? err : new Error(String(err));
suite.tracer = (0, phoenix_otel_1.createNoOpProvider)().getTracer("no-op");
suite.evaluatorTracer = suite.tracer;
return;
}
const baseUrl = client.config.baseUrl;
if (!baseUrl) {
suite.trackingDisabled = true;
suite.setupError = new Error("Phoenix base URL not found. Set PHOENIX_HOST or pass baseUrl on the client.");
suite.tracer = (0, phoenix_otel_1.createNoOpProvider)().getTracer("no-op");
suite.evaluatorTracer = suite.tracer;
return;
}
maybeWarnHttpScheme(baseUrl, client.config.headers);
let provider;
try {
provider = (0, phoenix_otel_1.register)({
projectName: suite.projectName,
url: baseUrl,
headers: client.config.headers
? (0, toObjectHeaders_1.toObjectHeaders)(client.config.headers)
: undefined,
batch: false,
global: false,
});
suite.tracerProvider = provider;
suite.globalRegistration = (0, phoenix_otel_1.attachGlobalTracerProvider)(provider);
}
catch (err) {
suite.trackingDisabled = true;
suite.setupError = err instanceof Error ? err : new Error(String(err));
suite.tracer = (0, phoenix_otel_1.createNoOpProvider)().getTracer("no-op");
suite.evaluatorTracer = suite.tracer;
return;
}
suite.tracer = provider.getTracer(suite.projectName);
suite.evaluatorTracer = provider.getTracer(`${suite.projectName}-evaluators`);
if (suite.datasetId && suite.experimentId) {
suite.links = buildLinks(client, suite.datasetId, suite.experimentId);
}
}
/** Lazily-created no-op tracer reused for dry runs / fallbacks. */
let noOpTracer;
function getNoOpTracer() {
if (!noOpTracer)
noOpTracer = (0, phoenix_otel_1.createNoOpProvider)().getTracer("no-op");
return noOpTracer;
}
/** The tracer to use for the currently-running test — no-op when dry. */
function taskTracer(suite) {
var _a;
var _b;
if ((_a = (0, state_1.currentRun)()) === null || _a === void 0 ? void 0 : _a.dryRun)
return getNoOpTracer();
return (_b = suite.tracer) !== null && _b !== void 0 ? _b : getNoOpTracer();
}
/**
* Wrap the user's test body in an OpenInference task span and return the
* trace id so we can submit it with the experiment run.
*/
async function runTaskWithTracing(suite, testName, fn) {
const tracer = taskTracer(suite);
return tracer.startActiveSpan(`Test: ${testName}`, async (span) => {
const traceId = span.spanContext().traceId;
const run = (0, state_1.currentRun)();
if (run) {
run.traceId = traceId;
taskSpansByRun.set(run, { span, traceId, ended: false });
}
try {
const result = await fn();
if (run) {
endTaskSpan({ run, fallbackOutput: result });
}
else {
endSpanAsTask({ span, input: undefined, output: result });
}
return { traceId, result };
}
catch (err) {
const error = err instanceof Error ? err : new Error(String(err));
const isTaskError = run ? !hasTaskSpanEnded(run) : true;
if (run && isTaskError) {
endTaskSpan({ run, error });
}
else if (!run) {
span.setStatus({ code: phoenix_otel_1.SpanStatusCode.ERROR, message: error.message });
span.end();
}
return { traceId, error, isTaskError };
}
finally {
if (run) {
taskSpansByRun.delete(run);
}
}
});
}
function hasTaskSpanEnded(run) {
var _a;
var _b;
return (_b = (_a = taskSpansByRun.get(run)) === null || _a === void 0 ? void 0 : _a.ended) !== null && _b !== void 0 ? _b : false;
}
/**
* End the current run's task span. `logOutput()` calls this immediately so
* evaluator work that follows is not included in the task span duration.
*/
function endTaskSpanForRun(run) {
endTaskSpan({ run });
}
function endTaskSpan({ run, fallbackOutput, error, }) {
const lifecycle = taskSpansByRun.get(run);
if (!lifecycle || lifecycle.ended)
return;
const output = run.outputSet ? run.output : fallbackOutput;
if (error) {
lifecycle.span.setStatus({
code: phoenix_otel_1.SpanStatusCode.ERROR,
message: error.message,
});
lifecycle.span.setAttributes({
[phoenix_otel_1.SemanticConventions.OPENINFERENCE_SPAN_KIND]: phoenix_otel_1.OpenInferenceSpanKind.CHAIN,
[phoenix_otel_1.SemanticConventions.INPUT_MIME_TYPE]: mimeTypeFor(run.params.input),
[phoenix_otel_1.SemanticConventions.INPUT_VALUE]: (0, ensureString_1.ensureString)(run.params.input),
});
lifecycle.span.end();
}
else {
endSpanAsTask({ span: lifecycle.span, input: run.params.input, output });
}
run.traceId = lifecycle.traceId;
run.taskEndTime = new Date();
lifecycle.ended = true;
}
function endSpanAsTask({ span, input, output, }) {
span.setAttributes({
[phoenix_otel_1.SemanticConventions.OPENINFERENCE_SPAN_KIND]: phoenix_otel_1.OpenInferenceSpanKind.CHAIN,
[phoenix_otel_1.SemanticConventions.INPUT_MIME_TYPE]: mimeTypeFor(input),
[phoenix_otel_1.SemanticConventions.INPUT_VALUE]: (0, ensureString_1.ensureString)(input),
[phoenix_otel_1.SemanticConventions.OUTPUT_MIME_TYPE]: mimeTypeFor(output),
[phoenix_otel_1.SemanticConventions.OUTPUT_VALUE]: (0, ensureString_1.ensureString)(output),
});
span.setStatus({ code: phoenix_otel_1.SpanStatusCode.OK });
span.end();
}
/**
* POST a single experiment run for one test case to Phoenix.
*
* Best-effort: failures are captured and surfaced via the test reporter but
* do not fail the test itself.
*/
async function postExperimentRun(suite, run) {
var _a;
var _b, _c, _d, _e, _f;
if (run.dryRun ||
suite.trackingDisabled ||
!isTrackingEnabled(suite).enabled ||
!suite.client ||
!suite.experimentId) {
return undefined;
}
const example = suite.exampleIdsByTest.get(run.logicalName);
if (!example) {
return undefined;
}
try {
const res = await suite.client.POST("/v1/experiments/{experiment_id}/runs", {
params: { path: { experiment_id: suite.experimentId } },
body: {
dataset_example_id: example.nodeId,
output: run.outputSet
? run.output
: null,
repetition_number: run.repetitionNumber,
start_time: run.startTime.toISOString(),
end_time: ((_c = (_b = run.taskEndTime) !== null && _b !== void 0 ? _b : run.endTime) !== null && _c !== void 0 ? _c : new Date()).toISOString(),
error: (_d = run.error) !== null && _d !== void 0 ? _d : null,
trace_id: (_e = run.traceId) !== null && _e !== void 0 ? _e : null,
},
});
return (_a = res.data) === null || _a === void 0 ? void 0 : _a.data.id;
}
catch (_g) {
suite.uploadFailureCount = ((_f = suite.uploadFailureCount) !== null && _f !== void 0 ? _f : 0) + 1;
return undefined;
}
}
/**
* POST one annotation (an "experiment_evaluation") for a run.
*/
async function postAnnotation(suite, runId, annotation) {
var _a, _b, _c, _d, _e, _f;
if (suite.trackingDisabled ||
!isTrackingEnabled(suite).enabled ||
!suite.client ||
!runId)
return;
const start = new Date();
const end = new Date();
try {
await suite.client.POST("/v1/experiment_evaluations", {
body: {
experiment_run_id: runId,
name: annotation.name,
annotator_kind: (_a = annotation.annotatorKind) !== null && _a !== void 0 ? _a : "CODE",
start_time: start.toISOString(),
end_time: end.toISOString(),
result: Object.assign({ score: typeof annotation.score === "boolean"
? annotation.score
? 1
: 0
: ((_b = annotation.score) !== null && _b !== void 0 ? _b : null), label: (_c = annotation.label) !== null && _c !== void 0 ? _c : null, explanation: (_d = annotation.explanation) !== null && _d !== void 0 ? _d : null }, (annotation.metadata ? { metadata: annotation.metadata } : {})),
error: null,
trace_id: (_e = annotation.traceId) !== null && _e !== void 0 ? _e : null,
},
});
}
catch (_g) {
suite.uploadFailureCount = ((_f = suite.uploadFailureCount) !== null && _f !== void 0 ? _f : 0) + 1;
}
}
/** Run an evaluator in an OpenInference evaluator span. */
async function runEvaluatorWithTracing(suite, name, params, fn) {
var _a;
var _b, _c, _d;
const isDryRun = (_b = (_a = (0, state_1.currentRun)()) === null || _a === void 0 ? void 0 : _a.dryRun) !== null && _b !== void 0 ? _b : false;
const tracer = isDryRun
? getNoOpTracer()
: ((_d = (_c = suite.evaluatorTracer) !== null && _c !== void 0 ? _c : suite.tracer) !== null && _d !== void 0 ? _d : getNoOpTracer());
const parentlessContext = phoenix_otel_1.trace.deleteSpan(phoenix_otel_1.context.active());
const evaluatorContext = isDryRun
? (0, phoenix_otel_1.suppressTracing)(parentlessContext)
: parentlessContext;
return phoenix_otel_1.context.with(evaluatorContext, () => tracer.startActiveSpan(`Evaluation: ${name}`, async (span) => {
const traceId = isDryRun ? null : span.spanContext().traceId;
try {
const result = await fn(params);
span.setAttributes({
[phoenix_otel_1.SemanticConventions.OPENINFERENCE_SPAN_KIND]: phoenix_otel_1.OpenInferenceSpanKind.EVALUATOR,
[phoenix_otel_1.SemanticConventions.INPUT_MIME_TYPE]: phoenix_otel_1.MimeType.JSON,
[phoenix_otel_1.SemanticConventions.INPUT_VALUE]: (0, ensureString_1.ensureString)(params),
[phoenix_otel_1.SemanticConventions.OUTPUT_MIME_TYPE]: phoenix_otel_1.MimeType.JSON,
[phoenix_otel_1.SemanticConventions.OUTPUT_VALUE]: (0, ensureString_1.ensureString)(result),
});
span.setStatus({ code: phoenix_otel_1.SpanStatusCode.OK });
return { result, traceId };
}
catch (err) {
const error = err instanceof Error ? err : new Error(String(err));
span.setStatus({ code: phoenix_otel_1.SpanStatusCode.ERROR, message: error.message });
throw error;
}
finally {
span.end();
}
}));
}
/**
* Tear down the tracer provider created for the suite, flushing any pending
* spans before the process exits.
*/
async function teardownSuite(suite) {
const provider = suite.tracerProvider;
if (!provider)
return;
try {
await (0, tracing_1.cleanupOwnedTracerProvider)({
provider,
globalRegistration: suite.globalRegistration,
});
}
finally {
suite.tracerProvider = undefined;
suite.globalRegistration = null;
}
}
/**
* Snapshot environment-derived metadata recorded on the experiment so users
* can filter experiments by env in the Phoenix UI.
*/
function envMetadata() {
var _a, _b;
const out = {};
const env = (_b = (_a = process.env.PHOENIX_ENVIRONMENT) !== null && _a !== void 0 ? _a : process.env.ENVIRONMENT) !== null && _b !== void 0 ? _b : process.env.NODE_ENV;
if (env)
out.environment = env;
return out;
}
//# sourceMappingURL=phoenix-test-tracking.js.map