axiom
Version:
Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals - online evals
783 lines (778 loc) • 27.7 kB
JavaScript
import {
getConsoleUrl,
loadConfig,
u
} from "./chunk-72PVEOMF.js";
import {
dotNotationToNested,
flattenObject,
getAxiomConfig
} from "./chunk-63VQQCZB.js";
import {
AxiomCLIError,
errorToString
} from "./chunk-ISSDOC43.js";
import {
initAxiomAI
} from "./chunk-PU64TWX4.js";
import {
__publicField
} from "./chunk-KEXKKQVW.js";
// src/evals/reporter.console-utils.ts
function scoreToNumber(score) {
if (typeof score === "boolean") return score ? 1 : 0;
return score ?? 0;
}
function formatPercentage(value) {
if (!Number.isFinite(value)) {
return "N/A";
}
return Number(value * 100).toFixed(2) + "%";
}
function formatDiff(current, baseline) {
if (!Number.isFinite(current) || !Number.isFinite(baseline)) {
return { text: "N/A", color: u.dim };
}
const diff = current - baseline;
const diffText = (diff >= 0 ? "+" : "") + formatPercentage(diff);
const color = diff > 0 ? u.green : diff < 0 ? u.red : u.dim;
return { text: diffText, color };
}
function truncate(str, max) {
return str.length > max ? str.slice(0, max) + "\u2026" : str;
}
function stringify(value) {
try {
if (typeof value === "string") return value;
return JSON.stringify(value);
} catch {
return String(value);
}
}
function getCaseFingerprint(input, expected) {
const inputStr = typeof input === "string" ? input : JSON.stringify(input);
const expectedStr = typeof expected === "string" ? expected : JSON.stringify(expected);
return JSON.stringify({ input: inputStr, expected: expectedStr });
}
function printEvalNameAndFileName(testSuite, meta, logger = console.log) {
const cwd = process.cwd();
logger(
" ",
u.bgCyan(u.black(` ${testSuite.project.name} `)),
u.bgBlue(u.black(` ${meta.evaluation.name}-${meta.evaluation.version} `)),
u.dim(`(${testSuite.children.size} cases)`)
);
logger(" ", u.dim(testSuite.module.moduleId.replace(cwd, "")));
}
function printBaselineNameAndVersion(testMeta, logger = console.log) {
if (testMeta.evaluation.baseline) {
logger(
" ",
" baseline ",
u.bgMagenta(
u.black(` ${testMeta.evaluation.baseline.name}-${testMeta.evaluation.baseline.version} `)
)
);
} else {
logger(" ", u.bgWhite(u.blackBright(" baseline: ")), "none");
}
logger("");
}
function printRuntimeFlags(testMeta, logger = console.log) {
if (testMeta.case.runtimeFlags && Object.keys(testMeta.case.runtimeFlags).length > 0) {
const entries = Object.entries(testMeta.case.runtimeFlags);
logger(" ", u.dim("runtime flags"));
for (const [k, v] of entries) {
switch (v.kind) {
case "replaced": {
const valText = truncate(stringify(v.value), 80);
const defText = truncate(stringify(v.default), 80);
logger(" ", `${k}: ${valText} (default: ${defText})`);
break;
}
case "introduced": {
const valText = truncate(stringify(v.value), 80);
logger(" ", `${k}: ${valText} (no default)`);
break;
}
}
}
}
}
function printTestCaseCountStartDuration(testSuite, startTime, duration, trials, logger = console.log) {
logger(" ");
const trialsLabel = trials && trials > 1 ? ` (${trials} trials each)` : "";
logger(" ", u.dim("Cases"), `${testSuite.children.size}${trialsLabel}`);
logger(" ", u.dim("Start at"), new Date(startTime).toTimeString());
logger(" ", u.dim("Duration"), `${duration}s`);
}
function printTestCaseSuccessOrFailed(testMeta, ok, logger = console.log) {
const index = testMeta.case.index;
if (ok) {
logger(" ", u.yellow(` \u2714 case ${index}:`));
} else {
logger(" ", u.red(` \u2716 case ${index}: failed`));
for (const e of testMeta.case.errors ?? []) {
logger("", e.message);
}
}
}
function printTestCaseScores(testMeta, baselineCase, logger = console.log) {
const scores = testMeta.case.scores;
const keys = Object.keys(scores);
if (keys.length === 0) {
return;
}
const maxNameLength = Math.max(...keys.map((k) => k.length));
keys.forEach((k) => {
const scoreData = scores[k];
const hasError = scoreData.metadata?.error;
const v = scoreToNumber(scoreData.score);
const rawCurrent = hasError ? "N/A" : formatPercentage(v);
const paddedCurrent = rawCurrent.padStart(7);
const coloredCurrent = hasError ? u.dim(paddedCurrent) : u.magentaBright(paddedCurrent);
const paddedName = k.padEnd(maxNameLength);
if (baselineCase?.scores[k]) {
const baselineScoreValue = baselineCase.scores[k].value;
const rawBaseline = formatPercentage(baselineScoreValue);
const paddedBaseline = rawBaseline === "N/A" ? rawBaseline : rawBaseline.padStart(7);
const coloredBaseline = rawBaseline === "N/A" ? u.dim(paddedBaseline) : u.blueBright(paddedBaseline);
const { text: diffText, color: diffColor } = formatDiff(v, baselineScoreValue);
const paddedDiff = diffText.padStart(8);
logger(
` ${paddedName} ${coloredBaseline} \u2192 ${coloredCurrent} ${hasError ? u.dim("(scorer not run)") : u.dim("(") + diffColor(paddedDiff) + u.dim(")")}`
);
} else {
logger(` ${paddedName} ${coloredCurrent} ${hasError ? u.dim("(scorer not run)") : ""}`);
}
});
}
function printOutOfScopeFlags(testMeta, logger = console.log) {
if (testMeta.case.outOfScopeFlags && testMeta.case.outOfScopeFlags.length > 0) {
const pickedFlagsText = testMeta.case.pickedFlags ? `(picked: ${testMeta.case.pickedFlags.map((f) => `'${f}'`).join(", ")})` : "(none)";
logger(" ", u.yellow(`\u26A0 Out-of-scope flags: ${pickedFlagsText}`));
testMeta.case.outOfScopeFlags.forEach((flag) => {
const timeStr = new Date(flag.accessedAt).toLocaleTimeString();
logger(" ", `${flag.flagPath} (at ${timeStr})`);
if (flag.stackTrace && flag.stackTrace.length > 0) {
flag.stackTrace.forEach((frame, i) => {
const prefix = i === flag.stackTrace.length - 1 ? " \u2514\u2500" : " \u251C\u2500";
logger(" ", u.dim(`${prefix} ${frame}`));
});
}
});
}
}
function printCaseResult(test, baselineCasesByFingerprint, matchedIndices, logger = console.log) {
const ok = test.ok();
const testMeta = test.meta();
if (!testMeta?.case) {
return;
}
printTestCaseSuccessOrFailed(testMeta, ok, logger);
const fingerprint = getCaseFingerprint(testMeta.case.input, testMeta.case.expected);
const baselineCases = baselineCasesByFingerprint.get(fingerprint);
const baselineCase = baselineCases?.shift();
if (baselineCase) {
matchedIndices.add(baselineCase.index);
}
printTestCaseScores(testMeta, baselineCase, logger);
printRuntimeFlags(testMeta, logger);
printOutOfScopeFlags(testMeta, logger);
}
function printOrphanedBaselineCases(baseline, matchedIndices, logger = console.log) {
const orphanedCases = baseline.cases.filter((c) => !matchedIndices.has(c.index));
if (orphanedCases.length === 0) {
return;
}
logger("");
logger(" ", u.yellow("Orphaned baseline cases:"));
for (const orphanedCase of orphanedCases) {
logger(
" ",
u.dim(
`case ${orphanedCase.index}: ${truncate(orphanedCase.input, 50)} (score: ${truncate(
JSON.stringify(orphanedCase.scores),
50
)})`
)
);
const keys = Object.keys(orphanedCase.scores);
if (keys.length > 0) {
const maxNameLength = Math.max(...keys.map((k) => k.length));
keys.forEach((k) => {
const scoreData = orphanedCase.scores[k];
const rawScore = formatPercentage(scoreData.value);
const paddedName = k.padEnd(maxNameLength);
const paddedScore = rawScore.padStart(7);
logger(` ${paddedName} ${u.blueBright(paddedScore)}`);
});
}
}
}
var reporterDate = (d) => {
const date = d.toISOString().slice(0, 10);
const hours = d.getUTCHours().toString().padStart(2, "0");
const minutes = d.getUTCMinutes().toString().padStart(2, "0");
return `${date}, ${hours}:${minutes} UTC`;
};
function printGlobalFlagOverrides(overrides, defaults, logger = console.log) {
if (Object.keys(overrides).length === 0) {
logger("");
logger(u.dim("Flag overrides: (none)"));
logger("");
return;
}
logger("");
logger("Flag overrides:");
for (const [key, value] of Object.entries(overrides)) {
const defaultValue = defaults[key];
const valueStr = JSON.stringify(value);
const defaultStr = defaultValue !== void 0 ? JSON.stringify(defaultValue) : "none";
logger(` \u2022 ${key}: ${valueStr} ${u.dim(`(default: ${defaultStr})`)}`);
}
logger("");
}
function printSuiteBox({
suite,
scorerAverages,
calculateBaselineScorerAverage: calculateBaselineScorerAverage2,
flagDiff,
logger = console.log
}) {
const filename = suite.file.split("/").pop();
logger("\u250C\u2500");
logger(`\u2502 ${u.blue(suite.name)} ${u.gray(`(${filename})`)}`);
logger("\u251C\u2500");
const scorerNames = Object.keys(scorerAverages);
const maxNameLength = Math.max(...scorerNames.map((name) => name.length));
const allCasesErrored = (scorerName) => {
return suite.cases.every((caseData) => caseData.scores[scorerName]?.metadata?.error);
};
for (const scorerName of scorerNames) {
const avg = scorerAverages[scorerName];
const paddedName = scorerName.padEnd(maxNameLength);
const hasAllErrors = allCasesErrored(scorerName);
const baselineAvg = suite.baseline ? calculateBaselineScorerAverage2(suite.baseline, scorerName) : null;
if (baselineAvg !== null) {
const currentPercent = hasAllErrors ? u.dim("N/A") : formatPercentage(avg);
const baselinePercent = formatPercentage(baselineAvg);
const { text: diffText, color: diffColor } = formatDiff(avg, baselineAvg);
const paddedBaseline = baselinePercent.padStart(7);
const paddedCurrent = hasAllErrors ? currentPercent : currentPercent.padStart(7);
const diffDisplay = hasAllErrors ? u.dim("all cases failed") : diffColor(diffText.padStart(8));
logger(
`\u2502 ${paddedName} ${u.blueBright(paddedBaseline)} \u2192 ${hasAllErrors ? paddedCurrent : u.magentaBright(paddedCurrent)} (${diffDisplay})`
);
} else {
const currentPercent = hasAllErrors ? u.red("N/A (all cases failed)") : formatPercentage(avg);
logger(`\u2502 \u2022 ${paddedName} ${currentPercent}`);
}
}
logger("\u251C\u2500");
if (suite.baseline) {
const baselineTimestamp = suite.baseline.runAt ? reporterDate(new Date(suite.baseline.runAt)) : "unknown time";
logger(
`\u2502 Baseline: ${suite.baseline.name}-${suite.baseline.version} ${u.gray(`(${baselineTimestamp})`)}`
);
} else {
logger(`\u2502 Baseline: ${u.gray("(none)")}`);
}
const hasConfigChanges = flagDiff.length > 0;
logger("\u2502 Config changes:", hasConfigChanges ? "" : u.gray("(none)"));
if (hasConfigChanges) {
for (const { flag, current, baseline, default: defaultVal } of flagDiff) {
logger(`\u2502 \u2022 ${flag}: ${current ?? "<not set>"}`);
if (defaultVal !== void 0) {
logger(`\u2502 ${u.gray(`default: ${defaultVal}`)}`);
}
if (suite.baseline) {
logger(`\u2502 ${u.gray(`baseline: ${baseline ?? "<not set>"}`)}`);
}
}
}
if (suite.outOfScopeFlags && suite.outOfScopeFlags.length > 0) {
const pickedFlagsText = suite.configFlags && suite.configFlags.length > 0 ? suite.configFlags.map((f) => `'${f}'`).join(", ") : "none";
logger("\u2502");
logger(`\u2502 ${u.yellow("\u26A0 Out-of-scope flags")} ${u.gray(`(picked: ${pickedFlagsText})`)}:`);
for (const flag of suite.outOfScopeFlags) {
const lastStackTraceFrame = flag.stackTrace[0];
const lastStackTraceFnName = lastStackTraceFrame.split(" ").shift();
const lastStackTraceFile = lastStackTraceFrame.split("/").pop()?.slice(0, -1);
logger(
`\u2502 \u2022 ${flag.flagPath} ${u.gray(`at ${lastStackTraceFnName} (${lastStackTraceFile})`)}`
);
}
}
logger("\u2514\u2500");
}
function calculateScorerAverages(suite) {
const scorerTotals = {};
for (const caseData of suite.cases) {
for (const [scorerName, score] of Object.entries(caseData.scores)) {
if (!scorerTotals[scorerName]) {
scorerTotals[scorerName] = { sum: 0, count: 0 };
}
if (!score.metadata?.error) {
scorerTotals[scorerName].sum += scoreToNumber(score.score);
scorerTotals[scorerName].count += 1;
}
}
}
const averages = {};
for (const [scorerName, totals] of Object.entries(scorerTotals)) {
averages[scorerName] = totals.count > 0 ? totals.sum / totals.count : 0;
}
return averages;
}
function calculateBaselineScorerAverage(baseline, scorerName) {
const scores = [];
for (const caseData of baseline.cases) {
if (caseData.scores[scorerName]) {
scores.push(caseData.scores[scorerName].value);
}
}
if (scores.length === 0) return null;
const sum = scores.reduce((acc, val) => acc + val, 0);
return sum / scores.length;
}
function calculateFlagDiff(suite) {
if (!suite.configFlags || suite.configFlags.length === 0) {
return [];
}
const diffs = [];
const currentConfig = suite.flagConfig || {};
const baselineConfig = suite.baseline?.flagConfig || {};
const defaultConfig = suite.defaultFlagConfig || {};
const currentFlat = flattenObject(currentConfig);
const baselineFlat = flattenObject(baselineConfig);
const defaultFlat = flattenObject(defaultConfig);
const allKeys = /* @__PURE__ */ new Set([
...Object.keys(currentFlat),
...Object.keys(baselineFlat),
...Object.keys(defaultFlat)
]);
for (const key of allKeys) {
const isInScope = suite.configFlags.some((pattern) => key.startsWith(pattern));
if (!isInScope) continue;
const currentValue = currentFlat[key];
const baselineValue = baselineFlat[key];
const defaultValue = defaultFlat[key];
const currentStr = currentValue !== void 0 ? JSON.stringify(currentValue) : void 0;
const baselineStr = baselineValue !== void 0 ? JSON.stringify(baselineValue) : void 0;
const defaultStr = defaultValue !== void 0 ? JSON.stringify(defaultValue) : void 0;
const diffFromBaseline = suite.baseline && currentStr !== baselineStr;
const diffFromDefault = currentStr !== defaultStr;
if (diffFromBaseline || diffFromDefault) {
diffs.push({
flag: key,
current: currentStr,
baseline: suite.baseline ? baselineStr : void 0,
default: defaultStr
});
}
}
return diffs;
}
function printFinalReport({
suiteData,
config,
registrationStatus,
isDebug,
logger = console.log
}) {
logger("");
logger(u.bgBlue(u.white(" FINAL EVALUATION REPORT ")));
logger("");
const runId = suiteData[0]?.runId;
const orgId = suiteData[0]?.orgId;
const anyRegistered = registrationStatus.some((s) => s.registered);
const anyFailed = registrationStatus.some((s) => !s.registered);
const allFailed = registrationStatus.length > 0 && registrationStatus.every((s) => !s.registered);
const hasAnyScores = suiteData.some(
(suite) => suite.cases.some((caseData) => Object.keys(caseData.scores ?? {}).length > 0)
);
const shouldPrintSuiteBoxes = isDebug || !allFailed || hasAnyScores;
if (shouldPrintSuiteBoxes) {
for (const suite of suiteData) {
const scorerAverages = calculateScorerAverages(suite);
const flagDiff = calculateFlagDiff(suite);
printSuiteBox({
suite,
scorerAverages,
calculateBaselineScorerAverage,
flagDiff,
logger
});
logger("");
}
}
if (anyRegistered && orgId && config?.consoleEndpointUrl) {
if (suiteData.length === 1) {
const suite = suiteData[0];
const baselineParam = suite.baseline?.traceId ? `?baselineId=${suite.baseline.traceId}` : "";
logger("View eval result:");
logger(
`${config.consoleEndpointUrl}/${orgId}/ai/evaluations/${suite.name}/${suite.version}${baselineParam}`
);
} else {
logger("View full report:");
logger(`${config.consoleEndpointUrl}/${orgId}/ai/evaluations?runId=${runId}`);
}
} else if (isDebug) {
logger(u.dim("Results not uploaded to Axiom (debug mode)"));
} else {
logger("Results not available in Axiom UI (registration failed)");
}
if (anyFailed) {
logger("");
for (const status of registrationStatus) {
if (!status.registered) {
logger(u.yellow(`\u26A0\uFE0F Warning: Failed to register "${status.name}" with Axiom`));
if (status.error) {
logger(u.dim(` Error: ${status.error}`));
}
logger(u.dim(` Results for this evaluation will not be available in the Axiom UI.`));
}
}
}
}
// src/config/resolver.ts
var buildConsoleUrl = (urlString) => {
const url = new URL(urlString);
return `${url.protocol}//app.${url.host.split("api.").at(-1)}`;
};
function resolveAxiomConnection(config, consoleUrlOverride) {
const consoleEndpointUrl = consoleUrlOverride ?? buildConsoleUrl(config.eval.url);
const edgeUrl = config.eval.edgeUrl || config.eval.url;
return {
url: config.eval.url,
edgeUrl,
consoleEndpointUrl,
token: config.eval.token,
dataset: config.eval.dataset,
orgId: config.eval.orgId
};
}
// src/evals/reporter.ts
var AxiomReporter = class {
constructor() {
__publicField(this, "startTime", 0);
__publicField(this, "start", 0);
__publicField(this, "_endOfRunConfigEnd");
__publicField(this, "_suiteData", []);
__publicField(this, "_printedFlagOverrides", false);
__publicField(this, "_config");
}
onTestRunStart() {
this.start = performance.now();
this.startTime = (/* @__PURE__ */ new Date()).getTime();
const config = getAxiomConfig();
if (config) {
this._config = resolveAxiomConnection(config, getConsoleUrl());
}
}
async onTestSuiteReady(_testSuite) {
const meta = _testSuite.meta();
if (_testSuite.state() === "skipped" || !meta?.evaluation) {
return;
}
if (!this._printedFlagOverrides) {
const defaultsFromConfigEnd = meta.evaluation.configEnd?.flags ?? {};
const overridesFromConfigEnd = meta.evaluation.configEnd?.overrides ?? {};
if (Object.keys(overridesFromConfigEnd).length > 0) {
printGlobalFlagOverrides(overridesFromConfigEnd, defaultsFromConfigEnd);
}
this._printedFlagOverrides = true;
}
if (meta.evaluation.configEnd && !this._endOfRunConfigEnd) {
this._endOfRunConfigEnd = meta.evaluation.configEnd;
}
}
onTestCaseReady(test) {
const meta = test.meta();
if (!meta.case) return;
}
async onTestSuiteResult(testSuite) {
const meta = testSuite.meta();
if (testSuite.state() === "skipped" || !meta?.evaluation) {
return;
}
if (meta.evaluation.configEnd && !this._endOfRunConfigEnd) {
this._endOfRunConfigEnd = meta.evaluation.configEnd;
}
const durationSeconds = Number((performance.now() - this.start) / 1e3).toFixed(2);
const cases = [];
for (const test of testSuite.children) {
if (test.type !== "test") continue;
const testMeta = test.meta();
if (!testMeta?.case) continue;
cases.push({
index: testMeta.case.index,
scores: testMeta.case.scores,
outOfScopeFlags: testMeta.case.outOfScopeFlags,
errors: testMeta.case.errors,
runtimeFlags: testMeta.case.runtimeFlags
});
}
const cwd = process.cwd();
const relativePath = testSuite.module.moduleId.replace(cwd, "").replace(/^\//, "");
let suiteBaseline = meta.evaluation.baseline;
let flagConfig = meta.evaluation.flagConfig;
if (meta.evaluation.configEnd) {
const defaults = meta.evaluation.configEnd.flags ?? {};
const overrides = meta.evaluation.configEnd.overrides ?? {};
const defaultsFlat = flattenObject(defaults);
const overridesFlat = flattenObject(overrides);
flagConfig = dotNotationToNested({ ...defaultsFlat, ...overridesFlat });
}
const defaultFlagConfig = meta.evaluation.configEnd?.flags;
this._suiteData.push({
version: meta.evaluation.version,
name: meta.evaluation.name,
file: relativePath,
duration: durationSeconds + "s",
baseline: suiteBaseline || null,
configFlags: meta.evaluation.configFlags,
flagConfig,
defaultFlagConfig,
runId: meta.evaluation.runId,
orgId: meta.evaluation.orgId,
cases,
outOfScopeFlags: meta.evaluation.outOfScopeFlags,
registrationStatus: meta.evaluation.registrationStatus
});
printEvalNameAndFileName(testSuite, meta);
printBaselineNameAndVersion(meta);
printTestCaseCountStartDuration(
testSuite,
this.startTime,
durationSeconds,
meta.evaluation.trials
);
const matchedBaselineIndices = /* @__PURE__ */ new Set();
const baselineCasesByFingerprint = /* @__PURE__ */ new Map();
if (suiteBaseline) {
for (const c of suiteBaseline.cases) {
const fp = getCaseFingerprint(c.input, c.expected);
const cases2 = baselineCasesByFingerprint.get(fp) || [];
cases2.push(c);
baselineCasesByFingerprint.set(fp, cases2);
}
}
for (const test of testSuite.children) {
if (test.type !== "test") continue;
printCaseResult(test, baselineCasesByFingerprint, matchedBaselineIndices);
}
if (suiteBaseline) {
printOrphanedBaselineCases(suiteBaseline, matchedBaselineIndices);
}
console.log("");
}
async onTestRunEnd(_testModules, _errors, _reason) {
const shouldClear = !process.env.CI && process.stdout.isTTY !== false;
if (shouldClear) {
process.stdout.write("\x1B[2J\x1B[0f");
}
const registrationStatus = this._suiteData.map((suite) => ({
name: suite.name,
registered: suite.registrationStatus?.status === "success",
error: suite.registrationStatus?.status === "failed" ? suite.registrationStatus.error : void 0
}));
const isDebug = process.env.AXIOM_DEBUG === "true";
printFinalReport({
suiteData: this._suiteData,
config: this._config,
registrationStatus,
isDebug
});
}
};
// src/evals/instrument.ts
import { BatchSpanProcessor, NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import {
context,
trace,
SpanStatusCode
} from "@opentelemetry/api";
var axiomProvider;
var axiomTracer;
var userProvider;
var initializationPromise = null;
var initialized = false;
async function resolveInstrumentationHook(config) {
if (config.eval.instrumentation) {
return config.eval.instrumentation;
}
try {
const { config: loadedConfig } = await loadConfig(process.cwd());
return loadedConfig.eval.instrumentation ?? null;
} catch (error) {
throw new AxiomCLIError(
`Failed to reload instrumentation from config: ${errorToString(error)}`
);
}
}
async function runInstrumentationHook(hook, options) {
try {
return await hook(options);
} catch (error) {
throw new AxiomCLIError(`Failed to execute instrumentation hook: ${errorToString(error)}`);
}
}
function setupEvalProvider(connection) {
const headers = {
"X-Axiom-Dataset": connection.dataset,
...connection.orgId ? { "X-AXIOM-ORG-ID": connection.orgId } : {}
};
if (connection.token) {
headers.Authorization = `Bearer ${connection.token}`;
}
const collectorOptions = {
url: `${connection.edgeUrl}/v1/traces`,
headers,
concurrencyLimit: 10
};
const exporter = new OTLPTraceExporter(collectorOptions);
const processor = new BatchSpanProcessor(exporter, {
maxQueueSize: 2048,
maxExportBatchSize: 512,
scheduledDelayMillis: 5e3,
exportTimeoutMillis: 3e4
});
axiomProvider = new NodeTracerProvider({
resource: resourceFromAttributes({
["service.name"]: "axiom",
["service.version"]: "0.51.1"
}),
spanProcessors: [processor]
});
axiomTracer = axiomProvider.getTracer("axiom", "0.51.1");
}
async function initInstrumentation(config) {
if (initialized) {
return;
}
if (initializationPromise) {
await initializationPromise;
return;
}
initializationPromise = (async () => {
if (!config.enabled) {
axiomTracer = trace.getTracer("axiom", "0.51.1");
initialized = true;
return;
}
const connection = resolveAxiomConnection(config.config);
const hook = await resolveInstrumentationHook(config.config);
let hookResult = void 0;
if (hook) {
config.config.eval.instrumentation = hook;
hookResult = await runInstrumentationHook(hook, {
dataset: connection.dataset,
token: connection.token,
url: connection.url,
edgeUrl: connection.edgeUrl,
orgId: connection.orgId
});
userProvider = hookResult?.provider ?? userProvider;
}
setupEvalProvider(connection);
if (!hook) {
axiomProvider?.register();
if (axiomTracer) {
initAxiomAI({ tracer: axiomTracer });
}
}
initialized = true;
})();
try {
await initializationPromise;
} finally {
initializationPromise = null;
}
}
var flush = async () => {
if (initializationPromise) {
await initializationPromise;
}
const tasks = [];
if (axiomProvider) {
tasks.push(axiomProvider.forceFlush());
}
const candidateProviders = /* @__PURE__ */ new Set();
if (userProvider) {
candidateProviders.add(userProvider);
}
const globalProvider = trace.getTracerProvider();
if (globalProvider) {
candidateProviders.add(globalProvider);
}
for (const provider of candidateProviders) {
const flushFn = provider.forceFlush;
if (typeof flushFn === "function") {
tasks.push(
flushFn.call(provider).catch((error) => {
console.warn("[AxiomAI] Failed to flush tracer provider:", errorToString(error));
})
);
}
}
if (tasks.length > 0) {
await Promise.all(tasks);
}
};
async function ensureInstrumentationInitialized(config, options = {}) {
if (initialized) {
return;
}
const enabled = options.enabled ?? true;
await initInstrumentation({ enabled, config });
}
var getEvalTracer = () => {
if (!axiomTracer) {
throw new Error(
"Eval tracer not initialized. Ensure ensureInstrumentationInitialized() was awaited before starting spans."
);
}
return axiomTracer;
};
var startSpan = (name, opts, context2) => {
if (!initialized || !axiomTracer) {
throw new Error(
"Instrumentation not initialized. This is likely a bug - instrumentation should be initialized before startSpan is called."
);
}
return getEvalTracer().startSpan(name, opts, context2);
};
var startActiveSpan = async (name, options, fn, parentContext) => {
const span = startSpan(name, options, parentContext);
return context.with(trace.setSpan(context.active(), span), async () => {
try {
const result = await fn(span);
span.setStatus({ code: SpanStatusCode.OK });
return result;
} catch (error) {
span.recordException(error);
span.setStatus({
code: SpanStatusCode.ERROR,
message: error instanceof Error ? error.message : String(error)
});
throw error;
} finally {
span.end();
}
});
};
export {
resolveAxiomConnection,
AxiomReporter,
initInstrumentation,
flush,
ensureInstrumentationInitialized,
startSpan,
startActiveSpan
};
//# sourceMappingURL=chunk-SQJ53C2N.js.map