axiom
Version:
Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals
1,578 lines (1,529 loc) • 105 kB
JavaScript
"use strict";
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __esm = (fn, res) => function __init() {
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
};
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
// ../../node_modules/.pnpm/tsup@8.5.0_jiti@2.6.1_postcss@8.5.6_tsx@4.20.4_typescript@5.9.2/node_modules/tsup/assets/cjs_shims.js
var getImportMetaUrl, importMetaUrl;
var init_cjs_shims = __esm({
"../../node_modules/.pnpm/tsup@8.5.0_jiti@2.6.1_postcss@8.5.6_tsx@4.20.4_typescript@5.9.2/node_modules/tsup/assets/cjs_shims.js"() {
"use strict";
getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.src || new URL("main.js", document.baseURI).href;
importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
}
});
// src/evals/context/manager.ts
function getGlobalContextManager() {
return globalThis[CONTEXT_MANAGER_SYMBOL];
}
function setGlobalContextManager(manager) {
globalThis[CONTEXT_MANAGER_SYMBOL] = manager;
}
function getContextManager() {
const existing = getGlobalContextManager();
if (existing) return existing;
let manager;
if (isNodeJS) {
try {
let AsyncLocalStorage;
const req = (0, import_node_module.createRequire)(importMetaUrl);
try {
AsyncLocalStorage = req("node:async_hooks").AsyncLocalStorage;
} catch {
AsyncLocalStorage = req("async_hooks").AsyncLocalStorage;
}
manager = new AsyncLocalStorage();
} catch (error) {
console.warn("AsyncLocalStorage not available, using fallback context manager:", error);
manager = createFallbackManager();
}
} else {
console.warn("AsyncLocalStorage not available, using fallback context manager");
manager = createFallbackManager();
}
setGlobalContextManager(manager);
return manager;
}
function createFallbackManager() {
let currentContext = null;
return {
getStore: () => currentContext,
run: (value, fn) => {
const prev = currentContext;
currentContext = value;
try {
return fn();
} finally {
currentContext = prev;
}
}
};
}
function createAsyncHook(_name) {
return {
get() {
const manager = getContextManager();
if (manager.getStore) {
return manager.getStore();
}
return void 0;
},
run(value, fn) {
const manager = getContextManager();
return manager.run(value, fn);
}
};
}
var import_node_module, CONTEXT_MANAGER_SYMBOL, isNodeJS;
var init_manager = __esm({
"src/evals/context/manager.ts"() {
"use strict";
init_cjs_shims();
import_node_module = require("module");
CONTEXT_MANAGER_SYMBOL = Symbol.for("axiom.context_manager");
isNodeJS = typeof process !== "undefined" && !!process.versions?.node;
}
});
// src/evals/context/global-flags.ts
function getRoot() {
return globalThis[GLOBAL_OVERRIDES_SYMBOL] ?? {};
}
function setRoot(val) {
globalThis[GLOBAL_OVERRIDES_SYMBOL] = val;
}
function setGlobalFlagOverrides(overrides) {
setRoot(overrides);
}
function getGlobalFlagOverrides() {
return getRoot();
}
var GLOBAL_OVERRIDES_SYMBOL;
var init_global_flags = __esm({
"src/evals/context/global-flags.ts"() {
"use strict";
init_cjs_shims();
GLOBAL_OVERRIDES_SYMBOL = Symbol.for("axiom.global_flag_overrides");
}
});
// src/cli/utils/format-zod-errors.ts
var import_zod;
var init_format_zod_errors = __esm({
"src/cli/utils/format-zod-errors.ts"() {
"use strict";
init_cjs_shims();
import_zod = require("zod");
}
});
// src/util/zod-internals.ts
var init_zod_internals = __esm({
"src/util/zod-internals.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/util/dot-path.ts
function parsePath(path) {
return path.split(".");
}
function dotNotationToNested(dotNotationObject) {
const result = {};
for (const [dotPath, value] of Object.entries(dotNotationObject)) {
const segments = parsePath(dotPath);
let current = result;
for (let i = 0; i < segments.length; i++) {
const segment = segments[i];
if (i === segments.length - 1) {
current[segment] = value;
} else {
if (!(segment in current) || typeof current[segment] !== "object") {
current[segment] = {};
}
current = current[segment];
}
}
}
return result;
}
function flattenObject(obj, prefix = "") {
const result = {};
for (const [key, value] of Object.entries(obj)) {
const newKey = prefix ? `${prefix}.${key}` : key;
if (value && typeof value === "object" && !Array.isArray(value)) {
Object.assign(result, flattenObject(value, newKey));
} else {
result[newKey] = value;
}
}
return result;
}
var import_zod2;
var init_dot_path = __esm({
"src/util/dot-path.ts"() {
"use strict";
init_cjs_shims();
import_zod2 = require("zod");
init_zod_internals();
}
});
// src/util/deep-partial-schema.ts
var import_zod3;
var init_deep_partial_schema = __esm({
"src/util/deep-partial-schema.ts"() {
"use strict";
init_cjs_shims();
import_zod3 = require("zod");
init_zod_internals();
}
});
// src/validate-flags.ts
var import_zod4;
var init_validate_flags = __esm({
"src/validate-flags.ts"() {
"use strict";
init_cjs_shims();
import_zod4 = require("zod");
init_global_flags();
init_format_zod_errors();
init_dot_path();
init_deep_partial_schema();
init_zod_internals();
}
});
// src/otel/utils/to-otel-attribute.ts
var import_api;
var init_to_otel_attribute = __esm({
"src/otel/utils/to-otel-attribute.ts"() {
"use strict";
init_cjs_shims();
import_api = require("@opentelemetry/api");
}
});
// src/otel/withSpanBaggageKey.ts
var init_withSpanBaggageKey = __esm({
"src/otel/withSpanBaggageKey.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/otel/semconv/eval_proposal.ts
var ATTR_EVAL_ID, ATTR_EVAL_NAME, ATTR_EVAL_VERSION, ATTR_EVAL_TYPE, ATTR_EVAL_TAGS, ATTR_EVAL_BASELINE_ID, ATTR_EVAL_BASELINE_NAME, ATTR_EVAL_BASELINE_VERSION, ATTR_EVAL_METADATA, ATTR_EVAL_CAPABILITY_NAME, ATTR_EVAL_STEP_NAME, ATTR_EVAL_COLLECTION_ID, ATTR_EVAL_COLLECTION_SIZE, ATTR_EVAL_COLLECTION_NAME, ATTR_EVAL_CONFIG_FLAGS, ATTR_EVAL_CASE_INDEX, ATTR_EVAL_CASE_INPUT, ATTR_EVAL_CASE_OUTPUT, ATTR_EVAL_CASE_EXPECTED, ATTR_EVAL_CASE_SCORES, ATTR_EVAL_CASE_METADATA, ATTR_EVAL_TASK_OUTPUT, ATTR_EVAL_TASK_NAME, ATTR_EVAL_TASK_TYPE, ATTR_EVAL_RUN_ID, ATTR_EVAL_SCORE_NAME, ATTR_EVAL_SCORE_VALUE, ATTR_EVAL_SCORE_THRESHOLD, ATTR_EVAL_SCORE_PASSED, ATTR_EVAL_SCORE_IS_BOOLEAN, ATTR_EVAL_SCORE_METADATA, ATTR_EVAL_USER_NAME, ATTR_EVAL_USER_EMAIL;
var init_eval_proposal = __esm({
"src/otel/semconv/eval_proposal.ts"() {
"use strict";
init_cjs_shims();
ATTR_EVAL_ID = "eval.id";
ATTR_EVAL_NAME = "eval.name";
ATTR_EVAL_VERSION = "eval.version";
ATTR_EVAL_TYPE = "eval.type";
ATTR_EVAL_TAGS = "eval.tags";
ATTR_EVAL_BASELINE_ID = "eval.baseline.id";
ATTR_EVAL_BASELINE_NAME = "eval.baseline.name";
ATTR_EVAL_BASELINE_VERSION = "eval.baseline.version";
ATTR_EVAL_METADATA = "eval.metadata";
ATTR_EVAL_CAPABILITY_NAME = "eval.capability.name";
ATTR_EVAL_STEP_NAME = "eval.step.name";
ATTR_EVAL_COLLECTION_ID = "eval.collection.id";
ATTR_EVAL_COLLECTION_SIZE = "eval.collection.size";
ATTR_EVAL_COLLECTION_NAME = "eval.collection.name";
ATTR_EVAL_CONFIG_FLAGS = "eval.config.flags";
ATTR_EVAL_CASE_INDEX = "eval.case.index";
ATTR_EVAL_CASE_INPUT = "eval.case.input";
ATTR_EVAL_CASE_OUTPUT = "eval.case.output";
ATTR_EVAL_CASE_EXPECTED = "eval.case.expected";
ATTR_EVAL_CASE_SCORES = "eval.case.scores";
ATTR_EVAL_CASE_METADATA = "eval.case.metadata";
ATTR_EVAL_TASK_OUTPUT = "eval.task.output";
ATTR_EVAL_TASK_NAME = "eval.task.name";
ATTR_EVAL_TASK_TYPE = "eval.task.type";
ATTR_EVAL_RUN_ID = "eval.run.id";
ATTR_EVAL_SCORE_NAME = "eval.score.name";
ATTR_EVAL_SCORE_VALUE = "eval.score.value";
ATTR_EVAL_SCORE_THRESHOLD = "eval.score.threshold";
ATTR_EVAL_SCORE_PASSED = "eval.score.passed";
ATTR_EVAL_SCORE_IS_BOOLEAN = "eval.score.is_boolean";
ATTR_EVAL_SCORE_METADATA = "eval.score.metadata";
ATTR_EVAL_USER_NAME = "eval.user.name";
ATTR_EVAL_USER_EMAIL = "eval.user.email";
}
});
// src/otel/semconv/attributes.ts
var import_semantic_conventions, import_incubating, ATTR_AXIOM_GEN_AI_SCHEMA_URL, ATTR_AXIOM_GEN_AI_SDK_NAME, ATTR_AXIOM_GEN_AI_SDK_VERSION, ATTR_GEN_AI_CAPABILITY_NAME, ATTR_GEN_AI_STEP_NAME, ATTR_GEN_AI_TOOL_ARGUMENTS, ATTR_GEN_AI_TOOL_MESSAGE, GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI, GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS, GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM, GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA, GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS, GEN_AI_PROVIDER_NAME_VALUE_FAL, GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS, GEN_AI_PROVIDER_NAME_VALUE_GLADIA, GEN_AI_PROVIDER_NAME_VALUE_HUME, GEN_AI_PROVIDER_NAME_VALUE_LMNT, GEN_AI_PROVIDER_NAME_VALUE_LUMA, GEN_AI_PROVIDER_NAME_VALUE_REPLICATE, GEN_AI_PROVIDER_NAME_VALUE_REVAI, GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI, GEN_AI_PROVIDER_NAME_VALUE_VERCEL, Attr;
var init_attributes = __esm({
"src/otel/semconv/attributes.ts"() {
"use strict";
init_cjs_shims();
import_semantic_conventions = require("@opentelemetry/semantic-conventions");
init_eval_proposal();
import_incubating = require("@opentelemetry/semantic-conventions/incubating");
ATTR_AXIOM_GEN_AI_SCHEMA_URL = "axiom.gen_ai.schema_url";
ATTR_AXIOM_GEN_AI_SDK_NAME = "axiom.gen_ai.sdk.name";
ATTR_AXIOM_GEN_AI_SDK_VERSION = "axiom.gen_ai.sdk.version";
ATTR_GEN_AI_CAPABILITY_NAME = "gen_ai.capability.name";
ATTR_GEN_AI_STEP_NAME = "gen_ai.step.name";
ATTR_GEN_AI_TOOL_ARGUMENTS = "gen_ai.tool.arguments";
ATTR_GEN_AI_TOOL_MESSAGE = "gen_ai.tool.message";
GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI = "assemblyai";
GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS = "cerebras";
GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM = "deepgram";
GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA = "deepinfra";
GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS = "elevenlabs";
GEN_AI_PROVIDER_NAME_VALUE_FAL = "fal";
GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS = "fireworks";
GEN_AI_PROVIDER_NAME_VALUE_GLADIA = "gladia";
GEN_AI_PROVIDER_NAME_VALUE_HUME = "hume";
GEN_AI_PROVIDER_NAME_VALUE_LMNT = "lmnt";
GEN_AI_PROVIDER_NAME_VALUE_LUMA = "luma";
GEN_AI_PROVIDER_NAME_VALUE_REPLICATE = "replicate";
GEN_AI_PROVIDER_NAME_VALUE_REVAI = "revai";
GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI = "togetherai";
GEN_AI_PROVIDER_NAME_VALUE_VERCEL = "vercel";
Attr = {
__EXPERIMENTAL_Flag: (flagName) => `flag.${flagName}`,
__EXPERIMENTAL_Fact: (factName) => `fact.${factName}`,
Axiom: {
GenAI: {
SchemaURL: ATTR_AXIOM_GEN_AI_SCHEMA_URL,
SDK: {
Name: ATTR_AXIOM_GEN_AI_SDK_NAME,
Version: ATTR_AXIOM_GEN_AI_SDK_VERSION
}
}
},
GenAI: {
PromptMetadata: {
ID: "axiom.gen_ai.prompt.id",
Name: "axiom.gen_ai.prompt.name",
Slug: "axiom.gen_ai.prompt.slug",
Version: "axiom.gen_ai.prompt.version"
},
/**
* These two are used to identify the span
*/
Capability: {
Name: ATTR_GEN_AI_CAPABILITY_NAME
},
Step: {
Name: ATTR_GEN_AI_STEP_NAME
},
Provider: {
Name: import_incubating.ATTR_GEN_AI_PROVIDER_NAME,
Name_Values: {
Anthropic: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC,
AssemblyAI: GEN_AI_PROVIDER_NAME_VALUE_ASSEMBLYAI,
AWSBedrock: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK,
AzureAIInference: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE,
AzureAIOpenAI: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI,
Cerebras: GEN_AI_PROVIDER_NAME_VALUE_CEREBRAS,
Cohere: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_COHERE,
Deepgram: GEN_AI_PROVIDER_NAME_VALUE_DEEPGRAM,
DeepInfra: GEN_AI_PROVIDER_NAME_VALUE_DEEPINFRA,
Deepseek: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK,
ElevenLabs: GEN_AI_PROVIDER_NAME_VALUE_ELEVENLABS,
Fal: GEN_AI_PROVIDER_NAME_VALUE_FAL,
Fireworks: GEN_AI_PROVIDER_NAME_VALUE_FIREWORKS,
GCPGemini: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI,
GCPGenAI: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI,
GCPVertexAI: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI,
Gladia: GEN_AI_PROVIDER_NAME_VALUE_GLADIA,
Groq: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_GROQ,
Hume: GEN_AI_PROVIDER_NAME_VALUE_HUME,
IBMWatsonxAI: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI,
Lmnt: GEN_AI_PROVIDER_NAME_VALUE_LMNT,
Luma: GEN_AI_PROVIDER_NAME_VALUE_LUMA,
MistralAI: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI,
OpenAI: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_OPENAI,
Perplexity: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY,
Replicate: GEN_AI_PROVIDER_NAME_VALUE_REPLICATE,
RevAI: GEN_AI_PROVIDER_NAME_VALUE_REVAI,
TogetherAI: GEN_AI_PROVIDER_NAME_VALUE_TOGETHERAI,
Vercel: GEN_AI_PROVIDER_NAME_VALUE_VERCEL,
XAI: import_incubating.GEN_AI_PROVIDER_NAME_VALUE_X_AI
}
},
/**
* Regular attributes
*/
Agent: {
Description: import_incubating.ATTR_GEN_AI_AGENT_DESCRIPTION,
// not yet used by axiom-ai
ID: import_incubating.ATTR_GEN_AI_AGENT_ID,
// not yet used by axiom-ai
Name: import_incubating.ATTR_GEN_AI_AGENT_NAME
// not yet used by axiom-ai
},
Conversation: {
ID: import_incubating.ATTR_GEN_AI_CONVERSATION_ID
// not yet used by axiom-ai, anyway probably needs to be provided by user
},
Input: {
Messages: import_incubating.ATTR_GEN_AI_INPUT_MESSAGES
},
Operation: {
Name: import_incubating.ATTR_GEN_AI_OPERATION_NAME,
Name_Values: {
/**
* Note that "text_completion" is deprecated in favor of "chat" for both OpenAI and Anthropic
*/
Chat: import_incubating.GEN_AI_OPERATION_NAME_VALUE_CHAT,
CreateAgent: import_incubating.GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT,
Embeddings: import_incubating.GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS,
ExecuteTool: import_incubating.GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL,
GenerateContent: import_incubating.GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT,
InvokeAgent: import_incubating.GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT
}
},
Output: {
Messages: import_incubating.ATTR_GEN_AI_OUTPUT_MESSAGES,
Type: import_incubating.ATTR_GEN_AI_OUTPUT_TYPE,
Type_Values: {
Text: import_incubating.GEN_AI_OUTPUT_TYPE_VALUE_TEXT,
Json: import_incubating.GEN_AI_OUTPUT_TYPE_VALUE_JSON,
Image: import_incubating.GEN_AI_OUTPUT_TYPE_VALUE_IMAGE,
Speech: import_incubating.GEN_AI_OUTPUT_TYPE_VALUE_SPEECH
}
},
/**
* The provider that is hosting the model, eg AWS Bedrock
* There doesn't seem to be a semconv for this
*/
Request: {
ChoiceCount: import_incubating.ATTR_GEN_AI_REQUEST_CHOICE_COUNT,
// not yet used by axiom-ai
EncodingFormats: import_incubating.ATTR_GEN_AI_REQUEST_ENCODING_FORMATS,
// not yet used by axiom-ai
FrequencyPenalty: import_incubating.ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY,
MaxTokens: import_incubating.ATTR_GEN_AI_REQUEST_MAX_TOKENS,
/**
* The model you asked for
*/
Model: import_incubating.ATTR_GEN_AI_REQUEST_MODEL,
PresencePenalty: import_incubating.ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY,
Seed: import_incubating.ATTR_GEN_AI_REQUEST_SEED,
StopSequences: import_incubating.ATTR_GEN_AI_REQUEST_STOP_SEQUENCES,
Temperature: import_incubating.ATTR_GEN_AI_REQUEST_TEMPERATURE,
TopK: import_incubating.ATTR_GEN_AI_REQUEST_TOP_K,
TopP: import_incubating.ATTR_GEN_AI_REQUEST_TOP_P
},
Response: {
FinishReasons: import_incubating.ATTR_GEN_AI_RESPONSE_FINISH_REASONS,
ID: import_incubating.ATTR_GEN_AI_RESPONSE_ID,
/**
* The model that was actually used (might be different bc routing) - only ever get this from the response, otherwise omit
*/
Model: import_incubating.ATTR_GEN_AI_RESPONSE_MODEL
// somehow not landing on the span for google models? check up on this...
},
Tool: {
CallID: import_incubating.ATTR_GEN_AI_TOOL_CALL_ID,
Description: import_incubating.ATTR_GEN_AI_TOOL_DESCRIPTION,
Name: import_incubating.ATTR_GEN_AI_TOOL_NAME,
Type: import_incubating.ATTR_GEN_AI_TOOL_TYPE,
/**
* Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span
* But we at least want to give users THE OPTION to put them on the tool spans themselves as well
* Because it enables a lot of things with querying
* @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0
*/
Arguments: ATTR_GEN_AI_TOOL_ARGUMENTS,
/**
* Note, OTel Semantic Convention suggest only putting tool inputs/outputs on the parent chat span
* But we at least want to give users THE OPTION to put them on the tool spans themselves as well
* Because it enables a lot of things with querying
* @see https://github.com/open-telemetry/semantic-conventions/releases/tag/v1.37.0
*/
Message: ATTR_GEN_AI_TOOL_MESSAGE
},
Usage: {
InputTokens: import_incubating.ATTR_GEN_AI_USAGE_INPUT_TOKENS,
OutputTokens: import_incubating.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS
}
},
Eval: {
ID: ATTR_EVAL_ID,
Name: ATTR_EVAL_NAME,
Version: ATTR_EVAL_VERSION,
Type: ATTR_EVAL_TYPE,
Baseline: {
ID: ATTR_EVAL_BASELINE_ID,
Name: ATTR_EVAL_BASELINE_NAME,
Version: ATTR_EVAL_BASELINE_VERSION
},
Capability: {
Name: ATTR_EVAL_CAPABILITY_NAME
},
Step: {
Name: ATTR_EVAL_STEP_NAME
},
Tags: ATTR_EVAL_TAGS,
Metadata: ATTR_EVAL_METADATA,
Collection: {
ID: ATTR_EVAL_COLLECTION_ID,
Name: ATTR_EVAL_COLLECTION_NAME,
Size: ATTR_EVAL_COLLECTION_SIZE
},
Config: {
Flags: ATTR_EVAL_CONFIG_FLAGS
},
Run: {
ID: ATTR_EVAL_RUN_ID
},
Case: {
Index: ATTR_EVAL_CASE_INDEX,
Input: ATTR_EVAL_CASE_INPUT,
Output: ATTR_EVAL_CASE_OUTPUT,
Expected: ATTR_EVAL_CASE_EXPECTED,
Scores: ATTR_EVAL_CASE_SCORES,
Metadata: ATTR_EVAL_CASE_METADATA
},
Task: {
Output: ATTR_EVAL_TASK_OUTPUT,
Name: ATTR_EVAL_TASK_NAME,
Type: ATTR_EVAL_TASK_TYPE
},
Score: {
Name: ATTR_EVAL_SCORE_NAME,
Value: ATTR_EVAL_SCORE_VALUE,
Threshold: ATTR_EVAL_SCORE_THRESHOLD,
Passed: ATTR_EVAL_SCORE_PASSED,
IsBoolean: ATTR_EVAL_SCORE_IS_BOOLEAN,
Metadata: ATTR_EVAL_SCORE_METADATA
},
User: {
Name: ATTR_EVAL_USER_NAME,
Email: ATTR_EVAL_USER_EMAIL
}
},
Error: {
Type: import_semantic_conventions.ATTR_ERROR_TYPE,
Message: import_incubating.ATTR_ERROR_MESSAGE
},
HTTP: {
Response: {
StatusCode: import_semantic_conventions.ATTR_HTTP_RESPONSE_STATUS_CODE
}
}
};
}
});
// src/schema.ts
var SCHEMA_VERSION, SCHEMA_BASE_URL, SCHEMA_URL;
var init_schema = __esm({
"src/schema.ts"() {
"use strict";
init_cjs_shims();
SCHEMA_VERSION = "0.0.2";
SCHEMA_BASE_URL = "https://axiom.co/ai/schemas/";
SCHEMA_URL = `${SCHEMA_BASE_URL}${SCHEMA_VERSION}`;
}
});
// src/otel/startActiveSpan.ts
var import_api2;
var init_startActiveSpan = __esm({
"src/otel/startActiveSpan.ts"() {
"use strict";
init_cjs_shims();
import_api2 = require("@opentelemetry/api");
}
});
// package.json
var package_default;
var init_package = __esm({
"package.json"() {
package_default = {
name: "axiom",
version: "0.37.0",
type: "module",
author: "Axiom, Inc.",
contributors: [
"Islam Shehata <islam@axiom.co>",
"Chris Ehrlich <chris@axiom.co>",
"Gabriel de Andrade <gabriel@axiom.co>"
],
scripts: {
dev: "tsup --watch",
build: "tsup && chmod +x dist/bin.js",
format: "prettier --write .",
"format:check": "prettier --check .",
lint: "eslint './**/*.{js,ts}'",
typecheck: "tsc --noEmit",
test: "vitest run",
"test:watch": "vitest --watch",
publint: "npx publint"
},
types: "./dist/index.d.ts",
main: "./dist/index.cjs",
module: "./dist/index.js",
bin: {
axiom: "./dist/bin.js"
},
exports: {
"./ai": {
import: {
types: "./dist/index.d.ts",
default: "./dist/index.js"
},
require: {
types: "./dist/index.d.cts",
default: "./dist/index.cjs"
}
},
"./ai/evals": {
import: {
types: "./dist/evals.d.ts",
default: "./dist/evals.js"
},
require: {
types: "./dist/evals.d.cts",
default: "./dist/evals.cjs"
}
},
"./ai/config": {
import: {
types: "./dist/config.d.ts",
default: "./dist/config.js"
},
require: {
types: "./dist/config.d.cts",
default: "./dist/config.cjs"
}
},
"./ai/feedback": {
import: {
types: "./dist/feedback.d.ts",
default: "./dist/feedback.js"
},
require: {
types: "./dist/feedback.d.cts",
default: "./dist/feedback.cjs"
}
}
},
keywords: [
"axiom",
"logging",
"ai",
"otel",
"opentelemetry"
],
repository: {
type: "git",
url: "git+https://github.com/axiomhq/ai.git",
directory: "packages/ai"
},
license: "MIT",
dependencies: {
"@next/env": "^15.4.2",
"@opentelemetry/auto-instrumentations-node": "^0.60.1",
"@opentelemetry/context-async-hooks": "^2.0.1",
"@opentelemetry/exporter-trace-otlp-http": "^0.202.0",
"@opentelemetry/resources": "^2.0.1",
"@opentelemetry/sdk-trace-node": "^2.0.1",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@sinclair/typebox": "^0.34.37",
c12: "^2.0.4",
commander: "^14.0.0",
defu: "^6.1.4",
handlebars: "^4.7.8",
nanoid: "^5.1.5",
open: "^10.1.0",
"vite-tsconfig-paths": "^5.1.4",
vitest: "^4.0.0"
},
peerDependencies: {
"@opentelemetry/api": "^1.9.0",
zod: "^3.25.0 || ^4.0.0"
},
devDependencies: {
"@ai-sdk/anthropicv1": "npm:@ai-sdk/anthropic@^1.2.12",
"@ai-sdk/anthropicv2": "npm:@ai-sdk/anthropic@^2.0.44",
"@ai-sdk/openaiv1": "npm:@ai-sdk/openai@^1.3.24",
"@ai-sdk/openaiv2": "npm:@ai-sdk/openai@^2.0.67",
"@ai-sdk/providerv1": "npm:@ai-sdk/provider@^1.1.3",
"@ai-sdk/providerv2": "npm:@ai-sdk/provider@^2.0.0",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/sdk-trace-base": "^2.0.1",
"@opentelemetry/sdk-trace-node": "^2.0.1",
"@repo/eslint-config": "workspace:*",
"@types/node": "^22.15.29",
"@vitest/coverage-v8": "^4.0.0",
aiv4: "npm:ai@^4.3.19",
aiv5: "npm:ai@^5.0.93",
esbuild: "^0.25.8",
eslint: "catalog:",
msw: "^2.12.2",
prettier: "catalog:",
tinyrainbow: "^2.0.0",
tsup: "catalog:",
typescript: "catalog:",
vitest: "catalog:",
zod: "catalog:"
},
files: [
"dist"
],
packageManager: "pnpm@10.16.1"
};
}
});
// src/otel/utils/redaction.ts
var import_api3, AXIOM_AI_REDACTION_KEY;
var init_redaction = __esm({
"src/otel/utils/redaction.ts"() {
"use strict";
init_cjs_shims();
import_api3 = require("@opentelemetry/api");
init_withSpanBaggageKey();
AXIOM_AI_REDACTION_KEY = Symbol.for("__axiom_ai_redaction__");
}
});
// src/otel/initAxiomAI.ts
function extractTracerScope(tracer) {
const tracerAny = tracer;
const name = tracerAny._instrumentationScope?.name || tracerAny.instrumentationLibrary?.name || package_default.name;
const version = tracerAny._instrumentationScope?.version || tracerAny.instrumentationLibrary?.version || package_default.version;
return { name, version };
}
function initAxiomAI(config) {
const newScope = extractTracerScope(config.tracer);
const existingScope = globalThis[AXIOM_AI_SCOPE_KEY];
if (existingScope && existingScope.name === newScope.name && existingScope.version === newScope.version) {
return;
}
if (existingScope) {
console.warn(
`[AxiomAI] initAxiomAI() called multiple times with different scopes. Previous: ${existingScope.name}@${existingScope.version}, New: ${newScope.name}@${newScope.version}`
);
}
globalThis[AXIOM_AI_SCOPE_KEY] = newScope;
if (config.redactionPolicy) {
globalThis[AXIOM_AI_REDACTION_KEY] = config.redactionPolicy;
}
}
var import_api4, AXIOM_AI_SCOPE_KEY;
var init_initAxiomAI = __esm({
"src/otel/initAxiomAI.ts"() {
"use strict";
init_cjs_shims();
import_api4 = require("@opentelemetry/api");
init_package();
init_redaction();
AXIOM_AI_SCOPE_KEY = Symbol.for("__axiom_ai_scope__");
}
});
// src/otel/utils/wrapperUtils.ts
var import_api5;
var init_wrapperUtils = __esm({
"src/otel/utils/wrapperUtils.ts"() {
"use strict";
init_cjs_shims();
import_api5 = require("@opentelemetry/api");
init_attributes();
init_schema();
init_withSpanBaggageKey();
init_startActiveSpan();
init_initAxiomAI();
}
});
// src/util/name-validation-runtime.ts
var init_name_validation_runtime = __esm({
"src/util/name-validation-runtime.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/otel/withSpan.ts
var import_api6;
var init_withSpan = __esm({
"src/otel/withSpan.ts"() {
"use strict";
init_cjs_shims();
import_api6 = require("@opentelemetry/api");
init_withSpanBaggageKey();
init_wrapperUtils();
init_name_validation_runtime();
}
});
// src/otel/utils/contentSanitizer.ts
var init_contentSanitizer = __esm({
"src/otel/utils/contentSanitizer.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/otel/completionUtils.ts
var init_completionUtils = __esm({
"src/otel/completionUtils.ts"() {
"use strict";
init_cjs_shims();
init_contentSanitizer();
}
});
// src/util/promptUtils.ts
var init_promptUtils = __esm({
"src/util/promptUtils.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/otel/utils/normalized.ts
var init_normalized = __esm({
"src/otel/utils/normalized.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/util/currentUnixTime.ts
var init_currentUnixTime = __esm({
"src/util/currentUnixTime.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/otel/streaming/aggregators.ts
var init_aggregators = __esm({
"src/otel/streaming/aggregators.ts"() {
"use strict";
init_cjs_shims();
init_currentUnixTime();
}
});
// src/otel/middleware.ts
var import_api7;
var init_middleware = __esm({
"src/otel/middleware.ts"() {
"use strict";
init_cjs_shims();
import_api7 = require("@opentelemetry/api");
init_attributes();
init_completionUtils();
init_promptUtils();
init_contentSanitizer();
init_wrapperUtils();
init_normalized();
init_aggregators();
init_redaction();
}
});
// src/otel/AxiomWrappedLanguageModelV1.ts
var init_AxiomWrappedLanguageModelV1 = __esm({
"src/otel/AxiomWrappedLanguageModelV1.ts"() {
"use strict";
init_cjs_shims();
init_middleware();
}
});
// src/otel/AxiomWrappedLanguageModelV2.ts
var init_AxiomWrappedLanguageModelV2 = __esm({
"src/otel/AxiomWrappedLanguageModelV2.ts"() {
"use strict";
init_cjs_shims();
init_middleware();
}
});
// src/otel/vercel.ts
var init_vercel = __esm({
"src/otel/vercel.ts"() {
"use strict";
init_cjs_shims();
init_AxiomWrappedLanguageModelV1();
init_AxiomWrappedLanguageModelV2();
}
});
// src/otel/index.ts
var init_otel = __esm({
"src/otel/index.ts"() {
"use strict";
init_cjs_shims();
init_withSpan();
init_vercel();
init_attributes();
}
});
// src/app-scope.ts
var import_api8, import_zod5;
var init_app_scope = __esm({
"src/app-scope.ts"() {
"use strict";
init_cjs_shims();
init_global_flags();
init_storage();
init_validate_flags();
init_dot_path();
init_zod_internals();
import_api8 = require("@opentelemetry/api");
import_zod5 = require("zod");
init_to_otel_attribute();
init_otel();
}
});
// src/evals/context/storage.ts
function getGlobalConfigScope() {
return globalThis[CONFIG_SCOPE_SYMBOL];
}
function getAxiomConfig() {
return globalThis[AXIOM_CONFIG_SYMBOL];
}
function getEvalContext() {
const ctx = EVAL_CONTEXT.get();
if (!ctx) {
return {
flags: {},
facts: {},
pickedFlags: void 0,
outOfScopeFlags: void 0
};
}
return {
flags: ctx.flags,
facts: ctx.facts,
pickedFlags: ctx.pickedFlags,
outOfScopeFlags: ctx.outOfScopeFlags,
parent: ctx.parent,
overrides: ctx.overrides,
accessedFlagKeys: ctx.accessedFlagKeys
};
}
function withEvalContext(options = {}, fn) {
const { initialFlags = {}, pickedFlags = [] } = options;
return EVAL_CONTEXT.run(
{
flags: { ...initialFlags },
facts: {},
pickedFlags,
outOfScopeFlags: [],
accessedFlagKeys: []
},
fn
);
}
function getConfigScope() {
const current = EVAL_CONTEXT.get();
return current?.configScope ?? getGlobalConfigScope();
}
var import_api9, CONFIG_SCOPE_SYMBOL, AXIOM_CONFIG_SYMBOL, EVAL_CONTEXT;
var init_storage = __esm({
"src/evals/context/storage.ts"() {
"use strict";
init_cjs_shims();
import_api9 = require("@opentelemetry/api");
init_manager();
init_app_scope();
CONFIG_SCOPE_SYMBOL = Symbol.for("axiom.eval.configScope");
AXIOM_CONFIG_SYMBOL = Symbol.for("axiom.eval.config");
EVAL_CONTEXT = createAsyncHook("eval-context");
}
});
// src/config/resolver.ts
function resolveAxiomConnection(config, consoleUrlOverride) {
const consoleEndpointUrl = consoleUrlOverride ?? buildConsoleUrl(config.eval.url);
return {
url: config.eval.url,
consoleEndpointUrl,
token: config.eval.token,
dataset: config.eval.dataset,
orgId: config.eval.orgId
};
}
var buildConsoleUrl;
var init_resolver = __esm({
"src/config/resolver.ts"() {
"use strict";
init_cjs_shims();
buildConsoleUrl = (urlString) => {
const url = new URL(urlString);
return `${url.protocol}//app.${url.host.split("api.").at(-1)}`;
};
}
});
// src/util/errors.ts
function getCircularReplacer() {
const seen = /* @__PURE__ */ new WeakSet();
return (_k, v) => {
if (typeof v === "object" && v !== null) {
if (seen.has(v)) return "[Circular]";
seen.add(v);
}
return v;
};
}
function safeJson(x) {
try {
return JSON.stringify(x, getCircularReplacer());
} catch {
return String(x);
}
}
function errorToString(err) {
try {
if (typeof err === "string") return err;
if (err instanceof Error) {
return err.stack ?? err.message;
}
if (typeof err === "object" && err !== null) {
const msg = err.message;
const json = safeJson(err);
return msg ? `${msg} (${json})` : json;
}
return String(err);
} catch {
return "[unserializable error]";
}
}
var AxiomCLIError;
var init_errors = __esm({
"src/util/errors.ts"() {
"use strict";
init_cjs_shims();
AxiomCLIError = class extends Error {
constructor(message) {
super(message);
this.name = "AxiomCLIError";
}
};
}
});
// src/cli/auth/types.ts
var init_types = __esm({
"src/cli/auth/types.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/cli/auth/config.ts
var init_config = __esm({
"src/cli/auth/config.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/cli/auth/oauth.ts
var init_oauth = __esm({
"src/cli/auth/oauth.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/cli/auth/api.ts
var init_api = __esm({
"src/cli/auth/api.ts"() {
"use strict";
init_cjs_shims();
init_errors();
}
});
// src/cli/auth/callback-server.ts
var init_callback_server = __esm({
"src/cli/auth/callback-server.ts"() {
"use strict";
init_cjs_shims();
}
});
// src/cli/auth/global-auth.ts
function getAuthContext() {
return authContext;
}
var authContext;
var init_global_auth = __esm({
"src/cli/auth/global-auth.ts"() {
"use strict";
init_cjs_shims();
init_config();
authContext = null;
}
});
// src/cli/auth/index.ts
var init_auth = __esm({
"src/cli/auth/index.ts"() {
"use strict";
init_cjs_shims();
init_types();
init_config();
init_oauth();
init_api();
init_callback_server();
init_global_auth();
}
});
// src/config/index.ts
function createPartialDefaults() {
let token;
let url;
let orgId;
try {
const authContext2 = getAuthContext();
if (authContext2) {
token = authContext2.token;
url = authContext2.url;
orgId = authContext2.orgId;
}
} catch {
}
token = token || process.env.AXIOM_TOKEN;
url = url || process.env.AXIOM_URL;
orgId = orgId || process.env.AXIOM_ORG_ID;
return {
eval: {
url: url || "https://api.axiom.co",
orgId,
token,
dataset: process.env.AXIOM_DATASET,
flagSchema: void 0,
instrumentation: null,
include: [...DEFAULT_EVAL_INCLUDE],
exclude: [],
timeoutMs: 6e4
}
};
}
function validateConfig(config) {
const errors = [];
const isDebug = process.env.AXIOM_DEBUG === "true";
if (!isDebug) {
if (!config.eval?.token) {
errors.push(
"eval.token is required (set in axiom.config.ts or AXIOM_TOKEN environment variable)"
);
}
if (!config.eval?.dataset) {
errors.push(
"eval.dataset is required (set in axiom.config.ts or AXIOM_DATASET environment variable)"
);
}
if (!config.eval?.url) {
console.log(
"eval.url was not specified. Defaulting to `https://api.axiom.co`. Please set it in axiom.config.ts or AXIOM_URL environment variable if you want to use a different endpoint."
);
}
}
const instrumentation = config.eval?.instrumentation;
if (instrumentation !== null && instrumentation !== void 0 && typeof instrumentation !== "function") {
errors.push(
"eval.instrumentation must be a function returning OTEL setup information or null."
);
}
if (errors.length > 0) {
throw new AxiomCLIError(`Invalid Axiom configuration:
- ${errors.join("\n - ")}`);
}
return config;
}
var import_zod6, DEFAULT_EVAL_INCLUDE;
var init_config2 = __esm({
"src/config/index.ts"() {
"use strict";
init_cjs_shims();
import_zod6 = require("zod");
init_errors();
init_auth();
DEFAULT_EVAL_INCLUDE = ["**/*.eval.{ts,js,mts,mjs,cts,cjs}"];
}
});
// src/config/loader.ts
function customMerger(target, source) {
const merged = (0, import_defu.defu)(source, target);
if (source?.eval && "include" in source.eval) {
merged.eval.include = source.eval.include;
}
if (source?.eval && "flagSchema" in source.eval) {
merged.eval.flagSchema = source.eval.flagSchema;
}
return merged;
}
async function loadConfig(cwd = process.cwd()) {
try {
const defaults = createPartialDefaults();
const result = await (0, import_c12.loadConfig)({
name: "axiom",
cwd,
// Support common config file extensions
configFile: "axiom.config",
// Don't use defaultConfig - we'll merge manually to control array behavior
// Disable configs other than .ts/.js/.mts/.mjs/.cts/.cjs
rcFile: false,
globalRc: false,
packageJson: false,
giget: false
});
const mergedConfig = customMerger(defaults, result.config);
const validatedConfig = validateConfig(mergedConfig);
return {
config: validatedConfig
};
} catch (error) {
if (error instanceof AxiomCLIError) {
throw error;
}
throw new AxiomCLIError(`Failed to load config file: ${errorToString(error)}`);
}
}
var import_c12, import_defu;
var init_loader = __esm({
"src/config/loader.ts"() {
"use strict";
init_cjs_shims();
import_c12 = require("c12");
import_defu = require("defu");
init_config2();
init_errors();
}
});
// src/evals/instrument.ts
async function resolveInstrumentationHook(config) {
if (config.eval.instrumentation) {
return config.eval.instrumentation;
}
try {
const { config: loadedConfig } = await loadConfig(process.cwd());
return loadedConfig.eval.instrumentation ?? null;
} catch (error) {
throw new AxiomCLIError(
`Failed to reload instrumentation from config: ${errorToString(error)}`
);
}
}
async function runInstrumentationHook(hook, options) {
try {
return await hook(options);
} catch (error) {
throw new AxiomCLIError(`Failed to execute instrumentation hook: ${errorToString(error)}`);
}
}
function setupEvalProvider(connection) {
const headers = {
"X-Axiom-Dataset": connection.dataset,
...connection.orgId ? { "X-AXIOM-ORG-ID": connection.orgId } : {}
};
if (connection.token) {
headers.Authorization = `Bearer ${connection.token}`;
}
const collectorOptions = {
url: `${connection.url}/v1/traces`,
headers,
concurrencyLimit: 10
};
const exporter = new import_exporter_trace_otlp_http.OTLPTraceExporter(collectorOptions);
const processor = new import_sdk_trace_node.BatchSpanProcessor(exporter, {
maxQueueSize: 2048,
maxExportBatchSize: 512,
scheduledDelayMillis: 5e3,
exportTimeoutMillis: 3e4
});
axiomProvider = new import_sdk_trace_node.NodeTracerProvider({
resource: (0, import_resources.resourceFromAttributes)({
["service.name"]: "axiom",
["service.version"]: "0.37.0"
}),
spanProcessors: [processor]
});
axiomTracer = axiomProvider.getTracer("axiom", "0.37.0");
}
async function initInstrumentation(config) {
if (initialized) {
return;
}
if (initializationPromise) {
await initializationPromise;
return;
}
initializationPromise = (async () => {
if (!config.enabled) {
axiomTracer = import_api10.trace.getTracer("axiom", "0.37.0");
initialized = true;
return;
}
const connection = resolveAxiomConnection(config.config);
const hook = await resolveInstrumentationHook(config.config);
let hookResult = void 0;
if (hook) {
config.config.eval.instrumentation = hook;
hookResult = await runInstrumentationHook(hook, {
dataset: connection.dataset,
token: connection.token,
url: connection.url,
orgId: connection.orgId
});
userProvider = hookResult?.provider ?? userProvider;
}
setupEvalProvider(connection);
if (!hook) {
axiomProvider?.register();
if (axiomTracer) {
initAxiomAI({ tracer: axiomTracer });
}
}
initialized = true;
})();
try {
await initializationPromise;
} finally {
initializationPromise = null;
}
}
async function ensureInstrumentationInitialized(config, options = {}) {
if (initialized) {
return;
}
const enabled = options.enabled ?? true;
await initInstrumentation({ enabled, config });
}
var import_sdk_trace_node, import_resources, import_exporter_trace_otlp_http, import_api10, axiomProvider, axiomTracer, userProvider, initializationPromise, initialized, flush, getEvalTracer, startSpan, startActiveSpan;
var init_instrument = __esm({
"src/evals/instrument.ts"() {
"use strict";
init_cjs_shims();
import_sdk_trace_node = require("@opentelemetry/sdk-trace-node");
import_resources = require("@opentelemetry/resources");
import_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otlp-http");
import_api10 = require("@opentelemetry/api");
init_initAxiomAI();
init_resolver();
init_errors();
init_loader();
initializationPromise = null;
initialized = false;
flush = async () => {
if (initializationPromise) {
await initializationPromise;
}
const tasks = [];
if (axiomProvider) {
tasks.push(axiomProvider.forceFlush());
}
const candidateProviders = /* @__PURE__ */ new Set();
if (userProvider) {
candidateProviders.add(userProvider);
}
const globalProvider = import_api10.trace.getTracerProvider();
if (globalProvider) {
candidateProviders.add(globalProvider);
}
for (const provider of candidateProviders) {
const flushFn = provider.forceFlush;
if (typeof flushFn === "function") {
tasks.push(
flushFn.call(provider).catch((error) => {
console.warn("[AxiomAI] Failed to flush tracer provider:", errorToString(error));
})
);
}
}
if (tasks.length > 0) {
await Promise.all(tasks);
}
};
getEvalTracer = () => {
if (!axiomTracer) {
throw new Error(
"Eval tracer not initialized. Ensure ensureInstrumentationInitialized() was awaited before starting spans."
);
}
return axiomTracer;
};
startSpan = (name, opts, context6) => {
if (!initialized || !axiomTracer) {
throw new Error(
"Instrumentation not initialized. This is likely a bug - instrumentation should be initialized before startSpan is called."
);
}
return getEvalTracer().startSpan(name, opts, context6);
};
startActiveSpan = async (name, options, fn, parentContext) => {
const span = startSpan(name, options, parentContext);
return import_api10.context.with(import_api10.trace.setSpan(import_api10.context.active(), span), async () => {
try {
const result = await fn(span);
span.setStatus({ code: import_api10.SpanStatusCode.OK });
return result;
} catch (error) {
span.recordException(error);
span.setStatus({
code: import_api10.SpanStatusCode.ERROR,
message: error instanceof Error ? error.message : String(error)
});
throw error;
} finally {
span.end();
}
});
};
}
});
// ../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/chunk-BVHSVHOK.js
function a(n) {
return String(n);
}
function C(n = false) {
let e = typeof process != "undefined" ? process : void 0, i = (e == null ? void 0 : e.env) || {}, g = (e == null ? void 0 : e.argv) || [];
return !("NO_COLOR" in i || g.includes("--no-color")) && ("FORCE_COLOR" in i || g.includes("--color") || (e == null ? void 0 : e.platform) === "win32" || n && i.TERM !== "dumb" || "CI" in i) || typeof window != "undefined" && !!window.chrome;
}
function p(n = false) {
let e = C(n), i = (r2, t, c, o) => {
let l = "", s2 = 0;
do
l += r2.substring(s2, o) + c, s2 = o + t.length, o = r2.indexOf(t, s2);
while (~o);
return l + r2.substring(s2);
}, g = (r2, t, c = r2) => {
let o = (l) => {
let s2 = String(l), b = s2.indexOf(t, r2.length);
return ~b ? r2 + i(s2, t, c, b) + t : r2 + s2 + t;
};
return o.open = r2, o.close = t, o;
}, u2 = {
isColorSupported: e
}, d = (r2) => `\x1B[${r2}m`;
for (let [r2, t] of h)
u2[r2] = e ? g(
d(t[0]),
d(t[1]),
t[2]
) : a;
return u2;
}
var f, h;
var init_chunk_BVHSVHOK = __esm({
"../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/chunk-BVHSVHOK.js"() {
"use strict";
init_cjs_shims();
f = {
reset: [0, 0],
bold: [1, 22, "\x1B[22m\x1B[1m"],
dim: [2, 22, "\x1B[22m\x1B[2m"],
italic: [3, 23],
underline: [4, 24],
inverse: [7, 27],
hidden: [8, 28],
strikethrough: [9, 29],
black: [30, 39],
red: [31, 39],
green: [32, 39],
yellow: [33, 39],
blue: [34, 39],
magenta: [35, 39],
cyan: [36, 39],
white: [37, 39],
gray: [90, 39],
bgBlack: [40, 49],
bgRed: [41, 49],
bgGreen: [42, 49],
bgYellow: [43, 49],
bgBlue: [44, 49],
bgMagenta: [45, 49],
bgCyan: [46, 49],
bgWhite: [47, 49],
blackBright: [90, 39],
redBright: [91, 39],
greenBright: [92, 39],
yellowBright: [93, 39],
blueBright: [94, 39],
magentaBright: [95, 39],
cyanBright: [96, 39],
whiteBright: [97, 39],
bgBlackBright: [100, 49],
bgRedBright: [101, 49],
bgGreenBright: [102, 49],
bgYellowBright: [103, 49],
bgBlueBright: [104, 49],
bgMagentaBright: [105, 49],
bgCyanBright: [106, 49],
bgWhiteBright: [107, 49]
};
h = Object.entries(f);
a.open = "";
a.close = "";
}
});
// ../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/node.js
var import_tty, r, u;
var init_node = __esm({
"../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/node.js"() {
"use strict";
init_cjs_shims();
init_chunk_BVHSVHOK();
import_tty = require("tty");
r = process.env.FORCE_TTY !== void 0 || (0, import_tty.isatty)(1);
u = p(r);
}
});
// src/evals/reporter.console-utils.ts
function formatPercentage(value) {
if (!Number.isFinite(value)) {
return "N/A";
}
return Number(value * 100).toFixed(2) + "%";
}
function formatDiff(current, baseline) {
if (!Number.isFinite(current) || !Number.isFinite(baseline)) {
return { text: "N/A", color: u.dim };
}
const diff = current - baseline;
const diffText = (diff >= 0 ? "+" : "") + formatPercentage(diff);
const color = diff > 0 ? u.green : diff < 0 ? u.red : u.dim;
return { text: diffText, color };
}
function truncate(str, max) {
return str.length > max ? str.slice(0, max) + "\u2026" : str;
}
function stringify(value) {
try {
if (typeof value === "string") return value;
return JSON.stringify(value);
} catch {
return String(value);
}
}
function getCaseFingerprint(input, expected) {
const inputStr = typeof input === "string" ? input : JSON.stringify(input);
const expectedStr = typeof expected === "string" ? expected : JSON.stringify(expected);
return JSON.stringify({ input: inputStr, expected: expectedStr });
}
function printEvalNameAndFileName(testSuite, meta, logger = console.log) {
const cwd = process.cwd();
logger(
" ",
u.bgCyan(u.black(` ${testSuite.project.name} `)),
u.bgBlue(u.black(` ${meta.evaluation.name}-${meta.evaluation.version} `)),
u.dim(`(${testSuite.children.size} cases)`)
);
logger(" ", u.dim(testSuite.module.moduleId.replace(cwd, "")));
}
function printBaselineNameAndVersion(testMeta, logger = console.log) {
if (testMeta.evaluation.baseline) {
logger(
" ",
" baseline ",
u.bgMagenta(
u.black(` ${testMeta.evaluation.baseline.name}-${testMeta.evaluation.baseline.version} `)
)
);
} else {
logger(" ", u.bgWhite(u.blackBright(" baseline: ")), "none");
}
logger("");
}
function printRuntimeFlags(testMeta, logger = console.log) {
if (testMeta.case.runtimeFlags && Object.keys(testMeta.case.runtimeFlags).length > 0) {
const entries = Object.entries(testMeta.case.runtimeFlags);
logger(" ", u.dim("runtime flags"));
for (const [k, v] of entries) {
switch (v.kind) {
case "replaced": {