axiom
Version:
Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals - online evals
468 lines (458 loc) • 15 kB
JavaScript
import {
createPartialDefaults,
getAuthContext,
validateConfig
} from "./chunk-3VKWOZAQ.js";
import {
EVAL_CONTEXT,
assertZodV4,
dotNotationToNested,
formatZodErrors,
generateFlagExamples,
getEvalContext,
isValidPath,
makeDeepPartial,
parsePath,
putOnSpan,
setGlobalFlagOverrides,
withEvalContext
} from "./chunk-63VQQCZB.js";
import {
AxiomCLIError,
errorToString
} from "./chunk-ISSDOC43.js";
// src/cli/utils/parse-flag-overrides.ts
import "zod";
import { readFileSync } from "fs";
import { resolve } from "path";
var FLAG_RE = /^--flag\.([^=]+)(?:=(.*))?$/;
var CONFIG_RE = /^--flags-config(?:=(.*))?$/;
function ensureNoSpaceSeparatedSyntax(flagName, value, nextToken, flagType) {
if (value === void 0 && nextToken !== void 0) {
if (flagType === "flag" && !nextToken.startsWith("-") && nextToken !== "true" && nextToken !== "false") {
console.error(`\u274C Invalid syntax: --flag.${flagName} ${nextToken}`);
console.error(`\u{1F4A1} Use: --flag.${flagName}=${nextToken}`);
process.exit(1);
} else if (flagType === "config" && !nextToken.startsWith("-")) {
console.error(`\u274C Invalid syntax: --flags-config ${nextToken}`);
console.error(`\u{1F4A1} Use: --flags-config=${nextToken}`);
process.exit(1);
}
}
}
function collectFlagValidationErrors(overrides, flagSchema) {
if (!flagSchema || Object.keys(overrides).length === 0) {
return { success: true, errors: [] };
}
assertZodV4(flagSchema, "flagSchema");
const schema = flagSchema;
const errors = [];
for (const dotPath of Object.keys(overrides)) {
const segments = parsePath(dotPath);
if (!isValidPath(schema, segments)) {
errors.push({ type: "invalid_path", path: dotPath });
}
}
if (errors.length > 0) {
return { success: false, errors };
}
const nestedObject = dotNotationToNested(overrides);
const deepPartialSchema = makeDeepPartial(schema);
const result = deepPartialSchema.safeParse(nestedObject);
if (!result.success) {
errors.push({ type: "invalid_value", zodError: result.error });
}
return { success: errors.length === 0, errors };
}
function printFlagValidationErrorsAndExit(errors) {
console.error("\u274C Invalid CLI flags:");
for (const error of errors) {
if (error.type === "invalid_path") {
console.error(` \u2022 flag '${error.path}': Invalid flag path`);
} else {
console.error(formatZodErrors(error.zodError));
const examples = generateFlagExamples(error.zodError);
if (examples.length > 0) {
console.error("\n\u{1F4A1} Valid examples:");
examples.forEach((example) => console.error(` ${example}`));
}
}
}
process.exit(1);
}
function validateFlagOverrides(overrides, flagSchema) {
const result = collectFlagValidationErrors(overrides, flagSchema);
if (!result.success) {
printFlagValidationErrorsAndExit(result.errors);
}
}
function coerceValue(raw) {
if (raw === "true") return true;
if (raw === "false") return false;
const num = Number(raw);
if (!Number.isNaN(num) && raw.trim() === num.toString()) {
return num;
}
try {
return JSON.parse(raw);
} catch {
return raw;
}
}
function loadConfigFile(path) {
const abs = resolve(process.cwd(), path);
try {
const contents = readFileSync(abs, "utf8");
const parsed = JSON.parse(contents);
if (typeof parsed !== "object" || Array.isArray(parsed) || parsed === null) {
console.error(
`\u274C Flags config must be a JSON object, got ${Array.isArray(parsed) ? "array" : typeof parsed}`
);
process.exit(1);
}
return parsed;
} catch (err) {
console.error(`\u274C Could not read or parse flags config "${path}": ${err.message}`);
process.exit(1);
}
}
function extractOverrides(argv) {
const cleanedArgv = [];
const overrides = {};
let configPath = null;
let hasCliFlags = false;
let configPathCount = 0;
for (let i = 0; i < argv.length; i++) {
const token = argv[i];
const configMatch = token.match(CONFIG_RE);
const flagMatch = token.match(FLAG_RE);
if (configMatch) {
configPathCount++;
if (configPathCount > 1) {
console.error("\u274C Only one --flags-config can be supplied.");
process.exit(1);
}
const value = configMatch[1];
const nextToken = argv.length > i + 1 ? argv[i + 1] : void 0;
ensureNoSpaceSeparatedSyntax("flags-config", value, nextToken, "config");
if (!value) {
console.error("\u274C --flags-config requires a file path");
console.error("\u{1F4A1} Use: --flags-config=path/to/config.json");
process.exit(1);
}
configPath = value;
} else if (flagMatch) {
hasCliFlags = true;
const key = flagMatch[1];
const value = flagMatch[2];
const nextToken = argv.length > i + 1 ? argv[i + 1] : void 0;
ensureNoSpaceSeparatedSyntax(key, value, nextToken, "flag");
const finalValue = value === void 0 ? "true" : value;
overrides[key] = coerceValue(finalValue);
} else {
cleanedArgv.push(token);
}
}
if (configPath && hasCliFlags) {
console.error("\u274C Cannot use both --flags-config and --flag.* arguments together.");
console.error("Choose one approach:");
console.error(" \u2022 Config file: --flags-config=my-flags.json");
console.error(" \u2022 CLI flags: --flag.temperature=0.9 --flag.model=gpt-4o");
process.exit(1);
}
if (configPath) {
const configOverrides = loadConfigFile(configPath);
return { cleanedArgv, overrides: configOverrides };
}
return { cleanedArgv, overrides };
}
// ../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/chunk-BVHSVHOK.js
var f = {
reset: [0, 0],
bold: [1, 22, "\x1B[22m\x1B[1m"],
dim: [2, 22, "\x1B[22m\x1B[2m"],
italic: [3, 23],
underline: [4, 24],
inverse: [7, 27],
hidden: [8, 28],
strikethrough: [9, 29],
black: [30, 39],
red: [31, 39],
green: [32, 39],
yellow: [33, 39],
blue: [34, 39],
magenta: [35, 39],
cyan: [36, 39],
white: [37, 39],
gray: [90, 39],
bgBlack: [40, 49],
bgRed: [41, 49],
bgGreen: [42, 49],
bgYellow: [43, 49],
bgBlue: [44, 49],
bgMagenta: [45, 49],
bgCyan: [46, 49],
bgWhite: [47, 49],
blackBright: [90, 39],
redBright: [91, 39],
greenBright: [92, 39],
yellowBright: [93, 39],
blueBright: [94, 39],
magentaBright: [95, 39],
cyanBright: [96, 39],
whiteBright: [97, 39],
bgBlackBright: [100, 49],
bgRedBright: [101, 49],
bgGreenBright: [102, 49],
bgYellowBright: [103, 49],
bgBlueBright: [104, 49],
bgMagentaBright: [105, 49],
bgCyanBright: [106, 49],
bgWhiteBright: [107, 49]
};
var h = Object.entries(f);
function a(n) {
return String(n);
}
a.open = "";
a.close = "";
function C(n = false) {
let e = typeof process != "undefined" ? process : void 0, i = (e == null ? void 0 : e.env) || {}, g = (e == null ? void 0 : e.argv) || [];
return !("NO_COLOR" in i || g.includes("--no-color")) && ("FORCE_COLOR" in i || g.includes("--color") || (e == null ? void 0 : e.platform) === "win32" || n && i.TERM !== "dumb" || "CI" in i) || typeof window != "undefined" && !!window.chrome;
}
function p(n = false) {
let e = C(n), i = (r2, t, c, o) => {
let l = "", s2 = 0;
do
l += r2.substring(s2, o) + c, s2 = o + t.length, o = r2.indexOf(t, s2);
while (~o);
return l + r2.substring(s2);
}, g = (r2, t, c = r2) => {
let o = (l) => {
let s2 = String(l), b = s2.indexOf(t, r2.length);
return ~b ? r2 + i(s2, t, c, b) + t : r2 + s2 + t;
};
return o.open = r2, o.close = t, o;
}, u2 = {
isColorSupported: e
}, d = (r2) => `\x1B[${r2}m`;
for (let [r2, t] of h)
u2[r2] = e ? g(
d(t[0]),
d(t[1]),
t[2]
) : a;
return u2;
}
// ../../node_modules/.pnpm/tinyrainbow@2.0.0/node_modules/tinyrainbow/dist/node.js
import { isatty as s } from "tty";
var r = process.env.FORCE_TTY !== void 0 || s(1);
var u = p(r);
// src/cli/commands/eval.command.ts
import { Command, Argument, Option } from "commander";
import { customAlphabet } from "nanoid";
import { lstatSync } from "fs";
// src/context.ts
function overrideFlags(partial) {
const current = getEvalContext();
if (!current) {
if (process.env.NODE_ENV !== "test") {
console.warn("overrideFlags called outside of evaluation context");
}
return;
}
const overlayContext = {
...current,
flags: { ...current.flags, ...partial },
// Merge for backwards compatibility
parent: current,
overrides: { ...partial }
};
const currentCtx = EVAL_CONTEXT.get();
if (currentCtx) {
Object.assign(currentCtx, overlayContext);
}
for (const [key, value] of Object.entries(partial)) {
putOnSpan("flag", key, value);
}
}
// src/cli/utils/eval-context-runner.ts
async function runEvalWithContext(overrides, runFn) {
setGlobalFlagOverrides(overrides);
return withEvalContext({ initialFlags: overrides }, async () => {
if (Object.keys(overrides).length > 0) {
overrideFlags(overrides);
}
return runFn();
});
}
// src/cli/utils/glob-utils.ts
function isGlob(str) {
return /[*?[\]{}!]/.test(str);
}
// src/config/loader.ts
import { loadConfig as c12LoadConfig } from "c12";
import { defu } from "defu";
function customMerger(target, source) {
const merged = defu(source, target);
if (source?.eval && "include" in source.eval) {
merged.eval.include = source.eval.include;
}
if (source?.eval && "flagSchema" in source.eval) {
merged.eval.flagSchema = source.eval.flagSchema;
}
return merged;
}
async function loadConfig(cwd = process.cwd()) {
try {
const defaults = createPartialDefaults();
const result = await c12LoadConfig({
name: "axiom",
cwd,
// Support common config file extensions
configFile: "axiom.config",
// Don't use defaultConfig - we'll merge manually to control array behavior
// Disable configs other than .ts/.js/.mts/.mjs/.cts/.cjs
rcFile: false,
globalRc: false,
packageJson: false,
giget: false
});
const mergedConfig = customMerger(defaults, result.config);
const validatedConfig = validateConfig(mergedConfig);
return {
config: validatedConfig
};
} catch (error) {
if (error instanceof AxiomCLIError) {
throw error;
}
throw new AxiomCLIError(`Failed to load config file: ${errorToString(error)}`);
}
}
// src/cli/commands/eval.command.ts
var createRunId = customAlphabet("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ", 10);
var consoleUrl;
function getConsoleUrl() {
return consoleUrl;
}
function getDefaultToken(value) {
if (typeof value === "string") {
return value;
}
const authContext = getAuthContext();
return authContext?.token || process.env.AXIOM_TOKEN;
}
function getDefaultUrl(value) {
if (typeof value === "string") {
return value;
}
const authContext = getAuthContext();
return authContext?.url || process.env.AXIOM_URL || "https://api.axiom.co";
}
function getDefaultOrgId(value) {
if (typeof value === "string") {
return value;
}
const authContext = getAuthContext();
return authContext?.orgId ?? process.env.AXIOM_ORG_ID;
}
var loadEvalCommand = (program, flagOverrides = {}) => {
return program.addCommand(
new Command("eval").description("run evals locally").addArgument(
new Argument("[target]", "file, directory, glob pattern, or eval name").default(
".",
"any *.eval.ts file in current directory"
)
).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", getDefaultToken).option("-d, --dataset <DATASET>", "axiom dataset name").option("-u, --url <AXIOM URL>", "axiom url", getDefaultUrl).option("-o, --org-id <ORG ID>", "axiom organization id", getDefaultOrgId).option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").option("--debug", "run locally without any network operations", false).option("--list", "list evaluations and test cases without running them", false).addOption(new Option("-c, --console-url <URL>", "console url override").hideHelp()).action(async (target, options) => {
try {
if (options.debug) {
process.env.AXIOM_DEBUG = "true";
}
let include = [];
let exclude;
let testNamePattern;
const isGlobPattern = isGlob(target);
const { config: loadedConfig } = await loadConfig(".");
validateFlagOverrides(flagOverrides, loadedConfig.eval.flagSchema);
const config = {
...loadedConfig,
eval: {
...loadedConfig.eval,
...options.token && { token: options.token },
...options.url && { url: options.url },
...options.dataset && { dataset: options.dataset },
...options.orgId && { orgId: options.orgId }
}
};
if (isGlobPattern) {
include = [target];
} else {
try {
const stat = lstatSync(target);
if (stat.isDirectory()) {
include = config?.eval?.include || [];
} else {
include = [target];
}
} catch {
testNamePattern = new RegExp(target, "i");
include = config?.eval?.include || [];
}
}
exclude = config?.eval?.exclude;
if (!config?.eval?.instrumentation) {
console.warn(
u.yellow(
"\u26A0 App instrumentation (`eval.instrumentation` in `axiom.config.ts`) not configured. Using default provider."
)
);
console.log("");
}
const runId = createRunId();
consoleUrl = options.consoleUrl;
let runVitestModule;
try {
runVitestModule = await import("./run-vitest-SA4SZBNK.js");
} catch (err) {
if (err && typeof err === "object" && "code" in err && (err.code === "ERR_MODULE_NOT_FOUND" || err.code === "MODULE_NOT_FOUND")) {
throw new AxiomCLIError("Failed to load vitest.");
}
throw err;
}
const { runVitest } = runVitestModule;
await runEvalWithContext(flagOverrides, async () => {
return runVitest(".", {
watch: options.watch,
baseline: options.baseline,
include,
exclude,
testNamePattern,
debug: options.debug,
list: options.list,
overrides: flagOverrides,
config,
runId,
consoleUrl: options.consoleUrl
});
});
} catch (error) {
if (error instanceof AxiomCLIError) {
console.error(`
\u274C ${error.message}
`);
process.exit(1);
}
throw error;
}
})
);
};
export {
extractOverrides,
loadConfig,
u,
getConsoleUrl,
loadEvalCommand
};
//# sourceMappingURL=chunk-72PVEOMF.js.map