UNPKG

axiom

Version:

Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals

1 lines 9.94 kB
{"version":3,"sources":["../src/config.ts","../src/config/index.ts"],"sourcesContent":["export { defineConfig } from './config/index';\nexport type { AxiomConfig } from './config/index';\nexport type { AxiomEvalInstrumentationHook } from './config/index';\n","import type { TracerProvider } from '@opentelemetry/api';\nimport { type z } from 'zod';\nimport { AxiomCLIError } from '../util/errors';\nimport { getAuthContext } from '../cli/auth';\n\n/**\n * Utility type to make all properties in T required recursively.\n * Keeps the types as-is but removes the optionality.\n */\ntype DeepRequired<T> =\n T extends Array<infer U>\n ? Array<U>\n : T extends (...args: any[]) => any\n ? T\n : T extends object\n ? {\n [P in keyof T]-?: DeepRequired<T[P]>;\n }\n : T;\n\n/**\n * Axiom API connection configuration\n */\nexport interface AxiomConnectionConfig {\n /**\n * Axiom API URL\n * @default 'https://api.axiom.co'\n * @example 'https://api.axiom.co'\n */\n url?: string;\n\n /**\n * Axiom API token (can be undefined if not set)\n * @example process.env.AXIOM_TOKEN\n */\n token?: string | undefined;\n\n /**\n * Axiom dataset name\n * @example process.env.AXIOM_DATASET\n */\n dataset?: string;\n\n /**\n * Axiom organization ID\n * @example process.env.AXIOM_ORG_ID\n */\n orgId?: string;\n}\n\n/**\n * Options passed to the instrumentation hook\n * - url: string\n * - token: string\n * - dataset: string\n */\nexport interface AxiomEvalInstrumentationOptions {\n url: string;\n token: string;\n dataset: string;\n orgId?: string;\n}\n\n/**\n * Result returned from the instrumentation hook\n */\nexport interface AxiomEvalInstrumentationResult {\n /**\n * TracerProvider to be flushed when eval finishes.\n *\n * If you use the NodeSDK or register your provider globally, you don't need to return it\n * here as the NodeSDK automatically flushes the global provider.\n *\n * Only return a provider if you want Axiom to explicitly flush it for you and it's not\n * registered as the global tracer provider.\n */\n provider?: TracerProvider;\n}\n\nexport type SyncAxiomEvalInstrumentationHook = (\n options: AxiomEvalInstrumentationOptions,\n) => AxiomEvalInstrumentationResult;\n\nexport type AsyncAxiomEvalInstrumentationHook = (\n options: AxiomEvalInstrumentationOptions,\n) => Promise<AxiomEvalInstrumentationResult>;\n\n/**\n * Hook function to initialize application OpenTelemetry instrumentation.\n * Called before eval execution with resolved Axiom connection details.\n *\n * @param options - Configuration options\n * @param options.url - Axiom API URL\n * @param options.token - Axiom API token\n * @param options.dataset - Axiom dataset name\n * @returns TracerProvider or Promise resolving to TracerProvider\n *\n * @example\n * ```typescript\n * instrumentation: ({ url, token, dataset }) => {\n * return setupAppInstrumentation({ url, token, dataset });\n * }\n * ```\n */\nexport type AxiomEvalInstrumentationHook = (\n options: AxiomEvalInstrumentationOptions,\n) => AxiomEvalInstrumentationResult | Promise<AxiomEvalInstrumentationResult>;\n\n/**\n * Axiom AI SDK base configuration (user-facing, all optional)\n */\nexport const DEFAULT_EVAL_INCLUDE = ['**/*.eval.{ts,js,mts,mjs,cts,cjs}'] as const;\n\nexport interface AxiomConfigBase {\n /**\n * Eval configuration settings\n *\n * @example\n * ```typescript\n * eval: {\n * url: process.env.AXIOM_URL,\n * token: process.env.AXIOM_TOKEN,\n * dataset: process.env.AXIOM_DATASET\n * }\n * ```\n */\n eval?: AxiomConnectionConfig & {\n /**\n * Zod schema for flag validation.\n * When provided, CLI flags (--flag.*) are validated against this schema\n * before eval execution begins.\n *\n * @example\n * ```typescript\n * import { z } from 'zod';\n *\n * export default defineConfig({\n * eval: {\n * flagSchema: z.object({\n * model: z.object({\n * temperature: z.number().min(0).max(2).default(0.7),\n * name: z.string().default('gpt-4o'),\n * }),\n * }),\n * }\n * });\n * ```\n */\n flagSchema?: z.ZodObject<any> | null;\n /**\n * Optional hook to initialize application OpenTelemetry instrumentation.\n * Called before eval execution with resolved Axiom connection details.\n * Return your configured tracer provider/tracer (or void) after registering them.\n */\n instrumentation?: AxiomEvalInstrumentationHook | null;\n /**\n * Timeout for eval execution in milliseconds\n * @default 60000\n */\n timeoutMs?: number;\n /**\n * Glob patterns to include when running evals\n * @default ['**\\/*.eval.{ts,js,mts,mjs,cts,cjs}']\n * @example ['**\\/*.eval.ts', 'tests/**\\/*.test.ts']\n */\n include?: string[];\n /**\n * Glob patterns to exclude when running evals\n * @default ['**\\/node_modules/**', '**\\/dist/**', '**\\/build/**']\n * @example ['**\\/node_modules/**', '**\\/.next/**']\n */\n exclude?: string[];\n };\n}\n\n/**\n * Resolved Axiom AI SDK configuration with all required keys.\n * This is the type returned after merging user config with defaults.\n *\n * Uses DeepRequired to ensure all optional properties from AxiomConfigBase\n * become required, preventing missing properties in the resolved config.\n */\nexport type ResolvedAxiomConfig = DeepRequired<AxiomConfigBase>;\n\n/**\n * Axiom AI SDK configuration with optional environment-specific overrides.\n *\n * Supports c12 environment overrides using $development, $production, etc.\n *\n * @example\n * ```typescript\n * export default defineConfig({\n * eval: {\n * url: process.env.AXIOM_URL,\n * token: process.env.AXIOM_TOKEN,\n * dataset: process.env.AXIOM_DATASET,\n * },\n * })\n * ```\n */\nexport interface AxiomConfig extends AxiomConfigBase {\n /**\n * Allow c12 environment-specific overrides ($development, $production, $test etc.)\n * but don't show them in autocomplete for now\n */\n [key: `$${string}`]: Partial<AxiomConfigBase> | undefined;\n}\n\n/**\n * Type-safe helper for defining Axiom configuration.\n *\n * @param config - The configuration object\n * @returns The same configuration object with type checking\n *\n * @example\n * ```typescript\n * import { defineConfig } from 'axiom/ai/config';\n *\n * export default defineConfig({\n * eval: {\n * url: process.env.AXIOM_URL,\n * token: process.env.AXIOM_TOKEN,\n * dataset: process.env.AXIOM_DATASET,\n * include: ['**\\/*.eval.{ts,js}'],\n * instrumentation: ({ url, token, dataset }) => setupAppInstrumentation({ url, token, dataset }),\n * },\n * });\n * ```\n */\nexport function defineConfig(config: AxiomConfig): AxiomConfig {\n return config;\n}\n\n/**\n * Create partial default configuration from auth context or environment variables.\n * Does not throw if required values are missing - validation happens after merge.\n *\n * @returns Partial configuration with defaults and auth/env var values\n * @internal\n */\nexport function createPartialDefaults(): Partial<AxiomConfigBase> {\n // Try to get auth context first (available when running via CLI)\n // Fall back to process.env for backward compatibility (e.g., when used outside CLI)\n let token: string | undefined;\n let url: string | undefined;\n let orgId: string | undefined;\n\n try {\n const authContext = getAuthContext();\n if (authContext) {\n token = authContext.token;\n url = authContext.url;\n orgId = authContext.orgId;\n }\n } catch {\n // Auth context not available, fall back to env vars\n }\n\n // Fall back to process.env if auth context not available\n token = token || process.env.AXIOM_TOKEN;\n url = url || process.env.AXIOM_URL;\n orgId = orgId || process.env.AXIOM_ORG_ID;\n\n return {\n eval: {\n url: url || 'https://api.axiom.co',\n orgId,\n token,\n dataset: process.env.AXIOM_DATASET,\n flagSchema: undefined,\n instrumentation: null,\n include: [...DEFAULT_EVAL_INCLUDE],\n exclude: [],\n timeoutMs: 60_000,\n },\n };\n}\n\n/**\n * Validates and returns a fully resolved Axiom configuration.\n *\n * @param config - Partial configuration to validate\n * @returns Fully resolved configuration with all required fields\n * @throws {AxiomCLIError} If required fields are missing or invalid\n * @internal\n */\nexport function validateConfig(config: Partial<AxiomConfigBase>): ResolvedAxiomConfig {\n const errors: string[] = [];\n const isDebug = process.env.AXIOM_DEBUG === 'true';\n\n if (!isDebug) {\n if (!config.eval?.token) {\n errors.push(\n 'eval.token is required (set in axiom.config.ts or AXIOM_TOKEN environment variable)',\n );\n }\n if (!config.eval?.dataset) {\n errors.push(\n 'eval.dataset is required (set in axiom.config.ts or AXIOM_DATASET environment variable)',\n );\n }\n\n if (!config.eval?.url) {\n console.log(\n 'eval.url was not specified. Defaulting to `https://api.axiom.co`. Please set it in axiom.config.ts or AXIOM_URL environment variable if you want to use a different endpoint.',\n );\n }\n }\n\n const instrumentation = config.eval?.instrumentation;\n if (\n instrumentation !== null &&\n instrumentation !== undefined &&\n typeof instrumentation !== 'function'\n ) {\n errors.push(\n 'eval.instrumentation must be a function returning OTEL setup information or null.',\n );\n }\n\n if (errors.length > 0) {\n throw new AxiomCLIError(`Invalid Axiom configuration:\\n - ${errors.join('\\n - ')}`);\n }\n\n return config as ResolvedAxiomConfig;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,iBAAuB;AAoOhB,SAAS,aAAa,QAAkC;AAC7D,SAAO;AACT;","names":[]}