axiom/dist/evals/custom-runner.cjs.map

Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals

{"version":3,"sources":["../../src/evals/custom-runner.ts","../../src/util/errors.ts","../../src/evals/name-validation-runtime.ts","../../src/util/name-validation-runtime.ts"],"sourcesContent":["import { VitestTestRunner } from 'vitest/runners';\nimport { existsSync, readFileSync, writeFileSync } from 'node:fs';\nimport { validateName } from './name-validation-runtime';\n\n/**\n * Custom Vitest runner that validates eval and scorer names before running any tests.\n *\n * The default runner doesn't give us a good way of doing this validation\n * before tests start, unfortunately.\n */\nexport default class AxiomEvalRunner extends VitestTestRunner {\n  private validationChecked = false;\n\n  /**\n   * Override onBeforeRunSuite to validate names before the first suite runs.\n   */\n  async onBeforeRunSuite(suite: any): Promise<void> {\n    if (!this.validationChecked) {\n      this.validationChecked = true;\n\n      const registryFile = process.env.AXIOM_NAME_REGISTRY_FILE;\n      const abortFile = process.env.AXIOM_ABORT_FILE;\n\n      // Validate all names from the registry file\n      if (registryFile && abortFile && existsSync(registryFile)) {\n        const errors: string[] = [];\n        const content = readFileSync(registryFile, 'utf8');\n        const lines = content.trim().split('\\n').filter(Boolean);\n        const seenEvals = new Set<string>();\n        const seenScorers = new Set<string>();\n\n        for (const line of lines) {\n          try {\n            const record = JSON.parse(line) as { kind: 'eval' | 'scorer'; name: string };\n            const seen = record.kind === 'eval' ? seenEvals : seenScorers;\n            if (seen.has(record.name)) continue;\n            seen.add(record.name);\n\n            try {\n              validateName(record.name, record.kind);\n            } catch (error) {\n              errors.push((error as Error).message);\n            }\n          } catch {\n            // Skip malformed lines\n          }\n        }\n\n        if (errors.length > 0) {\n          const message = [\n            'Validation failed. No tests will run due to the following errors:',\n            '',\n            ...errors,\n            '',\n          ].join('\\n');\n\n          writeFileSync(abortFile, message, 'utf8');\n          throw new Error('\\n' + message + '\\n');\n        }\n      }\n    }\n\n    await super.onBeforeRunSuite(suite);\n  }\n}\n","export class AxiomCLIError extends Error {\n  constructor(message: string) {\n    super(message);\n    this.name = 'AxiomCLIError';\n  }\n}\n\nfunction getCircularReplacer() {\n  const seen = new WeakSet();\n  return (_k: string, v: any) => {\n    if (typeof v === 'object' && v !== null) {\n      if (seen.has(v)) return '[Circular]';\n      seen.add(v);\n    }\n    return v;\n  };\n}\n\nfunction safeJson(x: any) {\n  try {\n    return JSON.stringify(x, getCircularReplacer());\n  } catch {\n    return String(x);\n  }\n}\n\nexport function errorToString(err: unknown) {\n  try {\n    if (typeof err === 'string') return err;\n\n    if (err instanceof Error) {\n      return err.stack ?? err.message;\n    }\n\n    if (typeof err === 'object' && err !== null) {\n      const msg = (err as any).message;\n      const json = safeJson(err);\n      return msg ? `${msg} (${json})` : json;\n    }\n\n    return String(err);\n  } catch {\n    return '[unserializable error]';\n  }\n}\n","import { AxiomCLIError } from '../util/errors';\nimport { appendFileSync } from 'node:fs';\nimport { isValidName } from '../util/name-validation-runtime';\n\n/**\n * Records an eval, scorer, capability, or step name\n * Uses a file to work cross-worker\n */\nexport function recordName(kind: 'eval' | 'scorer' | 'capability' | 'step', name: string): void {\n  const registryFile = process.env.AXIOM_NAME_REGISTRY_FILE;\n  if (registryFile) {\n    try {\n      appendFileSync(registryFile, JSON.stringify({ kind, name }) + '\\n', 'utf8');\n    } catch {\n      // Silently fail if we can't write to registry file\n    }\n  }\n}\n\n/**\n * Validates that a name contains only allowed characters (A-Z, a-z, 0-9, -, _)\n * and is not empty. Throws AxiomCLIError if validation fails.\n */\nexport function validateName(name: string, kind: 'eval' | 'scorer' | 'capability' | 'step'): void {\n  const validation = isValidName(name);\n  if (!validation.valid) {\n    throw new AxiomCLIError(`❌ ${kind} name: ${validation.error}`);\n  }\n}\n","/**\n * Validates that a name contains only allowed characters (A-Z, a-z, 0-9, -, _)\n * and is not empty.\n *\n * @returns Object with validation result and optional error message\n */\nexport function isValidName(name: string): { valid: true } | { valid: false; error: string } {\n  if (name === '') {\n    return { valid: false, error: 'Name cannot be empty' };\n  }\n\n  const validPattern = /^[A-Za-z0-9_-]+$/;\n  if (!validPattern.test(name)) {\n    return {\n      valid: false,\n      error: `Invalid character in \"${name}\". Only A-Z, a-z, 0-9, -, _ allowed`,\n    };\n  }\n\n  return { valid: true };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qBAAiC;AACjC,IAAAA,kBAAwD;;;ACDjD,IAAM,gBAAN,cAA4B,MAAM;AAAA,EACvC,YAAY,SAAiB;AAC3B,UAAM,OAAO;AACb,SAAK,OAAO;AAAA,EACd;AACF;;;ACJA,qBAA+B;;;ACKxB,SAAS,YAAY,MAAiE;AAC3F,MAAI,SAAS,IAAI;AACf,WAAO,EAAE,OAAO,OAAO,OAAO,uBAAuB;AAAA,EACvD;AAEA,QAAM,eAAe;AACrB,MAAI,CAAC,aAAa,KAAK,IAAI,GAAG;AAC5B,WAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO,yBAAyB,IAAI;AAAA,IACtC;AAAA,EACF;AAEA,SAAO,EAAE,OAAO,KAAK;AACvB;;;ADGO,SAAS,aAAa,MAAc,MAAuD;AAChG,QAAM,aAAa,YAAY,IAAI;AACnC,MAAI,CAAC,WAAW,OAAO;AACrB,UAAM,IAAI,cAAc,UAAK,IAAI,UAAU,WAAW,KAAK,EAAE;AAAA,EAC/D;AACF;;;AFlBA,IAAqB,kBAArB,cAA6C,gCAAiB;AAAA,EAA9D;AAAA;AACE,wBAAQ,qBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA,EAK5B,MAAM,iBAAiB,OAA2B;AAChD,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AAEzB,YAAM,eAAe,QAAQ,IAAI;AACjC,YAAM,YAAY,QAAQ,IAAI;AAG9B,UAAI,gBAAgB,iBAAa,4BAAW,YAAY,GAAG;AACzD,cAAM,SAAmB,CAAC;AAC1B,cAAM,cAAU,8BAAa,cAAc,MAAM;AACjD,cAAM,QAAQ,QAAQ,KAAK,EAAE,MAAM,IAAI,EAAE,OAAO,OAAO;AACvD,cAAM,YAAY,oBAAI,IAAY;AAClC,cAAM,cAAc,oBAAI,IAAY;AAEpC,mBAAW,QAAQ,OAAO;AACxB,cAAI;AACF,kBAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,kBAAM,OAAO,OAAO,SAAS,SAAS,YAAY;AAClD,gBAAI,KAAK,IAAI,OAAO,IAAI,EAAG;AAC3B,iBAAK,IAAI,OAAO,IAAI;AAEpB,gBAAI;AACF,2BAAa,OAAO,MAAM,OAAO,IAAI;AAAA,YACvC,SAAS,OAAO;AACd,qBAAO,KAAM,MAAgB,OAAO;AAAA,YACtC;AAAA,UACF,QAAQ;AAAA,UAER;AAAA,QACF;AAEA,YAAI,OAAO,SAAS,GAAG;AACrB,gBAAM,UAAU;AAAA,YACd;AAAA,YACA;AAAA,YACA,GAAG;AAAA,YACH;AAAA,UACF,EAAE,KAAK,IAAI;AAEX,6CAAc,WAAW,SAAS,MAAM;AACxC,gBAAM,IAAI,MAAM,OAAO,UAAU,IAAI;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAEA,UAAM,MAAM,iBAAiB,KAAK;AAAA,EACpC;AACF;","names":["import_node_fs"]}