axiom/dist/run-vitest-SA4SZBNK.js

Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals - online evals

import {
  AxiomReporter,
  flush,
  initInstrumentation
} from "./chunk-SQJ53C2N.js";
import {
  u
} from "./chunk-72PVEOMF.js";
import "./chunk-3VKWOZAQ.js";
import {
  setAxiomConfig
} from "./chunk-63VQQCZB.js";
import "./chunk-ISSDOC43.js";
import "./chunk-PU64TWX4.js";
import "./chunk-MM5FFQJT.js";
import "./chunk-4TKUTT24.js";
import "./chunk-KEXKKQVW.js";

// src/evals/run-vitest.ts
import { resolve, join } from "path";
import { mkdirSync, writeFileSync, unlinkSync, existsSync, readFileSync } from "fs";
import { tmpdir } from "os";
import { fileURLToPath } from "url";
import path from "path";
import tsconfigPaths from "vite-tsconfig-paths";
import { createVitest, registerConsoleShortcuts } from "vitest/node";
var VITEST_MODE = "eval";
var getCurrentDir = () => {
  if (typeof __dirname !== "undefined") {
    return __dirname;
  }
  return path.dirname(fileURLToPath(import.meta.url));
};
var evalsRunnerPath = resolve(getCurrentDir(), "evals", "custom-runner.js");
var printCollectedEvals = (result, rootDir) => {
  if (!result.testModules || result.testModules.length === 0) {
    console.log(u.yellow("\nNo evaluations found\n"));
    return;
  }
  console.log(u.bold("\nFound evaluations:\n"));
  let totalEvals = 0;
  let totalCases = 0;
  for (const module of result.testModules) {
    const relativePath = path.relative(rootDir, module.moduleId);
    for (const suite of module.children.suites()) {
      totalEvals++;
      const caseCount = suite.children.size;
      totalCases += caseCount;
      console.log(u.green(`\u2713 ${suite.name} (${caseCount} cases)`));
      console.log(u.dim(`  ${relativePath}`));
      console.log("");
    }
  }
  console.log(u.bold(`Total: ${totalEvals} evaluations, ${totalCases} test cases
`));
};
var hasVitestFailures = (result, failedTestCount, dangerouslyIgnoreUnhandledErrors) => {
  if (failedTestCount > 0) {
    return true;
  }
  if (result.unhandledErrors.length > 0 && !dangerouslyIgnoreUnhandledErrors) {
    return true;
  }
  return result.testModules.some(
    (testModule) => testModule.state() === "failed" || !testModule.ok()
  );
};
var runVitest = async (dir, opts) => {
  process.env.AXIOM_EVAL = "true";
  setAxiomConfig(opts.config);
  await initInstrumentation({
    enabled: !opts.debug && !opts.list,
    config: opts.config
  });
  const providedConfig = {
    ...opts.config,
    eval: {
      ...opts.config.eval,
      // These can't be serialized, so we need to remove them
      instrumentation: null,
      flagSchema: null
    }
  };
  if (opts.debug) {
    console.log(u.bgWhite(u.blackBright(" Debug mode enabled ")));
  }
  const tmpDir = join(tmpdir(), "axiom-eval", opts.runId);
  mkdirSync(tmpDir, { recursive: true });
  const nameRegistryFile = join(tmpDir, "names.jsonl");
  const abortFile = join(tmpDir, "abort.txt");
  writeFileSync(nameRegistryFile, "", "utf8");
  if (existsSync(abortFile)) {
    unlinkSync(abortFile);
  }
  process.env.AXIOM_NAME_REGISTRY_FILE = nameRegistryFile;
  process.env.AXIOM_ABORT_FILE = abortFile;
  if (opts.list) {
    console.log(u.bgWhite(u.blackBright(" List mode ")));
  }
  const evalTimeoutMs = opts.config?.eval?.timeoutMs || 6e4;
  const hookTimeoutMs = Math.max(evalTimeoutMs, 6e4);
  const vi = await createVitest(
    "test",
    {
      config: false,
      root: dir ? dir : process.cwd(),
      mode: VITEST_MODE,
      include: opts.include,
      exclude: opts.exclude,
      testNamePattern: opts.testNamePattern,
      reporters: ["verbose", new AxiomReporter()],
      environment: "node",
      browser: void 0,
      watch: opts.watch,
      setupFiles: [],
      name: "axiom:eval",
      printConsoleTrace: true,
      silent: false,
      disableConsoleIntercept: true,
      testTimeout: evalTimeoutMs,
      // Vitest default is 10s, which might not be enough to flush OTLP exporters and update the evaluation via API in `afterAll`
      hookTimeout: hookTimeoutMs,
      globals: true,
      runner: evalsRunnerPath,
      provide: {
        baseline: opts.baseline,
        debug: opts.debug,
        list: opts.list,
        overrides: opts.overrides,
        axiomConfig: providedConfig,
        runId: opts.runId,
        consoleUrl: opts.consoleUrl
      }
    },
    {
      plugins: [tsconfigPaths({ root: dir || process.cwd() })]
    }
  );
  if (opts.list) {
    const result2 = await vi.collect();
    printCollectedEvals(result2, dir || process.cwd());
    await vi.close();
    process.exit(0);
  }
  const result = await vi.start();
  if (existsSync(abortFile)) {
    const message = readFileSync(abortFile, "utf8");
    console.error("\n" + message);
    await vi.close();
    process.exit(1);
  }
  const dispose = registerConsoleShortcuts(vi, process.stdin, process.stdout);
  if (!vi.shouldKeepServer()) {
    const exitCode = hasVitestFailures(
      result,
      vi.state.getCountOfFailedTests(),
      vi.config.dangerouslyIgnoreUnhandledErrors
    ) ? 1 : 0;
    dispose();
    await flush();
    await vi.close();
    process.exit(exitCode);
  }
  await flush();
};
export {
  runVitest
};
//# sourceMappingURL=run-vitest-SA4SZBNK.js.map