axiom/dist/run-vitest-MIWLICHA.js

Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals

import {
  AxiomReporter,
  flush,
  initInstrumentation
} from "./chunk-EHMYAMMT.js";
import {
  u
} from "./chunk-3EAYJ4JJ.js";
import "./chunk-KX7Z2MF4.js";
import {
  setAxiomConfig
} from "./chunk-PZCJLQFO.js";
import "./chunk-S65FSMB3.js";
import "./chunk-X2LH7XLM.js";
import {
  __dirname,
  init_esm_shims
} from "./chunk-4VNFFUM5.js";

// src/evals/run-vitest.ts
init_esm_shims();
import { resolve, join } from "path";
import { mkdirSync, writeFileSync, unlinkSync, existsSync, readFileSync } from "fs";
import { tmpdir } from "os";
import path from "path";
import tsconfigPaths from "vite-tsconfig-paths";
import { createVitest, registerConsoleShortcuts } from "vitest/node";
var printCollectedEvals = (result, rootDir) => {
  if (!result.testModules || result.testModules.length === 0) {
    console.log(u.yellow("\nNo evaluations found\n"));
    return;
  }
  console.log(u.bold("\nFound evaluations:\n"));
  let totalEvals = 0;
  let totalCases = 0;
  for (const module of result.testModules) {
    const relativePath = path.relative(rootDir, module.moduleId);
    for (const suite of module.children.suites()) {
      totalEvals++;
      const caseCount = suite.children.size;
      totalCases += caseCount;
      console.log(u.green(`\u2713 ${suite.name} (${caseCount} cases)`));
      console.log(u.dim(`  ${relativePath}`));
      console.log("");
    }
  }
  console.log(u.bold(`Total: ${totalEvals} evaluations, ${totalCases} test cases
`));
};
var runVitest = async (dir, opts) => {
  setAxiomConfig(opts.config);
  await initInstrumentation({
    enabled: !opts.debug && !opts.list,
    config: opts.config
  });
  const providedConfig = {
    ...opts.config,
    eval: {
      ...opts.config.eval,
      // These can't be serialized, so we need to remove them
      instrumentation: null,
      flagSchema: null
    }
  };
  if (opts.debug) {
    console.log(u.bgWhite(u.blackBright(" Debug mode enabled ")));
  }
  const tmpDir = join(tmpdir(), "axiom-eval", opts.runId);
  mkdirSync(tmpDir, { recursive: true });
  const nameRegistryFile = join(tmpDir, "names.jsonl");
  const abortFile = join(tmpDir, "abort.txt");
  writeFileSync(nameRegistryFile, "", "utf8");
  if (existsSync(abortFile)) {
    unlinkSync(abortFile);
  }
  process.env.AXIOM_NAME_REGISTRY_FILE = nameRegistryFile;
  process.env.AXIOM_ABORT_FILE = abortFile;
  if (opts.list) {
    console.log(u.bgWhite(u.blackBright(" List mode ")));
  }
  const vi = await createVitest(
    "test",
    {
      root: dir ? dir : process.cwd(),
      mode: "test",
      include: opts.include,
      exclude: opts.exclude,
      testNamePattern: opts.testNamePattern,
      reporters: ["verbose", new AxiomReporter()],
      environment: "node",
      browser: void 0,
      watch: opts.watch,
      setupFiles: [],
      // ignore user vitest.config.ts etc
      name: "axiom:eval",
      printConsoleTrace: true,
      silent: false,
      disableConsoleIntercept: true,
      testTimeout: opts.config?.eval?.timeoutMs || 6e4,
      globals: true,
      runner: resolve(__dirname, "evals", "custom-runner.js"),
      provide: {
        baseline: opts.baseline,
        debug: opts.debug,
        list: opts.list,
        overrides: opts.overrides,
        axiomConfig: providedConfig,
        runId: opts.runId,
        consoleUrl: opts.consoleUrl
      }
    },
    {
      plugins: [tsconfigPaths({ root: dir || process.cwd() })]
    }
  );
  if (opts.list) {
    const result = await vi.collect();
    printCollectedEvals(result, dir || process.cwd());
    await vi.close();
    process.exit(0);
  }
  await vi.start();
  if (existsSync(abortFile)) {
    const message = readFileSync(abortFile, "utf8");
    console.error("\n" + message);
    await vi.close();
    process.exit(1);
  }
  const dispose = registerConsoleShortcuts(vi, process.stdin, process.stdout);
  if (!vi.shouldKeepServer()) {
    dispose();
    await flush();
    await vi.close();
    process.exit(0);
  }
  await flush();
};
export {
  runVitest
};
//# sourceMappingURL=run-vitest-MIWLICHA.js.map