axiom/dist/evals/custom-runner.js

Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals

import {
  validateName
} from "../chunk-CLH5OLB6.js";
import "../chunk-S65FSMB3.js";
import "../chunk-X2LH7XLM.js";
import {
  __publicField,
  init_esm_shims
} from "../chunk-4VNFFUM5.js";

// src/evals/custom-runner.ts
init_esm_shims();
import { VitestTestRunner } from "vitest/runners";
import { existsSync, readFileSync, writeFileSync } from "fs";
var AxiomEvalRunner = class extends VitestTestRunner {
  constructor() {
    super(...arguments);
    __publicField(this, "validationChecked", false);
  }
  /**
   * Override onBeforeRunSuite to validate names before the first suite runs.
   */
  async onBeforeRunSuite(suite) {
    if (!this.validationChecked) {
      this.validationChecked = true;
      const registryFile = process.env.AXIOM_NAME_REGISTRY_FILE;
      const abortFile = process.env.AXIOM_ABORT_FILE;
      if (registryFile && abortFile && existsSync(registryFile)) {
        const errors = [];
        const content = readFileSync(registryFile, "utf8");
        const lines = content.trim().split("\n").filter(Boolean);
        const seenEvals = /* @__PURE__ */ new Set();
        const seenScorers = /* @__PURE__ */ new Set();
        for (const line of lines) {
          try {
            const record = JSON.parse(line);
            const seen = record.kind === "eval" ? seenEvals : seenScorers;
            if (seen.has(record.name)) continue;
            seen.add(record.name);
            try {
              validateName(record.name, record.kind);
            } catch (error) {
              errors.push(error.message);
            }
          } catch {
          }
        }
        if (errors.length > 0) {
          const message = [
            "Validation failed. No tests will run due to the following errors:",
            "",
            ...errors,
            ""
          ].join("\n");
          writeFileSync(abortFile, message, "utf8");
          throw new Error("\n" + message + "\n");
        }
      }
    }
    await super.onBeforeRunSuite(suite);
  }
};
export {
  AxiomEvalRunner as default
};
//# sourceMappingURL=custom-runner.js.map