axiom/dist/evals/custom-runner.cjs

Axiom AI SDK provides - an API to wrap your AI calls with observability instrumentation. - offline evals

"use strict";
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __export = (target, all) => {
  for (var name in all)
    __defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
  if (from && typeof from === "object" || typeof from === "function") {
    for (let key of __getOwnPropNames(from))
      if (!__hasOwnProp.call(to, key) && key !== except)
        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
  }
  return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);

// src/evals/custom-runner.ts
var custom_runner_exports = {};
__export(custom_runner_exports, {
  default: () => AxiomEvalRunner
});
module.exports = __toCommonJS(custom_runner_exports);
var import_runners = require("vitest/runners");
var import_node_fs2 = require("fs");

// src/util/errors.ts
var AxiomCLIError = class extends Error {
  constructor(message) {
    super(message);
    this.name = "AxiomCLIError";
  }
};

// src/evals/name-validation-runtime.ts
var import_node_fs = require("fs");

// src/util/name-validation-runtime.ts
function isValidName(name) {
  if (name === "") {
    return { valid: false, error: "Name cannot be empty" };
  }
  const validPattern = /^[A-Za-z0-9_-]+$/;
  if (!validPattern.test(name)) {
    return {
      valid: false,
      error: `Invalid character in "${name}". Only A-Z, a-z, 0-9, -, _ allowed`
    };
  }
  return { valid: true };
}

// src/evals/name-validation-runtime.ts
function validateName(name, kind) {
  const validation = isValidName(name);
  if (!validation.valid) {
    throw new AxiomCLIError(`\u274C ${kind} name: ${validation.error}`);
  }
}

// src/evals/custom-runner.ts
var AxiomEvalRunner = class extends import_runners.VitestTestRunner {
  constructor() {
    super(...arguments);
    __publicField(this, "validationChecked", false);
  }
  /**
   * Override onBeforeRunSuite to validate names before the first suite runs.
   */
  async onBeforeRunSuite(suite) {
    if (!this.validationChecked) {
      this.validationChecked = true;
      const registryFile = process.env.AXIOM_NAME_REGISTRY_FILE;
      const abortFile = process.env.AXIOM_ABORT_FILE;
      if (registryFile && abortFile && (0, import_node_fs2.existsSync)(registryFile)) {
        const errors = [];
        const content = (0, import_node_fs2.readFileSync)(registryFile, "utf8");
        const lines = content.trim().split("\n").filter(Boolean);
        const seenEvals = /* @__PURE__ */ new Set();
        const seenScorers = /* @__PURE__ */ new Set();
        for (const line of lines) {
          try {
            const record = JSON.parse(line);
            const seen = record.kind === "eval" ? seenEvals : seenScorers;
            if (seen.has(record.name)) continue;
            seen.add(record.name);
            try {
              validateName(record.name, record.kind);
            } catch (error) {
              errors.push(error.message);
            }
          } catch {
          }
        }
        if (errors.length > 0) {
          const message = [
            "Validation failed. No tests will run due to the following errors:",
            "",
            ...errors,
            ""
          ].join("\n");
          (0, import_node_fs2.writeFileSync)(abortFile, message, "utf8");
          throw new Error("\n" + message + "\n");
        }
      }
    }
    await super.onBeforeRunSuite(suite);
  }
};
//# sourceMappingURL=custom-runner.cjs.map