evalite/dist/run-vitest.js

Test your LLM-powered apps with a TypeScript-native, Vitest-based eval runner. No API key required.

import { DB_LOCATION, DEFAULT_SERVER_PORT, FILES_LOCATION, } from "@evalite/core";
import { createDatabase } from "@evalite/core/db";
import { createServer } from "@evalite/core/server";
import { mkdir } from "fs/promises";
import path from "path";
import { Writable } from "stream";
import { createVitest, registerConsoleShortcuts } from "vitest/node";
import EvaliteReporter from "./reporter.js";
export const runVitest = async (opts) => {
    const dbLocation = path.join(opts.cwd ?? "", DB_LOCATION);
    const filesLocation = path.join(opts.cwd ?? "", FILES_LOCATION);
    await mkdir(path.dirname(dbLocation), { recursive: true });
    await mkdir(filesLocation, { recursive: true });
    const db = createDatabase(dbLocation);
    const filters = opts.path ? [opts.path] : undefined;
    process.env.EVALITE_REPORT_TRACES = "true";
    let server = undefined;
    if (opts.mode === "watch-for-file-changes") {
        server = createServer({
            db: db,
        });
        server.start(DEFAULT_SERVER_PORT);
    }
    let exitCode = undefined;
    const vitest = await createVitest("test", {
        // Everything passed here cannot be
        // overridden by the user
        root: opts.cwd,
        include: ["**/*.eval.ts"],
        watch: opts.mode === "watch-for-file-changes",
        reporters: [
            new EvaliteReporter({
                logNewState: (newState) => {
                    server?.updateState(newState);
                },
                port: DEFAULT_SERVER_PORT,
                isWatching: opts.mode === "watch-for-file-changes",
                db: db,
                scoreThreshold: opts.scoreThreshold,
                modifyExitCode: (code) => {
                    exitCode = code;
                },
            }),
        ],
        mode: "test",
        browser: undefined,
    }, {
        plugins: [
            {
                name: "evalite-config-plugin",
                // Everything inside this config CAN be overridden
                config(config) {
                    config.test ??= {};
                    config.test.testTimeout ??= 30_000;
                    config.test.sequence ??= {};
                    config.test.sequence.concurrent ??= true;
                },
            },
        ],
    }, {
        stdout: opts.testOutputWritable || process.stdout,
        stderr: opts.testOutputWritable || process.stderr,
    });
    vitest.provide("cwd", opts.cwd ?? "");
    await vitest.start(filters);
    const dispose = registerConsoleShortcuts(vitest, process.stdin, process.stdout);
    if (!vitest.shouldKeepServer()) {
        dispose();
        await vitest.close();
        if (typeof exitCode === "number") {
            process.exit(exitCode);
        }
    }
};
//# sourceMappingURL=run-vitest.js.map