magnitude-test
Version:
A TypeScript client for running automated UI tests through the Magnitude testing platform
1,247 lines (1,230 loc) • 41.3 kB
JavaScript
import { Command } from '@commander-js/extra-typings';
import path from 'node:path';
import fs from 'node:fs';
import { glob } from 'glob';
import { Worker } from 'node:worker_threads';
import { isBun, isDeno } from 'std-env';
import { logger } from 'magnitude-core';
import { p as processUrl, k as knownCostMap, V as VERSION, d as describeModel, l as logger$1 } from './version-DtiDJ6_4.js';
import * as dotenv from 'dotenv';
import { spawn, execSync } from 'node:child_process';
import { EventEmitter } from 'node:events';
import { setImmediate } from 'node:timers';
import logUpdate from 'log-update';
import { setTimeout as setTimeout$1 } from 'node:timers/promises';
import chalk from 'chalk';
import 'pino';
import 'node:module';
async function findProjectRoot(startDir = process.cwd()) {
let currentDir = startDir;
const rootDir = path.parse(currentDir).root;
while (currentDir !== rootDir) {
const packagePath = path.join(currentDir, "package.json");
try {
await fs.promises.access(packagePath, fs.constants.F_OK);
return currentDir;
} catch (error) {
const parentDir = path.dirname(currentDir);
if (parentDir === currentDir) {
return null;
}
currentDir = parentDir;
}
}
return null;
}
async function isProjectRoot(dir) {
const packagePath = path.join(dir, "package.json");
try {
await fs.promises.access(packagePath, fs.constants.F_OK);
return true;
} catch (error) {
return false;
}
}
async function discoverTestFiles(patterns, cwd = process.cwd()) {
try {
const files = await glob(patterns, {
cwd,
dot: true,
// Ignore dot files by default
nodir: true,
// Only return files, not directories
absolute: true
// Return paths relative to cwd
});
return files.map((file) => path.resolve(cwd, file));
} catch (error) {
console.error("Error discovering test files:", error);
return [];
}
}
function findConfig(searchRoot) {
try {
const configFiles = glob.sync("**/magnitude.config.{mts,ts}", {
cwd: searchRoot,
ignore: ["**/node_modules/**", "**/dist/**"],
absolute: true
});
return configFiles.length > 0 ? configFiles[0] : null;
} catch (error) {
console.error("Error finding config file:", error);
return null;
}
}
function readConfig(configPath) {
return new Promise((resolve, reject) => {
const worker = new Worker(
new URL(
import.meta.url.endsWith(".ts") ? "../worker/readConfig.ts" : "./worker/readConfig.js",
import.meta.url
),
{
env: { NODE_ENV: "test", ...process.env },
execArgv: !(isBun || isDeno) ? ["--import=jiti/register"] : []
}
);
worker.once("message", ({ success, config, error }) => {
worker.terminate();
if (success) {
resolve(config);
} else {
reject(new Error(error));
}
});
worker.once("error", (error) => {
worker.terminate();
reject(error);
});
worker.once("online", () => {
worker.postMessage({ configPath });
});
});
}
class WorkerPool {
concurrency;
/**
* Creates an instance of WorkerPool.
* @param concurrency The maximum number of tasks to run concurrently. Must be at least 1.
*/
constructor(concurrency) {
this.concurrency = Math.max(1, concurrency);
}
/**
* Runs the given asynchronous tasks with the specified concurrency.
*
* @template T The type of the result returned by each task.
* @param tasks An array of functions, each returning a Promise<T>. Each function receives an AbortSignal.
* @param checkResultForAbort An optional function that checks the result of a completed task. If it returns true, the pool will abort further processing.
* @returns A Promise resolving to a WorkerPoolResult<T> object.
*/
async runTasks(tasks, checkResultForAbort = () => false) {
const abortController = new AbortController();
const { signal } = abortController;
const taskQueue = tasks.map((task, index) => ({ task, index }));
const results = new Array(tasks.length).fill(void 0);
const runningWorkers = /* @__PURE__ */ new Set();
const runWorker = async () => {
while (taskQueue.length > 0) {
if (signal.aborted) {
break;
}
const taskItem = taskQueue.shift();
if (!taskItem) continue;
const { task, index } = taskItem;
try {
if (signal.aborted) {
continue;
}
const result = await task(signal);
results[index] = result;
if (!signal.aborted && checkResultForAbort(result)) {
abortController.abort();
}
} catch (error) {
results[index] = void 0;
if (!signal.aborted) {
abortController.abort();
}
}
}
};
const startWorkers = () => {
while (runningWorkers.size < this.concurrency && taskQueue.length > 0 && !signal.aborted) {
const workerPromise = runWorker().finally(() => {
runningWorkers.delete(workerPromise);
if (!signal.aborted && taskQueue.length > 0) {
startWorkers();
}
});
runningWorkers.add(workerPromise);
}
};
startWorkers();
while (runningWorkers.size > 0) {
try {
await Promise.race(runningWorkers);
} catch (e) {
console.error("Unexpected error waiting for worker:", e);
if (!signal.aborted) {
abortController.abort();
}
}
}
const completed = !signal.aborted && taskQueue.length === 0;
return { completed, results };
}
}
const TEST_FILE_LOADING_TIMEOUT = 3e4;
class TestSuiteRunner {
runnerConfig;
renderer;
config;
tests;
executors = /* @__PURE__ */ new Map();
workerStoppers = [];
constructor(config) {
this.tests = [];
this.runnerConfig = config;
this.config = config.config;
}
async runTest(test, signal) {
const executor = this.executors.get(test.id);
if (!executor) {
throw new Error(`Test worker not found for test ID: ${test.id}`);
}
try {
return await executor(
{
type: "execute",
test,
browserOptions: this.config.browser,
llm: this.config.llm,
grounding: this.config.grounding,
telemetry: this.config.telemetry
},
(state) => {
this.renderer?.onTestStateUpdated(test, state);
},
signal
);
} catch (err) {
if (signal.aborted) {
return {
passed: false,
failure: {
message: "Test execution aborted"
}
};
}
const errorMessage = err instanceof Error ? err.message : String(err);
console.error(`Unexpected error during test '${test.title}':
${errorMessage}`);
return {
passed: false,
failure: {
message: errorMessage
}
};
}
}
getActiveOptions() {
const envOptions = process.env.MAGNITUDE_TEST_URL ? {
url: process.env.MAGNITUDE_TEST_URL
} : {};
return {
...this.config,
...envOptions,
// env options take precedence over config options
url: processUrl(envOptions.url, this.config.url)
};
}
async loadTestFile(absoluteFilePath, relativeFilePath) {
try {
const workerData = {
relativeFilePath,
absoluteFilePath,
options: this.getActiveOptions()
};
const createWorker = isBun ? createBunTestWorker : createNodeTestWorker;
const result = await createWorker(workerData);
this.tests.push(...result.tests);
for (const test of result.tests) {
this.executors.set(test.id, result.executor);
}
this.workerStoppers.push(result.stopper);
} catch (error) {
console.error(`Failed to load test file ${relativeFilePath}:`, error);
throw error;
}
}
async runTests() {
if (!this.tests) throw new Error("No tests were registered");
this.renderer = this.runnerConfig.createRenderer(this.tests);
this.renderer.start?.();
const workerPool = new WorkerPool(this.runnerConfig.workerCount);
let overallSuccess = true;
try {
const poolResult = await workerPool.runTasks(
this.tests.map((test) => (signal) => this.runTest(test, signal)),
this.runnerConfig.failFast ? (taskOutcome) => !taskOutcome.passed : () => false
);
for (const result of poolResult.results) {
if (result === void 0 || !result.passed) {
overallSuccess = false;
break;
}
}
if (!poolResult.completed) {
overallSuccess = false;
}
} catch (error) {
overallSuccess = false;
}
const stopperResults = await Promise.allSettled(
this.workerStoppers.map((stopper) => stopper())
);
const stopperErrors = stopperResults.filter((result) => result.status === "rejected");
if (stopperErrors.length > 0) {
overallSuccess = false;
console.error(`${stopperErrors.length} workers failed to stop`);
for (const error of stopperErrors) {
console.error(error);
}
}
this.renderer.stop?.();
return overallSuccess;
}
}
const createNodeTestWorker = async (workerData) => new Promise((resolve, reject) => {
const { relativeFilePath } = workerData;
const worker = new Worker(
new URL(
import.meta.url.endsWith(".ts") ? "../worker/readTest.ts" : "./worker/readTest.js",
import.meta.url
),
{
workerData,
env: { NODE_ENV: "test", ...process.env },
execArgv: !(isBun || isDeno) ? ["--import=jiti/register"] : []
}
);
let hasRunTests = false;
const stopper = async () => {
if (!hasRunTests) {
worker.terminate();
return;
}
worker.postMessage({ type: "graceful_shutdown" });
return new Promise((resolve2, reject2) => {
const shutdownHandler = (msg) => {
if (msg.type === "graceful_shutdown_complete") {
worker.off("message", shutdownHandler);
worker.off("error", errorHandler);
worker.off("exit", exitHandler);
worker.terminate();
resolve2();
}
};
const errorHandler = (error) => {
worker.off("message", shutdownHandler);
worker.off("error", errorHandler);
worker.off("exit", exitHandler);
reject2(new Error(`Worker error during shutdown: ${error.message}`));
};
const exitHandler = (code) => {
worker.off("message", shutdownHandler);
worker.off("error", errorHandler);
worker.off("exit", exitHandler);
reject2(new Error(`Worker exited unexpectedly during shutdown with code: ${code}`));
};
worker.on("message", shutdownHandler);
worker.on("error", errorHandler);
worker.on("exit", exitHandler);
});
};
const executor = (executeMessage, onStateChange, signal) => new Promise((res, rej) => {
const messageHandler = (msg) => {
if ("test" in executeMessage && "testId" in msg && msg.testId !== executeMessage.test.id) return;
hasRunTests = true;
if (msg.type === "test_result") {
worker.off("message", messageHandler);
res(msg.result);
} else if (msg.type === "test_error") {
worker.off("message", messageHandler);
rej(new Error(msg.error));
} else if (msg.type === "test_state_change") {
onStateChange(msg.state);
}
};
signal.addEventListener("abort", () => {
worker.off("message", messageHandler);
rej(new Error("Test execution aborted"));
});
worker.on("message", messageHandler);
worker.postMessage(executeMessage);
});
const registeredTests = [];
worker.on("message", (message) => {
if (message.type === "registered") {
registeredTests.push(message.test);
} else if (message.type === "load_error") {
clearTimeout(timeout);
worker.terminate();
reject(new Error(`Failed to load ${relativeFilePath}: ${message.error}`));
} else if (message.type === "load_complete") {
clearTimeout(timeout);
if (!registeredTests.length) {
reject(new Error(`No tests registered for file ${relativeFilePath}`));
return;
}
resolve({ tests: registeredTests, executor, stopper });
}
});
const timeout = setTimeout(() => {
worker.terminate();
reject(new Error(`Test file loading timeout: ${relativeFilePath}`));
}, TEST_FILE_LOADING_TIMEOUT);
worker.on("error", (error) => {
clearTimeout(timeout);
worker.terminate();
reject(error);
});
});
const createBunTestWorker = async (workerData) => new Promise((resolve, reject) => {
const { relativeFilePath } = workerData;
const emit = new EventEmitter();
const proc = Bun.spawn({
cmd: [
"bun",
new URL(
import.meta.url.endsWith(".ts") ? "../worker/readTest.ts" : "./worker/readTest.js",
import.meta.url
).pathname
],
env: {
NODE_ENV: "test",
...process.env,
MAGNITUDE_WORKER_DATA: JSON.stringify(workerData)
},
cwd: process.cwd(),
stdin: "inherit",
stdout: "inherit",
stderr: "inherit",
// "advanced" serialization in Bun somehow isn't able to clone test state messages?
serialization: "json",
ipc(message) {
emit.emit("message", message);
},
onExit(_subprocess, exitCode) {
if (exitCode !== 0) {
clearTimeout(timeout);
reject(new Error(`Worker process exited with code ${exitCode}`));
}
}
});
let hasRunTests = false;
const stopper = async () => {
if (!hasRunTests) {
proc.kill("SIGKILL");
return;
}
proc.send({ type: "graceful_shutdown" });
return new Promise((resolve2, reject2) => {
const shutdownHandler = (msg) => {
if (msg.type === "graceful_shutdown_complete") {
emit.off("message", shutdownHandler);
proc.kill("SIGKILL");
resolve2();
}
};
const exitHandler = (code) => {
emit.off("message", shutdownHandler);
reject2(new Error(`Bun worker exited unexpectedly during shutdown with code: ${code}`));
};
emit.on("message", shutdownHandler);
proc.exited.then(exitHandler);
});
};
const executor = (executeMessage, onStateChange, signal) => new Promise((res, rej) => {
const messageHandler = (msg) => {
if ("test" in executeMessage && "testId" in msg && msg.testId !== executeMessage.test.id) return;
hasRunTests = true;
if (msg.type === "test_result") {
emit.off("message", messageHandler);
res(msg.result);
} else if (msg.type === "test_error") {
emit.off("message", messageHandler);
rej(new Error(msg.error));
} else if (msg.type === "test_state_change") {
onStateChange(msg.state);
}
};
emit.on("message", messageHandler);
signal.addEventListener("abort", () => {
emit.off("message", messageHandler);
rej(new Error("Test execution aborted"));
});
proc.send(executeMessage);
});
const registeredTests = [];
emit.on("message", (message) => {
if (message.type === "registered") {
registeredTests.push(message.test);
} else if (message.type === "load_error") {
clearTimeout(timeout);
proc.kill();
reject(new Error(`Failed to load ${relativeFilePath}: ${message.error}`));
} else if (message.type === "load_complete") {
clearTimeout(timeout);
if (!registeredTests.length) {
reject(new Error(`No tests registered for file ${relativeFilePath}`));
return;
}
resolve({ tests: registeredTests, executor, stopper });
}
});
const timeout = setTimeout(() => {
proc.kill("SIGKILL");
reject(new Error(`Test file loading timeout: ${relativeFilePath}`));
}, TEST_FILE_LOADING_TIMEOUT);
});
const ANSI_RESET = "\x1B[0m";
const ANSI_GREEN = "\x1B[32m";
const ANSI_BRIGHT_BLUE = "\x1B[94m";
const ANSI_GRAY = "\x1B[90m";
const ANSI_RED = "\x1B[31m";
const ANSI_BOLD = "\x1B[1m";
const ANSI_DIM = "\x1B[2m";
const spinnerChars$1 = ["\u280B", "\u2819", "\u2839", "\u2838", "\u283C", "\u2834", "\u2826", "\u2827", "\u2807", "\u280F"];
const spinnerChars = spinnerChars$1;
let redrawScheduled = false;
let renderSettings = { showActions: true };
let timerInterval = null;
let currentTestStates = {};
let allRegisteredTests = [];
let currentModel = "";
let elapsedTimes = {};
let isFinished = false;
let spinnerFrame = 0;
function resetState() {
redrawScheduled = false;
renderSettings = { showActions: true };
timerInterval = null;
currentTestStates = {};
allRegisteredTests = [];
currentModel = "";
elapsedTimes = {};
isFinished = false;
spinnerFrame = 0;
}
function setRedrawScheduled(value) {
redrawScheduled = value;
}
function setCurrentModel(model) {
currentModel = model;
}
function setAllRegisteredTests(tests) {
allRegisteredTests = tests;
}
function setCurrentTestStates(states) {
currentTestStates = states;
}
function setTimerInterval(interval) {
timerInterval = interval;
}
function setSpinnerFrame(frame) {
spinnerFrame = frame;
}
function setElapsedTimes(times) {
elapsedTimes = times;
}
function updateElapsedTime(testId, time) {
elapsedTimes[testId] = time;
}
function setIsFinished(value) {
isFinished = value;
}
function setRenderSettings(settings) {
renderSettings = settings;
}
function formatDuration(ms) {
if (ms === void 0 || ms === null) {
return "";
}
return `${(ms / 1e3).toFixed(1)}s`;
}
function getTestStatusIndicatorChar(status) {
switch (status) {
case "passed":
return "\u2713";
case "failed":
return "\u2715";
case "cancelled":
return "\u2298";
case "pending":
default:
return "\u25CC";
}
}
function getStepStatusIndicatorChar(status) {
switch (status) {
case "running":
return ">";
case "passed":
return "\u2691";
case "failed":
return "\u2715";
case "cancelled":
return "\u2298";
case "pending":
default:
return "\u2022";
}
}
function getCheckStatusIndicatorChar(status) {
switch (status) {
case "running":
return "?";
case "passed":
return "\u2713";
case "failed":
return "\u2715";
case "cancelled":
return "\u2298";
case "pending":
default:
return "\u2022";
}
}
function styleAnsi(status, text, type) {
let colorCode = ANSI_GRAY;
switch (type) {
case "test":
switch (status) {
case "running":
colorCode = ANSI_BRIGHT_BLUE;
break;
case "passed":
colorCode = ANSI_GREEN;
break;
case "failed":
colorCode = ANSI_RED;
break;
case "cancelled":
colorCode = ANSI_GRAY;
break;
}
break;
case "step":
switch (status) {
case "running":
colorCode = ANSI_GRAY;
break;
case "passed":
colorCode = ANSI_BRIGHT_BLUE;
break;
case "failed":
colorCode = ANSI_RED;
break;
case "cancelled":
colorCode = ANSI_GRAY;
break;
}
break;
case "check":
switch (status) {
case "running":
colorCode = ANSI_GRAY;
break;
case "passed":
colorCode = ANSI_BRIGHT_BLUE;
break;
case "failed":
colorCode = ANSI_RED;
break;
case "cancelled":
colorCode = ANSI_GRAY;
break;
}
break;
}
return `${colorCode}${text}${ANSI_RESET}`;
}
const UI_LEFT_PADDING = " ";
function generateTitleBarString() {
const titleText = `${ANSI_BRIGHT_BLUE}${ANSI_BOLD}Magnitude v${VERSION}${ANSI_RESET}`;
const modelText = `${ANSI_GRAY}${currentModel}${ANSI_RESET}`;
return [`${UI_LEFT_PADDING}${titleText} ${modelText}`];
}
function generateFailureString(failure, indent) {
const output = [];
const prefix = "\u21B3 ";
const prefixAnsi = `${ANSI_RED}${prefix}${ANSI_RESET}`;
const addLine = (text, styleCode = ANSI_RED, bold = false) => {
const fullStyleCode = `${styleCode}${bold ? ANSI_BOLD : ""}`;
output.push(UI_LEFT_PADDING + " ".repeat(indent) + prefixAnsi + `${fullStyleCode}${text}${ANSI_RESET}`);
};
if (failure && failure.message) {
addLine(failure.message);
} else {
addLine("Unknown error details");
}
return output;
}
function generateTestString(test, state, indent) {
const output = [];
const testId = test.id;
const stepIndent = indent + 2;
const actionIndent = stepIndent + 2;
const currentStatus = state.status;
const statusCharPlain = currentStatus === "running" ? spinnerChars[spinnerFrame] : getTestStatusIndicatorChar(currentStatus);
const statusStyled = styleAnsi(currentStatus, statusCharPlain, "test");
const timerText = currentStatus !== "pending" ? `${ANSI_GRAY} [${formatDuration(elapsedTimes[testId] ?? 0)}]${ANSI_RESET}` : "";
output.push(UI_LEFT_PADDING + " ".repeat(indent) + `${statusStyled} ${test.title}${timerText}`);
if (state.stepsAndChecks && state.stepsAndChecks.length > 0) {
state.stepsAndChecks.forEach((item) => {
let itemCharPlain = "";
let itemDesc = "";
let itemStyleType = "step";
if (item.variant === "step") {
itemCharPlain = getStepStatusIndicatorChar(item.status);
itemDesc = item.description;
itemStyleType = "step";
} else {
itemCharPlain = getCheckStatusIndicatorChar(item.status);
itemDesc = item.description;
itemStyleType = "check";
}
const styledChar = styleAnsi(item.status, itemCharPlain, itemStyleType);
output.push(UI_LEFT_PADDING + " ".repeat(stepIndent) + `${styledChar} ${itemDesc}`);
if (renderSettings.showActions && item.variant === "step" && item.actions.length > 0) {
item.actions.forEach((action) => {
output.push(UI_LEFT_PADDING + " ".repeat(actionIndent) + `${ANSI_GRAY}${action.pretty}${ANSI_RESET}`);
});
}
});
}
if (state.failure) {
const failureLines = generateFailureString(state.failure, stepIndent);
output.push(...failureLines);
}
return output;
}
function groupRegisteredTestsForDisplay(tests) {
const files = {};
for (const test of tests) {
if (!files[test.filepath]) {
files[test.filepath] = { ungrouped: [], groups: {} };
}
if (test.group) {
if (!files[test.filepath].groups[test.group]) {
files[test.filepath].groups[test.group] = [];
}
files[test.filepath].groups[test.group].push(test);
} else {
files[test.filepath].ungrouped.push(test);
}
}
return files;
}
function generateTestListString() {
const output = [];
const fileIndent = 0;
const groupIndent = fileIndent + 2;
const testBaseIndent = groupIndent;
const groupedDisplayTests = groupRegisteredTestsForDisplay(allRegisteredTests);
for (const [filepath, { ungrouped, groups }] of Object.entries(groupedDisplayTests)) {
const fileHeader = `${ANSI_BRIGHT_BLUE}${ANSI_BOLD}\u2630 ${filepath}${ANSI_RESET}`;
output.push(UI_LEFT_PADDING + " ".repeat(fileIndent) + fileHeader);
if (ungrouped.length > 0) {
for (const test of ungrouped) {
const state = currentTestStates[test.id];
if (state) {
const testLines = generateTestString(test, state, testBaseIndent);
output.push(...testLines);
}
}
}
if (Object.entries(groups).length > 0) {
for (const [groupName, groupTests] of Object.entries(groups)) {
const groupHeader = `${ANSI_BRIGHT_BLUE}${ANSI_BOLD}\u21B3 ${groupName}${ANSI_RESET}`;
output.push(UI_LEFT_PADDING + " ".repeat(groupIndent) + groupHeader);
for (const test of groupTests) {
const state = currentTestStates[test.id];
if (state) {
const testLines = generateTestString(test, state, testBaseIndent + 2);
output.push(...testLines);
}
}
}
}
output.push(UI_LEFT_PADDING);
}
return output;
}
function generateSummaryString() {
const output = [];
let totalInputTokens = 0;
let totalOutputTokens = 0;
const statusCounts = { pending: 0, running: 0, passed: 0, failed: 0, cancelled: 0, total: 0 };
const failuresWithContext = [];
const testContextMap = /* @__PURE__ */ new Map();
allRegisteredTests.forEach((test) => {
testContextMap.set(test.id, { filepath: test.filepath, groupName: test.group, testTitle: test.title });
});
Object.entries(currentTestStates).forEach(([testId, state]) => {
statusCounts.total++;
statusCounts[state.status]++;
if (state.modelUsage.length > 0) {
totalInputTokens += state.modelUsage[0].inputTokens;
totalOutputTokens += state.modelUsage[0].outputTokens;
}
if (state.failure) {
const context = testContextMap.get(testId);
failuresWithContext.push({
filepath: context?.filepath ?? "Unknown File",
groupName: context?.groupName,
testTitle: context?.testTitle ?? "Unknown Test",
failure: state.failure
});
}
});
const hasFailures = failuresWithContext.length > 0;
let statusLine = "";
if (statusCounts.passed > 0) statusLine += `${ANSI_GREEN}\u2713 ${statusCounts.passed} passed${ANSI_RESET} `;
if (statusCounts.failed > 0) statusLine += `${ANSI_RED}\u2717 ${statusCounts.failed} failed${ANSI_RESET} `;
if (statusCounts.running > 0) statusLine += `${ANSI_BRIGHT_BLUE}\u25B7 ${statusCounts.running} running${ANSI_RESET} `;
if (statusCounts.pending > 0) statusLine += `${ANSI_GRAY}\u25CC ${statusCounts.pending} pending${ANSI_RESET} `;
if (statusCounts.cancelled > 0) statusLine += `${ANSI_GRAY}\u2298 ${statusCounts.cancelled} cancelled${ANSI_RESET} `;
let costDescription = "";
for (const [model, costs] of Object.entries(knownCostMap)) {
if (currentModel.includes(model)) {
const inputCost = costs[0];
const outputCost = costs[1];
costDescription = ` ($${((totalInputTokens * inputCost + totalOutputTokens * outputCost) / 1e6).toFixed(2)})`;
}
}
let tokenText = `${ANSI_GRAY}tokens: ${totalInputTokens} in, ${totalOutputTokens} out${costDescription}${ANSI_RESET}`;
output.push(UI_LEFT_PADDING + statusLine.trimEnd() + (statusLine && tokenText ? " " : "") + tokenText.trimStart());
if (hasFailures) {
output.push(UI_LEFT_PADDING + `${ANSI_DIM}Failures:${ANSI_RESET}`);
for (const { filepath, groupName, testTitle, failure } of failuresWithContext) {
const contextString = `${ANSI_DIM}${filepath}${groupName ? ` > ${groupName}` : ""} > ${testTitle}${ANSI_RESET}`;
output.push(UI_LEFT_PADDING + UI_LEFT_PADDING + contextString);
const failureLines = generateFailureString(failure, 4);
output.push(...failureLines);
output.push(UI_LEFT_PADDING);
}
}
return output;
}
function calculateTestListHeight(tests, testStates) {
let height = 0;
const groupedDisplayTests = groupRegisteredTestsForDisplay(tests);
for (const [filepath, { ungrouped, groups }] of Object.entries(groupedDisplayTests)) {
height++;
if (ungrouped.length > 0) {
for (const test of ungrouped) {
const state = testStates[test.id];
if (state) {
height++;
if (state.stepsAndChecks) {
state.stepsAndChecks.forEach((item) => {
height++;
if (renderSettings.showActions && item.variant === "step" && item.actions.length > 0) {
item.actions.forEach(() => height++);
}
});
}
if (state.failure) {
height++;
}
}
}
}
if (Object.entries(groups).length > 0) {
for (const [groupName, groupTests] of Object.entries(groups)) {
height++;
for (const test of groupTests) {
const state = testStates[test.id];
if (state) {
height++;
if (state.stepsAndChecks) {
state.stepsAndChecks.forEach((item) => {
height++;
if (renderSettings.showActions && item.variant === "step" && item.actions.length > 0) {
item.actions.forEach(() => height++);
}
});
}
if (state.failure) {
height++;
}
}
}
}
}
height++;
}
return height;
}
function calculateSummaryHeight(testStates) {
let height = 0;
height++;
const failuresExist = Object.values(testStates).some((state) => !!state.failure);
if (failuresExist) {
height++;
Object.values(testStates).forEach((state) => {
if (state.failure) {
height++;
height++;
height++;
}
});
}
return height;
}
function redraw() {
setRedrawScheduled(false);
let testListLineCount = calculateTestListHeight(allRegisteredTests, currentTestStates);
let summaryLineCount = calculateSummaryHeight(currentTestStates);
if (Object.values(currentTestStates).length === 0) {
summaryLineCount = 0;
testListLineCount = 0;
}
const outputLines = [];
outputLines.push(...generateTitleBarString());
outputLines.push(UI_LEFT_PADDING);
if (testListLineCount > 0) {
outputLines.push(...generateTestListString());
}
if (summaryLineCount > 0) {
if (testListLineCount > 0) outputLines.push(UI_LEFT_PADDING);
outputLines.push(...generateSummaryString());
}
const frameContent = outputLines.join("\n");
logUpdate.clear();
logUpdate(frameContent);
if (isFinished) {
logUpdate.done();
process.stderr.write("\n");
}
}
function scheduleRedraw() {
if (!redrawScheduled) {
setRedrawScheduled(true);
setImmediate(redraw);
}
}
class TermAppRenderer {
magnitudeConfig;
initialTests;
firstModelReportedInUI = false;
// New flag
// To manage SIGINT listener
sigintListener = null;
constructor(config, initialTests) {
this.magnitudeConfig = config;
this.initialTests = [...initialTests];
if (this.magnitudeConfig.display?.showActions !== void 0) {
setRenderSettings({ showActions: this.magnitudeConfig.display.showActions });
}
setCurrentModel("");
}
start() {
process.stdout.write("\n");
resetState();
if (this.magnitudeConfig.display?.showActions !== void 0) {
setRenderSettings({ showActions: this.magnitudeConfig.display.showActions });
}
this.firstModelReportedInUI = false;
setAllRegisteredTests(this.initialTests);
const initialTestStates = {};
for (const test of this.initialTests) {
initialTestStates[test.id] = {
status: "pending",
// Add initial status
stepsAndChecks: [],
modelUsage: []
// macroUsage: { provider: '', model: '', inputTokens: 0, outputTokens: 0, numCalls: 0 },
// microUsage: { provider: '', numCalls: 0 },
};
}
setCurrentTestStates(initialTestStates);
setElapsedTimes({});
this.sigintListener = this.handleExitKeyPress.bind(this);
process.on("SIGINT", this.sigintListener);
if (!timerInterval) {
const interval = setInterval(() => {
if (isFinished) {
clearInterval(timerInterval);
setTimerInterval(null);
return;
}
let runningTestsExist = false;
setSpinnerFrame((spinnerFrame + 1) % spinnerChars.length);
Object.entries(currentTestStates).forEach(([testId, state]) => {
const liveState = state;
if (liveState.status === "running") {
runningTestsExist = true;
if (liveState.startedAt) {
updateElapsedTime(testId, Date.now() - liveState.startedAt);
}
}
});
if (runningTestsExist && !redrawScheduled) {
scheduleRedraw();
}
}, 100);
setTimerInterval(interval);
}
scheduleRedraw();
}
stop() {
if (isFinished) return;
setIsFinished(true);
if (timerInterval) {
clearInterval(timerInterval);
setTimerInterval(null);
}
if (this.sigintListener) {
process.removeListener("SIGINT", this.sigintListener);
this.sigintListener = null;
}
redraw();
}
onTestStateUpdated(test, newState) {
const currentStates = { ...currentTestStates };
const testId = test.id;
const existingState = currentStates[testId] || {};
const updatedTestState = {
...existingState,
...newState,
startedAt: newState.startedAt || existingState.startedAt
};
currentStates[testId] = updatedTestState;
setCurrentTestStates(currentStates);
if (!this.firstModelReportedInUI && newState.modelUsage && newState.modelUsage.length > 0) {
const firstModelEntry = newState.modelUsage[0];
let modelNameToReport = void 0;
if (firstModelEntry && firstModelEntry.llm) {
modelNameToReport = describeModel(firstModelEntry.llm);
}
if (modelNameToReport) {
setCurrentModel(modelNameToReport);
this.firstModelReportedInUI = true;
}
}
if (updatedTestState.startedAt && !updatedTestState.doneAt) {
if (!elapsedTimes[testId] || elapsedTimes[testId] === 0) {
updateElapsedTime(testId, Date.now() - updatedTestState.startedAt);
}
} else if (updatedTestState.startedAt && updatedTestState.doneAt) {
updateElapsedTime(testId, updatedTestState.doneAt - updatedTestState.startedAt);
}
scheduleRedraw();
}
// onResize method removed as per user request.
// Adapted from term-app/index.ts
handleExitKeyPress() {
this.stop();
}
}
async function isServerRunning(url) {
try {
await fetch(url, { method: "HEAD" });
return true;
} catch {
return false;
}
}
async function startWebServer(config) {
const { command, url, timeout = 6e4, reuseExistingServer = false } = config;
if (reuseExistingServer && await isServerRunning(url)) {
return null;
}
const child = spawn(command, { shell: true, stdio: "inherit" });
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
if (await isServerRunning(url)) {
return child;
}
await setTimeout$1(500);
}
child.kill();
throw new Error(`Timed out waiting for web server at ${url}`);
}
function stopWebServer(proc) {
if (proc) {
proc.kill();
}
}
async function startWebServers(configs) {
if (Array.isArray(configs)) {
const procs = [];
for (const config of configs) {
procs.push(await startWebServer(config));
}
return procs;
} else {
return [await startWebServer(configs)];
}
}
function stopWebServers(procs) {
for (const proc of procs) {
stopWebServer(proc);
}
}
function getRelativePath(projectRoot, absolutePath) {
const normalizedAbsolutePath = path.normalize(absolutePath);
const normalizedProjectRoot = path.normalize(projectRoot);
if (!normalizedAbsolutePath.startsWith(normalizedProjectRoot)) {
return absolutePath;
}
return path.relative(normalizedProjectRoot, normalizedAbsolutePath);
}
const configTemplate = `import { type MagnitudeConfig } from 'magnitude-test';
// Learn more about configuring Magnitude:
// https://docs.magnitude.run/customizing/configuration
export default {
url: "http://localhost:5173"
} satisfies MagnitudeConfig;
`;
const exampleTestTemplate = `import { test } from 'magnitude-test';
// Learn more about building test case:
// https://docs.magnitude.run/core-concepts/building-test-cases
const sampleTodos = [
"Take out the trash",
"Pay AWS bill",
"Build more test cases with Magnitude"
];
test('can add and complete todos', { url: 'https://magnitodo.com' }, async (agent) => {
await agent.act('create 3 todos', { data: sampleTodos.join(', ') });
await agent.check('should see all 3 todos');
await agent.act('mark each todo complete');
await agent.check('says 0 items left');
});
`;
async function initializeProject(force = false, destination = "tests/magnitude") {
const cwd = process.cwd();
const isNodeProject = await isProjectRoot(cwd);
if (!isNodeProject && !force) {
console.error("Couldn't find package.json in current directory, please initialize Magnitude in a node.js project");
console.error("To override this check, use --force option");
process.exit(1);
}
console.log(chalk.blueBright(`Initializing Magnitude tests in ${cwd}`));
const testsDir = path.join(cwd, destination);
const configPath = path.join(testsDir, "magnitude.config.ts");
if (fs.existsSync(configPath)) {
console.error("Already initialized, magnitude.config.ts already exists!");
process.exit(1);
}
try {
await fs.promises.mkdir(testsDir, { recursive: true });
await fs.promises.writeFile(configPath, configTemplate);
const examplePath = path.join(testsDir, "example.mag.ts");
await fs.promises.writeFile(examplePath, exampleTestTemplate);
console.log(`${chalk.blueBright("\u2713")} Created Magnitude test directory structure:
- ${path.relative(cwd, configPath)}
- ${path.relative(cwd, examplePath)}
`);
} catch (error) {
console.error("Error initializing Magnitude project:", error);
process.exit(1);
}
console.log(chalk.blueBright("Installing Playwright Chromium..."));
try {
execSync("npx playwright install chromium", { stdio: "inherit" });
console.log(`${chalk.blueBright("\u2713")} Playwright Chromium installed successfully`);
} catch (error) {
console.error("Error installing Playwright Chromium:", error);
console.log(chalk.blueBright("You may need to manually run: npx playwright install chromium"));
}
console.log(`You can now run tests with: ${chalk.blueBright("npx magnitude")}`);
console.log("Docs:", chalk.blueBright("https://docs.magnitude.run"));
}
const program = new Command();
program.name("magnitude").description("Run Magnitude test cases").argument("[filter]", "glob pattern for test files (quote if contains spaces or wildcards)").option("-w, --workers <number>", "number of parallel workers for test execution", "1").option("-p, --plain", "disable pretty output and print lines instead").option("-d, --debug", "enable debug logs").option("--no-fail-fast", "continue running tests even if some fail").action(async (filter, options) => {
dotenv.config();
let logLevel;
if (process.env.MAGNITUDE_LOG_LEVEL) {
logLevel = process.env.MAGNITUDE_LOG_LEVEL;
} else if (options.debug) {
logLevel = "trace";
} else if (options.plain) {
logLevel = "info";
} else {
logLevel = "warn";
}
logger.level = logLevel;
logger$1.level = logLevel;
const patterns = [
"!**/node_modules/**",
"!**/dist/**"
];
if (filter) {
patterns.push(filter);
} else {
patterns.push("**/*.{mag,magnitude}.{js,jsx,ts,tsx}");
}
const workerCount = options.workers ? parseInt(options.workers, 10) : 1;
if (isNaN(workerCount) || workerCount < 1) {
console.error("Invalid worker count. Using default of 1.");
}
const absoluteFilePaths = await discoverTestFiles(patterns);
if (absoluteFilePaths.length === 0) {
console.error(`No test files found matching patterns: ${patterns.join(", ")}`);
process.exit(1);
}
const projectRoot = await findProjectRoot() ?? process.cwd();
const configPath = findConfig(projectRoot);
const config = configPath ? await readConfig(configPath) : {};
let webServerProcesses = [];
if (config.webServer) {
try {
webServerProcesses = await startWebServers(config.webServer);
const cleanup = () => stopWebServers(webServerProcesses);
process.on("exit", cleanup);
process.on("SIGINT", () => {
cleanup();
process.exit(1);
});
} catch (err) {
console.error("Error starting web server(s):", err);
process.exit(1);
}
}
const showUI = !options.debug && !options.plain;
const testSuiteRunner = new TestSuiteRunner({
config,
workerCount,
failFast: options.failFast === false ? false : !config.continueAfterFailure,
createRenderer: (tests) => showUI ? new TermAppRenderer(config, tests) : {
// Plain/debug renderer
onTestStateUpdated: (test, state) => {
logger$1.info(`Test: ${test.title} (${test.id})`);
logger$1.info(` Status: ${state.failure ? "failed" : state.doneAt ? "passed" : state.startedAt ? "running" : "pending"}`);
if (state.failure) {
logger$1.error(` Failure: ${state.failure.message}`);
}
}
}
});
for (const filePath of absoluteFilePaths) {
await testSuiteRunner.loadTestFile(filePath, getRelativePath(projectRoot, filePath));
}
try {
const overallSuccess = await testSuiteRunner.runTests();
process.exit(overallSuccess ? 0 : 1);
} catch (error) {
logger$1.error("Test suite execution failed:", error);
process.exit(1);
}
});
program.command("init").description("Initialize Magnitude test directory structure").option("-f, --force", "force initialization even if no package.json is found").option("--dir, --destination <path>", "destination directory for Magnitude tests", "tests/magnitude").action(async (options) => {
await initializeProject(options.force, options.destination);
});
program.parse();