magnitude-test
Version:
A TypeScript client for running automated UI tests through the Magnitude testing platform
322 lines (321 loc) • 12.2 kB
JavaScript
import { processUrl } from '../util';
import { WorkerPool } from './workerPool';
import { Worker } from 'node:worker_threads';
import { isBun, isDeno } from 'std-env';
import { EventEmitter } from 'node:events';
const TEST_FILE_LOADING_TIMEOUT = 30000;
export class TestSuiteRunner {
runnerConfig;
renderer;
config;
tests;
executors = new Map();
workerStoppers = [];
constructor(config) {
this.tests = [];
this.runnerConfig = config;
this.config = config.config;
}
async runTest(test, signal) {
const executor = this.executors.get(test.id);
if (!executor) {
throw new Error(`Test worker not found for test ID: ${test.id}`);
}
try {
return await executor({
type: "execute",
test,
browserOptions: this.config.browser,
llm: this.config.llm,
grounding: this.config.grounding,
telemetry: this.config.telemetry
}, (state) => {
this.renderer?.onTestStateUpdated(test, state);
}, signal);
}
catch (err) {
if (signal.aborted) {
return {
passed: false,
failure: {
message: 'Test execution aborted'
}
};
}
const errorMessage = err instanceof Error ? err.message : String(err);
// user-facing, can happen e.g. when URL is not running
console.error(`Unexpected error during test '${test.title}':\n${errorMessage}`);
return {
passed: false,
failure: {
message: errorMessage
}
};
}
}
getActiveOptions() {
const envOptions = process.env.MAGNITUDE_TEST_URL ? {
url: process.env.MAGNITUDE_TEST_URL
} : {};
return {
...this.config,
...envOptions, // env options take precedence over config options
url: processUrl(envOptions.url, this.config.url),
};
}
async loadTestFile(absoluteFilePath, relativeFilePath) {
try {
const workerData = {
relativeFilePath,
absoluteFilePath,
options: this.getActiveOptions(),
};
const createWorker = isBun ? createBunTestWorker : createNodeTestWorker;
const result = await createWorker(workerData);
this.tests.push(...result.tests);
for (const test of result.tests) {
this.executors.set(test.id, result.executor);
}
this.workerStoppers.push(result.stopper);
}
catch (error) {
console.error(`Failed to load test file ${relativeFilePath}:`, error);
throw error;
}
}
async runTests() {
if (!this.tests)
throw new Error('No tests were registered');
this.renderer = this.runnerConfig.createRenderer(this.tests);
this.renderer.start?.();
const workerPool = new WorkerPool(this.runnerConfig.workerCount);
let overallSuccess = true;
try {
const poolResult = await workerPool.runTasks(this.tests.map((test) => (signal) => this.runTest(test, signal)), this.runnerConfig.failFast
? (taskOutcome) => !taskOutcome.passed
: () => false);
for (const result of poolResult.results) {
if (result === undefined || !result.passed) {
overallSuccess = false;
break;
}
}
if (!poolResult.completed) { // If pool aborted for any reason (incl. a task failure)
overallSuccess = false;
}
}
catch (error) {
overallSuccess = false;
}
const stopperResults = await Promise.allSettled(this.workerStoppers.map(stopper => stopper()));
const stopperErrors = stopperResults
.filter(result => result.status === 'rejected');
if (stopperErrors.length > 0) {
overallSuccess = false;
console.error(`${stopperErrors.length} workers failed to stop`);
for (const error of stopperErrors) {
console.error(error);
}
}
this.renderer.stop?.();
return overallSuccess;
}
}
const createNodeTestWorker = async (workerData) => new Promise((resolve, reject) => {
const { relativeFilePath } = workerData;
const worker = new Worker(new URL(import.meta.url.endsWith(".ts")
? '../worker/readTest.ts'
: './worker/readTest.js', import.meta.url), {
workerData,
env: { NODE_ENV: 'test', ...process.env },
execArgv: !(isBun || isDeno) ? ["--import=jiti/register"] : []
});
let hasRunTests = false;
const stopper = async () => {
if (!hasRunTests) {
worker.terminate();
return;
}
worker.postMessage({ type: 'graceful_shutdown' });
return new Promise((resolve, reject) => {
const shutdownHandler = (msg) => {
if (msg.type === 'graceful_shutdown_complete') {
worker.off('message', shutdownHandler);
worker.off('error', errorHandler);
worker.off('exit', exitHandler);
worker.terminate();
resolve();
}
};
const errorHandler = (error) => {
worker.off('message', shutdownHandler);
worker.off('error', errorHandler);
worker.off('exit', exitHandler);
reject(new Error(`Worker error during shutdown: ${error.message}`));
};
const exitHandler = (code) => {
worker.off('message', shutdownHandler);
worker.off('error', errorHandler);
worker.off('exit', exitHandler);
reject(new Error(`Worker exited unexpectedly during shutdown with code: ${code}`));
};
worker.on('message', shutdownHandler);
worker.on('error', errorHandler);
worker.on('exit', exitHandler);
});
};
const executor = (executeMessage, onStateChange, signal) => new Promise((res, rej) => {
const messageHandler = (msg) => {
if ("test" in executeMessage && "testId" in msg && msg.testId !== executeMessage.test.id)
return;
hasRunTests = true;
if (msg.type === "test_result") {
worker.off("message", messageHandler);
res(msg.result);
}
else if (msg.type === "test_error") {
worker.off("message", messageHandler);
rej(new Error(msg.error));
}
else if (msg.type === "test_state_change") {
onStateChange(msg.state);
}
};
signal.addEventListener('abort', () => {
worker.off("message", messageHandler);
rej(new Error('Test execution aborted'));
});
worker.on("message", messageHandler);
worker.postMessage(executeMessage);
});
const registeredTests = [];
worker.on('message', (message) => {
if (message.type === 'registered') {
registeredTests.push(message.test);
}
else if (message.type === 'load_error') {
clearTimeout(timeout);
worker.terminate();
reject(new Error(`Failed to load ${relativeFilePath}: ${message.error}`));
}
else if (message.type === 'load_complete') {
clearTimeout(timeout);
if (!registeredTests.length) {
reject(new Error(`No tests registered for file ${relativeFilePath}`));
return;
}
resolve({ tests: registeredTests, executor, stopper });
}
});
const timeout = setTimeout(() => {
worker.terminate();
reject(new Error(`Test file loading timeout: ${relativeFilePath}`));
}, TEST_FILE_LOADING_TIMEOUT);
worker.on('error', (error) => {
clearTimeout(timeout);
worker.terminate();
reject(error);
});
});
const createBunTestWorker = async (workerData) => new Promise((resolve, reject) => {
const { relativeFilePath } = workerData;
const emit = new EventEmitter();
const proc = Bun.spawn({
cmd: [
"bun",
new URL(import.meta.url.endsWith(".ts")
? '../worker/readTest.ts'
: './worker/readTest.js', import.meta.url).pathname
],
env: {
NODE_ENV: 'test',
...process.env,
MAGNITUDE_WORKER_DATA: JSON.stringify(workerData)
},
cwd: process.cwd(),
stdin: "inherit",
stdout: "inherit",
stderr: "inherit",
// "advanced" serialization in Bun somehow isn't able to clone test state messages?
serialization: 'json',
ipc(message) {
emit.emit('message', message);
},
onExit(_subprocess, exitCode) {
if (exitCode !== 0) {
clearTimeout(timeout);
reject(new Error(`Worker process exited with code ${exitCode}`));
}
},
});
let hasRunTests = false;
const stopper = async () => {
if (!hasRunTests) {
proc.kill("SIGKILL");
return;
}
proc.send({ type: 'graceful_shutdown' });
return new Promise((resolve, reject) => {
const shutdownHandler = (msg) => {
if (msg.type === 'graceful_shutdown_complete') {
emit.off('message', shutdownHandler);
proc.kill("SIGKILL");
resolve();
}
};
const exitHandler = (code) => {
emit.off('message', shutdownHandler);
reject(new Error(`Bun worker exited unexpectedly during shutdown with code: ${code}`));
};
emit.on('message', shutdownHandler);
proc.exited.then(exitHandler);
});
};
const executor = (executeMessage, onStateChange, signal) => new Promise((res, rej) => {
const messageHandler = (msg) => {
if ("test" in executeMessage && "testId" in msg && msg.testId !== executeMessage.test.id)
return;
hasRunTests = true;
if (msg.type === "test_result") {
emit.off('message', messageHandler);
res(msg.result);
}
else if (msg.type === "test_error") {
emit.off('message', messageHandler);
rej(new Error(msg.error));
}
else if (msg.type === "test_state_change") {
onStateChange(msg.state);
}
};
emit.on('message', messageHandler);
signal.addEventListener('abort', () => {
emit.off('message', messageHandler);
rej(new Error('Test execution aborted'));
});
proc.send(executeMessage);
});
const registeredTests = [];
emit.on('message', ((message) => {
if (message.type === 'registered') {
registeredTests.push(message.test);
}
else if (message.type === 'load_error') {
clearTimeout(timeout);
proc.kill();
reject(new Error(`Failed to load ${relativeFilePath}: ${message.error}`));
}
else if (message.type === 'load_complete') {
clearTimeout(timeout);
if (!registeredTests.length) {
reject(new Error(`No tests registered for file ${relativeFilePath}`));
return;
}
resolve({ tests: registeredTests, executor, stopper });
}
}));
const timeout = setTimeout(() => {
proc.kill("SIGKILL");
reject(new Error(`Test file loading timeout: ${relativeFilePath}`));
}, TEST_FILE_LOADING_TIMEOUT);
});