@boundless-oss/atlas
Version:
Atlas - MCP Server for comprehensive startup project management
796 lines • 30.6 kB
JavaScript
import { exec } from 'child_process';
import { promisify } from 'util';
import { promises as fs } from 'fs';
import fsSync from 'fs';
import path from 'path';
import crypto from 'crypto';
const execAsync = promisify(exec);
export class TestRunner {
config;
constructor(config) {
this.config = config || this.detectTestFramework();
}
async runTests(options = {}) {
const startTime = Date.now();
const command = this.buildCommand(options);
try {
// Execute tests
const { stdout, stderr } = await execAsync(command, {
maxBuffer: 10 * 1024 * 1024, // 10MB buffer
timeout: options.timeout || 300000, // 5 minutes default
});
// Parse results based on framework
const result = await this.parseTestOutput(stdout, stderr, this.config.framework);
// Add metadata
result.id = crypto.randomUUID();
result.timestamp = new Date().toISOString();
result.duration = Date.now() - startTime;
result.command = command;
result.framework = this.config.framework;
// Add coverage if requested
if (options.coverage) {
result.coverage = await this.parseCoverageReport();
}
return result;
}
catch (error) {
// Even if tests fail, we want to parse the output
if (error.stdout || error.stderr) {
const result = await this.parseTestOutput(error.stdout || '', error.stderr || '', this.config.framework);
result.id = crypto.randomUUID();
result.timestamp = new Date().toISOString();
result.duration = Date.now() - startTime;
result.command = command;
result.framework = this.config.framework;
if (options.coverage) {
result.coverage = await this.parseCoverageReport();
}
return result;
}
throw error;
}
}
detectTestFramework() {
// Try to detect test framework from package.json
try {
const packageJsonPath = path.join(process.cwd(), 'package.json');
const packageJson = JSON.parse(fsSync.readFileSync(packageJsonPath, 'utf-8'));
const deps = { ...packageJson.dependencies, ...packageJson.devDependencies };
if (deps.jest || deps['@jest/core']) {
return {
framework: 'jest',
command: 'npx jest',
configFile: 'jest.config.js',
testMatch: ['**/__tests__/**/*.{js,jsx,ts,tsx}', '**/*.{test,spec}.{js,jsx,ts,tsx}'],
};
}
if (deps.mocha) {
return {
framework: 'mocha',
command: 'npx mocha',
configFile: '.mocharc.js',
testMatch: ['test/**/*.{js,ts}', '**/*.test.{js,ts}'],
};
}
if (deps.vitest) {
return {
framework: 'vitest',
command: 'npx vitest',
configFile: 'vitest.config.js',
testMatch: ['**/*.{test,spec}.{js,jsx,ts,tsx}'],
};
}
// Check for Python
if (fsSync.existsSync('pytest.ini') || fsSync.existsSync('setup.cfg')) {
return {
framework: 'pytest',
command: 'pytest',
configFile: 'pytest.ini',
testMatch: ['test_*.py', '*_test.py'],
};
}
// Check for Ruby
if (fsSync.existsSync('Gemfile') && fsSync.readFileSync('Gemfile', 'utf-8').includes('rspec')) {
return {
framework: 'rspec',
command: 'rspec',
configFile: '.rspec',
testMatch: ['spec/**/*_spec.rb'],
};
}
}
catch (error) {
// Fall through to default
}
// Default to jest for JavaScript projects
return {
framework: 'unknown',
command: 'npm test',
testMatch: ['**/*.test.*', '**/*.spec.*'],
};
}
buildCommand(options) {
let command = this.config.command;
// Add framework-specific options
switch (this.config.framework) {
case 'jest':
if (options.coverage)
command += ' --coverage';
if (options.watch)
command += ' --watch';
if (options.bail)
command += ' --bail';
if (options.verbose)
command += ' --verbose';
if (options.updateSnapshots)
command += ' --updateSnapshot';
if (options.parallel !== false)
command += ' --runInBand=false';
if (options.maxWorkers)
command += ` --maxWorkers=${options.maxWorkers}`;
if (options.pattern)
command += ` --testNamePattern="${options.pattern}"`;
if (options.files?.length)
command += ' ' + options.files.join(' ');
break;
case 'mocha':
if (options.watch)
command += ' --watch';
if (options.bail)
command += ' --bail';
if (options.verbose)
command += ' --reporter spec';
if (options.timeout)
command += ` --timeout ${options.timeout}`;
if (options.grep)
command += ` --grep "${options.grep}"`;
if (options.parallel)
command += ' --parallel';
if (options.files?.length)
command += ' ' + options.files.join(' ');
break;
case 'vitest':
if (options.coverage)
command += ' --coverage';
if (options.watch)
command += ' --watch';
if (options.bail)
command += ' --bail';
if (options.verbose)
command += ' --reporter=verbose';
if (options.parallel !== false)
command += ' --no-threads=false';
if (options.pattern)
command += ` --testNamePattern="${options.pattern}"`;
if (options.files?.length)
command += ' ' + options.files.join(' ');
break;
case 'pytest':
if (options.coverage)
command += ' --cov --cov-report=json';
if (options.verbose)
command += ' -v';
if (options.bail)
command += ' -x';
if (options.parallel)
command += ' -n auto';
if (options.pattern)
command += ` -k "${options.pattern}"`;
if (options.files?.length)
command += ' ' + options.files.join(' ');
break;
case 'rspec':
if (options.bail)
command += ' --fail-fast';
if (options.verbose)
command += ' --format documentation';
if (options.pattern)
command += ` --example "${options.pattern}"`;
if (options.files?.length)
command += ' ' + options.files.join(' ');
break;
}
return command;
}
async parseTestOutput(stdout, stderr, framework) {
switch (framework) {
case 'jest':
return this.parseJestOutput(stdout);
case 'mocha':
return this.parseMochaOutput(stdout);
case 'vitest':
return this.parseVitestOutput(stdout);
case 'pytest':
return this.parsePytestOutput(stdout);
case 'rspec':
return this.parseRspecOutput(stdout);
default:
return this.parseGenericOutput(stdout, stderr);
}
}
parseJestOutput(output) {
const suites = [];
const lines = output.split('\n');
let currentSuite = null;
let inTestResults = false;
for (const line of lines) {
// Detect test results section
if (line.includes('PASS') || line.includes('FAIL')) {
const match = line.match(/(PASS|FAIL)\s+(.+?)(?:\s+\((\d+(?:\.\d+)?)\s*m?s\))?$/);
if (match) {
const [, status, suitePath, duration] = match;
currentSuite = {
id: crypto.randomUUID(),
name: path.basename(suitePath),
path: suitePath,
tests: [],
status: status === 'PASS' ? 'passed' : 'failed',
duration: duration ? parseFloat(duration) : undefined,
timestamp: new Date().toISOString(),
};
suites.push(currentSuite);
inTestResults = true;
}
}
// Parse individual test results
if (inTestResults && currentSuite) {
const testMatch = line.match(/^\s*(✓|✕|○)\s+(.+?)(?:\s+\((\d+)\s*ms\))?$/);
if (testMatch) {
const [, icon, testName, duration] = testMatch;
const status = icon === '✓' ? 'passed' : icon === '✕' ? 'failed' : 'skipped';
currentSuite.tests.push({
id: crypto.randomUUID(),
name: testName,
suite: currentSuite.name,
status,
duration: duration ? parseInt(duration) : undefined,
});
}
}
// End of current suite
if (line.trim() === '' && currentSuite && currentSuite.tests.length > 0) {
inTestResults = false;
currentSuite = null;
}
}
// Parse summary
const summary = this.parseJestSummary(output);
return {
id: '',
timestamp: '',
projectId: '',
framework: 'jest',
suites,
summary,
duration: 0,
command: '',
};
}
parseJestSummary(output) {
const summaryMatch = output.match(/Tests:\s+(\d+)\s+failed,\s+(\d+)\s+passed,\s+(\d+)\s+total/);
const timeMatch = output.match(/Time:\s+(\d+(?:\.\d+)?)\s*s/);
if (summaryMatch) {
const [, failed, passed, total] = summaryMatch.map(Number);
const duration = timeMatch ? parseFloat(timeMatch[1]) * 1000 : 0;
return {
total,
passed,
failed,
skipped: 0,
pending: 0,
duration,
successRate: total > 0 ? (passed / total) * 100 : 0,
};
}
// Fallback to counting from suites
return {
total: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
duration: 0,
successRate: 0,
};
}
parseMochaOutput(output) {
const suites = [];
const lines = output.split('\n');
let currentSuite = null;
let suiteName = '';
for (const line of lines) {
// Detect test suite
if (line.trim().match(/^\s*[a-zA-Z\d\s]+$/)) {
if (currentSuite) {
suites.push(currentSuite);
}
suiteName = line.trim();
currentSuite = {
id: crypto.randomUUID(),
name: suiteName,
tests: [],
status: 'passed',
timestamp: new Date().toISOString(),
path: 'mocha-test-suite'
};
}
// Parse test results
if (currentSuite && line.match(/^\s*[✓✗]\s+/)) {
const testMatch = line.match(/^\s*([✓✗])\s+(.+?)(?:\s+\((\d+)ms\))?$/);
if (testMatch) {
const [, icon, testName, duration] = testMatch;
const status = icon === '✓' ? 'passed' : 'failed';
currentSuite.tests.push({
id: crypto.randomUUID(),
name: testName.trim(),
suite: currentSuite.name,
status,
duration: duration ? parseInt(duration) : undefined,
});
if (status === 'failed') {
currentSuite.status = 'failed';
}
}
}
// Parse pending tests
if (currentSuite && line.match(/^\s*-\s+/)) {
const pendingMatch = line.match(/^\s*-\s+(.+)$/);
if (pendingMatch) {
currentSuite.tests.push({
id: crypto.randomUUID(),
name: pendingMatch[1].trim(),
suite: currentSuite.name,
status: 'pending',
});
}
}
}
// Add the last suite
if (currentSuite) {
suites.push(currentSuite);
}
// Calculate summary
const summary = this.calculateSummary(suites);
return {
id: '',
timestamp: '',
projectId: '',
framework: 'mocha',
suites,
summary,
duration: 0,
command: '',
};
}
parseVitestOutput(output) {
// Similar to Jest parsing with Vitest-specific adjustments
return this.parseJestOutput(output);
}
parsePytestOutput(output) {
const suites = [];
const lines = output.split('\n');
let currentSuite = null;
const suiteMap = new Map();
for (const line of lines) {
// Parse test results line
const testMatch = line.match(/^(.+\.py)::([\w_]+)(::[\w_]+)?\s+(PASSED|FAILED|SKIPPED|ERROR)\s*(?:\[(\d+)%\])?/);
if (testMatch) {
const [, filePath, className, methodName, status, percentage] = testMatch;
const suiteName = path.basename(filePath, '.py');
// Create suite if not exists
if (!suiteMap.has(suiteName)) {
const suite = {
id: crypto.randomUUID(),
name: suiteName,
path: filePath,
tests: [],
status: 'passed',
timestamp: new Date().toISOString(),
};
suiteMap.set(suiteName, suite);
suites.push(suite);
}
currentSuite = suiteMap.get(suiteName);
// Parse test status
let testStatus;
switch (status) {
case 'PASSED':
testStatus = 'passed';
break;
case 'FAILED':
case 'ERROR':
testStatus = 'failed';
currentSuite.status = 'failed';
break;
case 'SKIPPED':
testStatus = 'skipped';
break;
default:
testStatus = 'failed';
}
currentSuite.tests.push({
id: crypto.randomUUID(),
name: methodName ? `${className}${methodName}` : className,
suite: currentSuite.name,
status: testStatus,
});
}
}
// Parse summary from output
const summary = this.parsePytestSummary(output);
return {
id: '',
timestamp: '',
projectId: '',
framework: 'pytest',
suites,
summary,
duration: 0,
command: '',
};
}
parseRspecOutput(output) {
const suites = [];
const lines = output.split('\n');
let currentSuite = null;
let inExampleGroup = false;
for (const line of lines) {
// Detect example groups (describe blocks)
const groupMatch = line.match(/^(\s*)(.*?)$/);
if (groupMatch && !line.includes('✓') && !line.includes('✗') && !line.includes('*') && line.trim().length > 0) {
const [, indent, groupName] = groupMatch;
// New top-level group
if (indent.length === 0 && groupName.trim()) {
if (currentSuite) {
suites.push(currentSuite);
}
currentSuite = {
id: crypto.randomUUID(),
name: groupName.trim(),
tests: [],
status: 'passed',
timestamp: new Date().toISOString(),
path: 'rspec-test-suite'
};
inExampleGroup = true;
}
}
// Parse test results
if (currentSuite && line.match(/^\s*[✓✗\*]\s+/)) {
const testMatch = line.match(/^\s*([✓✗\*])\s+(.+?)(?:\s+\(FAILED - \d+\))?(?:\s+\(PENDING:.+?\))?$/);
if (testMatch) {
const [, icon, testName] = testMatch;
let status;
switch (icon) {
case '✓':
status = 'passed';
break;
case '✗':
status = 'failed';
currentSuite.status = 'failed';
break;
case '*':
status = 'pending';
break;
default:
status = 'failed';
}
currentSuite.tests.push({
id: crypto.randomUUID(),
name: testName.trim(),
suite: currentSuite.name,
status,
});
}
}
}
// Add the last suite
if (currentSuite) {
suites.push(currentSuite);
}
// Parse summary from output
const summary = this.parseRspecSummary(output);
return {
id: '',
timestamp: '',
projectId: '',
framework: 'rspec',
suites,
summary,
duration: 0,
command: '',
};
}
parseGenericOutput(stdout, stderr) {
// Generic parsing for unknown frameworks
const suites = [];
const summary = {
total: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
duration: 0,
successRate: 0,
};
// Look for common patterns
const passMatch = stdout.match(/(\d+)\s+(passed|passing|pass)/i);
const failMatch = stdout.match(/(\d+)\s+(failed|failing|fail)/i);
if (passMatch)
summary.passed = parseInt(passMatch[1]);
if (failMatch)
summary.failed = parseInt(failMatch[1]);
summary.total = summary.passed + summary.failed;
summary.successRate = summary.total > 0 ? (summary.passed / summary.total) * 100 : 0;
return {
id: '',
timestamp: '',
projectId: '',
framework: 'unknown',
suites,
summary,
duration: 0,
command: '',
};
}
async parseCoverageReport() {
// Try to find and parse coverage report
const coverageFiles = [
'coverage/coverage-summary.json',
'coverage/lcov-report/index.html',
'.coverage',
'htmlcov/index.html',
];
for (const file of coverageFiles) {
try {
const coveragePath = path.join(process.cwd(), file);
if (await this.fileExists(coveragePath)) {
if (file.endsWith('.json')) {
return this.parseJsonCoverage(coveragePath);
}
// TODO: Parse other coverage formats (LCOV, HTML, etc.)
}
}
catch (error) {
// Continue to next format
}
}
return undefined;
}
async parseJsonCoverage(filePath) {
const content = await fs.readFile(filePath, 'utf-8');
const data = JSON.parse(content);
const report = {
lines: { total: 0, covered: 0, skipped: 0, percentage: 0 },
statements: { total: 0, covered: 0, skipped: 0, percentage: 0 },
functions: { total: 0, covered: 0, skipped: 0, percentage: 0 },
branches: { total: 0, covered: 0, skipped: 0, percentage: 0 },
files: [],
};
// Parse Jest/NYC coverage format
if (data.total) {
report.lines = this.extractCoverageMetric(data.total.lines);
report.statements = this.extractCoverageMetric(data.total.statements);
report.functions = this.extractCoverageMetric(data.total.functions);
report.branches = this.extractCoverageMetric(data.total.branches);
}
// Parse individual files
for (const [filePath, fileData] of Object.entries(data)) {
if (filePath !== 'total' && typeof fileData === 'object') {
const fileCoverage = {
path: filePath,
lines: this.extractCoverageMetric(fileData.lines),
statements: this.extractCoverageMetric(fileData.statements),
functions: this.extractCoverageMetric(fileData.functions),
branches: this.extractCoverageMetric(fileData.branches),
};
report.files.push(fileCoverage);
}
}
return report;
}
extractCoverageMetric(data) {
return {
total: data?.total || 0,
covered: data?.covered || 0,
skipped: data?.skipped || 0,
percentage: data?.pct || 0,
};
}
async fileExists(path) {
try {
await fs.access(path);
return true;
}
catch {
return false;
}
}
async detectFlakyTests(history) {
const testResults = new Map();
// Collect test results across runs
for (const result of history) {
for (const suite of result.suites) {
for (const test of suite.tests) {
const testKey = `${suite.path}::${test.name}`;
if (!testResults.has(testKey)) {
testResults.set(testKey, []);
}
testResults.get(testKey).push(test.status);
}
}
}
// Identify flakey tests
const flakyTests = [];
for (const [testKey, results] of testResults.entries()) {
const failures = results.filter(r => r === 'failed').length;
const passes = results.filter(r => r === 'passed').length;
// Test is flakey if it has both passes and failures
if (failures > 0 && passes > 0) {
const failureRate = failures / results.length;
const [suitePath, testName] = testKey.split('::');
flakyTests.push({
testId: crypto.randomUUID(),
name: testName,
suite: path.basename(suitePath),
failureRate,
recentResults: results.slice(-10), // Last 10 results
lastFailed: new Date().toISOString(), // TODO: Get actual timestamp
errorPatterns: [], // TODO: Extract error patterns
});
}
}
// Sort by failure rate
return flakyTests.sort((a, b) => b.failureRate - a.failureRate);
}
generateTestRecommendations(result, history) {
const recommendations = [];
// Check test coverage
if (result.coverage) {
if (result.coverage.lines.percentage < 80) {
recommendations.push({
type: 'improve-coverage',
title: 'Low Test Coverage',
description: `Line coverage is ${result.coverage.lines.percentage.toFixed(1)}%, below recommended 80%`,
priority: 'high',
});
}
// Find files with low coverage
const lowCoverageFiles = result.coverage.files
.filter(f => f.lines.percentage < 50)
.map(f => f.path);
if (lowCoverageFiles.length > 0) {
recommendations.push({
type: 'add-tests',
title: 'Files Need More Tests',
description: `${lowCoverageFiles.length} files have less than 50% coverage`,
priority: 'medium',
affectedTests: lowCoverageFiles,
});
}
}
// Check for test failures
if (result.summary.failed > 0) {
recommendations.push({
type: 'fix-flaky',
title: 'Failing Tests Detected',
description: `${result.summary.failed} tests are failing and need attention`,
priority: 'high',
});
}
// Check test performance
const slowTests = result.suites
.flatMap(s => s.tests)
.filter(t => t.duration && t.duration > 5000); // Tests taking more than 5 seconds
if (slowTests.length > 0) {
recommendations.push({
type: 'optimize-performance',
title: 'Slow Tests Detected',
description: `${slowTests.length} tests take more than 5 seconds to run`,
priority: 'low',
affectedTests: slowTests.map(t => t.name),
});
}
return recommendations;
}
calculateSummary(suites) {
let total = 0;
let passed = 0;
let failed = 0;
let skipped = 0;
let pending = 0;
let duration = 0;
for (const suite of suites) {
for (const test of suite.tests) {
total++;
switch (test.status) {
case 'passed':
passed++;
break;
case 'failed':
failed++;
break;
case 'skipped':
skipped++;
break;
case 'pending':
pending++;
break;
}
if (test.duration) {
duration += test.duration;
}
}
}
return {
total,
passed,
failed,
skipped,
pending,
duration,
successRate: total > 0 ? (passed / total) * 100 : 0,
};
}
parsePytestSummary(output) {
// Look for pytest summary line like "= 5 passed, 2 failed, 1 skipped in 2.34s ="
const summaryMatch = output.match(/=+\s*(?:(\d+)\s+failed,?)?\s*(?:(\d+)\s+passed,?)?\s*(?:(\d+)\s+skipped,?)?\s*(?:(\d+)\s+error,?)?\s*(?:in\s+([\d.]+)s)?\s*=+/);
if (summaryMatch) {
const [, failedStr, passedStr, skippedStr, errorStr, durationStr] = summaryMatch;
const failed = failedStr ? parseInt(failedStr) : 0;
const passed = passedStr ? parseInt(passedStr) : 0;
const skipped = skippedStr ? parseInt(skippedStr) : 0;
const errors = errorStr ? parseInt(errorStr) : 0;
const duration = durationStr ? parseFloat(durationStr) * 1000 : 0;
const total = failed + passed + skipped + errors;
return {
total,
passed,
failed: failed + errors,
skipped,
pending: 0,
duration,
successRate: total > 0 ? (passed / total) * 100 : 0,
};
}
return {
total: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
duration: 0,
successRate: 0,
};
}
parseRspecSummary(output) {
// Look for RSpec summary line like "5 examples, 1 failure, 1 pending"
const summaryMatch = output.match(/(\d+)\s+examples?,\s*(?:(\d+)\s+failures?,?)?\s*(?:(\d+)\s+pending)?/);
if (summaryMatch) {
const [, totalStr, failedStr, pendingStr] = summaryMatch;
const total = parseInt(totalStr);
const failed = failedStr ? parseInt(failedStr) : 0;
const pending = pendingStr ? parseInt(pendingStr) : 0;
const passed = total - failed - pending;
return {
total,
passed,
failed,
skipped: 0,
pending,
duration: 0,
successRate: total > 0 ? (passed / total) * 100 : 0,
};
}
return {
total: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
duration: 0,
successRate: 0,
};
}
}
//# sourceMappingURL=test-runner.js.map