task-master-neo-sdlc
Version:
Enhanced task management system with Neo SDLC agents and MCP tools for comprehensive, AI-driven software development lifecycle management.
853 lines (742 loc) • 26 kB
JavaScript
/**
* Test Metrics and Reporting
*
* Collects, analyzes, and reports test metrics for quality assurance.
*/
import { log } from '../../../utils/logging.js';
import fs from 'fs';
import path from 'path';
import { execSync } from 'child_process';
/**
* Test metrics types
*/
export const METRIC_TYPES = {
COVERAGE: 'coverage',
PERFORMANCE: 'performance',
RELIABILITY: 'reliability',
MAINTAINABILITY: 'maintainability'
};
/**
* Test metrics thresholds
*/
export const DEFAULT_THRESHOLDS = {
[METRIC_TYPES.COVERAGE]: {
statements: 80,
branches: 70,
functions: 80,
lines: 80
},
[METRIC_TYPES.PERFORMANCE]: {
maxExecutionTime: 5000, // ms
maxMemoryUsage: 100 // MB
},
[METRIC_TYPES.RELIABILITY]: {
maxFailureRate: 0.05, // 5%
minUptime: 0.99 // 99%
},
[METRIC_TYPES.MAINTAINABILITY]: {
maxComplexity: 10,
maxDuplication: 0.1 // 10%
}
};
/**
* Collect test metrics
* @param {Object} options - Collection options
* @param {string} options.projectRoot - Project root directory
* @param {Array<string>} options.testFrameworks - Test frameworks to collect metrics from
* @param {Array<string>} options.metricTypes - Types of metrics to collect
* @returns {Promise<Object>} Collected metrics
*/
export async function collectTestMetrics(options) {
const { projectRoot, testFrameworks, metricTypes = Object.values(METRIC_TYPES) } = options;
log.info(`Collecting test metrics for ${testFrameworks.join(', ')}`);
const metrics = {};
try {
// Collect coverage metrics
if (metricTypes.includes(METRIC_TYPES.COVERAGE)) {
metrics.coverage = await collectCoverageMetrics(projectRoot, testFrameworks);
}
// Collect performance metrics
if (metricTypes.includes(METRIC_TYPES.PERFORMANCE)) {
metrics.performance = await collectPerformanceMetrics(projectRoot, testFrameworks);
}
// Collect reliability metrics
if (metricTypes.includes(METRIC_TYPES.RELIABILITY)) {
metrics.reliability = await collectReliabilityMetrics(projectRoot, testFrameworks);
}
// Collect maintainability metrics
if (metricTypes.includes(METRIC_TYPES.MAINTAINABILITY)) {
metrics.maintainability = await collectMaintainabilityMetrics(projectRoot);
}
// Add timestamp
metrics.timestamp = new Date().toISOString();
// Save metrics to file
const metricsPath = path.join(projectRoot, 'test-metrics.json');
fs.writeFileSync(metricsPath, JSON.stringify(metrics, null, 2), 'utf8');
log.info(`Test metrics collected and saved to ${metricsPath}`);
return metrics;
} catch (error) {
log.error(`Error collecting test metrics: ${error.message}`);
throw error;
}
}
/**
* Collect coverage metrics
* @param {string} projectRoot - Project root directory
* @param {Array<string>} testFrameworks - Test frameworks to collect metrics from
* @returns {Promise<Object>} Coverage metrics
*/
async function collectCoverageMetrics(projectRoot, testFrameworks) {
log.info('Collecting coverage metrics');
try {
// Check if coverage report exists
const coveragePath = path.join(projectRoot, 'coverage/coverage-summary.json');
if (fs.existsSync(coveragePath)) {
// Parse existing coverage report
const coverageReport = JSON.parse(fs.readFileSync(coveragePath, 'utf8'));
return {
statements: coverageReport.total.statements.pct,
branches: coverageReport.total.branches.pct,
functions: coverageReport.total.functions.pct,
lines: coverageReport.total.lines.pct,
timestamp: new Date().toISOString()
};
}
// Generate coverage report
if (testFrameworks.includes('jest')) {
execSync('npx jest --coverage', { cwd: projectRoot });
// Check if coverage report was generated
if (fs.existsSync(coveragePath)) {
const coverageReport = JSON.parse(fs.readFileSync(coveragePath, 'utf8'));
return {
statements: coverageReport.total.statements.pct,
branches: coverageReport.total.branches.pct,
functions: coverageReport.total.functions.pct,
lines: coverageReport.total.lines.pct,
timestamp: new Date().toISOString()
};
}
}
// If no coverage report found, return default values
return {
statements: 0,
branches: 0,
functions: 0,
lines: 0,
timestamp: new Date().toISOString()
};
} catch (error) {
log.error(`Error collecting coverage metrics: ${error.message}`);
// Return default values on error
return {
statements: 0,
branches: 0,
functions: 0,
lines: 0,
timestamp: new Date().toISOString(),
error: error.message
};
}
}
/**
* Collect performance metrics
* @param {string} projectRoot - Project root directory
* @param {Array<string>} testFrameworks - Test frameworks to collect metrics from
* @returns {Promise<Object>} Performance metrics
*/
async function collectPerformanceMetrics(projectRoot, testFrameworks) {
log.info('Collecting performance metrics');
try {
// This would typically use a performance testing tool
// For now, we'll return mock metrics
return {
executionTime: 1200, // ms
memoryUsage: 45, // MB
timestamp: new Date().toISOString()
};
} catch (error) {
log.error(`Error collecting performance metrics: ${error.message}`);
// Return default values on error
return {
executionTime: 0,
memoryUsage: 0,
timestamp: new Date().toISOString(),
error: error.message
};
}
}
/**
* Collect reliability metrics
* @param {string} projectRoot - Project root directory
* @param {Array<string>} testFrameworks - Test frameworks to collect metrics from
* @returns {Promise<Object>} Reliability metrics
*/
async function collectReliabilityMetrics(projectRoot, testFrameworks) {
log.info('Collecting reliability metrics');
try {
// This would typically analyze test results over time
// For now, we'll return mock metrics
return {
failureRate: 0.02, // 2%
uptime: 0.995, // 99.5%
timestamp: new Date().toISOString()
};
} catch (error) {
log.error(`Error collecting reliability metrics: ${error.message}`);
// Return default values on error
return {
failureRate: 1,
uptime: 0,
timestamp: new Date().toISOString(),
error: error.message
};
}
}
/**
* Collect maintainability metrics
* @param {string} projectRoot - Project root directory
* @returns {Promise<Object>} Maintainability metrics
*/
async function collectMaintainabilityMetrics(projectRoot) {
log.info('Collecting maintainability metrics');
try {
// This would typically use a code quality tool
// For now, we'll return mock metrics
return {
complexity: 5,
duplication: 0.05, // 5%
timestamp: new Date().toISOString()
};
} catch (error) {
log.error(`Error collecting maintainability metrics: ${error.message}`);
// Return default values on error
return {
complexity: 0,
duplication: 0,
timestamp: new Date().toISOString(),
error: error.message
};
}
}
/**
* Analyze test metrics
* @param {Object} metrics - Test metrics
* @param {Object} thresholds - Metric thresholds
* @returns {Object} Analysis result
*/
export function analyzeTestMetrics(metrics, thresholds = DEFAULT_THRESHOLDS) {
log.info('Analyzing test metrics');
const analysis = {
passed: true,
issues: [],
recommendations: []
};
try {
// Analyze coverage metrics
if (metrics.coverage) {
const coverageThresholds = thresholds[METRIC_TYPES.COVERAGE];
if (metrics.coverage.statements < coverageThresholds.statements) {
analysis.passed = false;
analysis.issues.push(`Statement coverage (${metrics.coverage.statements}%) is below threshold (${coverageThresholds.statements}%)`);
analysis.recommendations.push('Add more unit tests to increase statement coverage');
}
if (metrics.coverage.branches < coverageThresholds.branches) {
analysis.passed = false;
analysis.issues.push(`Branch coverage (${metrics.coverage.branches}%) is below threshold (${coverageThresholds.branches}%)`);
analysis.recommendations.push('Add tests for conditional branches');
}
if (metrics.coverage.functions < coverageThresholds.functions) {
analysis.passed = false;
analysis.issues.push(`Function coverage (${metrics.coverage.functions}%) is below threshold (${coverageThresholds.functions}%)`);
analysis.recommendations.push('Add tests for untested functions');
}
if (metrics.coverage.lines < coverageThresholds.lines) {
analysis.passed = false;
analysis.issues.push(`Line coverage (${metrics.coverage.lines}%) is below threshold (${coverageThresholds.lines}%)`);
analysis.recommendations.push('Add more tests to increase line coverage');
}
}
// Analyze performance metrics
if (metrics.performance) {
const performanceThresholds = thresholds[METRIC_TYPES.PERFORMANCE];
if (metrics.performance.executionTime > performanceThresholds.maxExecutionTime) {
analysis.passed = false;
analysis.issues.push(`Execution time (${metrics.performance.executionTime}ms) exceeds threshold (${performanceThresholds.maxExecutionTime}ms)`);
analysis.recommendations.push('Optimize test execution time');
}
if (metrics.performance.memoryUsage > performanceThresholds.maxMemoryUsage) {
analysis.passed = false;
analysis.issues.push(`Memory usage (${metrics.performance.memoryUsage}MB) exceeds threshold (${performanceThresholds.maxMemoryUsage}MB)`);
analysis.recommendations.push('Reduce memory usage in tests');
}
}
// Analyze reliability metrics
if (metrics.reliability) {
const reliabilityThresholds = thresholds[METRIC_TYPES.RELIABILITY];
if (metrics.reliability.failureRate > reliabilityThresholds.maxFailureRate) {
analysis.passed = false;
analysis.issues.push(`Failure rate (${metrics.reliability.failureRate * 100}%) exceeds threshold (${reliabilityThresholds.maxFailureRate * 100}%)`);
analysis.recommendations.push('Investigate and fix flaky tests');
}
if (metrics.reliability.uptime < reliabilityThresholds.minUptime) {
analysis.passed = false;
analysis.issues.push(`Uptime (${metrics.reliability.uptime * 100}%) is below threshold (${reliabilityThresholds.minUptime * 100}%)`);
analysis.recommendations.push('Improve system stability');
}
}
// Analyze maintainability metrics
if (metrics.maintainability) {
const maintainabilityThresholds = thresholds[METRIC_TYPES.MAINTAINABILITY];
if (metrics.maintainability.complexity > maintainabilityThresholds.maxComplexity) {
analysis.passed = false;
analysis.issues.push(`Complexity (${metrics.maintainability.complexity}) exceeds threshold (${maintainabilityThresholds.maxComplexity})`);
analysis.recommendations.push('Refactor complex code');
}
if (metrics.maintainability.duplication > maintainabilityThresholds.maxDuplication) {
analysis.passed = false;
analysis.issues.push(`Duplication (${metrics.maintainability.duplication * 100}%) exceeds threshold (${maintainabilityThresholds.maxDuplication * 100}%)`);
analysis.recommendations.push('Reduce code duplication');
}
}
return analysis;
} catch (error) {
log.error(`Error analyzing test metrics: ${error.message}`);
return {
passed: false,
issues: [`Error analyzing metrics: ${error.message}`],
recommendations: ['Fix metric analysis error']
};
}
}
/**
* Generate test report
* @param {Object} options - Report options
* @param {Object} options.metrics - Test metrics
* @param {Object} options.analysis - Metrics analysis
* @param {string} options.projectRoot - Project root directory
* @param {string} options.format - Report format (html, json, markdown)
* @returns {Promise<Object>} Generated report
*/
export async function generateTestReport(options) {
const { metrics, analysis, projectRoot, format = 'html' } = options;
log.info(`Generating ${format} test report`);
try {
let reportContent;
let reportPath;
switch (format) {
case 'html':
reportContent = generateHTMLReport(metrics, analysis);
reportPath = path.join(projectRoot, 'test-report.html');
break;
case 'json':
reportContent = JSON.stringify({ metrics, analysis }, null, 2);
reportPath = path.join(projectRoot, 'test-report.json');
break;
case 'markdown':
reportContent = generateMarkdownReport(metrics, analysis);
reportPath = path.join(projectRoot, 'test-report.md');
break;
default:
throw new Error(`Unsupported report format: ${format}`);
}
// Write report to file
fs.writeFileSync(reportPath, reportContent, 'utf8');
log.info(`Test report generated at ${reportPath}`);
return {
format,
path: reportPath,
content: reportContent
};
} catch (error) {
log.error(`Error generating test report: ${error.message}`);
throw error;
}
}
/**
* Generate HTML test report
* @param {Object} metrics - Test metrics
* @param {Object} analysis - Metrics analysis
* @returns {string} HTML report
*/
function generateHTMLReport(metrics, analysis) {
return `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Test Report</title>
<style>
body {
font-family: Arial, sans-serif;
line-height: 1.6;
max-width: 1200px;
margin: 0 auto;
padding: 20px;
}
h1, h2, h3 {
color: #333;
}
.summary {
background-color: ${analysis.passed ? '#e6ffe6' : '#ffe6e6'};
padding: 15px;
border-radius: 5px;
margin-bottom: 20px;
}
.metrics-section {
margin-bottom: 30px;
}
.metric-card {
background-color: #f5f5f5;
padding: 15px;
border-radius: 5px;
margin-bottom: 10px;
}
.metric-value {
font-size: 24px;
font-weight: bold;
}
.issues {
background-color: #fff0f0;
padding: 15px;
border-radius: 5px;
margin-bottom: 20px;
}
.recommendations {
background-color: #f0f8ff;
padding: 15px;
border-radius: 5px;
}
.progress-bar {
height: 20px;
background-color: #e0e0e0;
border-radius: 10px;
margin-top: 5px;
}
.progress-value {
height: 100%;
border-radius: 10px;
background-color: #4CAF50;
}
.progress-value.warning {
background-color: #FFC107;
}
.progress-value.danger {
background-color: #F44336;
}
</style>
</head>
<body>
<h1>Test Report</h1>
<div class="summary">
<h2>Summary</h2>
<p>Status: ${analysis.passed ? 'PASSED ✅' : 'FAILED ❌'}</p>
<p>Generated: ${new Date(metrics.timestamp).toLocaleString()}</p>
</div>
${metrics.coverage ? `
<div class="metrics-section">
<h2>Coverage Metrics</h2>
<div class="metric-card">
<h3>Statement Coverage</h3>
<div class="metric-value">${metrics.coverage.statements}%</div>
<div class="progress-bar">
<div class="progress-value ${getProgressClass(metrics.coverage.statements, 80)}" style="width: ${metrics.coverage.statements}%"></div>
</div>
</div>
<div class="metric-card">
<h3>Branch Coverage</h3>
<div class="metric-value">${metrics.coverage.branches}%</div>
<div class="progress-bar">
<div class="progress-value ${getProgressClass(metrics.coverage.branches, 70)}" style="width: ${metrics.coverage.branches}%"></div>
</div>
</div>
<div class="metric-card">
<h3>Function Coverage</h3>
<div class="metric-value">${metrics.coverage.functions}%</div>
<div class="progress-bar">
<div class="progress-value ${getProgressClass(metrics.coverage.functions, 80)}" style="width: ${metrics.coverage.functions}%"></div>
</div>
</div>
<div class="metric-card">
<h3>Line Coverage</h3>
<div class="metric-value">${metrics.coverage.lines}%</div>
<div class="progress-bar">
<div class="progress-value ${getProgressClass(metrics.coverage.lines, 80)}" style="width: ${metrics.coverage.lines}%"></div>
</div>
</div>
</div>
` : ''}
${metrics.performance ? `
<div class="metrics-section">
<h2>Performance Metrics</h2>
<div class="metric-card">
<h3>Execution Time</h3>
<div class="metric-value">${metrics.performance.executionTime}ms</div>
</div>
<div class="metric-card">
<h3>Memory Usage</h3>
<div class="metric-value">${metrics.performance.memoryUsage}MB</div>
</div>
</div>
` : ''}
${metrics.reliability ? `
<div class="metrics-section">
<h2>Reliability Metrics</h2>
<div class="metric-card">
<h3>Failure Rate</h3>
<div class="metric-value">${(metrics.reliability.failureRate * 100).toFixed(2)}%</div>
</div>
<div class="metric-card">
<h3>Uptime</h3>
<div class="metric-value">${(metrics.reliability.uptime * 100).toFixed(2)}%</div>
</div>
</div>
` : ''}
${metrics.maintainability ? `
<div class="metrics-section">
<h2>Maintainability Metrics</h2>
<div class="metric-card">
<h3>Complexity</h3>
<div class="metric-value">${metrics.maintainability.complexity}</div>
</div>
<div class="metric-card">
<h3>Duplication</h3>
<div class="metric-value">${(metrics.maintainability.duplication * 100).toFixed(2)}%</div>
</div>
</div>
` : ''}
${analysis.issues.length > 0 ? `
<div class="issues">
<h2>Issues</h2>
<ul>
${analysis.issues.map(issue => `<li>${issue}</li>`).join('')}
</ul>
</div>
` : ''}
${analysis.recommendations.length > 0 ? `
<div class="recommendations">
<h2>Recommendations</h2>
<ul>
${analysis.recommendations.map(recommendation => `<li>${recommendation}</li>`).join('')}
</ul>
</div>
` : ''}
</body>
</html>`;
}
/**
* Generate Markdown test report
* @param {Object} metrics - Test metrics
* @param {Object} analysis - Metrics analysis
* @returns {string} Markdown report
*/
function generateMarkdownReport(metrics, analysis) {
let report = `# Test Report\n\n`;
// Add summary
report += `## Summary\n\n`;
report += `Status: ${analysis.passed ? 'PASSED ✅' : 'FAILED ❌'}\n\n`;
report += `Generated: ${new Date(metrics.timestamp).toLocaleString()}\n\n`;
// Add coverage metrics
if (metrics.coverage) {
report += `## Coverage Metrics\n\n`;
report += `- Statement Coverage: ${metrics.coverage.statements}%\n`;
report += `- Branch Coverage: ${metrics.coverage.branches}%\n`;
report += `- Function Coverage: ${metrics.coverage.functions}%\n`;
report += `- Line Coverage: ${metrics.coverage.lines}%\n\n`;
}
// Add performance metrics
if (metrics.performance) {
report += `## Performance Metrics\n\n`;
report += `- Execution Time: ${metrics.performance.executionTime}ms\n`;
report += `- Memory Usage: ${metrics.performance.memoryUsage}MB\n\n`;
}
// Add reliability metrics
if (metrics.reliability) {
report += `## Reliability Metrics\n\n`;
report += `- Failure Rate: ${(metrics.reliability.failureRate * 100).toFixed(2)}%\n`;
report += `- Uptime: ${(metrics.reliability.uptime * 100).toFixed(2)}%\n\n`;
}
// Add maintainability metrics
if (metrics.maintainability) {
report += `## Maintainability Metrics\n\n`;
report += `- Complexity: ${metrics.maintainability.complexity}\n`;
report += `- Duplication: ${(metrics.maintainability.duplication * 100).toFixed(2)}%\n\n`;
}
// Add issues
if (analysis.issues.length > 0) {
report += `## Issues\n\n`;
analysis.issues.forEach(issue => {
report += `- ${issue}\n`;
});
report += '\n';
}
// Add recommendations
if (analysis.recommendations.length > 0) {
report += `## Recommendations\n\n`;
analysis.recommendations.forEach(recommendation => {
report += `- ${recommendation}\n`;
});
report += '\n';
}
return report;
}
/**
* Get progress bar class based on value and threshold
* @param {number} value - Metric value
* @param {number} threshold - Metric threshold
* @returns {string} Progress bar class
*/
function getProgressClass(value, threshold) {
if (value >= threshold) {
return '';
} else if (value >= threshold * 0.8) {
return 'warning';
} else {
return 'danger';
}
}
/**
* Track test metrics over time
* @param {Object} options - Tracking options
* @param {string} options.projectRoot - Project root directory
* @param {Object} options.metrics - Current metrics
* @returns {Promise<Object>} Tracking result
*/
export async function trackTestMetricsOverTime(options) {
const { projectRoot, metrics } = options;
log.info('Tracking test metrics over time');
try {
const historyPath = path.join(projectRoot, 'test-metrics-history.json');
// Load existing history
let history = [];
if (fs.existsSync(historyPath)) {
history = JSON.parse(fs.readFileSync(historyPath, 'utf8'));
}
// Add current metrics to history
history.push({
timestamp: metrics.timestamp,
coverage: metrics.coverage,
performance: metrics.performance,
reliability: metrics.reliability,
maintainability: metrics.maintainability
});
// Limit history size
if (history.length > 100) {
history = history.slice(-100);
}
// Save history
fs.writeFileSync(historyPath, JSON.stringify(history, null, 2), 'utf8');
log.info(`Test metrics history updated at ${historyPath}`);
// Calculate trends
const trends = calculateMetricTrends(history);
return {
history,
trends
};
} catch (error) {
log.error(`Error tracking test metrics: ${error.message}`);
throw error;
}
}
/**
* Calculate metric trends
* @param {Array<Object>} history - Metrics history
* @returns {Object} Metric trends
*/
function calculateMetricTrends(history) {
if (history.length < 2) {
return {
coverage: { trend: 'stable' },
performance: { trend: 'stable' },
reliability: { trend: 'stable' },
maintainability: { trend: 'stable' }
};
}
const current = history[history.length - 1];
const previous = history[history.length - 2];
const trends = {};
// Calculate coverage trends
if (current.coverage && previous.coverage) {
const statementsDiff = current.coverage.statements - previous.coverage.statements;
const branchesDiff = current.coverage.branches - previous.coverage.branches;
const functionsDiff = current.coverage.functions - previous.coverage.functions;
const linesDiff = current.coverage.lines - previous.coverage.lines;
trends.coverage = {
statements: {
diff: statementsDiff,
trend: getTrendDirection(statementsDiff)
},
branches: {
diff: branchesDiff,
trend: getTrendDirection(branchesDiff)
},
functions: {
diff: functionsDiff,
trend: getTrendDirection(functionsDiff)
},
lines: {
diff: linesDiff,
trend: getTrendDirection(linesDiff)
}
};
}
// Calculate performance trends
if (current.performance && previous.performance) {
const executionTimeDiff = current.performance.executionTime - previous.performance.executionTime;
const memoryUsageDiff = current.performance.memoryUsage - previous.performance.memoryUsage;
trends.performance = {
executionTime: {
diff: executionTimeDiff,
trend: getTrendDirection(-executionTimeDiff) // Lower is better
},
memoryUsage: {
diff: memoryUsageDiff,
trend: getTrendDirection(-memoryUsageDiff) // Lower is better
}
};
}
// Calculate reliability trends
if (current.reliability && previous.reliability) {
const failureRateDiff = current.reliability.failureRate - previous.reliability.failureRate;
const uptimeDiff = current.reliability.uptime - previous.reliability.uptime;
trends.reliability = {
failureRate: {
diff: failureRateDiff,
trend: getTrendDirection(-failureRateDiff) // Lower is better
},
uptime: {
diff: uptimeDiff,
trend: getTrendDirection(uptimeDiff)
}
};
}
// Calculate maintainability trends
if (current.maintainability && previous.maintainability) {
const complexityDiff = current.maintainability.complexity - previous.maintainability.complexity;
const duplicationDiff = current.maintainability.duplication - previous.maintainability.duplication;
trends.maintainability = {
complexity: {
diff: complexityDiff,
trend: getTrendDirection(-complexityDiff) // Lower is better
},
duplication: {
diff: duplicationDiff,
trend: getTrendDirection(-duplicationDiff) // Lower is better
}
};
}
return trends;
}
/**
* Get trend direction based on difference
* @param {number} diff - Metric difference
* @returns {string} Trend direction
*/
function getTrendDirection(diff) {
if (diff > 0) {
return 'improving';
} else if (diff < 0) {
return 'declining';
} else {
return 'stable';
}
}