UNPKG

task-master-neo-sdlc

Version:

Enhanced task management system with Neo SDLC agents and MCP tools for comprehensive, AI-driven software development lifecycle management.

853 lines (742 loc) 26 kB
/** * Test Metrics and Reporting * * Collects, analyzes, and reports test metrics for quality assurance. */ import { log } from '../../../utils/logging.js'; import fs from 'fs'; import path from 'path'; import { execSync } from 'child_process'; /** * Test metrics types */ export const METRIC_TYPES = { COVERAGE: 'coverage', PERFORMANCE: 'performance', RELIABILITY: 'reliability', MAINTAINABILITY: 'maintainability' }; /** * Test metrics thresholds */ export const DEFAULT_THRESHOLDS = { [METRIC_TYPES.COVERAGE]: { statements: 80, branches: 70, functions: 80, lines: 80 }, [METRIC_TYPES.PERFORMANCE]: { maxExecutionTime: 5000, // ms maxMemoryUsage: 100 // MB }, [METRIC_TYPES.RELIABILITY]: { maxFailureRate: 0.05, // 5% minUptime: 0.99 // 99% }, [METRIC_TYPES.MAINTAINABILITY]: { maxComplexity: 10, maxDuplication: 0.1 // 10% } }; /** * Collect test metrics * @param {Object} options - Collection options * @param {string} options.projectRoot - Project root directory * @param {Array<string>} options.testFrameworks - Test frameworks to collect metrics from * @param {Array<string>} options.metricTypes - Types of metrics to collect * @returns {Promise<Object>} Collected metrics */ export async function collectTestMetrics(options) { const { projectRoot, testFrameworks, metricTypes = Object.values(METRIC_TYPES) } = options; log.info(`Collecting test metrics for ${testFrameworks.join(', ')}`); const metrics = {}; try { // Collect coverage metrics if (metricTypes.includes(METRIC_TYPES.COVERAGE)) { metrics.coverage = await collectCoverageMetrics(projectRoot, testFrameworks); } // Collect performance metrics if (metricTypes.includes(METRIC_TYPES.PERFORMANCE)) { metrics.performance = await collectPerformanceMetrics(projectRoot, testFrameworks); } // Collect reliability metrics if (metricTypes.includes(METRIC_TYPES.RELIABILITY)) { metrics.reliability = await collectReliabilityMetrics(projectRoot, testFrameworks); } // Collect maintainability metrics if (metricTypes.includes(METRIC_TYPES.MAINTAINABILITY)) { metrics.maintainability = await collectMaintainabilityMetrics(projectRoot); } // Add timestamp metrics.timestamp = new Date().toISOString(); // Save metrics to file const metricsPath = path.join(projectRoot, 'test-metrics.json'); fs.writeFileSync(metricsPath, JSON.stringify(metrics, null, 2), 'utf8'); log.info(`Test metrics collected and saved to ${metricsPath}`); return metrics; } catch (error) { log.error(`Error collecting test metrics: ${error.message}`); throw error; } } /** * Collect coverage metrics * @param {string} projectRoot - Project root directory * @param {Array<string>} testFrameworks - Test frameworks to collect metrics from * @returns {Promise<Object>} Coverage metrics */ async function collectCoverageMetrics(projectRoot, testFrameworks) { log.info('Collecting coverage metrics'); try { // Check if coverage report exists const coveragePath = path.join(projectRoot, 'coverage/coverage-summary.json'); if (fs.existsSync(coveragePath)) { // Parse existing coverage report const coverageReport = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); return { statements: coverageReport.total.statements.pct, branches: coverageReport.total.branches.pct, functions: coverageReport.total.functions.pct, lines: coverageReport.total.lines.pct, timestamp: new Date().toISOString() }; } // Generate coverage report if (testFrameworks.includes('jest')) { execSync('npx jest --coverage', { cwd: projectRoot }); // Check if coverage report was generated if (fs.existsSync(coveragePath)) { const coverageReport = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); return { statements: coverageReport.total.statements.pct, branches: coverageReport.total.branches.pct, functions: coverageReport.total.functions.pct, lines: coverageReport.total.lines.pct, timestamp: new Date().toISOString() }; } } // If no coverage report found, return default values return { statements: 0, branches: 0, functions: 0, lines: 0, timestamp: new Date().toISOString() }; } catch (error) { log.error(`Error collecting coverage metrics: ${error.message}`); // Return default values on error return { statements: 0, branches: 0, functions: 0, lines: 0, timestamp: new Date().toISOString(), error: error.message }; } } /** * Collect performance metrics * @param {string} projectRoot - Project root directory * @param {Array<string>} testFrameworks - Test frameworks to collect metrics from * @returns {Promise<Object>} Performance metrics */ async function collectPerformanceMetrics(projectRoot, testFrameworks) { log.info('Collecting performance metrics'); try { // This would typically use a performance testing tool // For now, we'll return mock metrics return { executionTime: 1200, // ms memoryUsage: 45, // MB timestamp: new Date().toISOString() }; } catch (error) { log.error(`Error collecting performance metrics: ${error.message}`); // Return default values on error return { executionTime: 0, memoryUsage: 0, timestamp: new Date().toISOString(), error: error.message }; } } /** * Collect reliability metrics * @param {string} projectRoot - Project root directory * @param {Array<string>} testFrameworks - Test frameworks to collect metrics from * @returns {Promise<Object>} Reliability metrics */ async function collectReliabilityMetrics(projectRoot, testFrameworks) { log.info('Collecting reliability metrics'); try { // This would typically analyze test results over time // For now, we'll return mock metrics return { failureRate: 0.02, // 2% uptime: 0.995, // 99.5% timestamp: new Date().toISOString() }; } catch (error) { log.error(`Error collecting reliability metrics: ${error.message}`); // Return default values on error return { failureRate: 1, uptime: 0, timestamp: new Date().toISOString(), error: error.message }; } } /** * Collect maintainability metrics * @param {string} projectRoot - Project root directory * @returns {Promise<Object>} Maintainability metrics */ async function collectMaintainabilityMetrics(projectRoot) { log.info('Collecting maintainability metrics'); try { // This would typically use a code quality tool // For now, we'll return mock metrics return { complexity: 5, duplication: 0.05, // 5% timestamp: new Date().toISOString() }; } catch (error) { log.error(`Error collecting maintainability metrics: ${error.message}`); // Return default values on error return { complexity: 0, duplication: 0, timestamp: new Date().toISOString(), error: error.message }; } } /** * Analyze test metrics * @param {Object} metrics - Test metrics * @param {Object} thresholds - Metric thresholds * @returns {Object} Analysis result */ export function analyzeTestMetrics(metrics, thresholds = DEFAULT_THRESHOLDS) { log.info('Analyzing test metrics'); const analysis = { passed: true, issues: [], recommendations: [] }; try { // Analyze coverage metrics if (metrics.coverage) { const coverageThresholds = thresholds[METRIC_TYPES.COVERAGE]; if (metrics.coverage.statements < coverageThresholds.statements) { analysis.passed = false; analysis.issues.push(`Statement coverage (${metrics.coverage.statements}%) is below threshold (${coverageThresholds.statements}%)`); analysis.recommendations.push('Add more unit tests to increase statement coverage'); } if (metrics.coverage.branches < coverageThresholds.branches) { analysis.passed = false; analysis.issues.push(`Branch coverage (${metrics.coverage.branches}%) is below threshold (${coverageThresholds.branches}%)`); analysis.recommendations.push('Add tests for conditional branches'); } if (metrics.coverage.functions < coverageThresholds.functions) { analysis.passed = false; analysis.issues.push(`Function coverage (${metrics.coverage.functions}%) is below threshold (${coverageThresholds.functions}%)`); analysis.recommendations.push('Add tests for untested functions'); } if (metrics.coverage.lines < coverageThresholds.lines) { analysis.passed = false; analysis.issues.push(`Line coverage (${metrics.coverage.lines}%) is below threshold (${coverageThresholds.lines}%)`); analysis.recommendations.push('Add more tests to increase line coverage'); } } // Analyze performance metrics if (metrics.performance) { const performanceThresholds = thresholds[METRIC_TYPES.PERFORMANCE]; if (metrics.performance.executionTime > performanceThresholds.maxExecutionTime) { analysis.passed = false; analysis.issues.push(`Execution time (${metrics.performance.executionTime}ms) exceeds threshold (${performanceThresholds.maxExecutionTime}ms)`); analysis.recommendations.push('Optimize test execution time'); } if (metrics.performance.memoryUsage > performanceThresholds.maxMemoryUsage) { analysis.passed = false; analysis.issues.push(`Memory usage (${metrics.performance.memoryUsage}MB) exceeds threshold (${performanceThresholds.maxMemoryUsage}MB)`); analysis.recommendations.push('Reduce memory usage in tests'); } } // Analyze reliability metrics if (metrics.reliability) { const reliabilityThresholds = thresholds[METRIC_TYPES.RELIABILITY]; if (metrics.reliability.failureRate > reliabilityThresholds.maxFailureRate) { analysis.passed = false; analysis.issues.push(`Failure rate (${metrics.reliability.failureRate * 100}%) exceeds threshold (${reliabilityThresholds.maxFailureRate * 100}%)`); analysis.recommendations.push('Investigate and fix flaky tests'); } if (metrics.reliability.uptime < reliabilityThresholds.minUptime) { analysis.passed = false; analysis.issues.push(`Uptime (${metrics.reliability.uptime * 100}%) is below threshold (${reliabilityThresholds.minUptime * 100}%)`); analysis.recommendations.push('Improve system stability'); } } // Analyze maintainability metrics if (metrics.maintainability) { const maintainabilityThresholds = thresholds[METRIC_TYPES.MAINTAINABILITY]; if (metrics.maintainability.complexity > maintainabilityThresholds.maxComplexity) { analysis.passed = false; analysis.issues.push(`Complexity (${metrics.maintainability.complexity}) exceeds threshold (${maintainabilityThresholds.maxComplexity})`); analysis.recommendations.push('Refactor complex code'); } if (metrics.maintainability.duplication > maintainabilityThresholds.maxDuplication) { analysis.passed = false; analysis.issues.push(`Duplication (${metrics.maintainability.duplication * 100}%) exceeds threshold (${maintainabilityThresholds.maxDuplication * 100}%)`); analysis.recommendations.push('Reduce code duplication'); } } return analysis; } catch (error) { log.error(`Error analyzing test metrics: ${error.message}`); return { passed: false, issues: [`Error analyzing metrics: ${error.message}`], recommendations: ['Fix metric analysis error'] }; } } /** * Generate test report * @param {Object} options - Report options * @param {Object} options.metrics - Test metrics * @param {Object} options.analysis - Metrics analysis * @param {string} options.projectRoot - Project root directory * @param {string} options.format - Report format (html, json, markdown) * @returns {Promise<Object>} Generated report */ export async function generateTestReport(options) { const { metrics, analysis, projectRoot, format = 'html' } = options; log.info(`Generating ${format} test report`); try { let reportContent; let reportPath; switch (format) { case 'html': reportContent = generateHTMLReport(metrics, analysis); reportPath = path.join(projectRoot, 'test-report.html'); break; case 'json': reportContent = JSON.stringify({ metrics, analysis }, null, 2); reportPath = path.join(projectRoot, 'test-report.json'); break; case 'markdown': reportContent = generateMarkdownReport(metrics, analysis); reportPath = path.join(projectRoot, 'test-report.md'); break; default: throw new Error(`Unsupported report format: ${format}`); } // Write report to file fs.writeFileSync(reportPath, reportContent, 'utf8'); log.info(`Test report generated at ${reportPath}`); return { format, path: reportPath, content: reportContent }; } catch (error) { log.error(`Error generating test report: ${error.message}`); throw error; } } /** * Generate HTML test report * @param {Object} metrics - Test metrics * @param {Object} analysis - Metrics analysis * @returns {string} HTML report */ function generateHTMLReport(metrics, analysis) { return `<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>Test Report</title> <style> body { font-family: Arial, sans-serif; line-height: 1.6; max-width: 1200px; margin: 0 auto; padding: 20px; } h1, h2, h3 { color: #333; } .summary { background-color: ${analysis.passed ? '#e6ffe6' : '#ffe6e6'}; padding: 15px; border-radius: 5px; margin-bottom: 20px; } .metrics-section { margin-bottom: 30px; } .metric-card { background-color: #f5f5f5; padding: 15px; border-radius: 5px; margin-bottom: 10px; } .metric-value { font-size: 24px; font-weight: bold; } .issues { background-color: #fff0f0; padding: 15px; border-radius: 5px; margin-bottom: 20px; } .recommendations { background-color: #f0f8ff; padding: 15px; border-radius: 5px; } .progress-bar { height: 20px; background-color: #e0e0e0; border-radius: 10px; margin-top: 5px; } .progress-value { height: 100%; border-radius: 10px; background-color: #4CAF50; } .progress-value.warning { background-color: #FFC107; } .progress-value.danger { background-color: #F44336; } </style> </head> <body> <h1>Test Report</h1> <div class="summary"> <h2>Summary</h2> <p>Status: ${analysis.passed ? 'PASSED ✅' : 'FAILED ❌'}</p> <p>Generated: ${new Date(metrics.timestamp).toLocaleString()}</p> </div> ${metrics.coverage ? ` <div class="metrics-section"> <h2>Coverage Metrics</h2> <div class="metric-card"> <h3>Statement Coverage</h3> <div class="metric-value">${metrics.coverage.statements}%</div> <div class="progress-bar"> <div class="progress-value ${getProgressClass(metrics.coverage.statements, 80)}" style="width: ${metrics.coverage.statements}%"></div> </div> </div> <div class="metric-card"> <h3>Branch Coverage</h3> <div class="metric-value">${metrics.coverage.branches}%</div> <div class="progress-bar"> <div class="progress-value ${getProgressClass(metrics.coverage.branches, 70)}" style="width: ${metrics.coverage.branches}%"></div> </div> </div> <div class="metric-card"> <h3>Function Coverage</h3> <div class="metric-value">${metrics.coverage.functions}%</div> <div class="progress-bar"> <div class="progress-value ${getProgressClass(metrics.coverage.functions, 80)}" style="width: ${metrics.coverage.functions}%"></div> </div> </div> <div class="metric-card"> <h3>Line Coverage</h3> <div class="metric-value">${metrics.coverage.lines}%</div> <div class="progress-bar"> <div class="progress-value ${getProgressClass(metrics.coverage.lines, 80)}" style="width: ${metrics.coverage.lines}%"></div> </div> </div> </div> ` : ''} ${metrics.performance ? ` <div class="metrics-section"> <h2>Performance Metrics</h2> <div class="metric-card"> <h3>Execution Time</h3> <div class="metric-value">${metrics.performance.executionTime}ms</div> </div> <div class="metric-card"> <h3>Memory Usage</h3> <div class="metric-value">${metrics.performance.memoryUsage}MB</div> </div> </div> ` : ''} ${metrics.reliability ? ` <div class="metrics-section"> <h2>Reliability Metrics</h2> <div class="metric-card"> <h3>Failure Rate</h3> <div class="metric-value">${(metrics.reliability.failureRate * 100).toFixed(2)}%</div> </div> <div class="metric-card"> <h3>Uptime</h3> <div class="metric-value">${(metrics.reliability.uptime * 100).toFixed(2)}%</div> </div> </div> ` : ''} ${metrics.maintainability ? ` <div class="metrics-section"> <h2>Maintainability Metrics</h2> <div class="metric-card"> <h3>Complexity</h3> <div class="metric-value">${metrics.maintainability.complexity}</div> </div> <div class="metric-card"> <h3>Duplication</h3> <div class="metric-value">${(metrics.maintainability.duplication * 100).toFixed(2)}%</div> </div> </div> ` : ''} ${analysis.issues.length > 0 ? ` <div class="issues"> <h2>Issues</h2> <ul> ${analysis.issues.map(issue => `<li>${issue}</li>`).join('')} </ul> </div> ` : ''} ${analysis.recommendations.length > 0 ? ` <div class="recommendations"> <h2>Recommendations</h2> <ul> ${analysis.recommendations.map(recommendation => `<li>${recommendation}</li>`).join('')} </ul> </div> ` : ''} </body> </html>`; } /** * Generate Markdown test report * @param {Object} metrics - Test metrics * @param {Object} analysis - Metrics analysis * @returns {string} Markdown report */ function generateMarkdownReport(metrics, analysis) { let report = `# Test Report\n\n`; // Add summary report += `## Summary\n\n`; report += `Status: ${analysis.passed ? 'PASSED ✅' : 'FAILED ❌'}\n\n`; report += `Generated: ${new Date(metrics.timestamp).toLocaleString()}\n\n`; // Add coverage metrics if (metrics.coverage) { report += `## Coverage Metrics\n\n`; report += `- Statement Coverage: ${metrics.coverage.statements}%\n`; report += `- Branch Coverage: ${metrics.coverage.branches}%\n`; report += `- Function Coverage: ${metrics.coverage.functions}%\n`; report += `- Line Coverage: ${metrics.coverage.lines}%\n\n`; } // Add performance metrics if (metrics.performance) { report += `## Performance Metrics\n\n`; report += `- Execution Time: ${metrics.performance.executionTime}ms\n`; report += `- Memory Usage: ${metrics.performance.memoryUsage}MB\n\n`; } // Add reliability metrics if (metrics.reliability) { report += `## Reliability Metrics\n\n`; report += `- Failure Rate: ${(metrics.reliability.failureRate * 100).toFixed(2)}%\n`; report += `- Uptime: ${(metrics.reliability.uptime * 100).toFixed(2)}%\n\n`; } // Add maintainability metrics if (metrics.maintainability) { report += `## Maintainability Metrics\n\n`; report += `- Complexity: ${metrics.maintainability.complexity}\n`; report += `- Duplication: ${(metrics.maintainability.duplication * 100).toFixed(2)}%\n\n`; } // Add issues if (analysis.issues.length > 0) { report += `## Issues\n\n`; analysis.issues.forEach(issue => { report += `- ${issue}\n`; }); report += '\n'; } // Add recommendations if (analysis.recommendations.length > 0) { report += `## Recommendations\n\n`; analysis.recommendations.forEach(recommendation => { report += `- ${recommendation}\n`; }); report += '\n'; } return report; } /** * Get progress bar class based on value and threshold * @param {number} value - Metric value * @param {number} threshold - Metric threshold * @returns {string} Progress bar class */ function getProgressClass(value, threshold) { if (value >= threshold) { return ''; } else if (value >= threshold * 0.8) { return 'warning'; } else { return 'danger'; } } /** * Track test metrics over time * @param {Object} options - Tracking options * @param {string} options.projectRoot - Project root directory * @param {Object} options.metrics - Current metrics * @returns {Promise<Object>} Tracking result */ export async function trackTestMetricsOverTime(options) { const { projectRoot, metrics } = options; log.info('Tracking test metrics over time'); try { const historyPath = path.join(projectRoot, 'test-metrics-history.json'); // Load existing history let history = []; if (fs.existsSync(historyPath)) { history = JSON.parse(fs.readFileSync(historyPath, 'utf8')); } // Add current metrics to history history.push({ timestamp: metrics.timestamp, coverage: metrics.coverage, performance: metrics.performance, reliability: metrics.reliability, maintainability: metrics.maintainability }); // Limit history size if (history.length > 100) { history = history.slice(-100); } // Save history fs.writeFileSync(historyPath, JSON.stringify(history, null, 2), 'utf8'); log.info(`Test metrics history updated at ${historyPath}`); // Calculate trends const trends = calculateMetricTrends(history); return { history, trends }; } catch (error) { log.error(`Error tracking test metrics: ${error.message}`); throw error; } } /** * Calculate metric trends * @param {Array<Object>} history - Metrics history * @returns {Object} Metric trends */ function calculateMetricTrends(history) { if (history.length < 2) { return { coverage: { trend: 'stable' }, performance: { trend: 'stable' }, reliability: { trend: 'stable' }, maintainability: { trend: 'stable' } }; } const current = history[history.length - 1]; const previous = history[history.length - 2]; const trends = {}; // Calculate coverage trends if (current.coverage && previous.coverage) { const statementsDiff = current.coverage.statements - previous.coverage.statements; const branchesDiff = current.coverage.branches - previous.coverage.branches; const functionsDiff = current.coverage.functions - previous.coverage.functions; const linesDiff = current.coverage.lines - previous.coverage.lines; trends.coverage = { statements: { diff: statementsDiff, trend: getTrendDirection(statementsDiff) }, branches: { diff: branchesDiff, trend: getTrendDirection(branchesDiff) }, functions: { diff: functionsDiff, trend: getTrendDirection(functionsDiff) }, lines: { diff: linesDiff, trend: getTrendDirection(linesDiff) } }; } // Calculate performance trends if (current.performance && previous.performance) { const executionTimeDiff = current.performance.executionTime - previous.performance.executionTime; const memoryUsageDiff = current.performance.memoryUsage - previous.performance.memoryUsage; trends.performance = { executionTime: { diff: executionTimeDiff, trend: getTrendDirection(-executionTimeDiff) // Lower is better }, memoryUsage: { diff: memoryUsageDiff, trend: getTrendDirection(-memoryUsageDiff) // Lower is better } }; } // Calculate reliability trends if (current.reliability && previous.reliability) { const failureRateDiff = current.reliability.failureRate - previous.reliability.failureRate; const uptimeDiff = current.reliability.uptime - previous.reliability.uptime; trends.reliability = { failureRate: { diff: failureRateDiff, trend: getTrendDirection(-failureRateDiff) // Lower is better }, uptime: { diff: uptimeDiff, trend: getTrendDirection(uptimeDiff) } }; } // Calculate maintainability trends if (current.maintainability && previous.maintainability) { const complexityDiff = current.maintainability.complexity - previous.maintainability.complexity; const duplicationDiff = current.maintainability.duplication - previous.maintainability.duplication; trends.maintainability = { complexity: { diff: complexityDiff, trend: getTrendDirection(-complexityDiff) // Lower is better }, duplication: { diff: duplicationDiff, trend: getTrendDirection(-duplicationDiff) // Lower is better } }; } return trends; } /** * Get trend direction based on difference * @param {number} diff - Metric difference * @returns {string} Trend direction */ function getTrendDirection(diff) { if (diff > 0) { return 'improving'; } else if (diff < 0) { return 'declining'; } else { return 'stable'; } }