cqt-agent/expansion-packs/hubtel-workflow/utils/test-failure-analyzer.md

CQT Agent

# Test Failure Analyzer

This utility analyzes test failures and provides specific, actionable remediation steps to help users resolve issues quickly and effectively.

## Purpose
Automatically categorize test failures, identify root causes, and provide clear, step-by-step remediation instructions with code examples and scripts.

## Failure Analysis Framework

### 1. Failure Categorization System

```javascript
const FailureCategories = {
  SYNTAX_ERROR: {
    priority: 'HIGH',
    category: 'syntax',
    description: 'Code syntax or compilation errors',
    fixable: true
  },
  ENVIRONMENT_ERROR: {
    priority: 'HIGH', 
    category: 'environment',
    description: 'Missing dependencies, configuration, or runtime issues',
    fixable: true
  },
  LOGIC_ERROR: {
    priority: 'MEDIUM',
    category: 'logic', 
    description: 'Test logic errors or incorrect assertions',
    fixable: true
  },
  DATA_ERROR: {
    priority: 'MEDIUM',
    category: 'data',
    description: 'Test data issues or database problems',
    fixable: true
  },
  TIMING_ERROR: {
    priority: 'LOW',
    category: 'timing',
    description: 'Race conditions or timeout issues',
    fixable: true
  },
  EXTERNAL_ERROR: {
    priority: 'LOW',
    category: 'external',
    description: 'Third-party service or network issues',
    fixable: false
  }
};
```

### 2. Error Pattern Recognition

```javascript
const ErrorPatterns = {
  // Syntax Errors
  typescript: [
    {
      pattern: /Cannot find module ['"`](.+)['"`]/,
      category: 'SYNTAX_ERROR',
      message: 'Missing import or module not found',
      solution: 'install_missing_dependency'
    },
    {
      pattern: /Property ['"`](.+)['"`] does not exist on type/,
      category: 'SYNTAX_ERROR', 
      message: 'TypeScript type error',
      solution: 'fix_typescript_types'
    },
    {
      pattern: /Expected (\d+) arguments, but got (\d+)/,
      category: 'SYNTAX_ERROR',
      message: 'Function call argument mismatch',
      solution: 'fix_function_arguments'
    }
  ],
  
  // Environment Errors
  environment: [
    {
      pattern: /ECONNREFUSED.*:(\d+)/,
      category: 'ENVIRONMENT_ERROR',
      message: 'Cannot connect to service on port',
      solution: 'start_required_service'
    },
    {
      pattern: /Command ['"`](.+)['"`] not found/,
      category: 'ENVIRONMENT_ERROR',
      message: 'Required command or tool not installed',
      solution: 'install_missing_tool'
    },
    {
      pattern: /Permission denied/,
      category: 'ENVIRONMENT_ERROR',
      message: 'Insufficient permissions',
      solution: 'fix_permissions'
    }
  ],
  
  // Test Logic Errors
  playwright: [
    {
      pattern: /locator\.click: Timeout (\d+)ms exceeded/,
      category: 'LOGIC_ERROR',
      message: 'Element not found or not clickable within timeout',
      solution: 'fix_element_locator'
    },
    {
      pattern: /expect.*\.toHaveText.*received/,
      category: 'LOGIC_ERROR',
      message: 'Text content assertion failed',
      solution: 'update_text_assertion'
    },
    {
      pattern: /Navigation timeout exceeded/,
      category: 'TIMING_ERROR',
      message: 'Page navigation too slow',
      solution: 'increase_navigation_timeout'
    }
  ],
  
  // API Test Errors
  karate: [
    {
      pattern: /status code was: (\d+), expected: (\d+)/,
      category: 'LOGIC_ERROR',
      message: 'API response status code mismatch',
      solution: 'fix_api_expectation'
    },
    {
      pattern: /path not found: (.+)/,
      category: 'DATA_ERROR',
      message: 'JSON path assertion failed',
      solution: 'fix_json_path'
    },
    {
      pattern: /connection refused/i,
      category: 'ENVIRONMENT_ERROR',
      message: 'Cannot connect to API server',
      solution: 'start_api_server'
    }
  ]
};
```

## Failure Analysis Engine

### 1. Error Log Parser
```javascript
function parseTestFailures(testResults) {
  const failures = [];
  
  // Parse different test result formats
  if (testResults.type === 'playwright') {
    testResults.suites.forEach(suite => {
      suite.tests.forEach(test => {
        if (test.outcome === 'failed') {
          failures.push({
            testName: test.title,
            testFile: suite.file,
            error: test.error,
            framework: 'playwright',
            rawLog: test.stderr
          });
        }
      });
    });
  }
  
  if (testResults.type === 'karate') {
    testResults.scenarioResults.forEach(scenario => {
      if (!scenario.passed) {
        failures.push({
          testName: scenario.scenario.name,
          testFile: scenario.scenario.feature.resource.file,
          error: scenario.error,
          framework: 'karate', 
          rawLog: scenario.stepResults.map(s => s.error).filter(Boolean).join('\n')
        });
      }
    });
  }
  
  if (testResults.type === 'jest') {
    testResults.testResults.forEach(testFile => {
      testFile.assertionResults.forEach(test => {
        if (test.status === 'failed') {
          failures.push({
            testName: test.title,
            testFile: testFile.testFilePath,
            error: test.failureMessages.join('\n'),
            framework: 'jest',
            rawLog: test.failureMessages.join('\n')
          });
        }
      });
    });
  }
  
  return failures;
}
```

### 2. Root Cause Analysis
```javascript
function analyzeFailure(failure) {
  const analysis = {
    testName: failure.testName,
    testFile: failure.testFile,
    framework: failure.framework,
    category: 'UNKNOWN',
    rootCause: 'Unable to determine root cause',
    confidence: 0,
    remediation: []
  };
  
  // Match against known error patterns
  for (const [framework, patterns] of Object.entries(ErrorPatterns)) {
    if (framework === failure.framework || framework === 'environment') {
      for (const pattern of patterns) {
        const match = failure.error.match(pattern.pattern);
        if (match) {
          analysis.category = pattern.category;
          analysis.rootCause = pattern.message;
          analysis.confidence = 0.9;
          analysis.patternMatch = match;
          analysis.solutionKey = pattern.solution;
          break;
        }
      }
    }
  }
  
  // Additional heuristic analysis
  if (analysis.confidence < 0.5) {
    analysis = performHeuristicAnalysis(failure, analysis);
  }
  
  // Generate remediation steps
  analysis.remediation = generateRemediation(analysis);
  
  return analysis;
}

function performHeuristicAnalysis(failure, analysis) {
  const error = failure.error.toLowerCase();
  
  // Check for common indicators
  if (error.includes('timeout') || error.includes('exceeded')) {
    analysis.category = 'TIMING_ERROR';
    analysis.rootCause = 'Operation timed out';
    analysis.confidence = 0.7;
  } else if (error.includes('not found') || error.includes('404')) {
    analysis.category = 'LOGIC_ERROR';
    analysis.rootCause = 'Resource or element not found';
    analysis.confidence = 0.7;
  } else if (error.includes('permission') || error.includes('403')) {
    analysis.category = 'ENVIRONMENT_ERROR';
    analysis.rootCause = 'Permission denied';
    analysis.confidence = 0.7;
  } else if (error.includes('network') || error.includes('econnreset')) {
    analysis.category = 'EXTERNAL_ERROR';
    analysis.rootCause = 'Network connectivity issue';
    analysis.confidence = 0.6;
  }
  
  return analysis;
}
```

### 3. Remediation Generator
```javascript
const RemediationStrategies = {
  install_missing_dependency: (analysis) => ({
    priority: 'HIGH',
    estimatedTime: '2-5 minutes',
    steps: [
      {
        action: 'Install missing dependency',
        command: `npm install ${analysis.patternMatch[1]}`,
        description: `Install the missing module: ${analysis.patternMatch[1]}`
      },
      {
        action: 'Verify installation',
        command: `npm list ${analysis.patternMatch[1]}`,
        description: 'Confirm the dependency was installed correctly'
      },
      {
        action: 'Re-run tests',
        command: 'npm test',
        description: 'Execute tests again to verify fix'
      }
    ],
    preventionTips: [
      'Run "npm ci" before test execution',
      'Keep package.json dependencies up to date',
      'Use package-lock.json for consistent dependencies'
    ]
  }),

  fix_typescript_types: (analysis) => ({
    priority: 'HIGH',
    estimatedTime: '5-10 minutes',
    steps: [
      {
        action: 'Identify missing type',
        description: `Property '${analysis.patternMatch[1]}' is not recognized`,
        manual: true
      },
      {
        action: 'Add type definition',
        codeExample: `
interface ExpectedType {
  ${analysis.patternMatch[1]}: string; // Adjust type as needed
}`,
        description: 'Add the missing property to your interface'
      },
      {
        action: 'Update import statements',
        description: 'Ensure all required types are imported'
      }
    ],
    automaticFix: {
      available: true,
      script: generateTypeFixScript(analysis)
    }
  }),

  start_required_service: (analysis) => ({
    priority: 'HIGH',
    estimatedTime: '1-3 minutes',
    steps: [
      {
        action: 'Check service status',
        command: `netstat -tulpn | grep :${analysis.patternMatch[1]}`,
        description: `Verify if service is running on port ${analysis.patternMatch[1]}`
      },
      {
        action: 'Start database service',
        command: 'docker-compose up -d db',
        description: 'Start the required database service',
        condition: 'if database port (5432, 3306, 27017)'
      },
      {
        action: 'Start application server',
        command: 'npm run start:dev',
        description: 'Start the application server',
        condition: 'if application port (3000, 8080, etc.)'
      },
      {
        action: 'Wait for service ready',
        command: `until nc -z localhost ${analysis.patternMatch[1]}; do sleep 1; done`,
        description: 'Wait for service to be available'
      }
    ],
    healthCheck: {
      command: `curl -f http://localhost:${analysis.patternMatch[1]}/health || echo "Service not ready"`,
      description: 'Verify service health'
    }
  }),

  fix_element_locator: (analysis) => ({
    priority: 'MEDIUM',
    estimatedTime: '5-15 minutes',
    steps: [
      {
        action: 'Inspect element selectors',
        description: 'Open browser dev tools and verify element exists',
        manual: true
      },
      {
        action: 'Try alternative selectors',
        codeExample: `
// Instead of:
await page.locator('[data-testid=old-selector]').click();

// Try these alternatives:
await page.locator('text="Button Text"').click();
await page.locator('[aria-label="Button Label"]').click();
await page.locator('button:has-text("Submit")').click();`,
        description: 'Use more robust selector strategies'
      },
      {
        action: 'Add explicit wait',
        codeExample: `
await page.waitForSelector('[data-testid=element]', { state: 'visible' });
await page.locator('[data-testid=element]').click();`,
        description: 'Wait for element to be ready before interaction'
      }
    ],
    debuggingTips: [
      'Use page.screenshot() to capture current page state',
      'Add console.log to verify page navigation',
      'Check if element is in viewport with scrollIntoView()'
    ]
  })
};

function generateRemediation(analysis) {
  const strategy = RemediationStrategies[analysis.solutionKey];
  
  if (strategy) {
    return strategy(analysis);
  }
  
  // Generic remediation for unknown issues
  return {
    priority: 'MEDIUM',
    estimatedTime: '10-30 minutes',
    steps: [
      {
        action: 'Review test logs',
        description: 'Examine the full error message and stack trace'
      },
      {
        action: 'Check test environment',
        description: 'Verify all services and dependencies are running'
      },
      {
        action: 'Run test in isolation',
        description: 'Execute the failing test individually to isolate the issue'
      },
      {
        action: 'Consult documentation',
        description: 'Review framework documentation for similar issues'
      }
    ]
  };
}
```

## User-Friendly Report Generation

### 1. Failure Summary Report
```javascript
function generateFailureReport(failures) {
  const analyses = failures.map(analyzeFailure);
  
  const report = {
    summary: {
      totalFailures: failures.length,
      categorized: groupBy(analyses, 'category'),
      highPriority: analyses.filter(a => a.category === 'SYNTAX_ERROR' || a.category === 'ENVIRONMENT_ERROR').length,
      autoFixable: analyses.filter(a => a.remediation?.automaticFix?.available).length
    },
    detailedAnalysis: analyses.map(formatAnalysisForUser),
    quickActions: generateQuickActions(analyses),
    executionScript: generateFixScript(analyses)
  };
  
  return report;
}

function formatAnalysisForUser(analysis) {
  return {
    testFile: analysis.testFile,
    testName: analysis.testName,
    issue: {
      category: FailureCategories[analysis.category]?.description || 'Unknown issue',
      rootCause: analysis.rootCause,
      confidence: `${Math.round(analysis.confidence * 100)}% confident`
    },
    remedy: {
      priority: analysis.remediation?.priority || 'MEDIUM',
      estimatedTime: analysis.remediation?.estimatedTime || 'Unknown',
      steps: analysis.remediation?.steps || [],
      autoFixAvailable: !!analysis.remediation?.automaticFix
    }
  };
}
```

### 2. Interactive Fix Script Generator
```javascript
function generateFixScript(analyses) {
  const script = ['#!/bin/bash', '# Automated Test Failure Fixes', ''];
  
  // Group fixes by priority
  const highPriorityFixes = analyses.filter(a => a.category === 'SYNTAX_ERROR' || a.category === 'ENVIRONMENT_ERROR');
  const mediumPriorityFixes = analyses.filter(a => a.category === 'LOGIC_ERROR' || a.category === 'DATA_ERROR');
  
  if (highPriorityFixes.length > 0) {
    script.push('echo "🔧 Applying high-priority fixes..."');
    highPriorityFixes.forEach(analysis => {
      if (analysis.remediation?.steps) {
        analysis.remediation.steps.forEach(step => {
          if (step.command) {
            script.push(`echo "Executing: ${step.description}"`);
            script.push(step.command);
            script.push('');
          }
        });
      }
    });
  }
  
  script.push('echo "✅ High-priority fixes completed. Re-running tests..."');
  script.push('npm test');
  
  return script.join('\n');
}
```

### 3. HTML Report Generator
```javascript
function generateHTMLReport(failureReport) {
  return `
<!DOCTYPE html>
<html>
<head>
    <title>Test Failure Analysis Report</title>
    <style>
        body { font-family: Arial, sans-serif; margin: 20px; background: #f5f5f5; }
        .container { max-width: 1200px; margin: 0 auto; background: white; padding: 20px; border-radius: 8px; }
        .header { text-align: center; border-bottom: 2px solid #007bff; padding-bottom: 15px; }
        .summary-card { background: #f8f9fa; padding: 15px; border-radius: 5px; margin: 15px 0; }
        .failure-item { background: white; border-left: 4px solid #dc3545; padding: 15px; margin: 10px 0; }
        .high-priority { border-left-color: #dc3545; }
        .medium-priority { border-left-color: #ffc107; }
        .low-priority { border-left-color: #28a745; }
        .remediation-steps { background: #e9ecef; padding: 10px; border-radius: 3px; }
        .command { background: #2d3748; color: #e2e8f0; padding: 8px; border-radius: 3px; font-family: monospace; }
        .auto-fix-btn { background: #28a745; color: white; padding: 8px 16px; border: none; border-radius: 4px; cursor: pointer; }
    </style>
</head>
<body>
    <div class="container">
        <div class="header">
            <h1>🔍 Test Failure Analysis Report</h1>
            <p>Generated on ${new Date().toLocaleString()}</p>
        </div>
        
        <div class="summary-card">
            <h2>📊 Summary</h2>
            <p><strong>Total Failures:</strong> ${failureReport.summary.totalFailures}</p>
            <p><strong>High Priority Issues:</strong> ${failureReport.summary.highPriority}</p>
            <p><strong>Auto-fixable Issues:</strong> ${failureReport.summary.autoFixable}</p>
        </div>
        
        <div class="quick-actions">
            <h2>⚡ Quick Actions</h2>
            <button class="auto-fix-btn" onclick="downloadFixScript()">Download Fix Script</button>
            <button class="auto-fix-btn" onclick="runAutoFixes()">Run Auto-Fixes</button>
        </div>
        
        <div class="failures-list">
            <h2>🚨 Detailed Analysis</h2>
            ${failureReport.detailedAnalysis.map(renderFailureItem).join('')}
        </div>
    </div>
    
    <script>
        function downloadFixScript() {
            const script = \`${failureReport.executionScript}\`;
            const blob = new Blob([script], { type: 'text/plain' });
            const url = URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.href = url;
            a.download = 'test-fixes.sh';
            a.click();
        }
        
        function runAutoFixes() {
            alert('Auto-fix functionality would be implemented here');
        }
    </script>
</body>
</html>`;
}
```

## Main Analysis Function

```javascript
function analyzeTestFailures(testResultsPath) {
  console.log('🔍 Analyzing test failures...');
  
  // Load test results
  const testResults = loadTestResults(testResultsPath);
  
  // Parse failures from results
  const failures = parseTestFailures(testResults);
  
  if (failures.length === 0) {
    console.log('✅ No test failures detected!');
    return;
  }
  
  console.log(`Found ${failures.length} test failures. Analyzing...`);
  
  // Generate comprehensive failure report
  const failureReport = generateFailureReport(failures);
  
  // Save reports
  fs.writeFileSync('test-results/failure-analysis.json', JSON.stringify(failureReport, null, 2));
  fs.writeFileSync('test-results/failure-report.html', generateHTMLReport(failureReport));
  fs.writeFileSync('test-results/auto-fix.sh', failureReport.executionScript, { mode: 0o755 });
  
  // Display summary
  console.log('\n🎯 Failure Analysis Complete');
  console.log('============================');
  console.log(`📊 Total failures analyzed: ${failures.length}`);
  console.log(`🔥 High priority issues: ${failureReport.summary.highPriority}`);
  console.log(`🔧 Auto-fixable issues: ${failureReport.summary.autoFixable}`);
  console.log(`📋 Detailed report: test-results/failure-report.html`);
  console.log(`⚡ Auto-fix script: test-results/auto-fix.sh`);
  
  return failureReport;
}
```

This utility provides comprehensive failure analysis with specific, actionable remediation steps, automated fix scripts, and clear reporting to help users resolve test issues quickly and effectively.