ctrlshiftleft/scripts/compare-security-analysis.js

AI-powered toolkit for embedding QA and security testing into development workflows

#!/usr/bin/env node

/**
 * Security Analysis Comparison Tool
 * 
 * This script compares the traditional pattern-based security analysis
 * with the new AI-enhanced security analysis to demonstrate the improvements.
 */

const fs = require('fs');
const path = require('path');
const { exec } = require('child_process');
const { promisify } = require('util');

const execAsync = promisify(exec);

// ANSI color codes for terminal output
const COLORS = {
  reset: '\x1b[0m',
  bright: '\x1b[1m',
  dim: '\x1b[2m',
  underscore: '\x1b[4m',
  red: '\x1b[31m',
  green: '\x1b[32m',
  yellow: '\x1b[33m',
  blue: '\x1b[34m',
  magenta: '\x1b[35m',
  cyan: '\x1b[36m',
  white: '\x1b[37m'
};

// Get the project root directory
const projectRoot = path.resolve(__dirname, '..');
const binPath = path.join(projectRoot, 'bin');
const vscodeExtPath = path.join(projectRoot, 'vscode-ext-test');
const reportsPath = path.join(projectRoot, 'security-reports');
const demoPath = path.join(projectRoot, 'demo', 'samples');

// Default test file
const defaultTestFile = path.join(demoPath, 'vulnerable-auth.js');

// Function to run security analysis
async function runSecurityAnalysis(filePath, useAI = false) {
  console.log(`\n${COLORS.bright}${useAI ? 'AI-Enhanced' : 'Pattern-Based'} Security Analysis${COLORS.reset}\n`);
  console.log(`Analyzing: ${COLORS.cyan}${filePath}${COLORS.reset}\n`);
  
  const startTime = Date.now();
  
  // File paths for reports
  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
  const outputDir = path.join(reportsPath, `${path.basename(filePath, path.extname(filePath))}-${useAI ? 'ai' : 'pattern'}-${timestamp}`);
  
  // Create output directory
  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir, { recursive: true });
  }
  
  const reportPath = path.join(outputDir, 'security-report.md');
  
  try {
    // Choose the appropriate tool
    const command = useAI
      ? `${path.join(binPath, 'ctrlshiftleft-ai')} analyze --ai "${filePath}" --output="${reportPath}"`
      : `${path.join(binPath, 'ctrlshiftleft')} analyze "${filePath}" --output="${reportPath}"`;
    
    // Run the command
    const { stdout, stderr } = await execAsync(command, { cwd: projectRoot });
    
    // Calculate execution time
    const endTime = Date.now();
    const executionTime = (endTime - startTime) / 1000;
    
    // Report results
    console.log(`${COLORS.green}Analysis completed in ${executionTime.toFixed(2)}s${COLORS.reset}`);
    console.log(`Report saved to: ${COLORS.cyan}${reportPath}${COLORS.reset}\n`);
    
    // Return report path for further analysis
    return { reportPath, executionTime };
  } catch (error) {
    console.error(`${COLORS.red}Error running security analysis:${COLORS.reset}`);
    console.error(error.message);
    if (error.stdout) console.log(error.stdout);
    if (error.stderr) console.error(error.stderr);
    throw error;
  }
}

// Function to extract issue count from the report
function analyzeReport(reportPath) {
  try {
    if (!fs.existsSync(reportPath)) {
      return { total: 0, critical: 0, high: 0, medium: 0, low: 0, info: 0 };
    }
    
    const content = fs.readFileSync(reportPath, 'utf8');
    
    // Extract issue counts from summary section
    const results = {
      total: 0,
      critical: 0,
      high: 0,
      medium: 0,
      low: 0,
      info: 0
    };
    
    // Try to extract from AI format first (different formats possible)
    const criticalMatch = content.match(/critical:?\s*(\d+)/i);
    const highMatch = content.match(/high:?\s*(\d+)/i);
    const mediumMatch = content.match(/medium:?\s*(\d+)/i);
    const lowMatch = content.match(/low:?\s*(\d+)/i);
    const infoMatch = content.match(/info:?\s*(\d+)/i);
    
    if (criticalMatch) results.critical = parseInt(criticalMatch[1], 10);
    if (highMatch) results.high = parseInt(highMatch[1], 10);
    if (mediumMatch) results.medium = parseInt(mediumMatch[1], 10);
    if (lowMatch) results.low = parseInt(lowMatch[1], 10);
    if (infoMatch) results.info = parseInt(infoMatch[1], 10);
    
    // Calculate total
    results.total = results.critical + results.high + results.medium + results.low + results.info;
    
    return results;
  } catch (error) {
    console.error(`${COLORS.red}Error analyzing report:${COLORS.reset}`, error.message);
    return { total: 0, critical: 0, high: 0, medium: 0, low: 0, info: 0 };
  }
}

// Function to print comparison table
function printComparisonTable(patternResults, aiResults, patternTime, aiTime) {
  console.log(`\n${COLORS.bright}${COLORS.white}==== Security Analysis Comparison ====${COLORS.reset}\n`);
  
  // Headers
  console.log(`${COLORS.bright}Metric                Pattern-Based     AI-Enhanced     Difference${COLORS.reset}`);
  console.log(`${COLORS.dim}-----------------------------------------------------------------${COLORS.reset}`);
  
  // Execution time
  const timeDiff = aiTime - patternTime;
  const timeDiffColor = timeDiff > 0 ? COLORS.yellow : COLORS.green;
  console.log(`Execution Time      ${patternTime.toFixed(2)}s            ${aiTime.toFixed(2)}s         ${timeDiffColor}${(timeDiff > 0 ? '+' : '')}${timeDiff.toFixed(2)}s${COLORS.reset}`);
  
  // Issue counts
  const printIssueRow = (label, patternCount, aiCount) => {
    const diff = aiCount - patternCount;
    const diffColor = diff > 0 ? COLORS.green : (diff < 0 ? COLORS.red : COLORS.dim);
    const diffText = diff === 0 ? '0' : (diff > 0 ? `+${diff}` : diff);
    console.log(`${label.padEnd(20)}${patternCount.toString().padEnd(18)}${aiCount.toString().padEnd(15)}${diffColor}${diffText}${COLORS.reset}`);
  };
  
  printIssueRow('Total Issues', patternResults.total, aiResults.total);
  printIssueRow('Critical Issues', patternResults.critical, aiResults.critical);
  printIssueRow('High Issues', patternResults.high, aiResults.high);
  printIssueRow('Medium Issues', patternResults.medium, aiResults.medium);
  printIssueRow('Low Issues', patternResults.low, aiResults.low);
  printIssueRow('Info Issues', patternResults.info, aiResults.info);
  
  console.log(`\n${COLORS.dim}Note: ${COLORS.reset}${COLORS.yellow}AI analysis may take longer but typically provides more comprehensive results.${COLORS.reset}`);
  console.log(`${COLORS.dim}      Higher issue counts with AI analysis usually indicate better vulnerability detection.${COLORS.reset}`);
}

// Function to print conclusion
function printConclusion(patternResults, aiResults) {
  console.log(`\n${COLORS.bright}${COLORS.white}==== Conclusion ====${COLORS.reset}\n`);
  
  const totalDiff = aiResults.total - patternResults.total;
  const criticalDiff = aiResults.critical - patternResults.critical;
  const highDiff = aiResults.high - patternResults.high;
  
  if (totalDiff > 0) {
    console.log(`${COLORS.green}✓ AI-enhanced analysis identified ${totalDiff} more potential security issues.${COLORS.reset}`);
    
    if (criticalDiff > 0 || highDiff > 0) {
      console.log(`${COLORS.green}✓ AI-enhanced analysis found ${criticalDiff + highDiff} more Critical/High severity issues.${COLORS.reset}`);
    }
    
    console.log(`\n${COLORS.yellow}The AI-enhanced security analyzer provides:${COLORS.reset}`);
    console.log(`- More thorough vulnerability detection`);
    console.log(`- Context-aware analysis of security implications`);
    console.log(`- Detailed remediation suggestions with code examples`);
    console.log(`- Nuanced security scoring based on real-world impact`);
  } else if (totalDiff === 0) {
    console.log(`${COLORS.yellow}⚠ Both analyzers found the same number of issues (${patternResults.total}).${COLORS.reset}`);
    console.log(`However, the AI-enhanced analyzer likely provided more detailed remediation suggestions.`);
  } else {
    console.log(`${COLORS.yellow}⚠ Pattern-based analysis found ${-totalDiff} more issues than AI-enhanced analysis.${COLORS.reset}`);
    console.log(`This could indicate that the AI model is filtering out false positives or consolidating related issues.`);
  }
  
  console.log(`\n${COLORS.bright}For more information:${COLORS.reset}`);
  console.log(`- See the full reports in the security-reports directory`);
  console.log(`- Review AI-enhanced security documentation: ${COLORS.cyan}docs/AI_SECURITY_GUIDE.md${COLORS.reset}`);
}

// Main function
async function main() {
  console.log(`${COLORS.bright}${COLORS.cyan}ctrl.shift.left Security Analysis Comparison Tool${COLORS.reset}`);
  console.log(`${COLORS.dim}This tool compares pattern-based and AI-enhanced security analysis${COLORS.reset}\n`);
  
  try {
    // Check if a test file was provided
    const targetFile = process.argv[2] || defaultTestFile;
    
    // Make sure the file exists
    if (!fs.existsSync(targetFile)) {
      console.error(`${COLORS.red}Error: File not found: ${targetFile}${COLORS.reset}`);
      console.error(`Please provide a valid file path or use the default demo file.`);
      process.exit(1);
    }
    
    // Create directories if they don't exist
    if (!fs.existsSync(reportsPath)) {
      fs.mkdirSync(reportsPath, { recursive: true });
    }
    
    // Run pattern-based analysis
    const { reportPath: patternReportPath, executionTime: patternTime } = await runSecurityAnalysis(targetFile, false);
    
    // Check if OpenAI API key is set
    if (!process.env.OPENAI_API_KEY) {
      console.log(`\n${COLORS.yellow}⚠ OpenAI API key not found in environment variables.${COLORS.reset}`);
      console.log(`AI-enhanced analysis requires an OpenAI API key.`);
      console.log(`Please set the OPENAI_API_KEY environment variable and try again.`);
      process.exit(1);
    }
    
    // Run AI-enhanced analysis
    const { reportPath: aiReportPath, executionTime: aiTime } = await runSecurityAnalysis(targetFile, true);
    
    // Analyze reports
    const patternResults = analyzeReport(patternReportPath);
    const aiResults = analyzeReport(aiReportPath);
    
    // Print comparison
    printComparisonTable(patternResults, aiResults, patternTime, aiTime);
    printConclusion(patternResults, aiResults);
    
  } catch (error) {
    console.error(`\n${COLORS.red}Error running comparison:${COLORS.reset}`);
    console.error(error);
    process.exit(1);
  }
}

// Run the main function
main().catch(error => {
  console.error(`${COLORS.red}Unhandled error:${COLORS.reset}`, error);
  process.exit(1);
});