ctrlshiftleft/scripts/integration-test.js

AI-powered toolkit for embedding QA and security testing into development workflows

#!/usr/bin/env node

/**
 * ctrl.shift.left v1.4.0 Integration Test
 * 
 * This script tests all v1.4.0 features in an integrated workflow
 * to ensure they work properly together before release.
 */

const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const { PerformanceTracker } = require('./utils/performanceTracker');
const { PathUtils, FileUtils } = require('./utils/platformUtils');
const { errorHandler } = require('./utils/errorHandler');

// Create performance tracker for integration test
const tracker = new PerformanceTracker({
  projectName: 'integration-test',
  version: require('../package.json').version,
  outputDir: './integration-test-results',
  enabled: true
});

// Create test directories
const testRootDir = path.join(process.cwd(), 'integration-test');
const resultsDir = path.join(testRootDir, 'results');
const securityDir = path.join(resultsDir, 'security');
const testsDir = path.join(resultsDir, 'tests');
const reportsDir = path.join(resultsDir, 'reports');

// Test component content with intentional vulnerabilities
const vulnerableComponentContent = `
import React, { useState, useEffect, useRef } from 'react';

/**
 * Vulnerable Auth Component - For Testing Security Analysis
 */
function VulnerableAuthComponent({ onLogin }) {
  const [username, setUsername] = useState('');
  const [password, setPassword] = useState('');
  const [error, setError] = useState('');
  const formRef = useRef(null);
  
  // Vulnerable effect with direct DOM manipulation
  useEffect(() => {
    // XSS vulnerability: using innerHTML
    formRef.current.innerHTML += "<div class='status'>Please log in</div>";
    
    // Missing dependency array
  });
  
  // Vulnerable JWT verification without options
  const verifyToken = (token) => {
    try {
      // Vulnerable: No algorithm specified
      const decoded = jwt.verify(token);
      return decoded;
    } catch (err) {
      return null;
    }
  };
  
  // Vulnerable authentication function
  const handleLogin = (e) => {
    e.preventDefault();
    
    // Hardcoded credentials (vulnerability)
    const apiKey = "sk_test_51HV9hBGLkjdassda5saS1SWK";
    
    // SQL Injection vulnerability
    const query = \`SELECT * FROM users WHERE username = '\${username}' AND password = '\${password}'\`;
    
    // Store password in localStorage (vulnerability)
    localStorage.setItem('password', password);
    
    // Unsafe operation (vulnerability)
    const loginResult = eval('username === "admin" && password === "secret"');
    
    if (loginResult) {
      onLogin({ username });
    } else {
      setError('Invalid credentials');
    }
  };
  
  return (
    <div className="auth-form" ref={formRef}>
      <h2>Login</h2>
      <form onSubmit={handleLogin}>
        <div>
          <label>Username:</label>
          <input 
            type="text" 
            value={username}
            onChange={(e) => setUsername(e.target.value)}
          />
        </div>
        <div>
          <label>Password:</label>
          <input 
            type="password" 
            value={password}
            onChange={(e) => setPassword(e.target.value)}
          />
        </div>
        <button type="submit">Login</button>
        {error && <div dangerouslySetInnerHTML={{ __html: error }} />}
      </form>
    </div>
  );
}

export default VulnerableAuthComponent;
`;

// API route with vulnerabilities
const vulnerableApiContent = `
/**
 * Vulnerable API Route - For Testing API Security Analysis
 */

const express = require('express');
const app = express();

// Middleware setup - order issue
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(helmet());

// Vulnerable API route with unvalidated input
app.post('/api/login', (req, res) => {
  const { username, password } = req.body;
  
  // SQL Injection vulnerability
  const query = \`SELECT * FROM users WHERE username = '\${username}' AND password = '\${password}'\`;
  
  // Hardcoded credentials
  const dbPassword = "database_password_123";
  
  // Log sensitive data
  console.log('Login attempt:', req.body);
  
  if (username === 'admin' && password === 'secret') {
    // Create session with insecure cookie settings
    const session = new Session({
      cookie: {
        secure: false,
        httpOnly: false
      }
    });
    
    // Unsafe JWT generation
    const token = jwt.sign({ username }, 'secret');
    
    // Return sensitive data in response
    res.status(200).json({
      success: true,
      token,
      password,
      apiKey: "sk_test_51HV9hBGLkjdassda5saS1SWK"
    });
  } else {
    res.status(401).json({ error: 'Invalid credentials' });
  }
});

// Permissive CORS
app.use((req, res, next) => {
  res.setHeader('Access-Control-Allow-Origin', '*');
  next();
});

// Path traversal vulnerability
app.get('/api/files/:filename', (req, res) => {
  const filename = req.params.filename;
  const filePath = \`./data/\${filename}\`;
  
  fs.readFile(filePath, (err, data) => {
    if (err) {
      res.status(404).send('File not found');
    } else {
      res.send(data);
    }
  });
});

module.exports = app;
`;

// Clean up any previous test runs
function cleanup() {
  try {
    if (fs.existsSync(testRootDir)) {
      console.log(`🧹 Cleaning up previous test run...`);
      fs.rmSync(testRootDir, { recursive: true, force: true });
    }
  } catch (err) {
    console.error(`⚠️ Cleanup error: ${err.message}`);
  }
}

// Create test directories and files
function setupTestEnvironment() {
  console.log(`🔧 Setting up test environment...`);
  const timerId = tracker.startTimer('setup', 'test-environment');
  
  try {
    // Create directories
    PathUtils.ensureDir(testRootDir);
    PathUtils.ensureDir(resultsDir);
    PathUtils.ensureDir(securityDir);
    PathUtils.ensureDir(testsDir);
    PathUtils.ensureDir(reportsDir);
    
    // Create test component file
    const componentPath = path.join(testRootDir, 'VulnerableAuthComponent.jsx');
    FileUtils.writeFile(componentPath, vulnerableComponentContent);
    console.log(`  ✅ Created React component for testing`);
    
    // Create test API route file
    const apiPath = path.join(testRootDir, 'vulnerableApi.js');
    FileUtils.writeFile(apiPath, vulnerableApiContent);
    console.log(`  ✅ Created API route for testing`);
    
    tracker.stopTimer(timerId);
    return { componentPath, apiPath };
  } catch (error) {
    tracker.stopTimer(timerId, { success: false, error: error.message });
    throw error;
  }
}

// Run security analysis test
async function testSecurityAnalysis(componentPath, apiPath) {
  console.log(`\n🔒 Testing enhanced security analysis...`);
  const timerId = tracker.startTimer('security-analysis', 'both-files');
  
  try {
    // Test React security analysis using the enhanced analyzer
    console.log(`  🔍 Analyzing React component...`);
    try {
      execSync(
        `node ./vscode-ext-test/analyze-security-enhanced.js "${componentPath}" --output "${securityDir}/react"`,
        { stdio: 'inherit' }
      );
      console.log(`  ✅ React security analysis completed`);
      
      // Verify report was generated
      const reactReportPath = path.join(securityDir, 'react', 'security-report.md');
      if (fs.existsSync(reactReportPath)) {
        const reportContent = fs.readFileSync(reactReportPath, 'utf8');
        
        // Check for React-specific issues detected
        const reactIssuesFound = 
          reportContent.includes('dangerouslySetInnerHTML') || 
          reportContent.includes('useEffect') ||
          reportContent.includes('innerHTML');
          
        console.log(`  ${reactIssuesFound ? '✅' : '❌'} React-specific issues detected: ${reactIssuesFound}`);
      } else {
        console.log(`  ❌ React security report not generated`);
      }
    } catch (error) {
      console.error(`  ❌ React security analysis failed: ${error.message}`);
      throw error;
    }
    
    // Test API route security analysis
    console.log(`\n  🔍 Analyzing API route...`);
    try {
      execSync(
        `node ./vscode-ext-test/analyze-security-enhanced.js "${apiPath}" --output "${securityDir}/api"`,
        { stdio: 'inherit' }
      );
      console.log(`  ✅ API security analysis completed`);
      
      // Verify report was generated
      const apiReportPath = path.join(securityDir, 'api', 'security-report.md');
      if (fs.existsSync(apiReportPath)) {
        const reportContent = fs.readFileSync(apiReportPath, 'utf8');
        
        // Check for API-specific issues detected
        const apiIssuesFound = 
          reportContent.includes('SQL Injection') || 
          reportContent.includes('CORS') ||
          reportContent.includes('API');
          
        console.log(`  ${apiIssuesFound ? '✅' : '❌'} API-specific issues detected: ${apiIssuesFound}`);
      } else {
        console.log(`  ❌ API security report not generated`);
      }
    } catch (error) {
      console.error(`  ❌ API security analysis failed: ${error.message}`);
      throw error;
    }
    
    const duration = tracker.stopTimer(timerId);
    console.log(`  ⏱️ Security analysis completed in ${duration}ms`);
    return true;
  } catch (error) {
    tracker.stopTimer(timerId, { success: false, error: error.message });
    return false;
  }
}

// Test performance tracking
function testPerformanceTracking() {
  console.log(`\n📊 Testing performance tracking...`);
  const timerId = tracker.startTimer('performance-test', 'tracking');
  
  try {
    // Create simulated load with multiple operations
    for (let i = 1; i <= 5; i++) {
      const opTimerId = tracker.startTimer('test-operation', `operation-${i}`);
      
      // Simulate work with different durations
      const duration = Math.floor(Math.random() * 50) + 10;
      const startTime = Date.now();
      while (Date.now() - startTime < duration) {
        // Busy wait to simulate work
      }
      
      tracker.stopTimer(opTimerId);
      console.log(`  ✅ Operation ${i} completed`);
    }
    
    // Generate performance report
    console.log(`\n  📝 Generating performance report...`);
    const jsonReportPath = tracker.saveReport('json', 'integration-test');
    const markdownReportPath = tracker.saveReport('markdown', 'integration-test');
    
    console.log(`  ✅ JSON report saved: ${jsonReportPath}`);
    console.log(`  ✅ Markdown report saved: ${markdownReportPath}`);
    
    const duration = tracker.stopTimer(timerId);
    console.log(`  ⏱️ Performance tracking test completed in ${duration}ms`);
    return true;
  } catch (error) {
    tracker.stopTimer(timerId, { success: false, error: error.message });
    console.error(`  ❌ Performance tracking test failed: ${error.message}`);
    return false;
  }
}

// Test cross-platform compatibility
function testCrossPlatformCompatibility() {
  console.log(`\n🌐 Testing cross-platform compatibility...`);
  const timerId = tracker.startTimer('cross-platform', 'test');
  
  try {
    // Run the cross-platform tests
    execSync(`node ./scripts/test-cross-platform.js`, { stdio: 'inherit' });
    
    const duration = tracker.stopTimer(timerId);
    console.log(`  ⏱️ Cross-platform compatibility test completed in ${duration}ms`);
    return true;
  } catch (error) {
    tracker.stopTimer(timerId, { success: false, error: error.message });
    console.error(`  ❌ Cross-platform compatibility test failed: ${error.message}`);
    return false;
  }
}

// Test error handling and recovery
function testErrorHandling() {
  console.log(`\n🛠️ Testing error handling and recovery...`);
  const timerId = tracker.startTimer('error-handling', 'test');
  
  try {
    console.log(`  🧪 Testing filesystem error recovery...`);
    // Create a deliberate filesystem error
    const nonexistentDir = path.join(testRootDir, 'nonexistent-dir', 'file.txt');
    
    try {
      fs.readFileSync(nonexistentDir);
      console.log(`  ❌ Expected filesystem error did not occur`);
    } catch (error) {
      // Use our error handler
      const enhancedError = errorHandler.handleError(error, 'integration-test', {
        path: nonexistentDir
      });
      
      // Attempt recovery
      const recovery = errorHandler.attemptRecovery(enhancedError, 'test', nonexistentDir);
      
      if (recovery.success) {
        console.log(`  ✅ Successfully recovered from filesystem error`);
        
        // Verify the fallback file was created
        if (fs.existsSync(nonexistentDir)) {
          console.log(`  ✅ Fallback file was created at ${nonexistentDir}`);
        } else {
          console.log(`  ❌ Fallback file was not created`);
        }
      } else {
        console.log(`  ❌ Failed to recover from filesystem error`);
      }
    }
    
    const duration = tracker.stopTimer(timerId);
    console.log(`  ⏱️ Error handling test completed in ${duration}ms`);
    return true;
  } catch (error) {
    tracker.stopTimer(timerId, { success: false, error: error.message });
    console.error(`  ❌ Error handling test failed: ${error.message}`);
    return false;
  }
}

// Run the integration test
async function runIntegrationTest() {
  console.log(`\n🚀 Running ctrl.shift.left v1.4.0 integration test...\n`);
  
  try {
    // Record start time
    const startTime = Date.now();
    
    // Clean up previous test runs
    cleanup();
    
    // Set up test environment
    const { componentPath, apiPath } = setupTestEnvironment();
    
    // Run the tests
    const securityResults = await testSecurityAnalysis(componentPath, apiPath);
    const performanceResults = testPerformanceTracking();
    const platformResults = testCrossPlatformCompatibility();
    const errorHandlingResults = testErrorHandling();
    
    // Calculate total time
    const totalTime = Date.now() - startTime;
    
    // Print results
    console.log(`\n📋 Integration Test Results:`);
    console.log(`  ${securityResults ? '✅' : '❌'} Enhanced Security Analysis`);
    console.log(`  ${performanceResults ? '✅' : '❌'} Performance Tracking`);
    console.log(`  ${platformResults ? '✅' : '❌'} Cross-Platform Compatibility`);
    console.log(`  ${errorHandlingResults ? '✅' : '❌'} Error Handling & Recovery`);
    
    const allPassed = securityResults && performanceResults && platformResults && errorHandlingResults;
    
    console.log(`\n${allPassed ? '✅ All tests passed!' : '❌ Some tests failed!'}`);
    console.log(`⏱️ Total test time: ${totalTime}ms`);
    
    // Save final report
    const jsonReportPath = tracker.saveReport('json', 'final');
    console.log(`\n📊 Final performance report saved: ${jsonReportPath}`);
    
    return allPassed;
  } catch (error) {
    console.error(`\n❌ Integration test failed with an error:`);
    console.error(error);
    return false;
  }
}

// Run the test
runIntegrationTest()
  .then(success => {
    process.exit(success ? 0 : 1);
  })
  .catch(error => {
    console.error('Unhandled error:', error);
    process.exit(1);
  });