vaultace-cli/tests/regression/security-scanning.test.js

AI-powered security scanner that detects vulnerabilities in AI-generated code. Proactive scanning, autonomous fixing, and emergency response for modern development teams.

/**
 * Regression Tests for Security Scanning Features
 * Ensures security scanning accuracy and prevents feature regressions
 */

const { SecurityScanner } = require('../../src/utils/security-scanner');
const { VulnerabilityAnalyzer } = require('../../src/utils/vulnerability-analyzer');
const fs = require('fs').promises;
const path = require('path');
const crypto = require('crypto');

describe('Security Scanning Regression Tests', () => {
  let scanner;
  let analyzer;
  let testFixturesPath;

  beforeAll(async () => {
    scanner = new SecurityScanner();
    analyzer = new VulnerabilityAnalyzer();
    testFixturesPath = path.join(__dirname, '../fixtures/security-scanning');

    // Ensure test fixtures directory exists
    await fs.mkdir(testFixturesPath, { recursive: true });

    // Create test vulnerable files
    await createTestFixtures();
  });

  afterAll(async () => {
    await scanner.cleanup();
  });

  async function createTestFixtures() {
    const fixtures = {
      // JavaScript vulnerabilities
      'vulnerable-js.js': `
        // SQL Injection vulnerability
        const query = "SELECT * FROM users WHERE id = " + userId;

        // XSS vulnerability
        document.innerHTML = userInput;

        // Command injection
        const exec = require('child_process').exec;
        exec('ls ' + userInput);

        // Hardcoded credentials
        const apiKey = 'sk-1234567890abcdef';
        const password = 'admin123';
      `,

      'package-vulnerable.json': JSON.stringify({
        name: "test-vulnerable-app",
        version: "1.0.0",
        dependencies: {
          "lodash": "4.17.0",      // Known vulnerability
          "axios": "0.18.0",       // Known vulnerability
          "express": "4.16.0",     // Known vulnerability
          "moment": "2.19.0"       // Known vulnerability
        }
      }, null, 2),

      // Python vulnerabilities
      'vulnerable-python.py': `
# SQL Injection
query = "SELECT * FROM users WHERE name = '" + user_input + "'"

# Command injection
import os
os.system("ls " + user_input)

# Hardcoded secrets
API_KEY = "sk-1234567890abcdef"
DB_PASSWORD = "supersecret123"

# Weak cryptography
import hashlib
hashlib.md5(password.encode()).hexdigest()
      `,

      // Docker vulnerabilities
      'Dockerfile.vulnerable': `
FROM ubuntu:18.04

# Running as root (security issue)
USER root

# Installing packages without version pinning
RUN apt-get update && apt-get install -y curl wget

# Copying secrets
COPY secret.key /app/
COPY .env /app/

# Exposed ports
EXPOSE 22
EXPOSE 3389

# Running with privileged access
CMD ["./start.sh"]
      `,

      // Infrastructure as Code vulnerabilities
      'terraform-vulnerable.tf': `
resource "aws_s3_bucket" "example" {
  bucket = "my-vulnerable-bucket"

  # Public read access (security issue)
  acl = "public-read"
}

resource "aws_security_group" "web" {
  name = "web-sg"

  # Overly permissive ingress
  ingress {
    from_port   = 0
    to_port     = 65535
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_db_instance" "example" {
  allocated_storage = 20
  storage_type      = "gp2"
  engine            = "mysql"
  engine_version    = "5.7"  # Outdated version

  # No encryption
  storage_encrypted = false

  # Publicly accessible
  publicly_accessible = true
}
      `
    };

    for (const [filename, content] of Object.entries(fixtures)) {
      await fs.writeFile(path.join(testFixturesPath, filename), content);
    }
  }

  describe('Vulnerability Detection Accuracy', () => {
    test('should detect JavaScript security vulnerabilities', async () => {
      const filePath = path.join(testFixturesPath, 'vulnerable-js.js');
      const results = await scanner.scanFile(filePath);

      expect(results.vulnerabilities).toHaveLength(4);

      // SQL Injection
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'sql_injection',
          severity: 'high',
          line: expect.any(Number),
          description: expect.stringContaining('SQL injection')
        })
      );

      // XSS
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'xss',
          severity: 'high',
          line: expect.any(Number)
        })
      );

      // Command Injection
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'command_injection',
          severity: 'critical',
          line: expect.any(Number)
        })
      );

      // Hardcoded Credentials
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'hardcoded_credentials',
          severity: 'medium',
          line: expect.any(Number)
        })
      );
    });

    test('should detect dependency vulnerabilities in package.json', async () => {
      const filePath = path.join(testFixturesPath, 'package-vulnerable.json');
      const results = await scanner.scanDependencies(filePath);

      expect(results.vulnerabilities.length).toBeGreaterThan(0);

      // Check for known vulnerable packages
      const packageNames = results.vulnerabilities.map(v => v.package_name);
      expect(packageNames).toContain('lodash');
      expect(packageNames).toContain('axios');
      expect(packageNames).toContain('express');

      // Verify CVE information is included
      results.vulnerabilities.forEach(vuln => {
        expect(vuln.cve_id).toBeDefined();
        expect(vuln.cvss_score).toBeGreaterThan(0);
        expect(vuln.severity).toMatch(/^(low|medium|high|critical)$/);
      });
    });

    test('should detect Python security issues', async () => {
      const filePath = path.join(testFixturesPath, 'vulnerable-python.py');
      const results = await scanner.scanFile(filePath);

      expect(results.vulnerabilities.length).toBeGreaterThan(2);

      // SQL Injection
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'sql_injection',
          severity: 'high'
        })
      );

      // Command Injection
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'command_injection',
          severity: 'critical'
        })
      );

      // Weak Cryptography
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'weak_cryptography',
          severity: 'medium',
          description: expect.stringContaining('MD5')
        })
      );
    });

    test('should detect Docker security misconfigurations', async () => {
      const filePath = path.join(testFixturesPath, 'Dockerfile.vulnerable');
      const results = await scanner.scanDockerfile(filePath);

      expect(results.vulnerabilities.length).toBeGreaterThan(3);

      // Running as root
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'docker_root_user',
          severity: 'medium',
          description: expect.stringContaining('root')
        })
      );

      // Exposed sensitive ports
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'exposed_sensitive_port',
          severity: 'high',
          port: expect.any(Number)
        })
      );
    });

    test('should detect Infrastructure as Code vulnerabilities', async () => {
      const filePath = path.join(testFixturesPath, 'terraform-vulnerable.tf');
      const results = await scanner.scanTerraform(filePath);

      expect(results.vulnerabilities.length).toBeGreaterThan(3);

      // Public S3 bucket
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'public_s3_bucket',
          severity: 'high',
          resource: expect.stringContaining('s3_bucket')
        })
      );

      // Overly permissive security group
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'overly_permissive_sg',
          severity: 'critical',
          resource: expect.stringContaining('security_group')
        })
      );

      // Unencrypted database
      expect(results.vulnerabilities).toContainEqual(
        expect.objectContaining({
          type: 'unencrypted_database',
          severity: 'high',
          resource: expect.stringContaining('db_instance')
        })
      );
    });
  });

  describe('False Positive Prevention', () => {
    test('should not flag secure code patterns as vulnerabilities', async () => {
      const secureCode = `
        // Secure SQL query with parameterization
        const query = "SELECT * FROM users WHERE id = ?";
        const results = await db.execute(query, [userId]);

        // Secure HTML output with escaping
        const safeHTML = escapeHtml(userInput);
        element.textContent = safeHTML;

        // Secure command execution with validation
        const allowedCommands = ['ls', 'pwd', 'whoami'];
        if (allowedCommands.includes(command)) {
          exec(command);
        }
      `;

      const tempFile = path.join(testFixturesPath, 'secure-code.js');
      await fs.writeFile(tempFile, secureCode);

      const results = await scanner.scanFile(tempFile);

      // Should have very few or no vulnerabilities
      expect(results.vulnerabilities.length).toBeLessThan(2);

      // Should not flag parameterized queries as SQL injection
      const sqlInjectionVulns = results.vulnerabilities.filter(v => v.type === 'sql_injection');
      expect(sqlInjectionVulns).toHaveLength(0);

      await fs.unlink(tempFile);
    });

    test('should distinguish between actual secrets and test data', async () => {
      const testCode = `
        // Test data - should not be flagged as secrets
        const testApiKey = 'test-api-key-123';
        const mockPassword = 'mock-password';
        const exampleToken = 'example-token-xyz';

        // Constants that look like secrets but aren't
        const API_ENDPOINT = 'https://api.example.com';
        const DEFAULT_TIMEOUT = 30000;
      `;

      const tempFile = path.join(testFixturesPath, 'test-code.js');
      await fs.writeFile(tempFile, testCode);

      const results = await scanner.scanFile(tempFile);

      // Should not flag test/mock data as real secrets
      const secretVulns = results.vulnerabilities.filter(v => v.type === 'hardcoded_credentials');
      expect(secretVulns.length).toBeLessThan(2);

      await fs.unlink(tempFile);
    });
  });

  describe('Performance Regression Tests', () => {
    test('should scan large files within acceptable time limits', async () => {
      // Create a large JavaScript file
      const largeCode = Array.from({ length: 10000 }, (_, i) =>
        `function func${i}() { console.log("Function ${i}"); }`
      ).join('\n');

      const largeFile = path.join(testFixturesPath, 'large-file.js');
      await fs.writeFile(largeFile, largeCode);

      const startTime = Date.now();
      const results = await scanner.scanFile(largeFile);
      const endTime = Date.now();

      const scanTime = endTime - startTime;

      // Should complete within 5 seconds for large files
      expect(scanTime).toBeLessThan(5000);
      expect(results).toBeDefined();
      expect(results.scan_time_ms).toBeLessThan(5000);

      await fs.unlink(largeFile);
    });

    test('should handle concurrent scans efficiently', async () => {
      const testFiles = [];

      // Create multiple test files
      for (let i = 0; i < 10; i++) {
        const fileName = `concurrent-test-${i}.js`;
        const filePath = path.join(testFixturesPath, fileName);
        const content = `
          const vulnerability${i} = "SELECT * FROM table WHERE id = " + userInput${i};
          const secret${i} = "api-key-${i}-123456789";
        `;
        await fs.writeFile(filePath, content);
        testFiles.push(filePath);
      }

      const startTime = Date.now();

      // Scan all files concurrently
      const scanPromises = testFiles.map(filePath => scanner.scanFile(filePath));
      const results = await Promise.all(scanPromises);

      const endTime = Date.now();
      const totalTime = endTime - startTime;

      // Should complete concurrent scans efficiently
      expect(totalTime).toBeLessThan(10000); // 10 seconds max
      expect(results).toHaveLength(10);

      // Each result should contain vulnerabilities
      results.forEach(result => {
        expect(result.vulnerabilities.length).toBeGreaterThan(0);
      });

      // Cleanup
      await Promise.all(testFiles.map(file => fs.unlink(file)));
    });
  });

  describe('Scan Result Consistency', () => {
    test('should produce consistent results across multiple scans', async () => {
      const filePath = path.join(testFixturesPath, 'vulnerable-js.js');

      // Run the same scan multiple times
      const results1 = await scanner.scanFile(filePath);
      const results2 = await scanner.scanFile(filePath);
      const results3 = await scanner.scanFile(filePath);

      // Results should be identical
      expect(results1.vulnerabilities).toHaveLength(results2.vulnerabilities.length);
      expect(results2.vulnerabilities).toHaveLength(results3.vulnerabilities.length);

      // Vulnerability types and locations should match
      const sortVulns = (vulns) => vulns.sort((a, b) => a.line - b.line);

      const sorted1 = sortVulns([...results1.vulnerabilities]);
      const sorted2 = sortVulns([...results2.vulnerabilities]);
      const sorted3 = sortVulns([...results3.vulnerabilities]);

      for (let i = 0; i < sorted1.length; i++) {
        expect(sorted1[i].type).toBe(sorted2[i].type);
        expect(sorted2[i].type).toBe(sorted3[i].type);
        expect(sorted1[i].line).toBe(sorted2[i].line);
        expect(sorted2[i].line).toBe(sorted3[i].line);
      }
    });

    test('should maintain scan result format and structure', async () => {
      const filePath = path.join(testFixturesPath, 'vulnerable-js.js');
      const results = await scanner.scanFile(filePath);

      // Verify expected result structure
      expect(results).toHaveProperty('vulnerabilities');
      expect(results).toHaveProperty('scan_metadata');
      expect(results).toHaveProperty('file_info');

      // Verify vulnerability structure
      results.vulnerabilities.forEach(vuln => {
        expect(vuln).toHaveProperty('type');
        expect(vuln).toHaveProperty('severity');
        expect(vuln).toHaveProperty('line');
        expect(vuln).toHaveProperty('description');
        expect(vuln).toHaveProperty('recommendation');

        // Severity should be valid
        expect(['low', 'medium', 'high', 'critical']).toContain(vuln.severity);

        // Line number should be positive
        expect(vuln.line).toBeGreaterThan(0);
      });

      // Verify scan metadata
      expect(results.scan_metadata).toHaveProperty('scan_time_ms');
      expect(results.scan_metadata).toHaveProperty('scanner_version');
      expect(results.scan_metadata).toHaveProperty('timestamp');
    });
  });

  describe('Error Handling and Edge Cases', () => {
    test('should gracefully handle missing files', async () => {
      const nonExistentFile = path.join(testFixturesPath, 'does-not-exist.js');

      await expect(scanner.scanFile(nonExistentFile)).rejects.toThrow(/file not found/i);
    });

    test('should handle empty files without errors', async () => {
      const emptyFile = path.join(testFixturesPath, 'empty.js');
      await fs.writeFile(emptyFile, '');

      const results = await scanner.scanFile(emptyFile);

      expect(results).toBeDefined();
      expect(results.vulnerabilities).toHaveLength(0);
      expect(results.scan_metadata).toBeDefined();

      await fs.unlink(emptyFile);
    });

    test('should handle files with invalid syntax', async () => {
      const invalidFile = path.join(testFixturesPath, 'invalid-syntax.js');
      await fs.writeFile(invalidFile, 'const invalid = { unclosed object');

      const results = await scanner.scanFile(invalidFile);

      // Should still attempt to scan even with syntax errors
      expect(results).toBeDefined();
      expect(results.vulnerabilities).toBeDefined();

      await fs.unlink(invalidFile);
    });

    test('should handle very large files gracefully', async () => {
      const hugeContent = 'const data = "' + 'x'.repeat(1000000) + '";';
      const hugeFile = path.join(testFixturesPath, 'huge-file.js');
      await fs.writeFile(hugeFile, hugeContent);

      const results = await scanner.scanFile(hugeFile);

      expect(results).toBeDefined();
      expect(results.file_info.size_bytes).toBeGreaterThan(1000000);

      await fs.unlink(hugeFile);
    });
  });
});