web-vuln-scanner/lib/scanner.js

Advanced, lightweight web vulnerability scanner with smart detection and easy-to-use interface

/**
 * Production-Ready Web Vulnerability Scanner
 * Enterprise-grade vulnerability scanning with comprehensive security, monitoring, and resilience
 */

const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
const EventEmitter = require('events');

// Debug function with fallback
let debug;
try {
  debug = require('debug')('web-vuln-scanner');
} catch (error) {
  // Fallback debug function for environments without debug module
  debug = (msg) => {
    if (process.env.DEBUG || process.env.NODE_ENV === 'development') {
      console.log(`[DEBUG] ${msg}`);
    }
  };
}

// Production-ready modules
const { SecurityValidator } = require('./security/validator');
const { ConfigManager } = require('./config/config-manager');
const { CircuitBreaker, RetryManager, ErrorHandler, ResourceManager } = require('./resilience/error-handler');
const { Logger, MetricsCollector, PerformanceMonitor } = require('./monitoring/logger');

// Initialize JSDOM with fallback
let JSDOM, VirtualConsole;
try {
  const jsdomModule = require('jsdom');
  JSDOM = jsdomModule.JSDOM;
  VirtualConsole = jsdomModule.VirtualConsole;
} catch (error) {
  console.warn('JSDOM not available, DOM parsing features will be limited');
  // Provide minimal fallback
  JSDOM = class {
    constructor(html) {
      this.window = {
        document: {
          createElement: () => ({}),
          querySelectorAll: () => [],
          querySelector: () => null,
          getElementsByTagName: () => [],
          body: { innerHTML: html || '' }
        }
      };
    }
  };
  VirtualConsole = class {
    on() { return this; }
  };
}

// Handle both CommonJS and ES module versions of p-limit
let pLimit;
try {
  pLimit = require('p-limit');
} catch (error) {
  // Fallback for when p-limit is not available
  pLimit = (concurrency) => (fn) => fn();
}

const { Crawler } = require('./crawler');

// Import scanners with error handling
const importScanner = (path) => {
  try {
    return require(path);
  } catch (error) {
    console.warn(`Failed to import scanner: ${path} - ${error.message}`);
    return null;
  }
};

// Production-ready scanner imports
const headerScanner = importScanner('./scanners/header');
const xssScanner = importScanner('./scanners/xss');
const sqlScanner = importScanner('./scanners/sql-injection');
const sslScanner = importScanner('./scanners/ssl-tls');
const portScanner = importScanner('./scanners/port');
const dirTraversalScanner = importScanner('./scanners/dir-traversal');
const csrfScanner = importScanner('./scanners/csrf');
const cspScanner = importScanner('./scanners/csp');
const rceScanner = importScanner('./scanners/rce');
const idorScanner = importScanner('./scanners/idor');
const misconfigScanner = importScanner('./scanners/misconfigured-headers');
const wafScanner = importScanner('./scanners/waf');

class Scanner extends EventEmitter {
  constructor(url, options = {}) {
    super();
    
    // Initialize production components
    this.config = ConfigManager.getInstance();
    this.logger = new Logger(this.config.get('logging'));
    this.metrics = new MetricsCollector(this.config.get('monitoring'));
    this.monitor = new PerformanceMonitor(this.logger, this.metrics);
    this.validator = new SecurityValidator();
    this.errorHandler = new ErrorHandler({ logger: this.logger });
    this.resourceManager = new ResourceManager(this.config.get('scanner'));
    
    // Circuit breakers for external dependencies
    this.httpCircuitBreaker = new CircuitBreaker({
      failureThreshold: 5,
      timeout: 30000,
      resetTimeout: 300000
    });
    
    this.retryManager = new RetryManager({
      maxRetries: this.config.get('scanner.retryAttempts', 3),
      baseDelay: this.config.get('scanner.retryDelay', 1000),
      maxDelay: 30000
    });

    // Validate and sanitize URL
    const urlValidation = this.validator.validateUrl(url);
    if (!urlValidation.isValid) {
      throw new Error(`Invalid URL: ${urlValidation.errors.join(', ')}`);
    }
    this.url = urlValidation.sanitized;

    // Validate and merge options with config
    const configValidation = this.validator.validateScanConfig(options);
    if (!configValidation.isValid) {
      throw new Error(`Invalid configuration: ${configValidation.errors.join(', ')}`);
    }

    // Merge configuration from multiple sources
    this.options = {
      timeout: this.config.get('scanner.requestTimeout', 30000),
      userAgent: this.config.get('scanner.userAgent', 'WebVulnScanner/2.0'),
      scanModules: [
        'headers', 'xss', 'sql', 'ssl', 'ports',
        'dirTraversal', 'csrf', 'csp',
        'rce', 'idor', 'misconfiguredHeaders', 'waf'
      ],
      depth: this.config.get('scanner.maxDepth', 5),
      concurrency: this.config.get('scanner.maxConcurrency', 10),
      disableCrawler: false,
      enableJS: this.config.get('scanner.enableJavaScript', false),
      headers: {},
      maxRetries: this.config.get('scanner.retryAttempts', 3),
      retryDelay: this.config.get('scanner.retryDelay', 1000),
      ...configValidation.sanitized
    };

    // Validate headers
    const headerValidation = this.validator.validateHeaders(this.options.headers);
    if (!headerValidation.isValid) {
      this.logger.warn('Invalid headers provided', { errors: headerValidation.errors });
    }
    this.options.headers = headerValidation.sanitized;
    
    this.results = {
      scanId: this.validator.generateSecureId(),
      url: this.url,
      timestamp: new Date().toISOString(),
      scanDuration: 0,
      scannedUrls: [],
      summary: { total: 0, critical: 0, high: 0, medium: 0, low: 0, info: 0 },
      vulnerabilities: [],
      metadata: {
        scannerVersion: '2.0.0',
        configuration: this.config.getSafeConfig(),
        userAgent: this.options.userAgent
      }
    };
    
    // Initialize crawler with error handling and monitoring
    try {
      this.crawler = new Crawler({
        baseUrl: this.url,
        depth: this.options.depth,
        concurrency: this.options.concurrency,
        userAgent: this.options.userAgent,
        headers: this.options.headers
      });
    } catch (crawlerError) {
      debug(`Crawler initialization failed: ${crawlerError.message}`);
      this.crawler = null;
    }
  }

  normalizeUrl(url) {
    if (!url || typeof url !== 'string') {
      throw new Error('Invalid URL provided');
    }
    return url.startsWith('http') ? url : `https://${url}`;
  }

  delay(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  async fetchPage(targetUrl, method = 'GET', data = null, retries = 0) {
    const startTime = Date.now();
    
    try {
      debug(`Fetching ${targetUrl} (attempt ${retries + 1})`);
      
      const options = {
        method,
        headers: {
          'User-Agent': this.options.userAgent,
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.5',
          'Accept-Encoding': 'gzip, deflate',
          'Connection': 'keep-alive',
          ...this.options.headers
        },
        timeout: this.options.timeout,
        redirect: 'follow'
      };

      if (data && method !== 'GET') {
        options.body = typeof data === 'string' ? data : JSON.stringify(data);
        options.headers['Content-Type'] = typeof data === 'string'
          ? 'application/x-www-form-urlencoded'
          : 'application/json';
      }

      const response = await fetch(targetUrl, options);
      const contentType = response.headers.get('content-type') || '';
      const content = await response.text();
      let dom = null;

      if (contentType.includes('text/html')) {
        try { 
          const cleanedContent = this.stripResourceElements(content);
          dom = new JSDOM(cleanedContent, {
            resources: 'usable',
            runScripts: 'outside-only',
            pretendToBeVisual: false,
            virtualConsole: this.createSilentVirtualConsole()
          });
        } catch (domError) { 
          debug(`DOM parse failed for ${targetUrl}: ${domError.message}`); 
          // Continue without DOM - some scanners can still work with raw content
        }
      }

      return {
        status: response.status,
        headers: Object.fromEntries(response.headers),
        content,
        dom,
        url: response.url,
        responseTime: Date.now() - startTime
      };
      
    } catch (error) {
      if (retries < this.options.maxRetries && this.isRetryableError(error)) {
        await this.delay(this.options.retryDelay * (retries + 1));
        return this.fetchPage(targetUrl, method, data, retries + 1);
      }
      debug(`Failed to fetch ${targetUrl}: ${error.message}`);
      throw new Error(`Fetch error: ${error.message}`);
    }
  }

  isRetryableError(error) {
    const retryableErrors = ['ECONNRESET', 'ENOTFOUND', 'ECONNREFUSED', 'ETIMEDOUT'];
    return retryableErrors.some(code => error.message.includes(code)) || 
           error.message.includes('timeout');
  }

  createSilentVirtualConsole() {
    const virtualConsole = new VirtualConsole();
    
    // Completely silence all console output
    virtualConsole.sendTo = () => {};
    
    ['error', 'warn', 'info', 'log', 'debug', 'trace', 'jsdomError'].forEach(event => {
      virtualConsole.on(event, () => {});
    });
    
    return virtualConsole;
  }

  stripResourceElements(html) {
    if (!html || typeof html !== 'string') {
      return '';
    }

    return html
      // Remove all link elements (CSS, etc.)
      .replace(/<link[^>]*>/gi, '<!-- link removed -->')
      // Remove script elements with src attributes
      .replace(/<script[^>]*src[^>]*>.*?<\/script\b[^>]*>/gis, '<!-- script removed -->')
      // Remove img elements with external src
      .replace(/<img[^>]*src=["'][^"']*["'][^>]*>/gi, '<img>')
      // Remove iframe elements
      .replace(/<iframe[^>]*>.*?<\/iframe>/gis, '<!-- iframe removed -->')
      // Remove object and embed elements
      .replace(/<(object|embed)[^>]*>.*?<\/\1>/gis, '<!-- $1 removed -->')
      // Remove style imports
      .replace(/@import[^;]*;/gi, '/* import removed */')
      // Remove problematic CSS that might cause parsing issues
      .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '<style>/* CSS removed */</style>');
  }

  async runScan() {
    const startTime = Date.now();
    debug(`Starting scan: ${this.url}`);
    
    const limit = pLimit ? pLimit(this.options.concurrency) : (fn => fn());
    let pages = [this.url];

    // Crawling phase
    if (!this.options.disableCrawler && this.crawler) {
      try {
        const crawled = await this.crawler.crawl();
        if (Array.isArray(crawled)) {
          pages = [...new Set([this.url, ...crawled])];
        }
      } catch (crawlerError) {
        debug(`Crawler error: ${crawlerError.message}`);
        this.addResults([{
          type: 'crawler_error',
          url: this.url,
          severity: 'info',
          description: `Crawler failed: ${crawlerError.message}`
        }]);
      }
    }

    this.results.scannedUrls = pages;
    debug(`Scanning ${pages.length} pages`);

    // Scan all discovered pages
    const scanTasks = pages.map(url => limit(() => this.scanOneUrl(url)));
    
    try {
      await Promise.allSettled(scanTasks);
    } catch (scanError) {
      debug(`Scan tasks error: ${scanError.message}`);
    }

    // Run SSL scan if enabled
    if (this.options.scanModules.includes('ssl') && sslScanner) {
      try {
        const sslResults = await sslScanner.scan(this.url);
        this.addResults(sslResults);
      } catch (sslError) {
        debug(`SSL scan error: ${sslError.message}`);
        this.addResults([{
          type: 'ssl_scan_error',
          url: this.url,
          severity: 'info',
          description: `SSL scan failed: ${sslError.message}`
        }]);
      }
    }

    // Run port scan if enabled
    if (this.options.scanModules.includes('ports') && portScanner) {
      try {
        const portResults = await portScanner.scan(this.url);
        this.addResults(portResults);
      } catch (portError) {
        debug(`Port scan error: ${portError.message}`);
        this.addResults([{
          type: 'port_scan_error',
          url: this.url,
          severity: 'info',
          description: `Port scan failed: ${portError.message}`
        }]);
      }
    }

    // Calculate scan duration
    this.results.scanDuration = Date.now() - startTime;
    debug(`Scan completed in ${this.results.scanDuration}ms`);
    
    return this.results;
  }

  async scanOneUrl(url) {
    try {
      const page = await this.fetchPage(url);
      await this.scanUrl(page, url);
    } catch (err) {
      debug(`Error scanning ${url}: ${err.message}`);
      this.results.vulnerabilities.push({
        type: 'scan_error',
        url,
        severity: 'info',
        description: `Scan error: ${err.message}`,
        timestamp: new Date().toISOString()
      });
    }
  }

  async scanUrl(page, url) {
    // Only include scanners that were successfully imported
    const scanners = {};
    
    if (headerScanner) scanners.headers = headerScanner;
    if (xssScanner) scanners.xss = xssScanner;
    if (sqlScanner) scanners.sql = sqlScanner;
    if (dirTraversalScanner) scanners.dirTraversal = dirTraversalScanner;
    if (csrfScanner) scanners.csrf = csrfScanner;
    if (cspScanner) scanners.csp = cspScanner;
    if (rceScanner) scanners.rce = rceScanner;
    if (idorScanner) scanners.idor = idorScanner;
    if (misconfigScanner) scanners.misconfiguredHeaders = misconfigScanner;
    if (wafScanner) scanners.waf = wafScanner;

    const modulesToRun = this.options.scanModules
      .filter(name => scanners[name] && typeof scanners[name].scan === 'function');
    
    debug(`Running ${modulesToRun.length} scanners on ${url}`);

    try {
      const results = await Promise.allSettled(
        modulesToRun.map(async (name) => {
          try {
            const scanResult = await Promise.race([
              scanners[name].scan(page, url),
              new Promise((_, reject) => 
                setTimeout(() => reject(new Error('Scanner timeout')), this.options.timeout)
              )
            ]);
            
            return Array.isArray(scanResult) ? scanResult : 
              scanResult ? [scanResult] : [];
          } catch (scannerError) {
            debug(`Scanner ${name} error on ${url}: ${scannerError.message}`);
            return [{
              type: `${name}_scanner_error`,
              url,
              severity: 'info',
              description: `${name} scanner failed: ${scannerError.message}`,
              timestamp: new Date().toISOString()
            }];
          }
        })
      );

      results.forEach((result, index) => {
        if (result.status === 'fulfilled' && Array.isArray(result.value)) {
          result.value.forEach(vuln => {
            if (vuln && typeof vuln === 'object') {
              if (!vuln.url) vuln.url = url;
              if (!vuln.timestamp) vuln.timestamp = new Date().toISOString();
              this.addResults([vuln]);
            }
          });
        } else if (result.status === 'rejected') {
          debug(`Scanner ${modulesToRun[index]} rejected: ${result.reason?.message}`);
        }
      });

    } catch (scanError) {
      debug(`Scan URL error: ${scanError.message}`);
      this.addResults([{
        type: 'scan_url_error',
        url,
        severity: 'info',
        description: `URL scan failed: ${scanError.message}`,
        timestamp: new Date().toISOString()
      }]);
    }
  }

  addResults(vulns) {
    if (!Array.isArray(vulns) || vulns.length === 0) {
      return;
    }
    
    const validVulns = vulns.filter(v => 
      v && 
      typeof v === 'object' && 
      v.type && 
      v.severity &&
      ['critical', 'high', 'medium', 'low', 'info'].includes(v.severity)
    );
    
    if (validVulns.length === 0) return;
    
    this.results.vulnerabilities.push(...validVulns);
    
    validVulns.forEach(v => {
      this.results.summary.total++;
      this.results.summary[v.severity]++;
    });
    
    debug(`Added ${validVulns.length} vulnerabilities. Total: ${this.results.vulnerabilities.length}`);
  }

  getReport() {
    if (!this.results || !this.results.vulnerabilities) {
      return {
        error: 'No scan results available',
        timestamp: new Date().toISOString()
      };
    }
    
    const sortOrder = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
    
    const sortedVulnerabilities = [...this.results.vulnerabilities].sort((a, b) => {
      return (sortOrder[a.severity] || 999) - (sortOrder[b.severity] || 999);
    });
    
    return {
      ...this.results,
      vulnerabilities: sortedVulnerabilities,
      scannerInfo: {
        version: '1.2.1',
        timestamp: this.results.timestamp,
        scanDuration: this.results.scanDuration,
        urlsScanned: this.results.scannedUrls.length
      }
    };
  }

  // Generate professional HTML report
  generateHTMLReport(options = {}) {
    const enhancedHtmlReporter = require('./reporters/enhanced-html-reporter');
    const reportData = this.getReport();
    
    return enhancedHtmlReporter.generateReport(reportData, {
      title: options.title || 'Web Vulnerability Assessment Report',
      includeExecutiveSummary: options.includeExecutiveSummary !== false,
      includeRiskMatrix: options.includeRiskMatrix !== false,
      includeCompliance: options.includeCompliance !== false,
      customBranding: options.customBranding || null
    });
  }

  // Generate JSON report
  generateJSONReport() {
    return JSON.stringify(this.getReport(), null, 2);
  }

  // Save report to file
  async saveReport(filePath, format = 'html', options = {}) {
    const fs = require('fs').promises;
    const path = require('path');
    
    let content;
    const fileExtension = path.extname(filePath) || `.${format}`;
    const finalPath = filePath.endsWith(fileExtension) ? filePath : `${filePath}${fileExtension}`;
    
    switch (format.toLowerCase()) {
    case 'html':
      content = this.generateHTMLReport(options);
      break;
    case 'json':
      content = this.generateJSONReport();
      break;
    default:
      throw new Error(`Unsupported report format: ${format}`);
    }
    
    await fs.writeFile(finalPath, content, 'utf8');
    return finalPath;
  }

  // Cleanup method to properly dispose of resources
  async cleanup() {
    if (this.crawler && typeof this.crawler.cleanup === 'function') {
      await this.crawler.cleanup();
    }
    
    // Clear large objects
    this.results.vulnerabilities = [];
    this.results.scannedUrls = [];
  }
}

module.exports = Scanner;