UNPKG

aiwg

Version:

Cognitive architecture for AI-augmented software development with structured memory, ensemble validation, and closed-loop correction. FAIR-aligned artifacts, 84% cost reduction via human-in-the-loop, standards adopted by 100+ organizations.

1,064 lines (897 loc) 30.7 kB
/** * SecurityValidator - Comprehensive security validation system * * Enforces: * - NFR-SEC-001: Zero external API calls (100% offline operation) * - NFR-SEC-002: 100% rollback safety * - NFR-SEC-003: File permissions validation (644/755) * - NFR-SEC-004: 100% secret detection * - NFR-SEC-PERF-001: Security scan <10s for 100 files * * Features: * - External API call detection with whitelist support * - Secret detection (API keys, passwords, tokens, private keys) * - File permission validation * - Dependency vulnerability scanning * - Security gate enforcement for Construction/Production phases */ import * as fs from 'fs/promises'; import * as path from 'path'; import { glob } from 'glob'; import { ALL_SECRET_PATTERNS, calculateEntropy, isPlaceholder, shouldExcludeFile as shouldExcludeFileFromSecretScan, } from './secret-patterns.js'; import { ALL_API_PATTERNS, isWhitelisted, } from './api-patterns.js'; // ============================================================================ // Types // ============================================================================ export type SecurityIssueSeverity = 'critical' | 'high' | 'medium' | 'low'; export type SecurityIssueCategory = | 'external-api-call' | 'secret-exposure' | 'file-permission' | 'vulnerability' | 'insecure-dependency'; export interface SecurityIssue { severity: SecurityIssueSeverity; category: SecurityIssueCategory; file: string; lineNumber?: number; description: string; recommendation: string; cve?: string; } export interface SecurityScanResult { passed: boolean; issues: SecurityIssue[]; summary: { critical: number; high: number; medium: number; low: number; }; checkedFiles: number; scanDuration: number; // milliseconds } export interface DetectedSecret { type: 'api-key' | 'password' | 'token' | 'private-key' | 'credential'; file: string; lineNumber: number; snippet: string; // Masked confidence: number; // 0-1 } export interface SecretDetectionResult { foundSecrets: boolean; secrets: DetectedSecret[]; falsePositiveRate: number; } export interface ExternalAPICall { file: string; lineNumber: number; url: string; method: 'fetch' | 'axios' | 'http' | 'https' | 'XMLHttpRequest'; reason: string; } export interface PermissionViolation { file: string; actual: string; expected: string; reason: string; } export interface PermissionValidationResult { passed: boolean; violations: PermissionViolation[]; checkedFiles: number; } export interface DependencyVulnerability { package: string; version: string; severity: SecurityIssueSeverity; cve?: string; description: string; recommendation: string; } export interface DependencyScanResult { vulnerabilities: DependencyVulnerability[]; passed: boolean; } export interface VulnerabilityReport { dependencies: DependencyScanResult; summary: { critical: number; high: number; medium: number; low: number; }; } export interface GateEnforcementResult { passed: boolean; gate: 'construction' | 'production'; blockingIssues: SecurityIssue[]; warnings: SecurityIssue[]; timestamp: string; } export interface SecurityConfig { excludePaths?: string[]; customWhitelist?: RegExp[]; permissionRules?: Record<string, string>; failOnWarnings?: boolean; } export interface ScanOptions { checkExternalAPIs?: boolean; checkSecrets?: boolean; checkPermissions?: boolean; checkDependencies?: boolean; parallel?: boolean; } // ============================================================================ // SecurityValidator Class // ============================================================================ export class SecurityValidator { private projectPath: string; private config: SecurityConfig; constructor(projectPath: string, config: SecurityConfig = {}) { this.projectPath = path.resolve(projectPath); this.config = { excludePaths: [ '**/node_modules/**', '**/dist/**', '**/build/**', '**/.git/**', '**/coverage/**', '**/*.min.js', ...config.excludePaths || [], ], customWhitelist: config.customWhitelist || [], permissionRules: config.permissionRules || {}, failOnWarnings: config.failOnWarnings ?? false, }; } // ============================================================================ // Comprehensive Scanning // ============================================================================ /** * Comprehensive security scan */ async scan(options: ScanOptions = {}): Promise<SecurityScanResult> { const startTime = Date.now(); const opts: ScanOptions = { checkExternalAPIs: options.checkExternalAPIs ?? true, checkSecrets: options.checkSecrets ?? true, checkPermissions: options.checkPermissions ?? true, checkDependencies: options.checkDependencies ?? true, parallel: options.parallel ?? true, }; const allIssues: SecurityIssue[] = []; const files = await this.getFilesToScan(); // Run checks in parallel if enabled if (opts.parallel) { const promises: Promise<SecurityIssue[]>[] = []; if (opts.checkExternalAPIs) { promises.push(this.checkExternalAPIsInFiles(files)); } if (opts.checkSecrets) { promises.push(this.checkSecretsInFiles(files)); } if (opts.checkPermissions) { promises.push(this.checkPermissionsInFiles(files)); } if (opts.checkDependencies) { promises.push(this.checkDependenciesIssues()); } const results = await Promise.all(promises); results.forEach(issues => allIssues.push(...issues)); } else { // Sequential execution if (opts.checkExternalAPIs) { allIssues.push(...await this.checkExternalAPIsInFiles(files)); } if (opts.checkSecrets) { allIssues.push(...await this.checkSecretsInFiles(files)); } if (opts.checkPermissions) { allIssues.push(...await this.checkPermissionsInFiles(files)); } if (opts.checkDependencies) { allIssues.push(...await this.checkDependenciesIssues()); } } const scanDuration = Date.now() - startTime; const summary = { critical: allIssues.filter(i => i.severity === 'critical').length, high: allIssues.filter(i => i.severity === 'high').length, medium: allIssues.filter(i => i.severity === 'medium').length, low: allIssues.filter(i => i.severity === 'low').length, }; const passed = summary.critical === 0 && summary.high === 0; return { passed, issues: allIssues, summary, checkedFiles: files.length, scanDuration, }; } /** * Scan single file for security issues */ async scanFile(filePath: string): Promise<SecurityIssue[]> { const issues: SecurityIssue[] = []; try { const content = await fs.readFile(filePath, 'utf-8'); // const lines = content.split('\n'); // Check for external API calls const apiCalls = await this.detectExternalAPICallsInContent(content, filePath); apiCalls.forEach(call => { issues.push({ severity: 'high', category: 'external-api-call', file: filePath, lineNumber: call.lineNumber, description: `External API call detected: ${call.url}`, recommendation: 'Remove external API call or add to whitelist. System must operate 100% offline (NFR-SEC-001).', }); }); // Check for secrets const secrets = await this.detectSecretsInFile(filePath); secrets.forEach(secret => { issues.push({ severity: 'critical', category: 'secret-exposure', file: filePath, lineNumber: secret.lineNumber, description: `Potential ${secret.type} detected: ${secret.snippet}`, recommendation: 'Remove hardcoded secret. Use environment variables or secure vaults.', }); }); // Check file permissions const permIssue = await this.checkFilePermission(filePath); if (permIssue) { issues.push(permIssue); } } catch (error: any) { // Skip files that can't be read if (error.code !== 'ENOENT') { issues.push({ severity: 'low', category: 'vulnerability', file: filePath, description: `Failed to scan file: ${error.message}`, recommendation: 'Verify file permissions and accessibility.', }); } } return issues; } /** * Scan directory recursively */ async scanDirectory(dirPath: string, recursive: boolean = true): Promise<SecurityScanResult> { const startTime = Date.now(); const pattern = recursive ? '**/*' : '*'; const files = await glob(pattern, { cwd: dirPath, absolute: true, nodir: true, ignore: this.config.excludePaths, }); const allIssues: SecurityIssue[] = []; // Scan files in parallel const results = await Promise.all( files.map(file => this.scanFile(file)) ); results.forEach(issues => allIssues.push(...issues)); const scanDuration = Date.now() - startTime; const summary = { critical: allIssues.filter(i => i.severity === 'critical').length, high: allIssues.filter(i => i.severity === 'high').length, medium: allIssues.filter(i => i.severity === 'medium').length, low: allIssues.filter(i => i.severity === 'low').length, }; const passed = summary.critical === 0 && summary.high === 0; return { passed, issues: allIssues, summary, checkedFiles: files.length, scanDuration, }; } // ============================================================================ // External API Detection // ============================================================================ /** * Detect external API calls in code path */ async detectExternalAPICalls(codePath: string): Promise<ExternalAPICall[]> { const allCalls: ExternalAPICall[] = []; const files = await this.getFilesToScan(codePath); for (const file of files) { try { const content = await fs.readFile(file, 'utf-8'); const calls = await this.detectExternalAPICallsInContent(content, file); allCalls.push(...calls); } catch (error) { // Skip files that can't be read } } return allCalls; } /** * Detect external API calls in content string */ private async detectExternalAPICallsInContent( content: string, filePath: string ): Promise<ExternalAPICall[]> { const calls: ExternalAPICall[] = []; // const lines = content.split('\n'); // Extract all URLs from content // const urls = extractURLs(content); // Check each pattern for (const pattern of ALL_API_PATTERNS) { const matches = content.matchAll(pattern.regex); for (const match of matches) { const lineNumber = this.findLineNumber(content, match.index || 0); const url = match[1] || match[2] || ''; // Skip if whitelisted if (this.isWhitelistedAPI(url)) { continue; } calls.push({ file: filePath, lineNumber, url, method: pattern.method, reason: `${pattern.name}: ${pattern.description}`, }); } } return calls; } /** * Validate offline operation (no external API calls) */ async validateOfflineOperation(codePath: string): Promise<boolean> { const calls = await this.detectExternalAPICalls(codePath); return calls.length === 0; } /** * Check if API URL is whitelisted */ isWhitelistedAPI(url: string): boolean { if (isWhitelisted(url)) { return true; } // Check custom whitelist return this.config.customWhitelist?.some(pattern => pattern.test(url)) || false; } // ============================================================================ // Secret Detection // ============================================================================ /** * Detect secrets in files */ async detectSecrets(files: string[]): Promise<SecretDetectionResult> { const allSecrets: DetectedSecret[] = []; let totalChecked = 0; let falsePositives = 0; for (const file of files) { if (shouldExcludeFileFromSecretScan(file)) { continue; } totalChecked++; const secrets = await this.detectSecretsInFile(file); allSecrets.push(...secrets); } // Calculate false positive rate (rough estimate) // Secrets with confidence < 0.7 are likely false positives falsePositives = allSecrets.filter(s => s.confidence < 0.7).length; const falsePositiveRate = totalChecked > 0 ? falsePositives / totalChecked : 0; return { foundSecrets: allSecrets.length > 0, secrets: allSecrets, falsePositiveRate, }; } /** * Detect secrets in single file */ async detectSecretsInFile(filePath: string): Promise<DetectedSecret[]> { const secrets: DetectedSecret[] = []; try { const content = await fs.readFile(filePath, 'utf-8'); // const lines = content.split('\n'); // Check each secret pattern for (const pattern of ALL_SECRET_PATTERNS) { const matches = content.matchAll(pattern.regex); for (const match of matches) { const value = match[1] || match[0]; const lineNumber = this.findLineNumber(content, match.index || 0); // Skip placeholders if (isPlaceholder(value)) { continue; } // Analyze entropy if pattern requires it let confidence = pattern.confidence; if (pattern.entropy) { const entropy = calculateEntropy(value); if (entropy < pattern.entropy) { // Reduce confidence if entropy is too low confidence *= 0.5; } } // Only report if confidence threshold met if (confidence < 0.25) { continue; } secrets.push({ type: this.categorizeSecret(pattern.name), file: filePath, lineNumber, snippet: this.maskSecret(value), confidence, }); } } } catch (error) { // Skip files that can't be read } return secrets; } /** * Validate no secrets committed */ async validateNoSecretsCommitted(): Promise<boolean> { const files = await this.getFilesToScan(); const result = await this.detectSecrets(files); return !result.foundSecrets; } /** * Categorize secret type */ private categorizeSecret(patternName: string): DetectedSecret['type'] { const lower = patternName.toLowerCase(); if (lower.includes('password') || lower.includes('pass')) { return 'password'; } if (lower.includes('token') || lower.includes('bearer') || lower.includes('jwt')) { return 'token'; } if (lower.includes('private key')) { return 'private-key'; } if (lower.includes('api key') || lower.includes('secret key')) { return 'api-key'; } return 'credential'; } /** * Mask secret value for display */ private maskSecret(value: string): string { if (value.length <= 8) { return '***'; } const prefix = value.substring(0, 4); const suffix = value.substring(value.length - 4); const masked = '*'.repeat(Math.min(value.length - 8, 20)); return `${prefix}${masked}${suffix}`; } // ============================================================================ // File Permission Validation // ============================================================================ /** * Validate file permissions in directory */ async validateFilePermissions(dirPath: string): Promise<PermissionValidationResult> { const files = await glob('**/*', { cwd: dirPath, absolute: true, nodir: true, ignore: this.config.excludePaths, }); const violations: PermissionViolation[] = []; for (const file of files) { const issue = await this.checkFilePermission(file); if (issue) { const relPath = path.relative(this.projectPath, file); violations.push({ file: relPath, actual: issue.description.match(/actual: (\d+)/)?.[1] || 'unknown', expected: issue.description.match(/expected: (\d+)/)?.[1] || 'unknown', reason: issue.recommendation, }); } } return { passed: violations.length === 0, violations, checkedFiles: files.length, }; } /** * Check single file permission */ async checkPermission(filePath: string, expected: string): Promise<boolean> { try { const stats = await fs.stat(filePath); const actual = (stats.mode & parseInt('777', 8)).toString(8); return actual === expected; } catch (error) { return false; } } /** * Fix file permissions */ async fixPermissions(filePath: string, target: string): Promise<void> { const mode = parseInt(target, 8); await fs.chmod(filePath, mode); } /** * Check file permission and return issue if invalid */ private async checkFilePermission(filePath: string): Promise<SecurityIssue | null> { try { const stats = await fs.stat(filePath); const actual = (stats.mode & parseInt('777', 8)).toString(8); const expected = this.getExpectedPermission(filePath); if (actual !== expected) { return { severity: 'medium', category: 'file-permission', file: filePath, description: `Invalid file permissions (actual: ${actual}, expected: ${expected})`, recommendation: `Run: chmod ${expected} ${filePath}`, }; } } catch (error) { // Skip files that can't be accessed } return null; } /** * Get expected permission for file */ private getExpectedPermission(filePath: string): string { const basename = path.basename(filePath); const ext = path.extname(filePath); // Check custom rules if (this.config.permissionRules) { for (const [pattern, permission] of Object.entries(this.config.permissionRules)) { if (new RegExp(pattern).test(filePath)) { return permission; } } } // Sensitive files if (basename === '.env' || basename.includes('private') || basename.includes('secret')) { return '600'; } // Executables if (['.sh', '.bash', '.zsh'].includes(ext)) { return '755'; } // Check if file has shebang // (Would need to read file - skip for performance) // Default: regular files return '644'; } // ============================================================================ // Dependency Scanning // ============================================================================ /** * Scan dependencies for vulnerabilities */ async scanDependencies(): Promise<DependencyScanResult> { // In a real implementation, this would check against a vulnerability database // For offline operation (NFR-SEC-001), we use a cached/local database const vulnerabilities: DependencyVulnerability[] = []; try { const packageJsonPath = path.join(this.projectPath, 'package.json'); // Verify package.json exists and is valid JSON const content = await fs.readFile(packageJsonPath, 'utf-8'); JSON.parse(content); // Note: In production, this would query a local vulnerability database // against the parsed dependencies. For now, we return empty results. } catch (_error) { // No package.json or can't read it - skip dependency check } return { vulnerabilities, passed: vulnerabilities.length === 0, }; } /** * Check for known vulnerabilities */ async checkKnownVulnerabilities(): Promise<VulnerabilityReport> { const dependencies = await this.scanDependencies(); const summary = { critical: dependencies.vulnerabilities.filter(v => v.severity === 'critical').length, high: dependencies.vulnerabilities.filter(v => v.severity === 'high').length, medium: dependencies.vulnerabilities.filter(v => v.severity === 'medium').length, low: dependencies.vulnerabilities.filter(v => v.severity === 'low').length, }; return { dependencies, summary, }; } // ============================================================================ // Security Gate Enforcement // ============================================================================ /** * Enforce security gate (auto-detect phase) */ async enforceSecurityGate(): Promise<GateEnforcementResult> { const result = await this.scan(); const blockingIssues = result.issues.filter( i => i.severity === 'critical' ); const warnings = result.issues.filter( i => i.severity === 'high' || i.severity === 'medium' ); return { passed: result.summary.critical === 0, gate: 'construction', blockingIssues, warnings, timestamp: new Date().toISOString(), }; } /** * Validate Construction gate * * Requirements: * - Zero critical security issues * - Zero external API calls (except whitelisted) * - Zero committed secrets * - All file permissions valid */ async validateConstructionGate(): Promise<boolean> { const result = await this.scan(); return result.summary.critical === 0; } /** * Validate Production gate (stricter) * * Requirements: * - Zero critical or high security issues * - Zero external API calls (except whitelisted) * - Zero committed secrets * - All file permissions valid * - All dependencies patched */ async validateProductionGate(): Promise<boolean> { const result = await this.scan(); return result.summary.critical === 0 && result.summary.high === 0; } // ============================================================================ // Reporting // ============================================================================ /** * Generate security report */ async generateSecurityReport(): Promise<string> { const result = await this.scan(); let report = '# Security Scan Report\n\n'; report += `Generated: ${new Date().toISOString()}\n\n`; report += `## Summary\n\n`; report += `- Status: ${result.passed ? '✅ PASSED' : '❌ FAILED'}\n`; report += `- Files Checked: ${result.checkedFiles}\n`; report += `- Scan Duration: ${result.scanDuration}ms\n`; report += `- Critical Issues: ${result.summary.critical}\n`; report += `- High Issues: ${result.summary.high}\n`; report += `- Medium Issues: ${result.summary.medium}\n`; report += `- Low Issues: ${result.summary.low}\n\n`; if (result.issues.length > 0) { report += `## Issues\n\n`; const grouped = this.groupIssuesByCategory(result.issues); for (const [category, issues] of Object.entries(grouped)) { report += `### ${category}\n\n`; for (const issue of issues) { report += `**${issue.severity.toUpperCase()}**: ${issue.description}\n`; report += `- File: ${issue.file}${issue.lineNumber ? `:${issue.lineNumber}` : ''}\n`; report += `- Recommendation: ${issue.recommendation}\n\n`; } } } return report; } /** * Export report in different formats */ async exportReport(format: 'markdown' | 'json' | 'html'): Promise<string> { const result = await this.scan(); switch (format) { case 'json': return JSON.stringify(result, null, 2); case 'html': return this.generateHTMLReport(result); case 'markdown': default: return this.generateSecurityReport(); } } /** * Generate remediation plan */ async generateRemediationPlan(issues: SecurityIssue[]): Promise<string> { let plan = '# Security Remediation Plan\n\n'; const grouped = this.groupIssuesByCategory(issues); for (const [category, categoryIssues] of Object.entries(grouped)) { plan += `## ${category}\n\n`; const prioritized = [...categoryIssues].sort((a, b) => { const severityOrder = { critical: 0, high: 1, medium: 2, low: 3 }; return severityOrder[a.severity] - severityOrder[b.severity]; }); prioritized.forEach((issue, index) => { plan += `${index + 1}. **${issue.severity.toUpperCase()}**: ${issue.description}\n`; plan += ` - File: ${issue.file}${issue.lineNumber ? `:${issue.lineNumber}` : ''}\n`; plan += ` - Action: ${issue.recommendation}\n\n`; }); } return plan; } // ============================================================================ // Helper Methods // ============================================================================ /** * Get files to scan */ private async getFilesToScan(basePath?: string): Promise<string[]> { const searchPath = basePath || this.projectPath; return glob('**/*.{ts,js,tsx,jsx,mjs,cjs,json,yaml,yml,env}', { cwd: searchPath, absolute: true, nodir: true, ignore: this.config.excludePaths, }); } /** * Find line number from string index */ private findLineNumber(content: string, index: number): number { const lines = content.substring(0, index).split('\n'); return lines.length; } /** * Group issues by category */ private groupIssuesByCategory(issues: SecurityIssue[]): Record<string, SecurityIssue[]> { const grouped: Record<string, SecurityIssue[]> = {}; for (const issue of issues) { const category = issue.category; if (!grouped[category]) { grouped[category] = []; } grouped[category].push(issue); } return grouped; } /** * Generate HTML report */ private generateHTMLReport(result: SecurityScanResult): string { let html = '<!DOCTYPE html>\n<html>\n<head>\n'; html += '<title>Security Scan Report</title>\n'; html += '<style>\n'; html += 'body { font-family: Arial, sans-serif; margin: 20px; }\n'; html += '.summary { background: #f0f0f0; padding: 15px; border-radius: 5px; }\n'; html += '.issue { margin: 10px 0; padding: 10px; border-left: 4px solid; }\n'; html += '.critical { border-color: #d32f2f; background: #ffebee; }\n'; html += '.high { border-color: #f57c00; background: #fff3e0; }\n'; html += '.medium { border-color: #fbc02d; background: #fffde7; }\n'; html += '.low { border-color: #388e3c; background: #e8f5e9; }\n'; html += '</style>\n</head>\n<body>\n'; html += '<h1>Security Scan Report</h1>\n'; html += '<div class="summary">\n'; html += `<p><strong>Status:</strong> ${result.passed ? '✅ PASSED' : '❌ FAILED'}</p>\n`; html += `<p><strong>Files Checked:</strong> ${result.checkedFiles}</p>\n`; html += `<p><strong>Scan Duration:</strong> ${result.scanDuration}ms</p>\n`; html += `<p><strong>Critical:</strong> ${result.summary.critical}</p>\n`; html += `<p><strong>High:</strong> ${result.summary.high}</p>\n`; html += `<p><strong>Medium:</strong> ${result.summary.medium}</p>\n`; html += `<p><strong>Low:</strong> ${result.summary.low}</p>\n`; html += '</div>\n'; if (result.issues.length > 0) { html += '<h2>Issues</h2>\n'; result.issues.forEach(issue => { html += `<div class="issue ${issue.severity}">\n`; html += `<strong>${issue.severity.toUpperCase()}</strong>: ${issue.description}<br>\n`; html += `<em>File: ${issue.file}${issue.lineNumber ? `:${issue.lineNumber}` : ''}</em><br>\n`; html += `<strong>Recommendation:</strong> ${issue.recommendation}\n`; html += '</div>\n'; }); } html += '</body>\n</html>'; return html; } /** * Check external APIs in multiple files */ private async checkExternalAPIsInFiles(files: string[]): Promise<SecurityIssue[]> { const issues: SecurityIssue[] = []; for (const file of files) { try { const content = await fs.readFile(file, 'utf-8'); const calls = await this.detectExternalAPICallsInContent(content, file); calls.forEach(call => { issues.push({ severity: 'high', category: 'external-api-call', file: call.file, lineNumber: call.lineNumber, description: `External API call detected: ${call.url}`, recommendation: 'Remove external API call or add to whitelist. System must operate 100% offline (NFR-SEC-001).', }); }); } catch (error) { // Skip files that can't be read } } return issues; } /** * Check secrets in multiple files */ private async checkSecretsInFiles(files: string[]): Promise<SecurityIssue[]> { const issues: SecurityIssue[] = []; for (const file of files) { if (shouldExcludeFileFromSecretScan(file)) { continue; } const secrets = await this.detectSecretsInFile(file); secrets.forEach(secret => { issues.push({ severity: 'critical', category: 'secret-exposure', file: secret.file, lineNumber: secret.lineNumber, description: `Potential ${secret.type} detected: ${secret.snippet}`, recommendation: 'Remove hardcoded secret. Use environment variables or secure vaults.', }); }); } return issues; } /** * Check permissions in multiple files */ private async checkPermissionsInFiles(files: string[]): Promise<SecurityIssue[]> { const issues: SecurityIssue[] = []; for (const file of files) { const issue = await this.checkFilePermission(file); if (issue) { issues.push(issue); } } return issues; } /** * Check dependencies issues */ private async checkDependenciesIssues(): Promise<SecurityIssue[]> { const issues: SecurityIssue[] = []; const vulnReport = await this.checkKnownVulnerabilities(); vulnReport.dependencies.vulnerabilities.forEach(vuln => { issues.push({ severity: vuln.severity, category: 'insecure-dependency', file: 'package.json', description: vuln.description, recommendation: vuln.recommendation, cve: vuln.cve, }); }); return issues; } }