UNPKG

recoder-security

Version:

Enterprise-grade security and compliance layer for CodeCraft CLI

913 lines 36.1 kB
"use strict"; /** * AI-Powered Threat Detection System * Advanced threat detection using machine learning, behavioral analysis, and pattern recognition * to identify and respond to security threats in real-time */ Object.defineProperty(exports, "__esModule", { value: true }); exports.ThreatDetectionEngine = void 0; const events_1 = require("events"); const shared_1 = require("@recoder/shared"); class ThreatDetectionEngine extends events_1.EventEmitter { constructor(config) { super(); this.logger = new shared_1.Logger('ThreatDetection'); this.signatures = new Map(); this.threatIntel = new Map(); this.behaviorProfiles = new Map(); this.huntingQueries = new Map(); this.threatEvents = []; this.eventBuffer = []; this.mlModels = new Map(); this.config = { enabled: true, realTimeMonitoring: true, behaviorAnalysis: true, threatIntelligence: true, machineLearning: false, // Disabled by default due to complexity responseAutomation: false, // Disabled by default for safety huntingQueries: true, confidenceThreshold: 70, alertThresholds: { critical: 1, high: 5, medium: 10, low: 20, }, learningPeriod: 30, retentionPeriod: 365, integrations: {}, ...config, }; this.loadThreatSignatures(); this.loadThreatIntelligence(); this.loadHuntingQueries(); this.startThreatDetection(); } /** * Analyze event for threats */ async analyzeEvent(event) { if (!this.config.enabled) { return []; } this.eventBuffer.push(event); const threats = []; // Real-time signature matching for (const [signatureId, signature] of this.signatures) { const detection = await this.matchSignature(event, signature); if (detection) { threats.push(detection); } } // Threat intelligence correlation if (this.config.threatIntelligence) { const intelThreats = await this.correlateThreatIntelligence(event); threats.push(...intelThreats); } // Behavioral analysis if (this.config.behaviorAnalysis) { const behaviorThreats = await this.analyzeBehavior(event); threats.push(...behaviorThreats); } // Machine learning analysis if (this.config.machineLearning) { const mlThreats = await this.runMLAnalysis(event); threats.push(...mlThreats); } // Process threats for (const threat of threats) { if (threat.confidence >= this.config.confidenceThreshold) { await this.processThreatEvent(threat); } } return threats; } /** * Hunt for threats using custom queries */ async huntThreats(queryId) { this.logger.info('Starting threat hunting'); const results = []; const queriesToRun = queryId ? [this.huntingQueries.get(queryId)].filter(Boolean) : Array.from(this.huntingQueries.values()).filter(q => q.enabled); for (const query of queriesToRun) { try { this.logger.debug(`Running hunting query: ${query.name}`); const queryResults = await this.executeHuntingQuery(query); results.push({ query, results: queryResults }); // Update query metadata query.lastRun = Date.now(); query.results = { timestamp: Date.now(), matches: queryResults.length, findings: queryResults, }; // Create threat events for significant findings if (queryResults.length > 0) { await this.createThreatFromHunt(query, queryResults); } } catch (error) { this.logger.error(`Failed to execute hunting query ${query.name}:`, error); } } this.logger.info(`Threat hunting completed: ${results.length} queries executed`); return results; } /** * Update behavioral baseline for entity */ async updateBehaviorProfile(entityId, entityType, events) { let profile = this.behaviorProfiles.get(entityId); if (!profile) { profile = this.createNewBehaviorProfile(entityId, entityType); this.behaviorProfiles.set(entityId, profile); } // Update baseline from events for (const event of events) { this.updateBaselineFromEvent(profile, event); } // Calculate current risk score profile.current.riskScore = await this.calculateRiskScore(profile, events); profile.current.lastUpdated = Date.now(); // Update learning metadata profile.learning.samples += events.length; profile.learning.confidence = Math.min(profile.learning.samples / 1000, 1.0); this.logger.debug(`Updated behavior profile for ${entityId}: risk score ${profile.current.riskScore}`); } /** * Add custom threat signature */ addThreatSignature(signature) { this.signatures.set(signature.id, signature); this.logger.info(`Added threat signature: ${signature.name}`); } /** * Add threat intelligence indicator */ addThreatIntelligence(intel) { this.threatIntel.set(intel.id, intel); this.logger.info(`Added threat intelligence: ${intel.type} from ${intel.source}`); } /** * Get threat dashboard data */ getThreatDashboard() { const activeThreats = this.threatEvents.filter(t => ['new', 'investigating', 'contained'].includes(t.status)); const criticalThreats = activeThreats.filter(t => t.severity === 'critical'); const investigationsOpen = activeThreats.filter(t => t.status === 'investigating'); // Calculate overall risk score const riskScore = activeThreats.length > 0 ? Math.min(activeThreats.reduce((sum, t) => sum + t.confidence, 0) / activeThreats.length, 100) : 0; // Get recent threats (last 24 hours) const dayAgo = Date.now() - (24 * 60 * 60 * 1000); const recentThreats = this.threatEvents .filter(t => t.timestamp > dayAgo) .sort((a, b) => b.timestamp - a.timestamp) .slice(0, 10); // Count attack types const attackTypeCounts = new Map(); for (const threat of this.threatEvents) { const type = threat.signature.category; attackTypeCounts.set(type, (attackTypeCounts.get(type) || 0) + 1); } const topAttackTypes = Array.from(attackTypeCounts.entries()) .sort(([, a], [, b]) => b - a) .slice(0, 5) .map(([type, count]) => ({ type, count })); // Count behavior anomalies const behaviorAnomalies = Array.from(this.behaviorProfiles.values()) .reduce((sum, profile) => sum + profile.current.anomalies.length, 0); return { activeThreats: activeThreats.length, criticalThreats: criticalThreats.length, investigationsOpen: investigationsOpen.length, riskScore: Math.round(riskScore), recentThreats, topAttackTypes, threatTrends: [], // Would calculate from historical data behaviorAnomalies, }; } /** * Load built-in threat signatures */ loadThreatSignatures() { const signatures = [ // Brute Force Attack { id: 'brute-force-login', name: 'Brute Force Authentication Attack', description: 'Multiple failed authentication attempts from same source', severity: 'high', category: 'intrusion', patterns: { type: 'frequency', rules: [ { field: 'eventType', value: 'authentication' }, { field: 'outcome', value: 'failure' }, { field: 'actor.ip', operator: 'same' } ], threshold: 5, timeWindow: 300000, // 5 minutes }, indicators: [ { type: 'ip', values: [], weight: 0.8 }, { type: 'user', values: [], weight: 0.6 } ], mitre: { tactics: ['Initial Access', 'Credential Access'], techniques: ['T1110', 'T1078'], }, response: { automated: true, actions: ['block', 'alert'], escalation: false, }, }, // Privilege Escalation { id: 'privilege-escalation', name: 'Privilege Escalation Attempt', description: 'User attempting to access resources above their privilege level', severity: 'critical', category: 'privilege_escalation', patterns: { type: 'sequence', rules: [ { field: 'eventType', value: 'authorization' }, { field: 'outcome', value: 'denied' }, { field: 'resource.classification', value: 'restricted' } ], threshold: 3, timeWindow: 600000, // 10 minutes }, indicators: [ { type: 'user', values: [], weight: 0.9 }, { type: 'behavior', values: ['privilege_request'], weight: 0.7 } ], mitre: { tactics: ['Privilege Escalation', 'Defense Evasion'], techniques: ['T1068', 'T1548'], }, response: { automated: false, actions: ['alert', 'investigate'], escalation: true, }, }, // Data Exfiltration { id: 'data-exfiltration', name: 'Potential Data Exfiltration', description: 'Unusual large volume data access patterns', severity: 'critical', category: 'data_breach', patterns: { type: 'statistical', rules: [ { field: 'eventType', value: 'data_access' }, { field: 'details.metadata.size', operator: 'anomaly', baseline: 'user_average' } ], threshold: 2.5, // Standard deviations timeWindow: 3600000, // 1 hour }, indicators: [ { type: 'user', values: [], weight: 0.8 }, { type: 'file', values: [], weight: 0.9 } ], mitre: { tactics: ['Collection', 'Exfiltration'], techniques: ['T1005', 'T1041'], }, response: { automated: false, actions: ['alert', 'quarantine'], escalation: true, }, }, // Suspicious Command Execution { id: 'suspicious-commands', name: 'Suspicious Command Execution', description: 'Execution of potentially malicious commands', severity: 'medium', category: 'malware', patterns: { type: 'regex', rules: [ { field: 'action', pattern: /(powershell|cmd|bash).*\s+(download|wget|curl|invoke-expression)/i }, { field: 'details.metadata.command', pattern: /(base64|encoded|obfuscated)/i } ], threshold: 1, }, indicators: [ { type: 'command', values: ['powershell', 'cmd', 'bash'], weight: 0.7 }, { type: 'behavior', values: ['command_execution'], weight: 0.8 } ], mitre: { tactics: ['Execution', 'Defense Evasion'], techniques: ['T1059', 'T1027'], }, response: { automated: true, actions: ['alert', 'log'], escalation: false, }, }, // Anomalous Login Times { id: 'anomalous-login-time', name: 'Login Outside Normal Hours', description: 'User authentication outside their typical hours', severity: 'low', category: 'anomaly', patterns: { type: 'statistical', rules: [ { field: 'eventType', value: 'authentication' }, { field: 'outcome', value: 'success' }, { field: 'timestamp', operator: 'time_anomaly', baseline: 'user_profile' } ], threshold: 0.1, // Probability threshold }, indicators: [ { type: 'user', values: [], weight: 0.6 }, { type: 'behavior', values: ['time_anomaly'], weight: 0.5 } ], mitre: { tactics: ['Initial Access'], techniques: ['T1078'], }, response: { automated: false, actions: ['alert'], escalation: false, }, }, ]; for (const signature of signatures) { this.signatures.set(signature.id, signature); } this.logger.info(`Loaded ${signatures.length} threat signatures`); } /** * Load threat intelligence feeds */ loadThreatIntelligence() { // In production, this would load from external threat intel feeds const sampleIntel = [ { id: 'malicious-ip-1', type: 'ioc', source: 'internal', confidence: 95, severity: 'high', indicators: [ { type: 'ip', value: '192.168.1.100', context: 'Known malicious IP' } ], description: 'IP address associated with brute force attacks', firstSeen: Date.now() - (7 * 24 * 60 * 60 * 1000), lastSeen: Date.now() - (24 * 60 * 60 * 1000), tags: ['brute-force', 'malicious-ip'], references: ['internal-incident-001'], }, ]; for (const intel of sampleIntel) { this.threatIntel.set(intel.id, intel); } this.logger.info(`Loaded ${sampleIntel.length} threat intelligence indicators`); } /** * Load hunting queries */ loadHuntingQueries() { const queries = [ { id: 'failed-logins-hunt', name: 'Hunt for Failed Login Patterns', description: 'Search for patterns of failed authentication attempts', query: 'eventType:authentication AND outcome:failure', language: 'elasticsearch', frequency: 'hourly', enabled: true, tags: ['authentication', 'brute-force'], author: 'security-team', created: Date.now(), }, { id: 'privilege-escalation-hunt', name: 'Hunt for Privilege Escalation', description: 'Search for unauthorized privilege escalation attempts', query: 'action:*privilege* OR action:*admin* OR action:*sudo*', language: 'elasticsearch', frequency: 'daily', enabled: true, tags: ['privilege-escalation', 'admin'], author: 'security-team', created: Date.now(), }, { id: 'data-exfiltration-hunt', name: 'Hunt for Data Exfiltration', description: 'Search for unusual data access patterns', query: 'eventType:data_access AND details.metadata.size:>10000000', language: 'elasticsearch', frequency: 'daily', enabled: true, tags: ['data-exfiltration', 'data-access'], author: 'security-team', created: Date.now(), }, ]; for (const query of queries) { this.huntingQueries.set(query.id, query); } this.logger.info(`Loaded ${queries.length} threat hunting queries`); } /** * Start threat detection engine */ startThreatDetection() { if (!this.config.realTimeMonitoring) { return; } // Process event buffer periodically setInterval(async () => { if (this.eventBuffer.length > 0) { await this.processBatchedEvents(); } }, 10000); // Every 10 seconds // Run hunting queries periodically setInterval(async () => { await this.runScheduledHunting(); }, 3600000); // Every hour // Update behavior profiles setInterval(async () => { await this.updateBehaviorProfiles(); }, 300000); // Every 5 minutes this.logger.info('Threat detection engine started'); } /** * Match event against threat signature */ async matchSignature(event, signature) { const { patterns } = signature; switch (patterns.type) { case 'regex': return this.matchRegexPattern(event, signature); case 'frequency': return this.matchFrequencyPattern(event, signature); case 'sequence': return this.matchSequencePattern(event, signature); case 'statistical': return this.matchStatisticalPattern(event, signature); default: return null; } } /** * Match regex patterns */ async matchRegexPattern(event, signature) { for (const rule of signature.patterns.rules) { const fieldValue = this.getFieldValue(event, rule.field); if (fieldValue && rule.pattern) { const regex = new RegExp(rule.pattern, 'i'); if (regex.test(fieldValue)) { return this.createThreatEvent(event, signature, 0.8); } } } return null; } /** * Match frequency patterns */ async matchFrequencyPattern(event, signature) { const timeWindow = signature.patterns.timeWindow || 300000; // 5 minutes default const threshold = signature.patterns.threshold; const windowStart = Date.now() - timeWindow; // Count matching events in time window const matchingEvents = this.eventBuffer.filter(e => { if (e.timestamp < windowStart) return false; return signature.patterns.rules.every(rule => { const eventValue = this.getFieldValue(e, rule.field); const targetValue = this.getFieldValue(event, rule.field); if (rule.operator === 'same') { return eventValue === targetValue; } return eventValue === rule.value; }); }); if (matchingEvents.length >= threshold) { return this.createThreatEvent(event, signature, 0.9, matchingEvents); } return null; } /** * Match sequence patterns */ async matchSequencePattern(event, signature) { const timeWindow = signature.patterns.timeWindow || 600000; // 10 minutes default const windowStart = Date.now() - timeWindow; // Find events that match the sequence const sequenceEvents = []; for (const rule of signature.patterns.rules) { const matchingEvent = this.eventBuffer.find(e => { if (e.timestamp < windowStart) return false; return this.getFieldValue(e, rule.field) === rule.value; }); if (matchingEvent) { sequenceEvents.push(matchingEvent); } } if (sequenceEvents.length >= signature.patterns.threshold) { return this.createThreatEvent(event, signature, 0.85, sequenceEvents); } return null; } /** * Match statistical patterns (anomaly detection) */ async matchStatisticalPattern(event, signature) { // This would implement statistical anomaly detection // For now, return null as it requires historical data analysis return null; } /** * Correlate with threat intelligence */ async correlateThreatIntelligence(event) { const threats = []; for (const [intelId, intel] of this.threatIntel) { for (const indicator of intel.indicators) { const eventValue = this.extractIndicatorValue(event, indicator.type); if (eventValue && eventValue === indicator.value) { // Create threat signature from intelligence const signature = { id: `intel-${intelId}`, name: `Threat Intel Match: ${intel.description}`, description: intel.description, severity: intel.severity === 'info' ? 'low' : intel.severity, category: 'intrusion', patterns: { type: 'regex', rules: [], threshold: 1 }, indicators: [{ type: indicator.type, values: [indicator.value], weight: 1.0 }], mitre: { tactics: [], techniques: [] }, response: { automated: true, actions: ['alert'], escalation: true }, }; const threatEvent = this.createThreatEvent(event, signature, intel.confidence / 100); threatEvent.context.campaign = intel.tags.find(t => t.includes('campaign')); threats.push(threatEvent); } } } return threats; } /** * Analyze behavior for anomalies */ async analyzeBehavior(event) { const threats = []; const actorId = event.actor.id; const profile = this.behaviorProfiles.get(actorId); if (!profile || !profile.learning.enabled) { return threats; } // Check for behavioral anomalies const anomalies = this.detectBehaviorAnomalies(event, profile); for (const anomaly of anomalies) { // Add anomaly to profile profile.current.anomalies.push({ type: anomaly.type, severity: anomaly.severity, description: anomaly.description, timestamp: Date.now(), }); // Create threat event if significant if (anomaly.severity >= 0.7) { const signature = { id: `behavior-${anomaly.type}`, name: `Behavioral Anomaly: ${anomaly.type}`, description: anomaly.description, severity: anomaly.severity > 0.9 ? 'high' : 'medium', category: 'anomaly', patterns: { type: 'statistical', rules: [], threshold: 1 }, indicators: [{ type: 'behavior', values: [anomaly.type], weight: anomaly.severity }], mitre: { tactics: ['Initial Access'], techniques: ['T1078'] }, response: { automated: false, actions: ['alert'], escalation: false }, }; threats.push(this.createThreatEvent(event, signature, anomaly.severity)); } } return threats; } /** * Run machine learning analysis */ async runMLAnalysis(event) { // Placeholder for ML analysis // In production, this would use trained models for anomaly detection return []; } /** * Process threat event */ async processThreatEvent(threat) { this.threatEvents.push(threat); this.emit('threatDetected', threat); this.logger.warn(`Threat detected: ${threat.signature.name} (confidence: ${threat.confidence}%)`); // Execute automated response if enabled if (this.config.responseAutomation && threat.signature.response.automated) { await this.executeAutomatedResponse(threat); } // Check alert thresholds await this.checkAlertThresholds(); } /** * Execute automated response */ async executeAutomatedResponse(threat) { const actions = threat.signature.response.actions; const executedActions = []; for (const action of actions) { try { switch (action) { case 'block': await this.blockEntity(threat.target); executedActions.push('block'); break; case 'quarantine': await this.quarantineEntity(threat.target); executedActions.push('quarantine'); break; case 'alert': await this.sendAlert(threat); executedActions.push('alert'); break; case 'log': this.logger.info(`Threat logged: ${threat.signature.name}`); executedActions.push('log'); break; } } catch (error) { this.logger.error(`Failed to execute response action ${action}:`, error); } } threat.response = { automated: true, actions: executedActions, effectiveness: executedActions.length / actions.length * 100, timestamp: Date.now(), }; } /** * Create threat event from detection */ createThreatEvent(triggerEvent, signature, confidence, relatedEvents = []) { return { id: this.generateThreatId(), timestamp: Date.now(), type: 'detection', signature, confidence: Math.round(confidence * 100), severity: signature.severity, source: 'threat-detection', target: { type: triggerEvent.actor.type, id: triggerEvent.actor.id, name: triggerEvent.actor.name, ip: triggerEvent.actor.ip, }, evidence: { events: [triggerEvent, ...relatedEvents], artifacts: [], timeline: [{ timestamp: triggerEvent.timestamp, description: `${signature.name} detected`, }], indicators: signature.indicators.map(ind => ({ type: ind.type, value: ind.values.join(','), confidence: ind.weight, })), }, context: { relatedThreats: [], }, status: 'new', response: { automated: false, actions: [], effectiveness: 0, timestamp: 0, }, }; } /** * Helper functions */ getFieldValue(event, field) { const parts = field.split('.'); let value = event; for (const part of parts) { value = value?.[part]; } return value; } extractIndicatorValue(event, type) { switch (type) { case 'ip': return event.actor.ip || null; case 'user': return event.actor.id; case 'file': return event.resource.type === 'file' ? event.resource.id : null; case 'command': return event.details.metadata.command || null; default: return null; } } createNewBehaviorProfile(entityId, entityType) { return { entityId, entityType, baseline: { loginTimes: [], locations: [], devices: [], activities: [], networkTraffic: [], accessPatterns: [], }, current: { riskScore: 0, anomalies: [], lastUpdated: Date.now(), }, learning: { enabled: true, samples: 0, confidence: 0, lastTrained: Date.now(), }, }; } updateBaselineFromEvent(profile, event) { // Update login times if (event.eventType === 'authentication' && event.outcome === 'success') { const hour = new Date(event.timestamp).getHours(); if (!profile.baseline.loginTimes.includes(hour)) { profile.baseline.loginTimes.push(hour); } } // Update locations if (event.details.geolocation?.country) { const location = event.details.geolocation.country; if (!profile.baseline.locations.includes(location)) { profile.baseline.locations.push(location); } } // Update activities const activity = profile.baseline.activities.find(a => a.action === event.action); if (activity) { activity.frequency++; } else { profile.baseline.activities.push({ action: event.action, frequency: 1 }); } } async calculateRiskScore(profile, events) { let riskScore = 0; // Risk from anomalies const recentAnomalies = profile.current.anomalies.filter(a => a.timestamp > Date.now() - (24 * 60 * 60 * 1000)); riskScore += recentAnomalies.length * 10; // Risk from failed events const failedEvents = events.filter(e => e.outcome === 'failure'); riskScore += (failedEvents.length / events.length) * 30; // Risk from privilege escalation attempts const privilegeEvents = events.filter(e => e.action.includes('privilege') || e.action.includes('admin')); riskScore += privilegeEvents.length * 5; return Math.min(riskScore, 100); } detectBehaviorAnomalies(event, profile) { const anomalies = []; // Check login time anomaly if (event.eventType === 'authentication') { const hour = new Date(event.timestamp).getHours(); if (!profile.baseline.loginTimes.includes(hour)) { anomalies.push({ type: 'unusual_login_time', severity: 0.6, description: `Login at unusual hour: ${hour}:00`, }); } } // Check location anomaly if (event.details.geolocation?.country) { const location = event.details.geolocation.country; if (!profile.baseline.locations.includes(location)) { anomalies.push({ type: 'new_location', severity: 0.8, description: `Login from new location: ${location}`, }); } } // Check activity anomaly const activity = profile.baseline.activities.find(a => a.action === event.action); if (!activity && event.action.includes('admin')) { anomalies.push({ type: 'new_admin_activity', severity: 0.9, description: `New administrative activity: ${event.action}`, }); } return anomalies; } async executeHuntingQuery(query) { // This would execute the hunting query against the data store // For now, return empty results return []; } async createThreatFromHunt(query, results) { // Create threat events from hunting query results this.logger.info(`Hunting query ${query.name} found ${results.length} potential threats`); } async processBatchedEvents() { const events = [...this.eventBuffer]; this.eventBuffer.length = 0; // Process events in batch for efficiency for (const event of events) { await this.analyzeEvent(event); } } async runScheduledHunting() { const now = Date.now(); const queriesToRun = Array.from(this.huntingQueries.values()).filter(q => { if (!q.enabled) return false; const lastRun = q.lastRun || 0; const interval = this.getQueryInterval(q.frequency); return now - lastRun >= interval; }); if (queriesToRun.length > 0) { await this.huntThreats(); } } async updateBehaviorProfiles() { // Update behavior profiles with recent events for (const [entityId, profile] of this.behaviorProfiles) { const recentEvents = this.eventBuffer.filter(e => e.actor.id === entityId); if (recentEvents.length > 0) { await this.updateBehaviorProfile(entityId, profile.entityType, recentEvents); } } } getQueryInterval(frequency) { switch (frequency) { case 'continuous': return 0; case 'hourly': return 60 * 60 * 1000; case 'daily': return 24 * 60 * 60 * 1000; case 'weekly': return 7 * 24 * 60 * 60 * 1000; default: return 60 * 60 * 1000; } } async checkAlertThresholds() { const thresholds = this.config.alertThresholds; const recentThreats = this.threatEvents.filter(t => t.timestamp > Date.now() - (60 * 60 * 1000) // Last hour ); const criticalCount = recentThreats.filter(t => t.severity === 'critical').length; const highCount = recentThreats.filter(t => t.severity === 'high').length; if (criticalCount >= thresholds.critical) { this.logger.error(`ALERT: ${criticalCount} critical threats detected in the last hour`); } if (highCount >= thresholds.high) { this.logger.warn(`ALERT: ${highCount} high-severity threats detected in the last hour`); } } async blockEntity(target) { this.logger.warn(`Blocking entity: ${target.type} ${target.id}`); // Implement blocking logic } async quarantineEntity(target) { this.logger.warn(`Quarantining entity: ${target.type} ${target.id}`); // Implement quarantine logic } async sendAlert(threat) { this.logger.warn(`Sending alert for threat: ${threat.signature.name}`); // Implement alert sending logic } generateThreatId() { return `threat_${Date.now()}_${Math.random().toString(36).substring(7)}`; } } exports.ThreatDetectionEngine = ThreatDetectionEngine; //# sourceMappingURL=threat-detection.js.map