mira-consciousness
Version:
Memory & Intelligence Retention Archive - Preserving The Spark
350 lines • 15 kB
JavaScript
import fs from 'fs-extra';
import * as path from 'path';
import { glob } from 'glob';
export class EntropyAnalyzer {
projectRoot;
highEntropyThreshold = 4.5;
minStringLength = 16;
constructor(projectRoot) {
this.projectRoot = projectRoot;
}
async analyze() {
const suspiciousStrings = [];
let totalStringsAnalyzed = 0;
let totalEntropy = 0;
try {
// Scan various file types for potential secrets
const filePatterns = [
'**/*.{ts,js,jsx,tsx,vue,svelte}', // Source code
'**/*.{json,yaml,yml,toml}', // Config files
'**/*.{env,env.*}', // Environment files
'**/*.{config,conf}', // Configuration files
'**/*.{md,txt}', // Documentation (might contain examples)
'**/*.{sh,bash,bat,ps1}' // Scripts
];
for (const pattern of filePatterns) {
const files = await glob(pattern, {
cwd: this.projectRoot,
ignore: [
'node_modules/**',
'dist/**',
'build/**',
'.git/**',
'**/*.min.js',
'**/package-lock.json',
'**/yarn.lock'
],
absolute: false
});
for (const file of files) {
const fileResults = await this.analyzeFile(file);
suspiciousStrings.push(...fileResults.suspiciousStrings);
totalStringsAnalyzed += fileResults.totalStrings;
totalEntropy += fileResults.totalEntropy;
}
}
}
catch (error) {
// Continue with what we have
}
// Calculate overall risk
const overallRisk = this.calculateOverallRisk(suspiciousStrings);
// Generate recommendations
const recommendations = this.generateRecommendations(suspiciousStrings);
// Sort by risk and confidence
suspiciousStrings.sort((a, b) => {
const riskOrder = { critical: 4, high: 3, medium: 2, low: 1 };
const riskDiff = riskOrder[b.riskLevel] - riskOrder[a.riskLevel];
if (riskDiff !== 0)
return riskDiff;
return b.confidence - a.confidence;
});
const averageEntropy = totalStringsAnalyzed > 0 ? totalEntropy / totalStringsAnalyzed : 0;
return {
suspiciousStrings: suspiciousStrings.slice(0, 100), // Limit results for performance
overallRisk,
totalHighEntropyStrings: suspiciousStrings.length,
recommendations,
statistics: {
totalStringsAnalyzed,
averageEntropy,
highEntropyThreshold: this.highEntropyThreshold
}
};
}
async analyzeFile(file) {
const suspiciousStrings = [];
let totalStrings = 0;
let totalEntropy = 0;
try {
const filePath = path.join(this.projectRoot, file);
const content = await fs.readFile(filePath, 'utf-8');
const lines = content.split('\n');
for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
const line = lines[lineIndex];
const lineResults = this.analyzeLine(line, file, lineIndex + 1);
suspiciousStrings.push(...lineResults.suspiciousStrings);
totalStrings += lineResults.totalStrings;
totalEntropy += lineResults.totalEntropy;
}
}
catch (error) {
// Skip files that can't be read
}
return { suspiciousStrings, totalStrings, totalEntropy };
}
analyzeLine(line, file, lineNumber) {
const suspiciousStrings = [];
let totalStrings = 0;
let totalEntropy = 0;
// Extract different types of string literals
const stringPatterns = [
// Single and double quoted strings
/['"]([^'"\\]|\\.){16,}['"]/g,
// Template literals
/`([^`\\]|\\.){16,}`/g,
// Environment variable values
/=\s*(['"]?)([^'"\\]|\\.){16,}\1/g,
// URL-like strings
/https?:\/\/[^\s'"]{16,}/g,
// Base64-like strings (at least 16 chars, mostly alphanumeric with +/=)
/\b[A-Za-z0-9+/]{20,}={0,2}\b/g,
// Hex strings (at least 32 chars)
/\b[a-fA-F0-9]{32,}\b/g,
// JWT-like tokens (three base64 parts separated by dots)
/\b[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g
];
for (const pattern of stringPatterns) {
let match;
while ((match = pattern.exec(line)) !== null) {
const fullMatch = match[0];
const extractedString = this.extractStringContent(fullMatch);
if (extractedString.length >= this.minStringLength) {
totalStrings++;
const entropy = this.calculateEntropy(extractedString);
totalEntropy += entropy;
if (entropy >= this.highEntropyThreshold) {
const type = this.classifySecret(extractedString, line);
const confidence = this.calculateConfidence(extractedString, line, entropy, type);
const riskLevel = this.calculateRiskLevel(type, confidence, entropy);
// Only include high-confidence or high-risk findings
if (confidence >= 0.3 || riskLevel === 'critical') {
suspiciousStrings.push({
string: this.truncateForSecurity(extractedString),
file,
line: lineNumber,
entropy,
type,
confidence,
context: this.extractContext(line, match.index || 0),
riskLevel
});
}
}
}
}
}
return { suspiciousStrings, totalStrings, totalEntropy };
}
extractStringContent(str) {
// Remove quotes and other delimiters
if (str.match(/^['"`]/)) {
return str.slice(1, -1);
}
// For URLs, return as-is
if (str.startsWith('http')) {
return str;
}
// For key=value, extract value
const kvMatch = str.match(/=\s*(['"]?)(.+)\1$/);
if (kvMatch) {
return kvMatch[2];
}
return str;
}
calculateEntropy(str) {
if (str.length === 0)
return 0;
const frequencies = {};
// Count character frequencies
for (const char of str) {
frequencies[char] = (frequencies[char] || 0) + 1;
}
// Calculate Shannon entropy
let entropy = 0;
const length = str.length;
for (const count of Object.values(frequencies)) {
const probability = count / length;
entropy -= probability * Math.log2(probability);
}
return entropy;
}
classifySecret(str, context) {
const lowerStr = str.toLowerCase();
const lowerContext = context.toLowerCase();
// Context-based classification (most reliable)
if (lowerContext.match(/\b(api[_-]?key|apikey)\b/))
return 'api_key';
if (lowerContext.match(/\b(password|pwd|pass)\b/))
return 'password';
if (lowerContext.match(/\b(token|auth[_-]?token|access[_-]?token|bearer)\b/))
return 'token';
if (lowerContext.match(/\b(secret|private[_-]?key|secret[_-]?key)\b/))
return 'crypto_key';
if (lowerContext.match(/\b(database[_-]?url|db[_-]?url|connection[_-]?string)\b/))
return 'database_url';
if (lowerContext.match(/\b(jwt[_-]?secret|signing[_-]?key)\b/))
return 'jwt_secret';
// Pattern-based classification
if (str.match(/^[A-Za-z0-9+/]{40,}={0,2}$/)) {
// Base64-encoded, could be API key or secret
if (str.length >= 64)
return 'api_key';
return 'crypto_key';
}
if (str.match(/^[a-f0-9]{32,}$/)) {
// Hex string, likely crypto key
return 'crypto_key';
}
if (str.match(/^[A-Za-z0-9._-]{100,}$/)) {
// Very long alphanumeric, likely JWT or token
return 'token';
}
if (str.match(/^postgres:\/\/|^mysql:\/\/|^mongodb:\/\//)) {
return 'database_url';
}
// JWT pattern (three base64 parts)
if (str.match(/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/)) {
return 'token';
}
// Length-based heuristics
if (str.length >= 128)
return 'token';
if (str.length >= 64)
return 'api_key';
if (str.length >= 32)
return 'crypto_key';
return 'unknown';
}
calculateConfidence(str, context, entropy, type) {
let confidence = 0;
// Entropy-based confidence (higher entropy = more random = more likely to be secret)
confidence += Math.min(entropy / 6, 0.3);
// Length-based confidence
if (str.length >= 32)
confidence += 0.1;
if (str.length >= 64)
confidence += 0.1;
if (str.length >= 128)
confidence += 0.1;
// Context-based confidence (strong indicators)
const lowerContext = context.toLowerCase();
const strongIndicators = [
'secret', 'key', 'password', 'token', 'auth', 'api', 'private',
'credential', 'jwt', 'bearer', 'database', 'connection'
];
for (const indicator of strongIndicators) {
if (lowerContext.includes(indicator)) {
confidence += 0.15;
break;
}
}
// Pattern-based confidence
if (str.match(/^[A-Za-z0-9+/]{40,}={0,2}$/))
confidence += 0.15; // Base64
if (str.match(/^[a-f0-9]{32,}$/))
confidence += 0.15; // Hex
if (str.match(/^[A-Za-z0-9._-]{100,}$/))
confidence += 0.1; // Long token
// Type-specific confidence boosts
if (type !== 'unknown')
confidence += 0.1;
if (type === 'api_key' || type === 'password' || type === 'crypto_key')
confidence += 0.1;
// Reduce confidence for common patterns that are likely false positives
if (str.match(/^[0-9]+$/))
confidence -= 0.2; // Pure numbers
if (str.match(/^[a-zA-Z]+$/))
confidence -= 0.1; // Pure letters
if (str.includes('example') || str.includes('test') || str.includes('demo'))
confidence -= 0.3;
return Math.max(0, Math.min(1, confidence));
}
calculateRiskLevel(type, confidence, entropy) {
// Critical risk for high-confidence secrets
if (confidence >= 0.8 && ['password', 'api_key', 'crypto_key', 'database_url'].includes(type)) {
return 'critical';
}
// High risk for moderate confidence or critical types
if (confidence >= 0.6 || ['password', 'database_url'].includes(type)) {
return 'high';
}
// Medium risk for moderate entropy and confidence
if (confidence >= 0.4 && entropy >= 5.0) {
return 'medium';
}
return 'low';
}
extractContext(line, matchIndex) {
const contextLength = 30;
const start = Math.max(0, matchIndex - contextLength);
const end = Math.min(line.length, matchIndex + contextLength);
let context = line.substring(start, end);
// Add ellipsis if truncated
if (start > 0)
context = '...' + context;
if (end < line.length)
context = context + '...';
return context.trim();
}
truncateForSecurity(str) {
if (str.length <= 20)
return str;
return str.substring(0, 10) + '...' + str.substring(str.length - 6);
}
calculateOverallRisk(suspiciousStrings) {
const critical = suspiciousStrings.filter(s => s.riskLevel === 'critical').length;
const high = suspiciousStrings.filter(s => s.riskLevel === 'high').length;
const medium = suspiciousStrings.filter(s => s.riskLevel === 'medium').length;
if (critical > 0)
return 'critical';
if (high >= 3)
return 'high';
if (high >= 1 || medium >= 5)
return 'medium';
return 'low';
}
generateRecommendations(suspiciousStrings) {
const recommendations = [];
if (suspiciousStrings.length === 0) {
recommendations.push('✅ No high-entropy strings detected - good secret management practices');
return recommendations;
}
const critical = suspiciousStrings.filter(s => s.riskLevel === 'critical').length;
const high = suspiciousStrings.filter(s => s.riskLevel === 'high').length;
if (critical > 0) {
recommendations.push('🚨 CRITICAL: Remove hardcoded secrets from source code immediately', '🔐 Use environment variables or secret management services', '🔄 Rotate any exposed credentials as they may be compromised');
}
if (high > 0) {
recommendations.push('⚠️ Review and remove potential secrets from codebase', '📋 Implement pre-commit hooks to prevent secret commits', '🔍 Scan git history for previously committed secrets');
}
// Type-specific recommendations
const types = [...new Set(suspiciousStrings.map(s => s.type))];
if (types.includes('api_key')) {
recommendations.push('🔑 Store API keys in environment variables or secure vaults');
}
if (types.includes('password')) {
recommendations.push('🔒 Never hardcode passwords - use secure authentication flows');
}
if (types.includes('database_url')) {
recommendations.push('🗄️ Move database connection strings to environment configuration');
}
if (types.includes('jwt_secret')) {
recommendations.push('🎫 Use strong, randomly generated JWT signing keys stored securely');
}
// General recommendations
recommendations.push('🛡️ Implement secret scanning in CI/CD pipelines', '📚 Train team on secure coding practices', '🔄 Regular security audits and secret rotation');
return recommendations.slice(0, 10); // Limit recommendations
}
}
//# sourceMappingURL=EntropyAnalyzer.js.map