UNPKG

secure-scan-js

Version:

A JavaScript implementation of Yelp's detect-secrets tool - no Python required

963 lines (865 loc) 29.1 kB
#!/usr/bin/env node const detectSecrets = require("../src/index"); const { program } = require("commander"); const ora = require("ora"); const chalk = require("chalk"); const fs = require("fs"); const path = require("path"); const os = require("os"); const auth = require("../src/auth-web"); const { runGitleaksScan, scanRemoteRepository, scanGitHistory, } = require("../src/gitleaks"); // Read version from package.json dynamically function getVersion() { try { const packageJsonPath = path.join(__dirname, "../../package.json"); const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); return packageJson.version; } catch (error) { console.warn("Could not read version from package.json, using fallback"); return "unknown"; } } // Debug function to log file operations function debugLog(message, error = false) { const timestamp = new Date().toISOString(); const logMessage = `[${timestamp}] ${message}`; if (error) { console.error(chalk.red(logMessage)); } else { console.log(chalk.blue(logMessage)); } } // Format scan results for display function formatResults(results) { let output = ""; // Filter out any results from the output file itself if passed let filteredResults = results; if (results._outputPath) { const outputFileName = path.basename(results._outputPath); filteredResults = { ...results, secrets: results.secrets.filter((secret) => { // Filter out any results from the output file return !( secret.file === outputFileName || secret.file.endsWith(`/${outputFileName}`) || secret.file.endsWith(`\\${outputFileName}`) ); }), }; } if ( filteredResults.secrets.length === 0 && filteredResults.missed_secrets.length === 0 ) { return chalk.green("\nNo secrets detected!"); } if (filteredResults.truncated) { output += chalk.yellow( "\nNote: Some files were truncated due to size limits" ); } if (filteredResults.secrets.length > 0) { output += chalk.red( `\nDetected ${filteredResults.secrets.length} secret(s):\n` ); // Group secrets by file const fileGroups = filteredResults.secrets.reduce((groups, secret) => { if (!groups[secret.file]) { groups[secret.file] = []; } groups[secret.file].push(secret); return groups; }, {}); // Sort files alphabetically Object.keys(fileGroups) .sort() .forEach((file) => { output += chalk.cyan(`\nFile: ${file}`); // Sort secrets by line number fileGroups[file] .sort((a, b) => a.line - b.line) .forEach((secret) => { output += `\n Line ${secret.line}:`; output += `\n Types: ${chalk.yellow(secret.types.join(", "))}`; if (secret.hashed_secret) { output += `\n Fingerprint: ${chalk.gray( secret.hashed_secret )}`; } if (secret.author) { output += `\n Author: ${chalk.gray(secret.author)}`; } if (secret.email) { output += `\n Email: ${chalk.gray(secret.email)}`; } if (secret.date) { output += `\n Date: ${chalk.gray(secret.date)}`; } if (secret.commit) { output += `\n Commit: ${chalk.gray(secret.commit)}`; } if (secret.message) { output += `\n Message: ${chalk.gray(secret.message)}`; } output += `\n DetectedBy: ${chalk.gray(secret.detectedBy || 'unknown')}`; if (secret.is_false_positive) { output += chalk.gray("\n [Likely False Positive]"); } if (secret.file.includes("node_modules")) { output += chalk.yellow( "\n [Dependency Code - Not Your Source]" ); } if (secret.file.includes(".next")) { output += chalk.yellow( "\n [Next.js Build Output - Not Your Source]" ); } }); }); } if (filteredResults.missed_secrets.length > 0) { output += chalk.yellow( `\n\nPotentially Missed Secrets (${filteredResults.missed_secrets.length}):\n` ); const groupedMissed = filteredResults.missed_secrets.reduce( (groups, secret) => { if (!groups[secret.file]) { groups[secret.file] = []; } groups[secret.file].push(secret); return groups; }, {} ); Object.keys(groupedMissed) .sort() .forEach((file) => { output += chalk.cyan(`\nFile: ${file}`); groupedMissed[file] .sort((a, b) => a.line - b.line) .forEach((secret) => { output += `\n Line ${secret.line}: ${secret.type}`; }); }); } return output; } // Save results to file with requested format function saveResults(results, outputPath) { try { // Get file extension to determine format const ext = path.extname(outputPath).toLowerCase(); // Make sure we filter out any results from the output file itself const outputFileName = path.basename(outputPath); const filteredResults = { ...results, _outputPath: outputPath, // Add output path to results for formatResults secrets: results.secrets.filter((secret) => { // Filter out any results from the output file if ( secret.file === outputFileName || secret.file.endsWith(`/${outputFileName}`) || secret.file.endsWith(`\\${outputFileName}`) ) { debugLog( `Filtered out secret from output file: ${secret.file}:${secret.line}` ); return false; } return true; }), }; // Report filtered findings if (results.secrets.length !== filteredResults.secrets.length) { console.log( chalk.yellow( `\nRemoved ${results.secrets.length - filteredResults.secrets.length } false positives from ${outputFileName}` ) ); } // Create directory if it doesn't exist const outputDir = path.dirname(outputPath); if (!fs.existsSync(outputDir)) { fs.mkdirSync(outputDir, { recursive: true }); } let outputContent = ""; // Format based on file extension switch (ext) { case ".json": outputContent = JSON.stringify(filteredResults, null, 2); break; case ".csv": // Create CSV header outputContent = "File,Line,Type,Fingerprint,Commit,Author,Email,Date,Message,DetectedBy,Is False Positive,Is Dependency,Is Build File\n"; // Add each secret as a row filteredResults.secrets.forEach((secret) => { const isDependency = secret.file.includes("node_modules"); const isBuildFile = secret.file.includes(".next"); outputContent += `"${secret.file}",${secret.line},"${secret.types.join("; ")}"`; outputContent += `,"${secret.hashed_secret || ""}","${secret.commit || ""}","${secret.author || "" }","${secret.email || ""}","${secret.date || ""}","${secret.message || "" }","${secret.detectedBy || ""}","${secret.is_false_positive}","${isDependency}","${isBuildFile}"\n`; }); break; case ".txt": // Simple text format outputContent = `Secrets Detection Results\n${"=".repeat(25)}\n\n`; outputContent += `Total secrets found: ${filteredResults.secrets.length}\n`; // Count node_modules secrets separately const dependencySecrets = filteredResults.secrets.filter((s) => s.file.includes("node_modules") ).length; const nextjsSecrets = filteredResults.secrets.filter((s) => s.file.includes(".next") ).length; const totalNonSourceSecrets = dependencySecrets + nextjsSecrets; if (totalNonSourceSecrets > 0) { outputContent += `Secrets in dependencies/build files: ${totalNonSourceSecrets}\n`; if (dependencySecrets > 0) { outputContent += ` - Secrets in node_modules: ${dependencySecrets}\n`; } if (nextjsSecrets > 0) { outputContent += ` - Secrets in Next.js build files: ${nextjsSecrets}\n`; } outputContent += `Secrets in your code: ${filteredResults.secrets.length - totalNonSourceSecrets }\n`; } outputContent += `Potentially missed secrets: ${filteredResults.missed_secrets.length}\n\n`; filteredResults.secrets.forEach((secret) => { outputContent += `File: ${secret.file}\n`; outputContent += `Line: ${secret.line}\n`; outputContent += `Types: ${secret.types.join(", ")}\n`; if (secret.hashed_secret) outputContent += `Fingerprint: ${secret.hashed_secret}\n`; if (secret.commit) outputContent += `Commit: ${secret.commit}\n`; if (secret.author) outputContent += `Author: ${secret.author}\n`; if (secret.date) outputContent += `Date: ${secret.date}\n`; outputContent += `Is False Positive: ${secret.is_false_positive}\n\n`; }); break; default: // Default to JSON if extension not recognized outputContent = JSON.stringify(filteredResults, null, 2); } fs.writeFileSync(outputPath, outputContent); console.log(chalk.green(`\nResults saved to: ${outputPath}`)); } catch (outputError) { debugLog(`Output error: ${outputError.message}`, true); console.error(chalk.red(`Failed to save results: ${outputError.message}`)); } } // Authentication middleware function checkAuth() { const status = auth.getTokenStatus(); if (!status.valid) { console.log(chalk.yellow(`\n⚠️ ${status.message}`)); console.log( chalk.cyan('\nPlease run "yarn custom:login" to authenticate.') ); process.exit(1); } console.log( chalk.green( `\n✅ Authentication verified. Token valid until ${status.expiresAt}` ) ); return true; } program .name("secure-scan-js") .description( "A JavaScript implementation of detect-secrets with Gitleaks integration" ) .version(getVersion()); program .command("login") .description("Authenticate with the service") .action(() => { // Use the new auth-web login flow auth .login() .then((tokenData) => { console.log(chalk.green("\n✅ Authentication successful!")); console.log( chalk.green( `Token will expire in ${auth.config.tokenExpirationSeconds} seconds` ) ); console.log( chalk.green( `Expiration: ${new Date( tokenData.expiresAt * 1000 ).toLocaleString()}` ) ); process.exit(0); }) .catch((error) => { console.error(chalk.red("\n❌ Authentication failed:"), error.message); process.exit(1); }); }); program .command("logout") .description("Log out from the service") .action(() => { const result = auth.logout(); if (result) { console.log(chalk.green("\n✅ Successfully logged out!")); console.log(chalk.green("Token has been removed.")); } else { console.log(chalk.yellow("\n⚠️ No active session found.")); } process.exit(0); }); program .command("scan <target>") .description( "Scan a local directory, file, remote repository, or git commits for secrets" ) .option( "-o, --output <path>", "Output path for results (default: ./scan-results.json)" ) .option( "-s, --scanner <scanner>", "Scanner to use (detect-secrets, gitleaks, or both)", "both" ) .option( "--max-file-size <size>", "Maximum file size in bytes (0 for no limit)", "0" ) .option("--exclude-dirs <dirs...>", "Directories to exclude") .option("--exclude-files <files...>", "File patterns to exclude") .option("--check-missed", "Check for potentially missed secrets") .option("--verbose", "Show additional information") .option("--remote", "Scan a remote repository (target should be a git URL)") .option( "--branch <branch>", "Branch to check out for remote repository scanning" ) .option("--commit <hash>", "Scan a specific commit hash") .option("--all-commits", "Scan all git commit history") .option("--from-commit <hash>", "Starting commit hash for git history scan") .option("--to-commit <hash>", "Ending commit hash for git history scan") .option( "--disable-git-blame", "Disable git blame information gathering", false ) .option( "--git-repo-path <path>", "Specify git repository path for external scans" ) .option( "--include-node-modules", "Include node_modules in the scan (not recommended)", false ) .option( "--skip-auth-check", "Skip authentication check (not recommended)", false ) .action(async (target, options) => { // Add authentication check unless explicitly skipped if (!options.skipAuthCheck) { checkAuth(); } else { console.log( chalk.yellow( "\n⚠️ Authentication check skipped. Proceeding without verification." ) ); } const spinner = ora("Initializing scan...").start(); try { // Determine output path (default to scan-results.json in current directory) const outputPath = options.output || "./scan-results.json"; // Clean scan-results.json file if it exists before starting the scan if (fs.existsSync(outputPath)) { try { debugLog(`Cleaning existing scan results file: ${outputPath}`); fs.unlinkSync(outputPath); console.log( chalk.blue(`Cleaned existing scan results file: ${outputPath}`) ); } catch (cleanError) { debugLog( `Failed to clean scan results file: ${cleanError.message}`, true ); console.warn( chalk.yellow( `Warning: Failed to clean scan results file: ${cleanError.message}` ) ); } } // Check if target contains node_modules if (target.includes("node_modules") && !options.includeNodeModules) { console.log( chalk.yellow( "\nWarning: You are attempting to scan a node_modules directory." ) ); console.log( chalk.yellow("Scanning node_modules is generally not recommended as:") ); console.log( chalk.yellow("1. These files are not part of your source code") ); console.log( chalk.yellow( "2. They may contain false positives from test/example files" ) ); console.log( chalk.yellow( "3. Scanning can be very slow due to the large number of files" ) ); console.log( chalk.yellow("\nFiles in node_modules will be excluded by default.") ); console.log( chalk.yellow( "If you want to scan node_modules, use the --include-node-modules flag." ) ); } // Logging scan configuration debugLog(`Current working directory: ${process.cwd()}`); debugLog(`Target: ${target}`); debugLog(`Using scanner(s): ${options.scanner}`); debugLog(`Output path: ${outputPath}`); // Git blame configuration if (options.disableGitBlame) { debugLog("Git blame information gathering is disabled"); } else if (options.gitRepoPath) { debugLog(`Using custom git repository path: ${options.gitRepoPath}`); } // Build scan options const scanOptions = { maxFileSize: parseInt(options.maxFileSize), excludeDirs: options.excludeDirs || [], excludeFiles: options.excludeFiles || [], checkMissed: options.checkMissed, verbose: options.verbose, enrichWithGitInfo: !options.disableGitBlame, gitRepoPath: options.gitRepoPath, includeNodeModules: options.includeNodeModules, }; // Always exclude node_modules unless explicitly included if ( !options.includeNodeModules && !scanOptions.excludeDirs.includes("node_modules") ) { scanOptions.excludeDirs.push("node_modules"); debugLog("Automatically excluding node_modules directory from scan"); } // Always exclude scan-results.json file const outputFileName = path.basename(outputPath); if (!scanOptions.excludeFiles.includes(outputFileName)) { scanOptions.excludeFiles.push(outputFileName); debugLog( `Automatically excluding scan results file (${outputFileName}) from scan` ); } // Mode detection const isRemote = options.remote || target.startsWith("http://") || target.startsWith("https://") || target.startsWith("git@"); const isSpecificCommit = options.commit && !options.allCommits; const isAllCommits = options.allCommits || (options.fromCommit && !options.commit); if (isRemote) { spinner.text = `Cloning and scanning repository: ${target}`; if (options.branch) { debugLog(`Branch: ${options.branch}`); spinner.text = `Cloning and scanning branch: ${options.branch}`; } if (options.allCommits) { debugLog(`Scanning all commits`); spinner.text = `Cloning and scanning all commit history`; } else if (options.commit) { debugLog(`Scanning specific commit: ${options.commit}`); spinner.text = `Cloning and scanning commit: ${options.commit}`; } // Create temporary directory for cloning const tmpDir = path.join(os.tmpdir(), `secure-scan-js-${Date.now()}`); try { // Clone the repository await new Promise((resolve, reject) => { const cloneArgs = ["clone", target, tmpDir]; if (options.branch) { cloneArgs.push("--branch", options.branch); } debugLog(`Cloning repository to ${tmpDir}`); const { spawn } = require("child_process"); const gitClone = spawn("git", cloneArgs); gitClone.on("close", (code) => { if (code === 0) resolve(); else reject( new Error(`Failed to clone repository with code ${code}`) ); }); }); // If both scanners are requested, run detect-secrets first let detectSecretsResults = null; if (options.scanner === "both") { try { debugLog("Running detect-secrets on cloned repository..."); spinner.text = "Running detect-secrets on cloned repository..."; detectSecretsResults = await detectSecrets.scanDirectory( tmpDir, scanOptions ); debugLog( `detect-secrets found ${detectSecretsResults.secrets.length} secrets` ); // Normalize file paths from detect-secrets detectSecretsResults.secrets.forEach((secret) => { // Remove the temporary directory prefix from the file path if (secret.file.startsWith(tmpDir)) { // Replace OS path separator with forward slash for consistency secret.file = secret.file .substring(tmpDir.length) .replace(/^[\\\/]+/, "") // Remove leading slashes .replace(/\\/g, "/"); // Replace Windows backslashes with forward slashes } }); } catch (detectSecretsError) { debugLog( `Failed to run detect-secrets: ${detectSecretsError.message}`, true ); console.warn( chalk.yellow( `Warning: Failed to run detect-secrets: ${detectSecretsError.message}` ) ); console.warn( chalk.yellow("Will continue with Gitleaks scan only") ); } } // Run Gitleaks for the remote scan debugLog("Running Gitleaks on remote repository"); spinner.text = `Scanning with Gitleaks: ${target}`; // Remote repository scanning with Gitleaks const remoteResults = await scanRemoteRepository( target, options.branch, scanOptions ); // If specific commit or all commits requested, add git history scan if (isSpecificCommit || isAllCommits) { let gitResults; if (isSpecificCommit) { // Scan specific commit spinner.text = `Scanning specific commit: ${options.commit}`; gitResults = await scanGitHistory( tmpDir, options.commit, options.commit, scanOptions ); } else { // Scan all commits or commit range spinner.text = `Scanning git history`; gitResults = await scanGitHistory( tmpDir, options.fromCommit, options.toCommit, scanOptions ); } // Merge results from git history scan remoteResults.secrets = [ ...remoteResults.secrets, ...gitResults.secrets, ]; } // Merge results if we have detect-secrets results if (detectSecretsResults) { // Merge results, preserving uniqueness const uniqueSecrets = new Map(); // Add Gitleaks results first remoteResults.secrets.forEach((secret) => { const key = `${secret.file}:${secret.line}`; uniqueSecrets.set(key, secret); }); // Add detect-secrets results detectSecretsResults.secrets.forEach((secret) => { const key = `${secret.file}:${secret.line}`; if (uniqueSecrets.has(key)) { // Merge with existing secret const existing = uniqueSecrets.get(key); existing.types = [ ...new Set([...existing.types, ...secret.types]), ]; // If the existing secret doesn't have a hash but this one does, use it if (!existing.hashed_secret && secret.hashed_secret) { existing.hashed_secret = secret.hashed_secret; } } else { // Add new secret uniqueSecrets.set(key, secret); } }); // Update results remoteResults.secrets = Array.from(uniqueSecrets.values()); remoteResults.missed_secrets = [ ...remoteResults.missed_secrets, ...detectSecretsResults.missed_secrets, ]; debugLog("Successfully merged results from both scanners"); } // Enrich with git blame information for remote repositories if (!options.disableGitBlame && remoteResults.secrets.length > 0) { spinner.text = "Enriching results with git blame information..."; debugLog("Adding git blame information to results"); try { const { execSync } = require("child_process"); // Process each secret to add git blame info for (let secret of remoteResults.secrets) { try { // Only process if file exists in the repo and commit/author are unknown if ( secret.author === "Unknown" || secret.commit === "Unknown" ) { const filePath = path.join(tmpDir, secret.file); if (fs.existsSync(filePath)) { // Get git blame information for the specific line const blameCommand = `git -C "${tmpDir}" blame -L ${secret.line},${secret.line} --porcelain "${secret.file}"`; const blameOutput = execSync(blameCommand, { encoding: "utf8", }); // Parse blame output const authorMatch = blameOutput.match(/author (.+)/); const emailMatch = blameOutput.match(/author-mail <(.+)>/); const dateMatch = blameOutput.match(/author-time ([0-9]+)/); const commitMatch = blameOutput.match(/^([a-f0-9]{40})/m); const summaryCommand = `git -C "${tmpDir}" show -s --format=%s ${commitMatch ? commitMatch[1] : "" }`; const summaryOutput = commitMatch ? execSync(summaryCommand, { encoding: "utf8" }).trim() : "Unknown"; // Update secret with git information if (authorMatch) secret.author = authorMatch[1]; if (emailMatch) secret.email = emailMatch[1]; if (dateMatch) { const timestamp = parseInt(dateMatch[1]) * 1000; secret.date = new Date(timestamp).toISOString(); } if (commitMatch) secret.commit = commitMatch[1]; if (summaryOutput !== "Unknown") secret.message = summaryOutput; debugLog( `Added git blame info for ${secret.file}:${secret.line}` ); } } } catch (blameError) { debugLog( `Failed to get blame info for ${secret.file}:${secret.line}: ${blameError.message}`, true ); } } debugLog("Completed adding git blame information"); } catch (gitError) { debugLog( `Error enriching results with git info: ${gitError.message}`, true ); console.warn( chalk.yellow( `Warning: Could not enrich results with git information: ${gitError.message}` ) ); } } spinner.stop(); // Save results saveResults(remoteResults, outputPath); // Add output path for filtering in formatResults remoteResults._outputPath = outputPath; // Display results console.log(formatResults(remoteResults)); // Exit with error code if secrets were found if (remoteResults.secrets.length > 0) { process.exit(1); } } finally { // Clean up try { if (fs.existsSync(tmpDir)) { fs.rmSync(tmpDir, { recursive: true, force: true }); } } catch (cleanupError) { console.warn( `Failed to clean up temporary directory: ${cleanupError}` ); } } } else if (isSpecificCommit || isAllCommits) { // Git history scanning of local repository spinner.text = isSpecificCommit ? `Scanning specific commit: ${options.commit}` : `Scanning git history`; // Log commit range if specified if (options.fromCommit) debugLog(`From commit: ${options.fromCommit}`); if (options.toCommit) debugLog(`To commit: ${options.toCommit}`); let results; if (options.scanner === "both" || options.scanner === "gitleaks") { // Scan with Gitleaks if (isSpecificCommit) { results = await scanGitHistory( target, options.commit, options.commit ); } else { results = await scanGitHistory( target, options.fromCommit, options.toCommit ); } // Add detect-secrets scan if both scanners are requested if (options.scanner === "both") { const detectSecretsResults = await detectSecrets.scanDirectory( target, scanOptions ); // Merge results, preserving uniqueness const uniqueSecrets = new Map(); // Add existing secrets first results.secrets.forEach((secret) => { const key = `${secret.file}:${secret.line}`; uniqueSecrets.set(key, secret); }); // Add detect-secrets results detectSecretsResults.secrets.forEach((secret) => { const key = `${secret.file}:${secret.line}`; if (uniqueSecrets.has(key)) { // Merge with existing secret const existing = uniqueSecrets.get(key); existing.types = [ ...new Set([...existing.types, ...secret.types]), ]; } else { // Add new secret uniqueSecrets.set(key, secret); } }); // Update results results.secrets = Array.from(uniqueSecrets.values()); results.missed_secrets = [ ...results.missed_secrets, ...detectSecretsResults.missed_secrets, ]; } } else { // Scan with detect-secrets only results = await detectSecrets.scanDirectory(target, scanOptions); } spinner.stop(); // Save results saveResults(results, outputPath); // Add output path for filtering in formatResults results._outputPath = outputPath; // Display results console.log(formatResults(results)); // Exit with error code if secrets were found if (results.secrets.length > 0) { process.exit(1); } } else { // Local file/directory scanning spinner.text = `Scanning ${target}`; let results; let gitleaksError = null; try { if (options.scanner === "both") { // Use both scanners console.log( chalk.blue( "Starting scan with both detect-secrets and Gitleaks..." ) ); results = await detectSecrets.scanWithBothScanners( target, scanOptions ); console.log(chalk.green("Successfully used both scanners")); } else if (options.scanner === "gitleaks") { // Use Gitleaks only console.log(chalk.blue("Starting scan with Gitleaks only...")); results = await runGitleaksScan(target); } else { // Use detect-secrets only console.log( chalk.blue("Starting scan with detect-secrets only...") ); results = await detectSecrets.scanDirectory(target, scanOptions); } } catch (scanError) { gitleaksError = scanError; // Check if the error is related to gitleaks not being installed if (scanError.message.includes("Gitleaks is not installed")) { console.log(chalk.yellow(`\nWarning: ${scanError.message}`)); console.log( chalk.yellow( "Please install Gitleaks from https://github.com/zricethezav/gitleaks#installation" ) ); } else { console.log(chalk.yellow(`\nWarning: ${scanError.message}`)); } // If the combined scanner failed, try using just detect-secrets if (options.scanner === "both") { console.log(chalk.yellow("Falling back to detect-secrets only...")); results = await detectSecrets.scanDirectory(target, scanOptions); } else { // Re-throw if it wasn't trying to use both scanners throw scanError; } } spinner.stop(); // Save results saveResults(results, outputPath); // Add output path for filtering in formatResults results._outputPath = outputPath; // Display results console.log(formatResults(results)); // Exit with error code if secrets were found if (results.secrets.length > 0) { process.exit(1); } } } catch (error) { spinner.fail("Scan failed"); debugLog(`Scan error: ${error.message}`, true); console.error(chalk.red(error.message)); process.exit(1); } }); program.parse(process.argv);