detect-secrets-js
Version:
A JavaScript implementation of Yelp's detect-secrets tool - no Python required
815 lines (728 loc) • 28.3 kB
JavaScript
const { program } = require("commander");
const chalk = require("chalk");
const ora = require("ora");
const path = require("path");
const fs = require("fs");
const detectSecrets = require("../dist");
const {
runGitleaksScan,
scanRemoteRepository,
scanGitHistory,
} = require("../dist/gitleaks");
// Debug function to log file operations
function debugLog(message, error = false) {
if (!process.env.DETECT_SECRETS_DEBUG) return;
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] ${message}`;
if (error) {
console.error(chalk.red(logMessage));
} else {
console.log(chalk.blue(logMessage));
}
}
// Format results for display
function formatResults(results) {
let output = "";
if (results.secrets.length === 0 && results.missed_secrets.length === 0) {
return chalk.green("\nNo secrets detected!");
}
if (results.truncated) {
output += chalk.yellow(
"\nNote: Some files were truncated due to size limits"
);
}
if (results.secrets.length > 0) {
output += chalk.red(`\nDetected ${results.secrets.length} secret(s):\n`);
// Group secrets by file
const fileGroups = results.secrets.reduce((groups, secret) => {
if (!groups[secret.file]) {
groups[secret.file] = [];
}
groups[secret.file].push(secret);
return groups;
}, {});
// Sort files alphabetically
Object.keys(fileGroups)
.sort()
.forEach((file) => {
output += chalk.cyan(`\nFile: ${file}`);
// Sort secrets by line number
fileGroups[file]
.sort((a, b) => a.line - b.line)
.forEach((secret) => {
output += `\n Line ${secret.line}:`;
output += `\n Types: ${chalk.yellow(secret.types.join(", "))}`;
if (secret.hashed_secret) {
output += `\n Fingerprint: ${chalk.gray(
secret.hashed_secret
)}`;
}
if (secret.author) {
output += `\n Author: ${chalk.gray(secret.author)}`;
}
if (secret.email) {
output += `\n Email: ${chalk.gray(secret.email)}`;
}
if (secret.date) {
output += `\n Date: ${chalk.gray(secret.date)}`;
}
if (secret.commit) {
output += `\n Commit: ${chalk.gray(secret.commit)}`;
}
if (secret.message) {
output += `\n Message: ${chalk.gray(secret.message)}`;
}
if (secret.is_false_positive) {
output += chalk.gray("\n [Likely False Positive]");
}
if (secret.file.includes("node_modules")) {
output += chalk.yellow(
"\n [Dependency Code - Not Your Source]"
);
}
if (secret.file.includes(".next")) {
output += chalk.yellow(
"\n [Next.js Build Output - Not Your Source]"
);
}
});
});
}
if (results.missed_secrets.length > 0) {
output += chalk.yellow(
`\n\nPotentially Missed Secrets (${results.missed_secrets.length}):\n`
);
const groupedMissed = results.missed_secrets.reduce((groups, secret) => {
if (!groups[secret.file]) {
groups[secret.file] = [];
}
groups[secret.file].push(secret);
return groups;
}, {});
Object.keys(groupedMissed)
.sort()
.forEach((file) => {
output += chalk.cyan(`\nFile: ${file}`);
groupedMissed[file]
.sort((a, b) => a.line - b.line)
.forEach((secret) => {
output += `\n Line ${secret.line}: ${secret.type}`;
});
});
}
return output;
}
// Save results to file with requested format
function saveResults(results, outputPath) {
try {
// Get file extension to determine format
const ext = path.extname(outputPath).toLowerCase();
// Create directory if it doesn't exist
const outputDir = path.dirname(outputPath);
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
let outputContent = "";
// Format based on file extension
switch (ext) {
case ".json":
outputContent = JSON.stringify(results, null, 2);
break;
case ".csv":
// Create CSV header
outputContent =
"File,Line,Type,Fingerprint,Commit,Author,Email,Date,Message,Is False Positive,Is Dependency,Is Build File\n";
// Add each secret as a row
results.secrets.forEach((secret) => {
const isDependency = secret.file.includes("node_modules");
const isBuildFile = secret.file.includes(".next");
outputContent += `"${secret.file}",${
secret.line
},"${secret.types.join("; ")}"`;
outputContent += `,"${secret.hashed_secret || ""}","${
secret.commit || ""
}","${secret.author || ""}","${secret.email || ""}","${
secret.date || ""
}","${secret.message || ""}","${
secret.is_false_positive
}","${isDependency}","${isBuildFile}"\n`;
});
break;
case ".txt":
// Simple text format
outputContent = `Secrets Detection Results\n${"=".repeat(25)}\n\n`;
outputContent += `Total secrets found: ${results.secrets.length}\n`;
// Count node_modules secrets separately
const dependencySecrets = results.secrets.filter((s) =>
s.file.includes("node_modules")
).length;
const nextjsSecrets = results.secrets.filter((s) =>
s.file.includes(".next")
).length;
const totalNonSourceSecrets = dependencySecrets + nextjsSecrets;
if (totalNonSourceSecrets > 0) {
outputContent += `Secrets in dependencies/build files: ${totalNonSourceSecrets}\n`;
if (dependencySecrets > 0) {
outputContent += ` - Secrets in node_modules: ${dependencySecrets}\n`;
}
if (nextjsSecrets > 0) {
outputContent += ` - Secrets in Next.js build files: ${nextjsSecrets}\n`;
}
outputContent += `Secrets in your code: ${
results.secrets.length - totalNonSourceSecrets
}\n`;
}
outputContent += `Potentially missed secrets: ${results.missed_secrets.length}\n\n`;
results.secrets.forEach((secret) => {
outputContent += `File: ${secret.file}\n`;
outputContent += `Line: ${secret.line}\n`;
outputContent += `Types: ${secret.types.join(", ")}\n`;
if (secret.hashed_secret)
outputContent += `Fingerprint: ${secret.hashed_secret}\n`;
if (secret.commit) outputContent += `Commit: ${secret.commit}\n`;
if (secret.author) outputContent += `Author: ${secret.author}\n`;
if (secret.email) outputContent += `Email: ${secret.email}\n`;
if (secret.date) outputContent += `Date: ${secret.date}\n`;
if (secret.message) outputContent += `Message: ${secret.message}\n`;
outputContent += `Is False Positive: ${secret.is_false_positive}\n\n`;
});
break;
default:
// Default to JSON if extension not recognized
outputContent = JSON.stringify(results, null, 2);
}
fs.writeFileSync(outputPath, outputContent);
console.log(chalk.green(`\nResults saved to: ${outputPath}`));
} catch (outputError) {
console.error(chalk.red(`Failed to save results: ${outputError.message}`));
}
}
// Set up the CLI
program
.name("detect-secrets-js")
.description("JavaScript secret scanner with Gitleaks integration")
.version(require("../package.json").version);
program
.command("scan [target]")
.description(
"Scan a local directory, file, remote repository, or git commits for secrets"
)
.option(
"-o, --output <path>",
"Output path for results (default: ./scan-results.json)"
)
.option(
"-s, --scanner <scanner>",
"Scanner to use (detect-secrets, gitleaks, or both)",
"both"
)
.option(
"--max-file-size <size>",
"Maximum file size in bytes (0 for no limit)",
"0"
)
.option("--exclude-dirs <dirs...>", "Directories to exclude")
.option("--exclude-files <files...>", "File patterns to exclude")
.option("--check-missed", "Check for potentially missed secrets")
.option("--verbose", "Show additional information")
.option("--remote", "Scan a remote repository (target should be a git URL)")
.option(
"--branch <branch>",
"Branch to check out for remote repository scanning"
)
.option("--commit <hash>", "Scan a specific commit hash")
.option("--all-commits", "Scan all git commit history")
.option("--from-commit <hash>", "Starting commit hash for git history scan")
.option("--to-commit <hash>", "Ending commit hash for git history scan")
.option(
"--disable-git-blame",
"Disable git blame information gathering",
false
)
.option(
"--git-repo-path <path>",
"Specify git repository path for external scans"
)
.option(
"--include-node-modules",
"Include node_modules in the scan (not recommended)",
false
)
.action(async (target = process.cwd(), options) => {
const spinner = ora("Initializing scan...").start();
try {
// Initialize WebAssembly module if required
await detectSecrets.initialize();
// Determine output path (default to scan-results.json in current directory)
const outputPath = options.output || "./scan-results.json";
// Check if target contains node_modules
if (target.includes("node_modules") && !options.includeNodeModules) {
console.log(
chalk.yellow(
"\nWarning: You are attempting to scan a node_modules directory."
)
);
console.log(
chalk.yellow("Scanning node_modules is generally not recommended as:")
);
console.log(
chalk.yellow("1. These files are not part of your source code")
);
console.log(
chalk.yellow(
"2. They may contain false positives from test/example files"
)
);
console.log(
chalk.yellow(
"3. Scanning can be very slow due to the large number of files"
)
);
console.log(
chalk.yellow("\nFiles in node_modules will be excluded by default.")
);
console.log(
chalk.yellow(
"If you want to scan node_modules, use the --include-node-modules flag."
)
);
}
// Logging scan configuration
debugLog(`Current working directory: ${process.cwd()}`);
debugLog(`Target: ${target}`);
debugLog(`Using scanner(s): ${options.scanner}`);
debugLog(`Output path: ${outputPath}`);
// Git blame configuration
if (options.disableGitBlame) {
debugLog("Git blame information gathering is disabled");
} else if (options.gitRepoPath) {
debugLog(`Using custom git repository path: ${options.gitRepoPath}`);
}
// Build scan options
const scanOptions = {
maxFileSize: parseInt(options.maxFileSize),
excludeDirs: options.excludeDirs || [],
excludeFiles: options.excludeFiles || [],
checkMissed: options.checkMissed,
verbose: options.verbose,
enrichWithGitInfo: !options.disableGitBlame,
gitRepoPath: options.gitRepoPath,
includeNodeModules: options.includeNodeModules,
};
// Always exclude node_modules unless explicitly included
if (
!options.includeNodeModules &&
!scanOptions.excludeDirs.includes("node_modules")
) {
scanOptions.excludeDirs.push("node_modules");
debugLog("Automatically excluding node_modules directory from scan");
}
// Mode detection
const isRemote =
options.remote ||
target.startsWith("http://") ||
target.startsWith("https://") ||
target.startsWith("git@");
const isSpecificCommit = options.commit && !options.allCommits;
const isAllCommits =
options.allCommits || (options.fromCommit && !options.commit);
if (isRemote) {
spinner.text = `Cloning and scanning repository: ${target}`;
if (options.branch) {
debugLog(`Branch: ${options.branch}`);
spinner.text = `Cloning and scanning branch: ${options.branch}`;
}
if (options.allCommits) {
debugLog(`Scanning all commits`);
spinner.text = `Cloning and scanning all commit history`;
} else if (options.commit) {
debugLog(`Scanning specific commit: ${options.commit}`);
spinner.text = `Cloning and scanning commit: ${options.commit}`;
}
// Create temporary directory for cloning
const tmpDir = path.join(
require("os").tmpdir(),
`detect-secrets-js-${Date.now()}`
);
try {
// Clone the repository
await new Promise((resolve, reject) => {
const cloneArgs = ["clone", target, tmpDir];
if (options.branch) {
cloneArgs.push("--branch", options.branch);
}
debugLog(`Cloning repository to ${tmpDir}`);
const gitClone = require("child_process").spawn("git", cloneArgs);
gitClone.on("close", (code) => {
if (code === 0) resolve();
else
reject(
new Error(`Failed to clone repository with code ${code}`)
);
});
});
// If both scanners are requested, run detect-secrets first
let detectSecretsResults = null;
if (options.scanner === "both") {
try {
debugLog("Running detect-secrets on cloned repository...");
spinner.text = "Running detect-secrets on cloned repository...";
detectSecretsResults = await detectSecrets.scanDirectory(
tmpDir,
scanOptions
);
debugLog(
`detect-secrets found ${detectSecretsResults.secrets.length} secrets`
);
// Normalize file paths from detect-secrets
detectSecretsResults.secrets.forEach((secret) => {
// Remove the temporary directory prefix from the file path
if (secret.file.startsWith(tmpDir)) {
// Replace OS path separator with forward slash for consistency
secret.file = secret.file
.substring(tmpDir.length)
.replace(/^[\\\/]+/, "") // Remove leading slashes
.replace(/\\/g, "/"); // Replace Windows backslashes with forward slashes
}
});
} catch (detectSecretsError) {
debugLog(
`Failed to run detect-secrets: ${detectSecretsError.message}`,
true
);
console.warn(
chalk.yellow(
`Warning: Failed to run detect-secrets: ${detectSecretsError.message}`
)
);
console.warn(
chalk.yellow("Will continue with Gitleaks scan only")
);
}
}
// Run Gitleaks for the remote scan
debugLog("Running Gitleaks on remote repository");
spinner.text = `Scanning with Gitleaks: ${target}`;
// Remote repository scanning with Gitleaks
const remoteResults = await scanRemoteRepository(
target,
options.branch,
scanOptions
);
// If specific commit or all commits requested, add git history scan
if (isSpecificCommit || isAllCommits) {
let gitResults;
if (isSpecificCommit) {
// Scan specific commit
spinner.text = `Scanning specific commit: ${options.commit}`;
gitResults = await scanGitHistory(
tmpDir,
options.commit,
options.commit,
scanOptions
);
} else {
// Scan all commits or commit range
spinner.text = `Scanning git history`;
gitResults = await scanGitHistory(
tmpDir,
options.fromCommit,
options.toCommit,
scanOptions
);
}
// Merge results from git history scan
remoteResults.secrets = [
...remoteResults.secrets,
...gitResults.secrets,
];
}
// Merge results if we have detect-secrets results
if (detectSecretsResults) {
// Merge results, preserving uniqueness
const uniqueSecrets = new Map();
// Add Gitleaks results first
remoteResults.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
uniqueSecrets.set(key, secret);
});
// Add detect-secrets results
detectSecretsResults.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
if (uniqueSecrets.has(key)) {
// Merge with existing secret
const existing = uniqueSecrets.get(key);
existing.types = [
...new Set([...existing.types, ...secret.types]),
];
// If the existing secret doesn't have a hash but this one does, use it
if (!existing.hashed_secret && secret.hashed_secret) {
existing.hashed_secret = secret.hashed_secret;
}
} else {
// Add new secret
uniqueSecrets.set(key, secret);
}
});
// Update results
remoteResults.secrets = Array.from(uniqueSecrets.values());
remoteResults.missed_secrets = [
...remoteResults.missed_secrets,
...detectSecretsResults.missed_secrets,
];
debugLog("Successfully merged results from both scanners");
}
spinner.stop();
// Save results
saveResults(remoteResults, outputPath);
// Display results
console.log(formatResults(remoteResults));
// Exit with error code if secrets were found
if (remoteResults.secrets.length > 0) {
process.exit(1);
}
} finally {
// Clean up
try {
if (fs.existsSync(tmpDir)) {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
} catch (cleanupError) {
console.warn(
`Failed to clean up temporary directory: ${cleanupError}`
);
}
}
} else if (isSpecificCommit || isAllCommits) {
// Git history scanning of local repository
spinner.text = isSpecificCommit
? `Scanning specific commit: ${options.commit}`
: `Scanning git history`;
// Log commit range if specified
if (options.fromCommit) debugLog(`From commit: ${options.fromCommit}`);
if (options.toCommit) debugLog(`To commit: ${options.toCommit}`);
let results;
// Run appropriate scanner(s) based on option
if (options.scanner === "gitleaks" || options.scanner === "both") {
// Scan with Gitleaks
debugLog("Running Gitleaks for git history scan");
if (isSpecificCommit) {
results = await scanGitHistory(
target,
options.commit,
options.commit,
scanOptions
);
} else {
results = await scanGitHistory(
target,
options.fromCommit,
options.toCommit,
scanOptions
);
}
// Add detect-secrets scan if both scanners are requested
if (options.scanner === "both") {
debugLog("Running detect-secrets for git history scan");
try {
const detectSecretsResults = await detectSecrets.scanDirectory(
target,
scanOptions
);
// Normalize file paths from detect-secrets results
detectSecretsResults.secrets.forEach((secret) => {
// If it's a full path, normalize it to a relative path
// that matches the format from Gitleaks
if (path.isAbsolute(secret.file)) {
try {
// Calculate relative path from target directory
const relativePath = path.relative(target, secret.file);
// Replace Windows backslashes with forward slashes for consistency
secret.file = relativePath.replace(/\\/g, "/");
} catch (e) {
// If we can't get a relative path, keep the original path
debugLog(`Could not normalize path: ${secret.file}`, true);
}
}
});
// Merge results, preserving uniqueness
const uniqueSecrets = new Map();
// Add existing secrets first
results.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
uniqueSecrets.set(key, secret);
});
// Add detect-secrets results
detectSecretsResults.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
if (uniqueSecrets.has(key)) {
// Merge with existing secret
const existing = uniqueSecrets.get(key);
existing.types = [
...new Set([...existing.types, ...secret.types]),
];
// If the existing secret doesn't have a hash but this one does, use it
if (!existing.hashed_secret && secret.hashed_secret) {
existing.hashed_secret = secret.hashed_secret;
}
} else {
// Add new secret
uniqueSecrets.set(key, secret);
}
});
// Update results
results.secrets = Array.from(uniqueSecrets.values());
results.missed_secrets = [
...results.missed_secrets,
...detectSecretsResults.missed_secrets,
];
debugLog(
"Successfully merged results from both scanners for git history"
);
} catch (detectSecretsError) {
debugLog(
`detect-secrets scan failed: ${detectSecretsError.message}`,
true
);
console.log(
chalk.yellow(
`Warning: detect-secrets scan failed: ${detectSecretsError.message}`
)
);
console.log(
chalk.yellow("Continuing with Gitleaks results only")
);
}
}
} else {
// Scan with detect-secrets only
debugLog("Using detect-secrets only for git history scan");
results = await detectSecrets.scanDirectory(target, scanOptions);
}
spinner.stop();
// Save results
saveResults(results, outputPath);
// Display results
console.log(formatResults(results));
// Exit with error code if secrets were found
if (results.secrets.length > 0) {
process.exit(1);
}
} else {
// Local file/directory scanning
spinner.text = `Scanning ${target}`;
let results;
let gitleaksError = null;
try {
if (options.scanner === "both") {
// Use both scanners
console.log(
chalk.blue(
"Starting scan with both detect-secrets and Gitleaks..."
)
);
results = await detectSecrets.scanWithBothScanners(
target,
scanOptions
);
console.log(chalk.green("Successfully used both scanners"));
} else if (options.scanner === "gitleaks") {
// Use Gitleaks only
console.log(chalk.blue("Starting scan with Gitleaks only..."));
results = await runGitleaksScan(target, scanOptions);
} else {
// Use detect-secrets only
console.log(
chalk.blue("Starting scan with detect-secrets only...")
);
results = await detectSecrets.scanDirectory(target, scanOptions);
}
} catch (scanError) {
gitleaksError = scanError;
// Check if the error is related to gitleaks not being installed
if (scanError.message.includes("Gitleaks is not installed")) {
console.log(chalk.yellow(`\nWarning: ${scanError.message}`));
console.log(
chalk.yellow(
"Please install Gitleaks from https://github.com/zricethezav/gitleaks#installation"
)
);
} else {
console.log(chalk.yellow(`\nWarning: ${scanError.message}`));
}
// If the combined scanner failed, try using just detect-secrets
if (options.scanner === "both") {
console.log(chalk.yellow("Falling back to detect-secrets only..."));
results = await detectSecrets.scanDirectory(target, scanOptions);
} else {
// Re-throw if it wasn't trying to use both scanners
throw scanError;
}
}
spinner.stop();
// Save results
saveResults(results, outputPath);
// Display results
console.log(formatResults(results));
// Exit with error code if secrets were found
if (results.secrets.length > 0) {
process.exit(1);
}
}
} catch (error) {
spinner.fail("Scan failed");
debugLog(`Scan error: ${error.message}`, true);
console.error(chalk.red(error.message));
process.exit(1);
}
});
// Add backward compatibility for users of previous versions
program
.option(
"-d, --directory <path>",
"Directory to scan (default: current directory)"
)
.option("-o, --output <file>", "Output file path")
.option("-v, --verbose", "Show additional information")
.option(
"-x, --exclude-dirs <patterns>",
"Directory patterns to exclude (comma-separated)"
)
.option(
"-e, --exclude-files <patterns>",
"File patterns to exclude (comma-separated)"
)
.option("-m, --check-missed", "Check for potentially missed secrets")
.on("--help", () => {
console.log("");
console.log("For more options and commands, use:");
console.log(" detect-secrets-js scan --help");
});
// Handle the case where no command is specified (backward compatibility)
const noCommandProvided =
process.argv.length <= 2 ||
(process.argv.length > 2 && process.argv[2].startsWith("-"));
if (noCommandProvided) {
const options = program.opts();
const directory = options.directory || process.cwd();
const args = ["scan", directory];
if (options.output) args.push("--output", options.output);
if (options.verbose) args.push("--verbose");
if (options.excludeDirs) args.push("--exclude-dirs", options.excludeDirs);
if (options.excludeFiles) args.push("--exclude-files", options.excludeFiles);
if (options.checkMissed) args.push("--check-missed");
process.argv = [
...process.argv.slice(0, 2),
...args,
...process.argv.slice(3),
];
console.log(
chalk.yellow("Notice: You are using the legacy CLI format. Consider using:")
);
console.log(chalk.cyan(`detect-secrets-js scan ${directory} [options]`));
console.log("");
program.parse(process.argv);
} else {
program.parse(process.argv);
}