detect-secrets-js
Version:
A JavaScript implementation of Yelp's detect-secrets tool - no Python required
710 lines (633 loc) • 24.2 kB
JavaScript
const detectSecrets = require("../src/index");
const { program } = require("commander");
const ora = require("ora");
const chalk = require("chalk");
const fs = require("fs");
const path = require("path");
const {
runGitleaksScan,
scanRemoteRepository,
scanGitHistory,
} = require("../src/gitleaks");
// Debug function to log file operations
function debugLog(message, error = false) {
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] ${message}`;
if (error) {
console.error(chalk.red(logMessage));
} else {
console.log(chalk.blue(logMessage));
}
}
// Format scan results for display
function formatResults(results) {
let output = "";
if (results.secrets.length === 0 && results.missed_secrets.length === 0) {
return chalk.green("\nNo secrets detected!");
}
if (results.truncated) {
output += chalk.yellow(
"\nNote: Some files were truncated due to size limits"
);
}
if (results.secrets.length > 0) {
output += chalk.red(`\nDetected ${results.secrets.length} secret(s):\n`);
// Group secrets by file
const fileGroups = results.secrets.reduce((groups, secret) => {
if (!groups[secret.file]) {
groups[secret.file] = [];
}
groups[secret.file].push(secret);
return groups;
}, {});
// Sort files alphabetically
Object.keys(fileGroups)
.sort()
.forEach((file) => {
output += chalk.cyan(`\nFile: ${file}`);
// Sort secrets by line number
fileGroups[file]
.sort((a, b) => a.line - b.line)
.forEach((secret) => {
output += `\n Line ${secret.line}:`;
output += `\n Types: ${chalk.yellow(secret.types.join(", "))}`;
if (secret.hashed_secret) {
output += `\n Fingerprint: ${chalk.gray(
secret.hashed_secret
)}`;
}
if (secret.author) {
output += `\n Author: ${chalk.gray(secret.author)}`;
}
if (secret.email) {
output += `\n Email: ${chalk.gray(secret.email)}`;
}
if (secret.date) {
output += `\n Date: ${chalk.gray(secret.date)}`;
}
if (secret.commit) {
output += `\n Commit: ${chalk.gray(secret.commit)}`;
}
if (secret.message) {
output += `\n Message: ${chalk.gray(secret.message)}`;
}
if (secret.is_false_positive) {
output += chalk.gray("\n [Likely False Positive]");
}
if (secret.file.includes("node_modules")) {
output += chalk.yellow(
"\n [Dependency Code - Not Your Source]"
);
}
if (secret.file.includes(".next")) {
output += chalk.yellow(
"\n [Next.js Build Output - Not Your Source]"
);
}
});
});
}
if (results.missed_secrets.length > 0) {
output += chalk.yellow(
`\n\nPotentially Missed Secrets (${results.missed_secrets.length}):\n`
);
const groupedMissed = results.missed_secrets.reduce((groups, secret) => {
if (!groups[secret.file]) {
groups[secret.file] = [];
}
groups[secret.file].push(secret);
return groups;
}, {});
Object.keys(groupedMissed)
.sort()
.forEach((file) => {
output += chalk.cyan(`\nFile: ${file}`);
groupedMissed[file]
.sort((a, b) => a.line - b.line)
.forEach((secret) => {
output += `\n Line ${secret.line}: ${secret.type}`;
});
});
}
return output;
}
// Save results to file with requested format
function saveResults(results, outputPath) {
try {
// Get file extension to determine format
const ext = path.extname(outputPath).toLowerCase();
// Create directory if it doesn't exist
const outputDir = path.dirname(outputPath);
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
let outputContent = "";
// Format based on file extension
switch (ext) {
case ".json":
outputContent = JSON.stringify(results, null, 2);
break;
case ".csv":
// Create CSV header
outputContent =
"File,Line,Type,Fingerprint,Commit,Author,Email,Date,Message,Is False Positive,Is Dependency,Is Build File\n";
// Add each secret as a row
results.secrets.forEach((secret) => {
const isDependency = secret.file.includes("node_modules");
const isBuildFile = secret.file.includes(".next");
outputContent += `"${secret.file}",${
secret.line
},"${secret.types.join("; ")}"`;
outputContent += `,"${secret.hashed_secret || ""}","${
secret.commit || ""
}","${secret.author || ""}","${secret.email || ""}","${
secret.date || ""
}","${secret.message || ""}","${
secret.is_false_positive
}","${isDependency}","${isBuildFile}"\n`;
});
break;
case ".txt":
// Simple text format
outputContent = `Secrets Detection Results\n${"=".repeat(25)}\n\n`;
outputContent += `Total secrets found: ${results.secrets.length}\n`;
// Count node_modules secrets separately
const dependencySecrets = results.secrets.filter((s) =>
s.file.includes("node_modules")
).length;
const nextjsSecrets = results.secrets.filter((s) =>
s.file.includes(".next")
).length;
const totalNonSourceSecrets = dependencySecrets + nextjsSecrets;
if (totalNonSourceSecrets > 0) {
outputContent += `Secrets in dependencies/build files: ${totalNonSourceSecrets}\n`;
if (dependencySecrets > 0) {
outputContent += ` - Secrets in node_modules: ${dependencySecrets}\n`;
}
if (nextjsSecrets > 0) {
outputContent += ` - Secrets in Next.js build files: ${nextjsSecrets}\n`;
}
outputContent += `Secrets in your code: ${
results.secrets.length - totalNonSourceSecrets
}\n`;
}
outputContent += `Potentially missed secrets: ${results.missed_secrets.length}\n\n`;
results.secrets.forEach((secret) => {
outputContent += `File: ${secret.file}\n`;
outputContent += `Line: ${secret.line}\n`;
outputContent += `Types: ${secret.types.join(", ")}\n`;
if (secret.hashed_secret)
outputContent += `Fingerprint: ${secret.hashed_secret}\n`;
if (secret.commit) outputContent += `Commit: ${secret.commit}\n`;
if (secret.author) outputContent += `Author: ${secret.author}\n`;
if (secret.date) outputContent += `Date: ${secret.date}\n`;
outputContent += `Is False Positive: ${secret.is_false_positive}\n\n`;
});
break;
default:
// Default to JSON if extension not recognized
outputContent = JSON.stringify(results, null, 2);
}
fs.writeFileSync(outputPath, outputContent);
console.log(chalk.green(`\nResults saved to: ${outputPath}`));
} catch (outputError) {
debugLog(`Output error: ${outputError.message}`, true);
console.error(chalk.red(`Failed to save results: ${outputError.message}`));
}
}
program
.name("detect-secrets-js")
.description(
"A JavaScript implementation of detect-secrets with Gitleaks integration"
)
.version("2.1.1");
program
.command("scan <target>")
.description(
"Scan a local directory, file, remote repository, or git commits for secrets"
)
.option(
"-o, --output <path>",
"Output path for results (default: ./scan-results.json)"
)
.option(
"-s, --scanner <scanner>",
"Scanner to use (detect-secrets, gitleaks, or both)",
"both"
)
.option(
"--max-file-size <size>",
"Maximum file size in bytes (0 for no limit)",
"0"
)
.option("--exclude-dirs <dirs...>", "Directories to exclude")
.option("--exclude-files <files...>", "File patterns to exclude")
.option("--check-missed", "Check for potentially missed secrets")
.option("--verbose", "Show additional information")
.option("--remote", "Scan a remote repository (target should be a git URL)")
.option(
"--branch <branch>",
"Branch to check out for remote repository scanning"
)
.option("--commit <hash>", "Scan a specific commit hash")
.option("--all-commits", "Scan all git commit history")
.option("--from-commit <hash>", "Starting commit hash for git history scan")
.option("--to-commit <hash>", "Ending commit hash for git history scan")
.option(
"--disable-git-blame",
"Disable git blame information gathering",
false
)
.option(
"--git-repo-path <path>",
"Specify git repository path for external scans"
)
.option(
"--include-node-modules",
"Include node_modules in the scan (not recommended)",
false
)
.action(async (target, options) => {
const spinner = ora("Initializing scan...").start();
try {
// Determine output path (default to scan-results.json in current directory)
const outputPath = options.output || "./scan-results.json";
// Check if target contains node_modules
if (target.includes("node_modules") && !options.includeNodeModules) {
console.log(
chalk.yellow(
"\nWarning: You are attempting to scan a node_modules directory."
)
);
console.log(
chalk.yellow("Scanning node_modules is generally not recommended as:")
);
console.log(
chalk.yellow("1. These files are not part of your source code")
);
console.log(
chalk.yellow(
"2. They may contain false positives from test/example files"
)
);
console.log(
chalk.yellow(
"3. Scanning can be very slow due to the large number of files"
)
);
console.log(
chalk.yellow("\nFiles in node_modules will be excluded by default.")
);
console.log(
chalk.yellow(
"If you want to scan node_modules, use the --include-node-modules flag."
)
);
}
// Logging scan configuration
debugLog(`Current working directory: ${process.cwd()}`);
debugLog(`Target: ${target}`);
debugLog(`Using scanner(s): ${options.scanner}`);
debugLog(`Output path: ${outputPath}`);
// Git blame configuration
if (options.disableGitBlame) {
debugLog("Git blame information gathering is disabled");
} else if (options.gitRepoPath) {
debugLog(`Using custom git repository path: ${options.gitRepoPath}`);
}
// Build scan options
const scanOptions = {
maxFileSize: parseInt(options.maxFileSize),
excludeDirs: options.excludeDirs || [],
excludeFiles: options.excludeFiles || [],
checkMissed: options.checkMissed,
verbose: options.verbose,
enrichWithGitInfo: !options.disableGitBlame,
gitRepoPath: options.gitRepoPath,
includeNodeModules: options.includeNodeModules,
};
// Always exclude node_modules unless explicitly included
if (
!options.includeNodeModules &&
!scanOptions.excludeDirs.includes("node_modules")
) {
scanOptions.excludeDirs.push("node_modules");
debugLog("Automatically excluding node_modules directory from scan");
}
// Mode detection
const isRemote =
options.remote ||
target.startsWith("http://") ||
target.startsWith("https://") ||
target.startsWith("git@");
const isSpecificCommit = options.commit && !options.allCommits;
const isAllCommits =
options.allCommits || (options.fromCommit && !options.commit);
if (isRemote) {
spinner.text = `Cloning and scanning repository: ${target}`;
if (options.branch) {
debugLog(`Branch: ${options.branch}`);
spinner.text = `Cloning and scanning branch: ${options.branch}`;
}
if (options.allCommits) {
debugLog(`Scanning all commits`);
spinner.text = `Cloning and scanning all commit history`;
} else if (options.commit) {
debugLog(`Scanning specific commit: ${options.commit}`);
spinner.text = `Cloning and scanning commit: ${options.commit}`;
}
// Create temporary directory for cloning
const tmpDir = path.join(
require("os").tmpdir(),
`detect-secrets-js-${Date.now()}`
);
try {
// Clone the repository
await new Promise((resolve, reject) => {
const cloneArgs = ["clone", target, tmpDir];
if (options.branch) {
cloneArgs.push("--branch", options.branch);
}
debugLog(`Cloning repository to ${tmpDir}`);
const gitClone = require("child_process").spawn("git", cloneArgs);
gitClone.on("close", (code) => {
if (code === 0) resolve();
else
reject(
new Error(`Failed to clone repository with code ${code}`)
);
});
});
// If both scanners are requested, run detect-secrets first
let detectSecretsResults = null;
if (options.scanner === "both") {
try {
debugLog("Running detect-secrets on cloned repository...");
spinner.text = "Running detect-secrets on cloned repository...";
detectSecretsResults = await detectSecrets.scanDirectory(
tmpDir,
scanOptions
);
debugLog(
`detect-secrets found ${detectSecretsResults.secrets.length} secrets`
);
// Normalize file paths from detect-secrets
detectSecretsResults.secrets.forEach((secret) => {
// Remove the temporary directory prefix from the file path
if (secret.file.startsWith(tmpDir)) {
// Replace OS path separator with forward slash for consistency
secret.file = secret.file
.substring(tmpDir.length)
.replace(/^[\\\/]+/, "") // Remove leading slashes
.replace(/\\/g, "/"); // Replace Windows backslashes with forward slashes
}
});
} catch (detectSecretsError) {
debugLog(
`Failed to run detect-secrets: ${detectSecretsError.message}`,
true
);
console.warn(
chalk.yellow(
`Warning: Failed to run detect-secrets: ${detectSecretsError.message}`
)
);
console.warn(
chalk.yellow("Will continue with Gitleaks scan only")
);
}
}
// Run Gitleaks for the remote scan
debugLog("Running Gitleaks on remote repository");
spinner.text = `Scanning with Gitleaks: ${target}`;
// Remote repository scanning with Gitleaks
const remoteResults = await scanRemoteRepository(
target,
options.branch,
scanOptions
);
// If specific commit or all commits requested, add git history scan
if (isSpecificCommit || isAllCommits) {
let gitResults;
if (isSpecificCommit) {
// Scan specific commit
spinner.text = `Scanning specific commit: ${options.commit}`;
gitResults = await scanGitHistory(
tmpDir,
options.commit,
options.commit,
scanOptions
);
} else {
// Scan all commits or commit range
spinner.text = `Scanning git history`;
gitResults = await scanGitHistory(
tmpDir,
options.fromCommit,
options.toCommit,
scanOptions
);
}
// Merge results from git history scan
remoteResults.secrets = [
...remoteResults.secrets,
...gitResults.secrets,
];
}
// Merge results if we have detect-secrets results
if (detectSecretsResults) {
// Merge results, preserving uniqueness
const uniqueSecrets = new Map();
// Add Gitleaks results first
remoteResults.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
uniqueSecrets.set(key, secret);
});
// Add detect-secrets results
detectSecretsResults.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
if (uniqueSecrets.has(key)) {
// Merge with existing secret
const existing = uniqueSecrets.get(key);
existing.types = [
...new Set([...existing.types, ...secret.types]),
];
// If the existing secret doesn't have a hash but this one does, use it
if (!existing.hashed_secret && secret.hashed_secret) {
existing.hashed_secret = secret.hashed_secret;
}
} else {
// Add new secret
uniqueSecrets.set(key, secret);
}
});
// Update results
remoteResults.secrets = Array.from(uniqueSecrets.values());
remoteResults.missed_secrets = [
...remoteResults.missed_secrets,
...detectSecretsResults.missed_secrets,
];
debugLog("Successfully merged results from both scanners");
}
spinner.stop();
// Save results
saveResults(remoteResults, outputPath);
// Display results
console.log(formatResults(remoteResults));
// Exit with error code if secrets were found
if (remoteResults.secrets.length > 0) {
process.exit(1);
}
} finally {
// Clean up
try {
if (fs.existsSync(tmpDir)) {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
} catch (cleanupError) {
console.warn(
`Failed to clean up temporary directory: ${cleanupError}`
);
}
}
} else if (isSpecificCommit || isAllCommits) {
// Git history scanning of local repository
spinner.text = isSpecificCommit
? `Scanning specific commit: ${options.commit}`
: `Scanning git history`;
// Log commit range if specified
if (options.fromCommit) debugLog(`From commit: ${options.fromCommit}`);
if (options.toCommit) debugLog(`To commit: ${options.toCommit}`);
let results;
if (options.scanner === "both" || options.scanner === "gitleaks") {
// Scan with Gitleaks
if (isSpecificCommit) {
results = await scanGitHistory(
target,
options.commit,
options.commit
);
} else {
results = await scanGitHistory(
target,
options.fromCommit,
options.toCommit
);
}
// Add detect-secrets scan if both scanners are requested
if (options.scanner === "both") {
const detectSecretsResults = await detectSecrets.scanDirectory(
target,
scanOptions
);
// Merge results, preserving uniqueness
const uniqueSecrets = new Map();
// Add existing secrets first
results.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
uniqueSecrets.set(key, secret);
});
// Add detect-secrets results
detectSecretsResults.secrets.forEach((secret) => {
const key = `${secret.file}:${secret.line}`;
if (uniqueSecrets.has(key)) {
// Merge with existing secret
const existing = uniqueSecrets.get(key);
existing.types = [
...new Set([...existing.types, ...secret.types]),
];
} else {
// Add new secret
uniqueSecrets.set(key, secret);
}
});
// Update results
results.secrets = Array.from(uniqueSecrets.values());
results.missed_secrets = [
...results.missed_secrets,
...detectSecretsResults.missed_secrets,
];
}
} else {
// Scan with detect-secrets only
results = await detectSecrets.scanDirectory(target, scanOptions);
}
spinner.stop();
// Save results
saveResults(results, outputPath);
// Display results
console.log(formatResults(results));
// Exit with error code if secrets were found
if (results.secrets.length > 0) {
process.exit(1);
}
} else {
// Local file/directory scanning
spinner.text = `Scanning ${target}`;
let results;
let gitleaksError = null;
try {
if (options.scanner === "both") {
// Use both scanners
console.log(
chalk.blue(
"Starting scan with both detect-secrets and Gitleaks..."
)
);
results = await detectSecrets.scanWithBothScanners(
target,
scanOptions
);
console.log(chalk.green("Successfully used both scanners"));
} else if (options.scanner === "gitleaks") {
// Use Gitleaks only
console.log(chalk.blue("Starting scan with Gitleaks only..."));
results = await runGitleaksScan(target);
} else {
// Use detect-secrets only
console.log(
chalk.blue("Starting scan with detect-secrets only...")
);
results = await detectSecrets.scanDirectory(target, scanOptions);
}
} catch (scanError) {
gitleaksError = scanError;
// Check if the error is related to gitleaks not being installed
if (scanError.message.includes("Gitleaks is not installed")) {
console.log(chalk.yellow(`\nWarning: ${scanError.message}`));
console.log(
chalk.yellow(
"Please install Gitleaks from https://github.com/zricethezav/gitleaks#installation"
)
);
} else {
console.log(chalk.yellow(`\nWarning: ${scanError.message}`));
}
// If the combined scanner failed, try using just detect-secrets
if (options.scanner === "both") {
console.log(chalk.yellow("Falling back to detect-secrets only..."));
results = await detectSecrets.scanDirectory(target, scanOptions);
} else {
// Re-throw if it wasn't trying to use both scanners
throw scanError;
}
}
spinner.stop();
// Save results
saveResults(results, outputPath);
// Display results
console.log(formatResults(results));
// Exit with error code if secrets were found
if (results.secrets.length > 0) {
process.exit(1);
}
}
} catch (error) {
spinner.fail("Scan failed");
debugLog(`Scan error: ${error.message}`, true);
console.error(chalk.red(error.message));
process.exit(1);
}
});
program.parse(process.argv);