forge-mutation-tester
Version:
Mutation testing tool for Solidity smart contracts using Gambit
891 lines (890 loc) ā¢ 77.4 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.GambitService = void 0;
const child_process_1 = require("child_process");
const util_1 = require("util");
const fs_1 = require("fs");
const path_1 = __importDefault(require("path"));
const chalk_1 = __importDefault(require("chalk"));
const ora_1 = __importDefault(require("ora"));
const minimatch_1 = require("minimatch");
const execAsync = (0, util_1.promisify)(child_process_1.exec);
class GambitService {
gambitPath = 'gambit';
async checkGambitInstalled() {
try {
await execAsync(`${this.gambitPath} --help`);
return true;
}
catch {
return false;
}
}
async installGambit() {
const spinner = (0, ora_1.default)('Installing Gambit...').start();
try {
// For now, we'll use a direct approach - download from releases
spinner.text = 'Downloading Gambit binary...';
// Create a local bin directory
const localBinDir = path_1.default.join(process.cwd(), '.bin');
await fs_1.promises.mkdir(localBinDir, { recursive: true });
const gambitBinPath = path_1.default.join(localBinDir, 'gambit');
// Try to build from source using cargo if available
try {
spinner.text = 'Checking if Rust/Cargo is available...';
await execAsync('cargo --version');
spinner.text = 'Building Gambit from source...';
// Clone and build Gambit
const tempDir = path_1.default.join(process.cwd(), '.gambit-temp');
await fs_1.promises.rm(tempDir, { recursive: true, force: true });
await execAsync(`git clone https://github.com/Certora/gambit.git ${tempDir}`);
await execAsync('cargo build --release', { cwd: tempDir });
// Copy the built binary
const builtBinary = path_1.default.join(tempDir, 'target', 'release', 'gambit');
await fs_1.promises.copyFile(builtBinary, gambitBinPath);
// Clean up
await fs_1.promises.rm(tempDir, { recursive: true, force: true });
}
catch (cargoError) {
// If cargo is not available, provide instructions
throw new Error('Gambit installation requires Rust/Cargo or manual download.\n' +
'Please either:\n' +
'1. Install Rust from https://rustup.rs/ and run this command again\n' +
'2. Download Gambit manually from https://github.com/Certora/gambit/releases\n' +
' and place it in your PATH or in .bin/gambit');
}
// Make it executable
await fs_1.promises.chmod(gambitBinPath, 0o755);
// Update the gambit path to use the local binary
this.gambitPath = gambitBinPath;
// Verify installation
await execAsync(`${this.gambitPath} --help`);
spinner.succeed(chalk_1.default.green(`Gambit installed successfully`));
}
catch (error) {
spinner.fail(chalk_1.default.red('Failed to install Gambit'));
throw error;
}
}
async compileProject(projectPath) {
const spinner = (0, ora_1.default)('Compiling project...').start();
try {
// Check for Foundry
const foundryConfigPath = path_1.default.join(projectPath, 'foundry.toml');
const isForgeProject = await fs_1.promises.access(foundryConfigPath).then(() => true).catch(() => false);
if (isForgeProject) {
spinner.text = 'Compiling Forge contracts...';
console.log(chalk_1.default.dim(' Running: forge build'));
try {
await execAsync('forge build', {
cwd: projectPath,
timeout: 180000 // 3 minutes
});
spinner.succeed(chalk_1.default.green('Project compiled successfully'));
}
catch (compileError) {
if (compileError.killed && compileError.signal === 'SIGTERM') {
spinner.fail(chalk_1.default.red('Forge build timed out after 3 minutes'));
throw new Error('Compilation timeout - project may be too large or have dependency issues');
}
// Log the actual error
console.error(chalk_1.default.red('\nCompilation error:'));
console.error(chalk_1.default.dim(compileError.stdout || ''));
console.error(chalk_1.default.yellow(compileError.stderr || ''));
// Try with --force flag
spinner.text = 'Retrying with --force flag...';
console.log(chalk_1.default.dim(' Trying: forge build --force'));
await execAsync('forge build --force', {
cwd: projectPath,
timeout: 180000
});
spinner.succeed(chalk_1.default.green('Project compiled with --force flag'));
}
}
else {
spinner.warn(chalk_1.default.yellow('Could not detect Forge project. Assuming project is already compiled.'));
console.log(chalk_1.default.dim(' No foundry.toml found'));
console.log(chalk_1.default.dim(' Make sure your contracts are compiled before proceeding.'));
}
}
catch (error) {
spinner.fail(chalk_1.default.red('Failed to compile project'));
throw error;
}
}
async prepareProject(projectPath) {
const spinner = (0, ora_1.default)('Preparing project for mutation testing...').start();
try {
// Check if it's a Forge project
const foundryTomlPath = path_1.default.join(projectPath, 'foundry.toml');
const isForgeProject = await fs_1.promises.access(foundryTomlPath).then(() => true).catch(() => false);
if (isForgeProject) {
spinner.text = 'Detected Forge project, checking dependencies...';
console.log(chalk_1.default.dim(` Project path: ${projectPath}`));
// Install dependencies if needed
try {
spinner.text = 'Installing Forge dependencies...';
console.log(chalk_1.default.dim(' Running: forge install'));
const { stdout: installOutput } = await execAsync('forge install', {
cwd: projectPath,
timeout: 60000 // 60 second timeout
});
if (installOutput) {
console.log(chalk_1.default.dim(` Install output: ${installOutput.trim()}`));
}
}
catch (e) {
console.log(chalk_1.default.yellow(` Forge install skipped: ${e.message?.split('\n')[0] || 'Already installed'}`));
}
// Build the project
spinner.text = 'Building Forge contracts...';
console.log(chalk_1.default.dim(' Running: forge build'));
console.log(chalk_1.default.dim(' This may take a while for large projects...'));
try {
const { stdout, stderr } = await execAsync('forge build', {
cwd: projectPath,
maxBuffer: 1024 * 1024 * 10,
timeout: 300000 // 5 minute timeout for large projects
});
if (stdout) {
const lines = stdout.trim().split('\n');
console.log(chalk_1.default.dim(` Build output: ${lines[lines.length - 1]}`));
}
if (stderr && !stderr.includes('Warning')) {
throw new Error(`Forge build failed: ${stderr}`);
}
spinner.succeed(chalk_1.default.green('Project built successfully'));
}
catch (buildError) {
if (buildError.code === 'ETIMEDOUT') {
spinner.fail(chalk_1.default.red('Forge build timed out after 5 minutes'));
throw new Error('Build timeout - project may be too large or have network dependencies');
}
throw buildError;
}
}
else {
spinner.warn(chalk_1.default.yellow('Could not detect Forge project (no foundry.toml found).'));
console.log(chalk_1.default.dim(' Assuming project is already compiled.'));
console.log(chalk_1.default.dim(' Make sure you have run "forge build" before proceeding.'));
}
}
catch (error) {
spinner.fail(chalk_1.default.red('Failed to prepare project'));
console.error(chalk_1.default.red(` Error details: ${error.message}`));
throw error;
}
}
async setupGambitConfig(projectPath) {
const spinner = (0, ora_1.default)('Setting up Gambit configuration...').start();
try {
// Detect project type and adjust config accordingly
const foundryTomlPath = path_1.default.join(projectPath, 'foundry.toml');
const isForgeProject = await fs_1.promises.access(foundryTomlPath).then(() => true).catch(() => false);
// Try to find the contracts directory
let sourceRoot = "contracts";
const possibleDirs = ["contracts", "src", "contract", "sol"];
for (const dir of possibleDirs) {
try {
await fs_1.promises.access(path_1.default.join(projectPath, dir));
sourceRoot = dir;
break;
}
catch {
// Continue checking
}
}
// Try to find solc - check common locations
let solcPath = "solc";
try {
await execAsync('which solc');
}
catch {
// Try to find solc in node_modules
try {
const { stdout } = await execAsync('find node_modules -name solc -type f -perm +111 | head -1', { cwd: projectPath });
if (stdout.trim()) {
solcPath = stdout.trim();
}
}
catch {
console.warn(chalk_1.default.yellow('Warning: solc not found in node_modules, Gambit may fail'));
}
}
// Configure to process all Solidity files in the source directory
const config = {
filename: `${sourceRoot}/**/*.sol`, // Process all .sol files recursively
sourceroot: ".",
skip_validate: true,
mutations: [
"binary-op-mutation",
"unary-operator-mutation",
"require-mutation",
"assignment-mutation",
"delete-expression-mutation",
"swap-arguments-operator-mutation",
"elim-delegate-mutation",
],
outdir: "gambit_out",
solc: solcPath,
num_mutants: 50, // Increased limit since we're processing all files
// Add solc remappings for common imports
solc_remappings: [
"@openzeppelin/=node_modules/@openzeppelin/",
"@uniswap/=node_modules/@uniswap/"
]
};
const configPath = path_1.default.join(projectPath, 'gambit.conf.json');
await fs_1.promises.writeFile(configPath, JSON.stringify(config, null, 2));
spinner.succeed(chalk_1.default.green('Gambit configuration created'));
}
catch (error) {
spinner.fail(chalk_1.default.red('Failed to create Gambit configuration'));
throw error;
}
}
async setupGambitConfigWithoutDependencies(projectPath, includePatterns, excludePatterns) {
const spinner = (0, ora_1.default)('Setting up Gambit configuration...').start();
try {
// Detect project type and adjust config accordingly
const foundryTomlPath = path_1.default.join(projectPath, 'foundry.toml');
const isForgeProject = await fs_1.promises.access(foundryTomlPath).then(() => true).catch(() => false);
// Try to find the contracts directory
let sourceRoot = "contracts";
const possibleDirs = ["contracts", "src", "contract", "sol"];
for (const dir of possibleDirs) {
try {
await fs_1.promises.access(path_1.default.join(projectPath, dir));
sourceRoot = dir;
break;
}
catch {
// Continue checking
}
}
// Find source files and filter out test files
const glob = require('glob');
const allSolFiles = glob.sync(`${sourceRoot}/**/*.sol`, { cwd: projectPath });
// Filter out test files based on common patterns
const sourceFiles = allSolFiles.filter((file) => {
const fileName = path_1.default.basename(file);
const filePath = file.toLowerCase();
// Exclude common test patterns
return !(filePath.includes('/test/') ||
filePath.includes('/tests/') ||
fileName.endsWith('.test.sol') ||
fileName.endsWith('.t.sol') ||
fileName.startsWith('Test') ||
fileName.endsWith('Test.sol') ||
filePath.includes('mock') ||
filePath.includes('Mock'));
});
// Apply custom include/exclude patterns if provided
const filteredFiles = this.filterFilesByPatterns(sourceFiles, includePatterns, excludePatterns);
if (filteredFiles.length === 0) {
const patternsMsg = includePatterns || excludePatterns
? ` (after applying custom patterns)`
: ` (after excluding test files)`;
throw new Error(`No source files found in ${sourceRoot} directory${patternsMsg}`);
}
console.log(chalk_1.default.blue(`šÆ Running mutation testing on ${filteredFiles.length} source files...`));
if (includePatterns && includePatterns.length > 0) {
console.log(chalk_1.default.dim(` Include patterns: ${includePatterns.join(', ')}`));
}
if (excludePatterns && excludePatterns.length > 0) {
console.log(chalk_1.default.dim(` Exclude patterns: ${excludePatterns.join(', ')}`));
}
console.log(chalk_1.default.dim(' Excluding test files from mutation testing'));
// Auto-detect essential remappings (full list causes Gambit to crash)
let solcRemappings = [];
try {
if (isForgeProject) {
const { stdout: remappingsOutput } = await execAsync('forge remappings', { cwd: projectPath });
const allRemappings = remappingsOutput
.trim()
.split('\n')
.filter(line => line.trim() && !line.startsWith('Warning:'))
.map(line => line.trim());
// Filter to only essential remappings that don't crash Gambit
solcRemappings = allRemappings.filter(mapping => {
return mapping.includes('@openzeppelin/contracts/=') ||
mapping.includes('@openzeppelin/contracts-upgradeable/=') ||
mapping.includes('v4-core/=') ||
mapping.includes('v4-periphery/=') ||
mapping.includes('@uniswap/v4-core/=') ||
mapping.includes('forge-std/=') ||
mapping.includes('solady/=');
});
console.log(chalk_1.default.dim(` Using ${solcRemappings.length} essential remappings (${allRemappings.length} total available)`));
}
}
catch (e) {
console.log(chalk_1.default.dim(' Could not get forge remappings, using defaults'));
// Add default remappings if forge remappings fails
solcRemappings = isForgeProject ? [
"@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/",
"@uniswap/v4-core/=lib/v4-core/",
"v4-core/=lib/v4-core/src/",
"forge-std/=lib/forge-std/src/",
"ds-test/=lib/ds-test/src/",
"solady/=lib/solady/src/",
] : [
"@openzeppelin/=node_modules/@openzeppelin/",
"@uniswap/=node_modules/@uniswap/",
"solady/=node_modules/solady/",
];
}
console.log(chalk_1.default.dim(` Will process all ${sourceFiles.length} files individually during mutation testing`));
console.log(chalk_1.default.dim(` Gambit configurations will be generated dynamically for each file`));
spinner.succeed(chalk_1.default.green(`Setup complete. Found ${sourceFiles.length} source files ready for mutation testing`));
}
catch (error) {
spinner.fail(chalk_1.default.red('Failed to create Gambit configuration'));
throw error;
}
}
async ensureSolcAvailable(projectPath) {
// Check if solc is available
try {
await execAsync('solc --version');
return 'solc';
}
catch {
// Try to install solc-select and use it
try {
console.log(chalk_1.default.yellow('Solc not found, attempting to install...'));
// Check if Python/pip is available
try {
await execAsync('python3 -m pip --version');
// Install solc-select
await execAsync('python3 -m pip install solc-select');
// Install a specific solc version
await execAsync('solc-select install 0.8.19');
await execAsync('solc-select use 0.8.19');
return 'solc';
}
catch {
// Python not available, try npm
await execAsync('npm install -g solc', { cwd: projectPath });
return 'solcjs';
}
}
catch {
// If all else fails, try to use the project's solc
const { stdout } = await execAsync('find . -path ./node_modules -prune -o -name solc -type f -print | head -1', { cwd: projectPath });
if (stdout.trim()) {
return stdout.trim();
}
throw new Error('Could not find or install Solidity compiler (solc)');
}
}
}
async runMutationTesting(projectPath) {
const spinner = (0, ora_1.default)('Running mutation testing...').start();
try {
// Ensure Gambit is installed
const isInstalled = await this.checkGambitInstalled();
if (!isInstalled) {
spinner.text = 'Gambit not found, installing...';
await this.installGambit();
}
// Ensure solc is available
// spinner.text = 'Checking Solidity compiler...';
// const solcPath = await this.ensureSolcAvailable(projectPath);
// Update the config with the correct solc path
const configPath = path_1.default.join(projectPath, 'gambit.conf.json');
const config = JSON.parse(await fs_1.promises.readFile(configPath, 'utf-8'));
// config.solc = solcPath;
await fs_1.promises.writeFile(configPath, JSON.stringify(config, null, 2));
// Prepare the project (compile, install deps, etc.)
await this.prepareProject(projectPath);
// Run Gambit mutation testing
spinner.text = 'Generating mutants...';
const { stdout: mutateOutput } = await execAsync(`${this.gambitPath} mutate --json gambit.conf.json`, {
cwd: projectPath,
maxBuffer: 1024 * 1024 * 10 // 10MB buffer
});
// Parse the results
const results = await this.parseGambitResults(projectPath);
spinner.succeed(chalk_1.default.green(`Mutation testing completed. Found ${results.length} mutations`));
return results;
}
catch (error) {
spinner.fail(chalk_1.default.red('Mutation testing failed'));
throw error;
}
}
/**
* Filter source files based on include/exclude patterns
* @param files List of files to filter
* @param includePatterns Optional glob patterns to include
* @param excludePatterns Optional glob patterns to exclude
* @returns Filtered list of files
*/
filterFilesByPatterns(files, includePatterns, excludePatterns) {
let filteredFiles = files;
// Apply include patterns if specified
if (includePatterns && includePatterns.length > 0) {
filteredFiles = filteredFiles.filter(file => {
// Check if file matches any include pattern
return includePatterns.some(pattern => (0, minimatch_1.minimatch)(file, pattern, { matchBase: true }));
});
}
// Apply exclude patterns if specified
if (excludePatterns && excludePatterns.length > 0) {
filteredFiles = filteredFiles.filter(file => {
// Check if file doesn't match any exclude pattern
return !excludePatterns.some(pattern => (0, minimatch_1.minimatch)(file, pattern, { matchBase: true }));
});
}
return filteredFiles;
}
async runMutationTestingWithoutSetup(projectPath, numMutants = 25, includePatterns, excludePatterns, customRemappings) {
const spinner = (0, ora_1.default)('Running mutation testing...').start();
try {
// Ensure Gambit is installed
const isInstalled = await this.checkGambitInstalled();
if (!isInstalled) {
spinner.text = 'Gambit not found, installing...';
await this.installGambit();
}
// Get all source files to process (excluding test files)
const foundryTomlPath = path_1.default.join(projectPath, 'foundry.toml');
const isForgeProject = await fs_1.promises.access(foundryTomlPath).then(() => true).catch(() => false);
// Find the contracts directory
let sourceRoot = "contracts";
const possibleDirs = ["contracts", "src", "contract", "sol"];
for (const dir of possibleDirs) {
try {
await fs_1.promises.access(path_1.default.join(projectPath, dir));
sourceRoot = dir;
break;
}
catch {
// Continue checking
}
}
// Find source files and filter out test files
const glob = require('glob');
const allSolFiles = glob.sync(`${sourceRoot}/**/*.sol`, { cwd: projectPath });
// Filter out test files based on common patterns
const sourceFiles = allSolFiles.filter((file) => {
const fileName = path_1.default.basename(file);
const filePath = file.toLowerCase();
// Exclude common test patterns
return !(filePath.includes('/test/') ||
filePath.includes('/tests/') ||
fileName.endsWith('.test.sol') ||
fileName.endsWith('.t.sol') ||
fileName.startsWith('Test') ||
fileName.endsWith('Test.sol') ||
filePath.includes('mock') ||
filePath.includes('Mock'));
});
// Apply custom include/exclude patterns if provided
const filteredFiles = this.filterFilesByPatterns(sourceFiles, includePatterns, excludePatterns);
if (filteredFiles.length === 0) {
const patternsMsg = includePatterns || excludePatterns
? ` (after applying custom patterns)`
: ` (after excluding test files)`;
throw new Error(`No source files found in ${sourceRoot} directory${patternsMsg}`);
}
console.log(chalk_1.default.blue(`šÆ Running mutation testing on ${filteredFiles.length} source files...`));
if (includePatterns && includePatterns.length > 0) {
console.log(chalk_1.default.dim(` Include patterns: ${includePatterns.join(', ')}`));
}
if (excludePatterns && excludePatterns.length > 0) {
console.log(chalk_1.default.dim(` Exclude patterns: ${excludePatterns.join(', ')}`));
}
console.log(chalk_1.default.dim(' Excluding test files from mutation testing'));
// Auto-detect essential remappings (full list causes Gambit to crash)
let solcRemappings = [];
try {
if (isForgeProject) {
const { stdout: remappingsOutput } = await execAsync('forge remappings', { cwd: projectPath });
const allRemappings = remappingsOutput
.trim()
.split('\n')
.filter(line => line.trim() && !line.startsWith('Warning:'))
.map(line => line.trim());
// Filter to only essential remappings that don't crash Gambit
solcRemappings = allRemappings.filter(mapping => {
return mapping.includes('@openzeppelin/contracts/=') ||
mapping.includes('@openzeppelin/contracts-upgradeable/=') ||
mapping.includes('v4-core/=') ||
mapping.includes('v4-periphery/=') ||
mapping.includes('@uniswap/v4-core/=') ||
mapping.includes('forge-std/=') ||
mapping.includes('solady/=');
});
console.log(chalk_1.default.dim(` Using ${solcRemappings.length} essential remappings (${allRemappings.length} total available)`));
}
}
catch (e) {
console.log(chalk_1.default.dim(' Could not get forge remappings, using defaults'));
}
// Add custom remappings if provided
if (customRemappings && customRemappings.length > 0) {
console.log(chalk_1.default.dim(` Adding ${customRemappings.length} custom remappings from configuration`));
// Merge custom remappings with auto-detected ones, custom ones take precedence
const customRemappingPrefixes = customRemappings.map(r => r.split('=')[0]);
const filteredAutoRemappings = solcRemappings.filter(r => {
const prefix = r.split('=')[0];
return !customRemappingPrefixes.includes(prefix);
});
solcRemappings = [...filteredAutoRemappings, ...customRemappings];
console.log(chalk_1.default.dim(` Total remappings: ${solcRemappings.length}`));
}
// Check if solc is available and get the correct PATH
let solcPath = 'solc';
// Build comprehensive PATH with all common solc-select locations
const homeDir = process.env.HOME || process.env.USERPROFILE;
const possibleSolcPaths = [
// Python user installs (solc-select common locations)
`${homeDir}/Library/Python/3.9/bin`, // macOS Python 3.9
`${homeDir}/Library/Python/3.10/bin`, // macOS Python 3.10
`${homeDir}/Library/Python/3.11/bin`, // macOS Python 3.11
`${homeDir}/Library/Python/3.12/bin`, // macOS Python 3.12
`${homeDir}/.local/bin`, // Linux user installs
`/usr/local/bin`, // System installs
`/opt/homebrew/bin`, // Homebrew (M1 Mac)
`/usr/bin`, // Default system
].filter(Boolean);
const envPath = `${possibleSolcPaths.join(':')}:${process.env.PATH}`;
console.log(chalk_1.default.dim(` Using enhanced PATH with multiple solc-select locations`));
try {
const { stdout: whichSolc } = await execAsync('which solc', {
env: { ...process.env, PATH: envPath }
});
solcPath = whichSolc.trim();
console.log(chalk_1.default.dim(` Found solc at: ${solcPath}`));
}
catch {
throw new Error('Solidity compiler (solc) not found in PATH.\n' +
'Please ensure solc-select is properly installed and in PATH:\n' +
' ā¢ pip3 install solc-select\n' +
' ā¢ solc-select install <your-version>\n' +
' ā¢ solc-select use <your-version>\n' +
' ā¢ export PATH="$HOME/Library/Python/3.9/bin:$PATH"\n' +
' ā¢ Or install globally: brew install solidity');
}
// Step 1: Generate all mutants first
spinner.text = 'Generating mutants for all source files...';
const allMutants = [];
for (let i = 0; i < filteredFiles.length; i++) {
const sourceFile = filteredFiles[i];
console.log(chalk_1.default.cyan(`\nš Generating mutants for file ${i + 1}/${filteredFiles.length}: ${sourceFile}`));
try {
// Configure to process ONE file at a time
const config = {
filename: sourceFile, // Single file only!
sourceroot: ".",
skip_validate: true,
mutations: [
"binary-op-mutation",
"unary-operator-mutation",
"require-mutation",
"assignment-mutation",
"delete-expression-mutation",
"swap-arguments-operator-mutation",
"elim-delegate-mutation",
],
outdir: `gambit_out_${path_1.default.basename(sourceFile, '.sol')}`, // Unique output dir per file
solc: "solc", // Use just "solc" and rely on PATH
num_mutants: numMutants // Use the provided numMutants
};
// Auto-detect and use ALL remappings for proper import resolution
if (solcRemappings.length > 0) {
config.solc_remappings = solcRemappings;
}
// Run gambit with the config
const configPath = path_1.default.join(projectPath, `gambit-config-${i}.json`);
await fs_1.promises.writeFile(configPath, JSON.stringify(config, null, 2));
console.log(chalk_1.default.dim(` Running: gambit mutate --json ${path_1.default.basename(configPath)}`));
const { stdout: gambitOutput, stderr: gambitError } = await execAsync(`${this.gambitPath} mutate --json ${path_1.default.basename(configPath)}`, {
cwd: projectPath,
maxBuffer: 1024 * 1024 * 10, // 10MB buffer
env: { ...process.env, PATH: envPath },
timeout: 300000 // 5 minute timeout
});
// Debug: Show what Gambit actually output
if (gambitOutput && gambitOutput.trim()) {
console.log(chalk_1.default.dim(` Gambit stdout: ${gambitOutput.trim().substring(0, 500)}${gambitOutput.trim().length > 500 ? '...' : ''}`));
}
if (gambitError && gambitError.trim()) {
console.log(chalk_1.default.yellow(` Gambit stderr: ${gambitError.trim().substring(0, 500)}${gambitError.trim().length > 500 ? '...' : ''}`));
}
console.log(chalk_1.default.green(` ā
Gambit completed for ${sourceFile}`));
// Parse mutants from this file's output directory
let fileMutants = [];
try {
fileMutants = await this.parseGambitMutants(projectPath, config.outdir, sourceFile);
console.log(chalk_1.default.dim(` Found ${fileMutants.length} mutants for ${sourceFile}`));
allMutants.push(...fileMutants);
}
catch (parseError) {
console.log(chalk_1.default.yellow(` Warning: Could not parse mutants for ${sourceFile}: ${parseError}`));
}
// If no mutants were generated, try with simplified config
if (fileMutants.length === 0) {
console.log(chalk_1.default.yellow(` No mutants generated, trying simplified approach for ${sourceFile}...`));
const simpleConfig = {
filename: sourceFile,
sourceroot: ".",
skip_validate: true,
mutations: ["binary-op-mutation", "require-mutation"], // Just 2 simple mutation types
outdir: `gambit_out_simple_${path_1.default.basename(sourceFile, '.sol')}`,
solc: "solc",
num_mutants: numMutants, // Use the provided numMutants
// No remappings - let it fail gracefully if imports don't work
};
const simpleConfigPath = path_1.default.join(projectPath, `gambit-simple-${i}.json`);
await fs_1.promises.writeFile(simpleConfigPath, JSON.stringify(simpleConfig, null, 2));
try {
console.log(chalk_1.default.dim(` Running simplified: gambit mutate --json ${path_1.default.basename(simpleConfigPath)}`));
const { stdout: simpleOutput, stderr: simpleError } = await execAsync(`${this.gambitPath} mutate --json ${path_1.default.basename(simpleConfigPath)}`, {
cwd: projectPath,
maxBuffer: 1024 * 1024 * 10,
env: { ...process.env, PATH: envPath },
timeout: 120000 // 2 minute timeout for simple version
});
if (simpleOutput && simpleOutput.trim()) {
console.log(chalk_1.default.dim(` Simple Gambit stdout: ${simpleOutput.trim().substring(0, 700)}${simpleOutput.trim().length > 500 ? '...' : ''}`));
}
if (simpleError && simpleError.trim()) {
console.log(chalk_1.default.yellow(` Simple Gambit stderr: ${simpleError.trim().substring(0, 700)}${simpleError.trim().length > 500 ? '...' : ''}`));
}
const simpleMutants = await this.parseGambitMutants(projectPath, simpleConfig.outdir, sourceFile);
console.log(chalk_1.default.dim(` Found ${simpleMutants.length} mutants with simple config for ${sourceFile}`));
allMutants.push(...simpleMutants);
await fs_1.promises.unlink(simpleConfigPath);
}
catch (simpleError) {
console.log(chalk_1.default.red(` Even simplified config failed for ${sourceFile}: ${simpleError.message}`));
await fs_1.promises.unlink(simpleConfigPath).catch(() => { });
}
}
// Clean up config file
await fs_1.promises.unlink(configPath);
}
catch (error) {
console.log(chalk_1.default.red(` ā Failed to generate mutants for ${sourceFile}: ${error.message}`));
}
}
if (allMutants.length === 0) {
console.log(chalk_1.default.red('\nšØ No mutants were generated from any source files!'));
console.log(chalk_1.default.yellow('\nPossible causes:'));
console.log(chalk_1.default.yellow(' ā¢ Solidity version 0.8.28 may not be fully supported by Gambit'));
console.log(chalk_1.default.yellow(' ā¢ Complex imports/dependencies preventing Gambit from parsing files'));
console.log(chalk_1.default.yellow(' ā¢ Files may be interfaces/abstract contracts with no implementation'));
console.log(chalk_1.default.yellow(' ā¢ Solc compilation issues preventing mutation generation'));
console.log(chalk_1.default.cyan('\nTroubleshooting suggestions:'));
console.log(chalk_1.default.cyan(' ā¢ Try with a project using Solidity 0.8.19 or earlier'));
console.log(chalk_1.default.cyan(' ā¢ Ensure all imports resolve correctly with forge'));
console.log(chalk_1.default.cyan(' ā¢ Check that solc can compile individual files'));
console.log(chalk_1.default.cyan(' ā¢ Focus on files with actual logic (not just interfaces)'));
throw new Error(`No mutants were generated from ${filteredFiles.length} source files. See troubleshooting suggestions above.`);
}
console.log(chalk_1.default.blue(`\nš§¬ Generated ${allMutants.length} mutants across ${filteredFiles.length} files`));
console.log(chalk_1.default.blue(`\nš§Ŗ Now testing each mutant to see which ones are killed...`));
// Step 2: Test each mutant to determine if it's killed or survived
const results = [];
for (let i = 0; i < allMutants.length; i++) {
const mutant = allMutants[i];
console.log(chalk_1.default.cyan(`\nš§¬ Testing mutant ${i + 1}/${allMutants.length} (ID: ${mutant.id}) in ${mutant.sourceFile}...`));
console.log(chalk_1.default.dim(` Mutation: ${mutant.mutationType} at line ${mutant.line}`));
console.log(chalk_1.default.dim(` Original: "${mutant.original}" ā Mutated: "${mutant.mutated}"`));
const mutationResult = await this.testSingleMutant(projectPath, mutant, mutant.sourceFile, mutant.outputDir, isForgeProject);
// Store the mutant file location in the result for future iterations
mutationResult.mutantId = mutant.id;
mutationResult.outputDir = mutant.outputDir;
results.push(mutationResult);
// Show progress
const killed = results.filter(r => r.status === 'killed').length;
const survived = results.filter(r => r.status === 'survived').length;
console.log(chalk_1.default.dim(` Progress: ${killed} killed, ${survived} survived so far`));
}
// Summary
const totalMutations = results.length;
const killedMutations = results.filter(r => r.status === 'killed').length;
const survivedMutations = results.filter(r => r.status === 'survived').length;
const mutationScore = totalMutations > 0 ? (killedMutations / totalMutations) * 100 : 0;
console.log(chalk_1.default.blue(`\nš Mutation Testing Results:`));
console.log(chalk_1.default.green(` ā
Total mutants tested: ${totalMutations}`));
console.log(chalk_1.default.green(` š Mutants killed: ${killedMutations}`));
console.log(chalk_1.default.red(` š§ Mutants survived: ${survivedMutations}`));
console.log(chalk_1.default.cyan(` š Mutation score: ${mutationScore.toFixed(2)}%`));
if (survivedMutations > 0) {
console.log(chalk_1.default.yellow(`\nā ļø Survived mutants indicate gaps in your test suite:`));
const survivedResults = results.filter(r => r.status === 'survived');
survivedResults.forEach(r => {
console.log(chalk_1.default.yellow(` ā¢ ${r.file}:${r.line} - ${r.mutationType}: "${r.original}" ā "${r.mutated}"`));
});
}
spinner.succeed(chalk_1.default.green(`Mutation testing completed! Score: ${mutationScore.toFixed(2)}% (${killedMutations}/${totalMutations} killed)`));
return results;
}
catch (error) {
spinner.fail(chalk_1.default.red('Mutation testing failed'));
console.error(chalk_1.default.red('Error details:'));
console.error(chalk_1.default.red(` Message: ${error.message}`));
if (error.stderr) {
console.error(chalk_1.default.red(` Stderr: ${error.stderr}`));
}
if (error.cmd) {
console.error(chalk_1.default.red(` Command: ${error.cmd}`));
}
throw error;
}
}
/**
* Test a single mutant by replacing the original file, running tests, and restoring
*/
async testSingleMutant(projectPath, mutant, originalSourceFile, mutantOutputDir, isForgeProject) {
const originalFilePath = path_1.default.join(projectPath, originalSourceFile);
const mutantFilePath = path_1.default.join(projectPath, mutantOutputDir, 'mutants', mutant.id.toString(), originalSourceFile);
const backupFilePath = `${originalFilePath}.backup`;
// Set up cleanup handler for interruptions
let cleanupHandler = null;
const cleanup = async () => {
try {
// Always try to restore the original file
await fs_1.promises.copyFile(backupFilePath, originalFilePath);
await fs_1.promises.unlink(backupFilePath);
console.log(chalk_1.default.green(` ā Restored original file: ${originalSourceFile}`));
}
catch (restoreError) {
console.error(chalk_1.default.red(` ā Failed to restore original file: ${restoreError}`));
}
};
// Register cleanup handler for process interruption
cleanupHandler = cleanup;
const handleInterrupt = async () => {
console.log(chalk_1.default.yellow('\nā ļø Interrupted! Restoring original files...'));
if (cleanupHandler) {
await cleanupHandler();
}
process.exit(1);
};
process.once('SIGINT', handleInterrupt);
process.once('SIGTERM', handleInterrupt);
try {
// Step 1: Backup original file
await fs_1.promises.copyFile(originalFilePath, backupFilePath);
// Step 2: Replace original with mutant
await fs_1.promises.copyFile(mutantFilePath, originalFilePath);
// Step 3: Run tests
const testCommand = 'forge test';
console.log(chalk_1.default.dim(` Running: ${testCommand}`));
try {
const { stdout, stderr } = await execAsync(testCommand, {
cwd: projectPath,
timeout: 120000, // 2 minute timeout per test
maxBuffer: 1024 * 1024 * 5 // 5MB buffer
});
// If tests pass, mutant survived
console.log(chalk_1.default.red(` š§ SURVIVED - Tests still pass with this mutant`));
return {
file: originalSourceFile,
line: mutant.line || 0,
column: mutant.column || 0,
mutationType: mutant.mutationType || 'unknown',
original: mutant.original || '',
mutated: mutant.mutated || '',
status: 'survived',
testOutput: stdout || ''
};
}
catch (testError) {
// If tests fail, mutant was killed
console.log(chalk_1.default.green(` š KILLED - Tests failed with this mutant`));
// Log why it was killed
if (testError.code === 'ETIMEDOUT') {
console.log(chalk_1.default.dim(` Reason: Test execution timed out after 2 minutes`));
}
else if (testError.stderr) {
// Look for compilation errors first
if (testError.stderr.includes('Compilation failed') || testError.stderr.includes('CompilerError')) {
console.log(chalk_1.default.dim(` Reason: Compilation failed (invalid mutation)`));
}
else {
// Extract meaningful error lines, skipping warnings
const errorLines = testError.stderr.split('\n')
.filter((line) => line.trim() &&
!line.includes('Warning:') &&
!line.includes('nightly build') &&
(line.includes('Error:') || line.includes('error:') || line.includes('failed') || line.includes('FAILED')))
.slice(0, 3);
if (errorLines.length > 0) {
console.log(chalk_1.default.dim(` Reason: ${errorLines.join(' | ')}`));
}
else {
// If no error lines found, check stdout for test failures
const testFailures = (testError.stdout || '').split('\n')
.filter((line) => line.includes('FAILED') ||
line.includes('[FAIL') ||
line.includes('failing') ||
line.includes('Test result:'))
.slice(0, 2);
if (testFailures.length > 0) {
console.log(chalk_1.default.dim(` Reason: Test failures - ${testFailures.join(' | ')}`));
}
else {
// Show exit code and any other info
console.log(chalk_1.default.dim(` Reason: Exit code ${testError.code} (check full output for details)`));
}
}
}
}
else {
// No stderr, check stdout
const output = testError.stdout || testError.message || '';
const meaningfulLine = output.split('\n')
.find((line) => line.trim() &&
!line.includes('Warning:') &&
!line.includes('nightly build')) || 'Unknown error';
console.log(chalk_1.default.dim(` Reason: ${meaningfulLine.substring(0, 100)}`));
}
return {
file: originalSourceFile,
line: mutant.line || 0,
column: mutant.column || 0,
mutationType: mutant.mutationType || 'unknown',
original: mutant.original || '',
mutated: mutant.mutated || '',
status: 'killed',
testOutput: testError.stdout || testError.stderr || ''
};
}
}
catch (error) {
console.log(chalk_1.default.yellow(` ā ļø ERROR - Could not test mutant: ${error.message}`));
return {
file: originalSourceFile,
line: mutant.line || 0,
column: mutant.column || 0,
mutationType: mutant.mutationType || 'unknown',
original: mutant.original || '',
mutated: mutant.mutated || '',
status: 'error',
testOutput: error.message || ''
};
}
finally {
// Always restore original file
await cleanup();
// Remove signal handlers
process.removeListener('SIGINT', handleInterrupt);
process.removeListener('SIGTERM', handleInterrupt);
}
}
/**
* Parse mutants from Gambit output directory
*/
async parseGambitMutants(projectPath, outputDir, sourceFile) {
const mutants = [];
const gambitOutputDir = path_1.default.join(projectPath, outputDir);
try {