vaultace-cli
Version:
AI-powered security scanner that detects vulnerabilities in AI-generated code. Proactive scanning, autonomous fixing, and emergency response for modern development teams.
561 lines (459 loc) • 21.7 kB
JavaScript
/**
* Functional Tests for CLI Commands
* End-to-end testing of all Vaultace CLI commands and user workflows
*/
const { execSync, exec } = require('child_process');
const fs = require('fs').promises;
const path = require('path');
const os = require('os');
describe('Vaultace CLI Functional Tests', () => {
let testDir;
let originalCwd;
let testConfigPath;
beforeAll(async () => {
originalCwd = process.cwd();
testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'vaultace-test-'));
testConfigPath = path.join(testDir, '.vaultace-config.json');
// Setup test environment
process.chdir(testDir);
// Create test config
const testConfig = {
api_endpoint: 'https://api-test.vaultace.co',
api_key: 'test_api_key_123',
organization: 'test-org',
user_id: 'test-user'
};
await fs.writeFile(testConfigPath, JSON.stringify(testConfig, null, 2));
});
afterAll(async () => {
process.chdir(originalCwd);
await fs.rmdir(testDir, { recursive: true });
});
function execCLI(command, options = {}) {
const fullCommand = `node ${path.join(originalCwd, 'src/index.js')} ${command}`;
return execSync(fullCommand, {
encoding: 'utf8',
cwd: testDir,
env: { ...process.env, VAULTACE_CONFIG: testConfigPath },
...options
});
}
function execCLIAsync(command, options = {}) {
const fullCommand = `node ${path.join(originalCwd, 'src/index.js')} ${command}`;
return new Promise((resolve, reject) => {
exec(fullCommand, {
encoding: 'utf8',
cwd: testDir,
env: { ...process.env, VAULTACE_CONFIG: testConfigPath },
...options
}, (error, stdout, stderr) => {
resolve({ error, stdout, stderr });
});
});
}
describe('CLI Help and Version Commands', () => {
test('should display help information', () => {
const output = execCLI('--help');
expect(output).toContain('vaultace');
expect(output).toContain('Commands:');
expect(output).toContain('scan');
expect(output).toContain('fix');
expect(output).toContain('workflow');
expect(output).toContain('auth');
expect(output).toContain('config');
});
test('should display version information', () => {
const output = execCLI('--version');
expect(output).toMatch(/\d+\.\d+\.\d+/);
});
test('should display command-specific help', () => {
const scanHelp = execCLI('scan --help');
expect(scanHelp).toContain('scan');
expect(scanHelp).toContain('Options:');
expect(scanHelp).toContain('--target');
expect(scanHelp).toContain('--format');
const workflowHelp = execCLI('workflow --help');
expect(workflowHelp).toContain('workflow');
expect(workflowHelp).toContain('Commands:');
expect(workflowHelp).toContain('list');
expect(workflowHelp).toContain('run');
});
});
describe('Configuration Management', () => {
test('should manage configuration settings', async () => {
// Test config set
execCLI('config set api_endpoint https://api-new.vaultace.co');
execCLI('config set timeout 30000');
// Test config get
const endpoint = execCLI('config get api_endpoint').trim();
expect(endpoint).toBe('https://api-new.vaultace.co');
const timeout = execCLI('config get timeout').trim();
expect(timeout).toBe('30000');
// Test config list
const configList = execCLI('config list');
expect(configList).toContain('api_endpoint');
expect(configList).toContain('timeout');
expect(configList).toContain('organization');
});
test('should validate configuration values', async () => {
// Test invalid API endpoint
const result = await execCLIAsync('config set api_endpoint invalid-url');
expect(result.error).toBeTruthy();
expect(result.stderr).toContain('invalid URL');
// Test invalid timeout
const result2 = await execCLIAsync('config set timeout -100');
expect(result2.error).toBeTruthy();
expect(result2.stderr).toContain('timeout must be positive');
});
test('should handle configuration file operations', async () => {
// Test config export
execCLI('config export config-backup.json');
const backupExists = await fs.access(path.join(testDir, 'config-backup.json'))
.then(() => true)
.catch(() => false);
expect(backupExists).toBe(true);
// Test config import
const newConfig = { api_endpoint: 'https://imported.vaultace.co', organization: 'imported-org' };
await fs.writeFile(path.join(testDir, 'new-config.json'), JSON.stringify(newConfig));
execCLI('config import new-config.json');
const importedEndpoint = execCLI('config get api_endpoint').trim();
expect(importedEndpoint).toBe('https://imported.vaultace.co');
});
});
describe('Authentication Commands', () => {
test('should handle authentication flow', async () => {
// Test login command
const loginResult = await execCLIAsync('auth login --api-key test_api_key_456');
expect(loginResult.error).toBeFalsy();
expect(loginResult.stdout).toContain('Successfully authenticated');
// Test status command
const statusOutput = execCLI('auth status');
expect(statusOutput).toContain('Authenticated');
expect(statusOutput).toContain('test-org');
// Test logout command
execCLI('auth logout');
const statusAfterLogout = execCLI('auth status');
expect(statusAfterLogout).toContain('Not authenticated');
});
test('should handle invalid authentication', async () => {
const invalidLoginResult = await execCLIAsync('auth login --api-key invalid_key');
expect(invalidLoginResult.error).toBeTruthy();
expect(invalidLoginResult.stderr).toContain('Authentication failed');
});
test('should manage API keys', () => {
// Test API key validation
const validKeyResult = execCLI('auth validate-key sk-valid-key-12345678901234567890');
expect(validKeyResult).toContain('API key format: valid');
// Test invalid key format
const invalidKeyResult = execCLI('auth validate-key invalid-key');
expect(invalidKeyResult).toContain('API key format: invalid');
});
});
describe('Security Scanning Commands', () => {
beforeEach(async () => {
// Create test files for scanning
await fs.writeFile(path.join(testDir, 'vulnerable.js'), `
const query = "SELECT * FROM users WHERE id = " + userId;
const apiKey = "sk-1234567890abcdef";
`);
await fs.writeFile(path.join(testDir, 'package.json'), JSON.stringify({
name: "test-app",
dependencies: {
"lodash": "4.17.0",
"express": "4.16.0"
}
}, null, 2));
await fs.writeFile(path.join(testDir, 'Dockerfile'), `
FROM ubuntu:18.04
USER root
EXPOSE 22
`);
});
test('should perform basic security scan', () => {
const scanOutput = execCLI('scan --target . --format json');
const scanResult = JSON.parse(scanOutput);
expect(scanResult).toHaveProperty('vulnerabilities');
expect(scanResult).toHaveProperty('summary');
expect(scanResult.vulnerabilities.length).toBeGreaterThan(0);
// Should detect JavaScript vulnerabilities
const jsVulns = scanResult.vulnerabilities.filter(v => v.file.includes('vulnerable.js'));
expect(jsVulns.length).toBeGreaterThan(0);
// Should detect dependency vulnerabilities
const depVulns = scanResult.vulnerabilities.filter(v => v.type === 'dependency');
expect(depVulns.length).toBeGreaterThan(0);
});
test('should support different output formats', () => {
// Test JSON format
const jsonOutput = execCLI('scan --target vulnerable.js --format json');
expect(() => JSON.parse(jsonOutput)).not.toThrow();
// Test table format
const tableOutput = execCLI('scan --target vulnerable.js --format table');
expect(tableOutput).toContain('│');
expect(tableOutput).toContain('Vulnerability');
expect(tableOutput).toContain('Severity');
// Test CSV format
const csvOutput = execCLI('scan --target vulnerable.js --format csv');
expect(csvOutput).toContain(',');
expect(csvOutput.split('\n')[0]).toContain('File,Type,Severity');
});
test('should filter scan results by severity', () => {
// Test high severity filter
const highSeverityOutput = execCLI('scan --target . --severity high --format json');
const highSeverityResult = JSON.parse(highSeverityOutput);
highSeverityResult.vulnerabilities.forEach(vuln => {
expect(['high', 'critical']).toContain(vuln.severity);
});
// Test medium and above filter
const mediumPlusOutput = execCLI('scan --target . --severity medium --format json');
const mediumPlusResult = JSON.parse(mediumPlusOutput);
mediumPlusResult.vulnerabilities.forEach(vuln => {
expect(['medium', 'high', 'critical']).toContain(vuln.severity);
});
});
test('should handle scan output to file', async () => {
const outputFile = path.join(testDir, 'scan-results.json');
execCLI(`scan --target . --output ${outputFile} --format json`);
const fileExists = await fs.access(outputFile).then(() => true).catch(() => false);
expect(fileExists).toBe(true);
const fileContent = await fs.readFile(outputFile, 'utf8');
const scanResult = JSON.parse(fileContent);
expect(scanResult).toHaveProperty('vulnerabilities');
});
test('should support incremental scanning', async () => {
// Initial scan
execCLI('scan --target . --save-baseline baseline.json');
// Create new vulnerable file
await fs.writeFile(path.join(testDir, 'new-vulnerable.js'), `
eval(userInput);
`);
// Incremental scan
const incrementalOutput = execCLI('scan --target . --baseline baseline.json --format json');
const incrementalResult = JSON.parse(incrementalOutput);
// Should only show new vulnerabilities
expect(incrementalResult.new_vulnerabilities.length).toBeGreaterThan(0);
expect(incrementalResult.new_vulnerabilities.some(v => v.file.includes('new-vulnerable.js'))).toBe(true);
});
});
describe('Vulnerability Fixing Commands', () => {
beforeEach(async () => {
await fs.writeFile(path.join(testDir, 'package.json'), JSON.stringify({
name: "test-app",
dependencies: {
"lodash": "4.17.0"
}
}, null, 2));
});
test('should suggest vulnerability fixes', () => {
const fixOutput = execCLI('fix --target package.json --dry-run --format json');
const fixResult = JSON.parse(fixOutput);
expect(fixResult).toHaveProperty('fixes');
expect(fixResult.fixes.length).toBeGreaterThan(0);
const lodashFix = fixResult.fixes.find(f => f.package === 'lodash');
expect(lodashFix).toBeDefined();
expect(lodashFix.recommended_version).toBeDefined();
expect(lodashFix.fix_type).toBe('dependency_update');
});
test('should apply automatic fixes', async () => {
// Apply fixes
execCLI('fix --target package.json --auto-apply');
// Verify package.json was updated
const updatedPackageJson = JSON.parse(await fs.readFile(path.join(testDir, 'package.json'), 'utf8'));
expect(updatedPackageJson.dependencies.lodash).not.toBe('4.17.0');
// Should create backup
const backupExists = await fs.access(path.join(testDir, 'package.json.backup'))
.then(() => true)
.catch(() => false);
expect(backupExists).toBe(true);
});
test('should support different fix strategies', () => {
// Conservative strategy
const conservativeOutput = execCLI('fix --target package.json --strategy conservative --dry-run --format json');
const conservativeResult = JSON.parse(conservativeOutput);
// Aggressive strategy
const aggressiveOutput = execCLI('fix --target package.json --strategy aggressive --dry-run --format json');
const aggressiveResult = JSON.parse(aggressiveOutput);
// Conservative should suggest fewer/safer changes
expect(conservativeResult.fixes.length).toBeLessThanOrEqual(aggressiveResult.fixes.length);
});
});
describe('Workflow Commands', () => {
test('should list available workflows', () => {
const workflowListOutput = execCLI('workflow list --format json');
const workflowList = JSON.parse(workflowListOutput);
expect(Array.isArray(workflowList.workflows)).toBe(true);
expect(workflowList.workflows.length).toBeGreaterThan(0);
// Should contain built-in workflows
const workflowNames = workflowList.workflows.map(w => w.id);
expect(workflowNames).toContain('vulnerability_response');
expect(workflowNames).toContain('security_scan_automated');
});
test('should show workflow details', () => {
const workflowInfoOutput = execCLI('workflow info vulnerability_response --format json');
const workflowInfo = JSON.parse(workflowInfoOutput);
expect(workflowInfo).toHaveProperty('id');
expect(workflowInfo).toHaveProperty('name');
expect(workflowInfo).toHaveProperty('description');
expect(workflowInfo).toHaveProperty('steps');
expect(workflowInfo.id).toBe('vulnerability_response');
});
test('should execute workflows', async () => {
// Create test input data
const workflowInput = {
vulnerability_id: 'CVE-2024-TEST',
severity: 'high',
affected_files: ['test.js']
};
await fs.writeFile(path.join(testDir, 'workflow-input.json'), JSON.stringify(workflowInput));
const executionResult = await execCLIAsync('workflow run vulnerability_response --input workflow-input.json --format json');
expect(executionResult.error).toBeFalsy();
const result = JSON.parse(executionResult.stdout);
expect(result).toHaveProperty('execution_id');
expect(result).toHaveProperty('status');
expect(['running', 'completed', 'pending']).toContain(result.status);
});
test('should monitor workflow executions', async () => {
// Start a workflow
const executionResult = await execCLIAsync('workflow run security_scan_automated --async');
const execution = JSON.parse(executionResult.stdout);
// Monitor execution
const monitorOutput = execCLI(`workflow monitor ${execution.execution_id} --format json`);
const monitorResult = JSON.parse(monitorOutput);
expect(monitorResult).toHaveProperty('execution_id');
expect(monitorResult).toHaveProperty('status');
expect(monitorResult).toHaveProperty('progress');
expect(monitorResult.execution_id).toBe(execution.execution_id);
});
test('should create custom workflows from templates', () => {
const createOutput = execCLI('workflow create --template basic_security_check --name my_custom_workflow');
expect(createOutput).toContain('Workflow created successfully');
// Verify workflow was created
const listOutput = execCLI('workflow list --format json');
const workflowList = JSON.parse(listOutput);
const customWorkflow = workflowList.workflows.find(w => w.name === 'my_custom_workflow');
expect(customWorkflow).toBeDefined();
});
});
describe('Team and Organization Commands', () => {
test('should manage team information', () => {
const teamInfoOutput = execCLI('team info --format json');
const teamInfo = JSON.parse(teamInfoOutput);
expect(teamInfo).toHaveProperty('organization');
expect(teamInfo).toHaveProperty('members');
expect(teamInfo).toHaveProperty('permissions');
expect(teamInfo.organization).toBe('test-org');
});
test('should list team members', () => {
const membersOutput = execCLI('team members --format json');
const members = JSON.parse(membersOutput);
expect(Array.isArray(members.members)).toBe(true);
members.members.forEach(member => {
expect(member).toHaveProperty('user_id');
expect(member).toHaveProperty('role');
expect(member).toHaveProperty('permissions');
});
});
test('should manage team permissions', async () => {
const permissionsOutput = execCLI('team permissions --format json');
const permissions = JSON.parse(permissionsOutput);
expect(permissions).toHaveProperty('user_permissions');
expect(permissions).toHaveProperty('role_permissions');
expect(Array.isArray(permissions.user_permissions)).toBe(true);
});
});
describe('Analytics and Reporting Commands', () => {
test('should generate security reports', () => {
const reportOutput = execCLI('analytics report --type security --format json');
const report = JSON.parse(reportOutput);
expect(report).toHaveProperty('summary');
expect(report).toHaveProperty('vulnerabilities_by_severity');
expect(report).toHaveProperty('trends');
expect(report).toHaveProperty('recommendations');
});
test('should show vulnerability trends', () => {
const trendsOutput = execCLI('analytics trends --period 30d --format json');
const trends = JSON.parse(trendsOutput);
expect(trends).toHaveProperty('period');
expect(trends).toHaveProperty('vulnerability_counts');
expect(trends).toHaveProperty('fix_rates');
expect(trends.period).toBe('30d');
});
test('should export analytics data', async () => {
const exportFile = path.join(testDir, 'analytics-export.csv');
execCLI(`analytics export --format csv --output ${exportFile}`);
const fileExists = await fs.access(exportFile).then(() => true).catch(() => false);
expect(fileExists).toBe(true);
const exportContent = await fs.readFile(exportFile, 'utf8');
expect(exportContent).toContain(','); // CSV format
expect(exportContent.split('\n').length).toBeGreaterThan(1);
});
});
describe('CI/CD Integration Commands', () => {
test('should support CI mode operations', () => {
const ciScanOutput = execCLI('scan --target . --ci-mode --format json');
const ciResult = JSON.parse(ciScanOutput);
expect(ciResult).toHaveProperty('ci_metadata');
expect(ciResult).toHaveProperty('exit_code');
expect(ciResult).toHaveProperty('summary');
// Should include CI-specific metadata
expect(ciResult.ci_metadata).toHaveProperty('execution_time');
expect(ciResult.ci_metadata).toHaveProperty('environment');
});
test('should generate CI reports', async () => {
const ciReportFile = path.join(testDir, 'ci-report.xml');
execCLI(`scan --target . --ci-report ${ciReportFile}`);
const reportExists = await fs.access(ciReportFile).then(() => true).catch(() => false);
expect(reportExists).toBe(true);
const reportContent = await fs.readFile(ciReportFile, 'utf8');
expect(reportContent).toContain('<testsuite');
expect(reportContent).toContain('</testsuite>');
});
test('should handle CI environment variables', () => {
const envVars = {
CI: 'true',
GITHUB_ACTIONS: 'true',
GITHUB_SHA: 'abc123',
GITHUB_REF: 'refs/heads/main'
};
const ciOutput = execCLI('scan --target . --format json', {
env: { ...process.env, ...envVars, VAULTACE_CONFIG: testConfigPath }
});
const ciResult = JSON.parse(ciOutput);
expect(ciResult.ci_metadata.detected_ci).toBe('github_actions');
expect(ciResult.ci_metadata.commit_sha).toBe('abc123');
expect(ciResult.ci_metadata.branch).toBe('main');
});
});
describe('Error Handling and Edge Cases', () => {
test('should handle invalid commands gracefully', async () => {
const invalidResult = await execCLIAsync('invalid-command');
expect(invalidResult.error).toBeTruthy();
expect(invalidResult.stderr).toContain('Unknown command');
});
test('should handle missing required parameters', async () => {
const missingParamResult = await execCLIAsync('scan');
expect(missingParamResult.error).toBeTruthy();
expect(missingParamResult.stderr).toContain('target is required');
});
test('should handle network connectivity issues', async () => {
// Test with invalid API endpoint
execCLI('config set api_endpoint https://invalid.nonexistent.domain');
const networkErrorResult = await execCLIAsync('auth status');
expect(networkErrorResult.error).toBeTruthy();
expect(networkErrorResult.stderr).toContain('network error');
});
test('should handle file permission issues', async () => {
const restrictedFile = path.join(testDir, 'restricted.js');
await fs.writeFile(restrictedFile, 'console.log("test");');
await fs.chmod(restrictedFile, 0o000); // No permissions
const permissionResult = await execCLIAsync(`scan --target ${restrictedFile}`);
expect(permissionResult.error).toBeTruthy();
expect(permissionResult.stderr).toContain('permission denied');
});
test('should provide helpful error messages', async () => {
const helpfulErrorResult = await execCLIAsync('workflow run nonexistent_workflow');
expect(helpfulErrorResult.error).toBeTruthy();
expect(helpfulErrorResult.stderr).toContain('Workflow not found');
expect(helpfulErrorResult.stderr).toContain('Available workflows:');
});
});
});