UNPKG

citty-test-utils

Version:

Unified testing framework for CLI applications with auto-detecting local/cleanroom execution, vitest config integration, and simplified scenario DSL.

981 lines (867 loc) โ€ข 29.3 kB
#!/usr/bin/env node /** * @fileoverview AST-Based CLI Coverage Analyzer - Innovation Implementation * @description Uses AST parsing to analyze actual CLI definition files for accurate coverage */ import { readFileSync, existsSync, readdirSync, statSync } from 'fs' import { join, basename } from 'path' import { parse } from 'acorn' import { simple as walk } from 'acorn-walk' /** * AST-Based CLI Coverage Analyzer - Innovation Implementation * Uses AST parsing to analyze actual CLI definition files for accurate coverage */ export class ASTCLIAnalyzer { constructor(options = {}) { this.options = { cliPath: options.cliPath || 'src/cli.mjs', testDir: options.testDir || 'test', includePatterns: options.includePatterns || [ '.test.mjs', '.test.js', '.spec.mjs', '.spec.js', ], excludePatterns: options.excludePatterns || ['node_modules', '.git', 'coverage'], verbose: options.verbose || false, ...options, } } /** * Analyze CLI coverage using AST parsing * @param {Object} options - Analysis options * @returns {Promise<Object>} Coverage analysis results */ async analyze(options = {}) { const analysisOptions = { ...this.options, ...options } if (analysisOptions.verbose) { console.log('๐Ÿš€ Starting AST-based CLI coverage analysis...') } try { // Step 1: Discover CLI structure using AST parsing const cliStructure = await this.discoverCLIStructureAST(analysisOptions) // Step 2: Discover and analyze test files using AST const testPatterns = await this.discoverTestPatternsAST(analysisOptions) // Step 3: Calculate coverage const coverage = this.calculateCoverage(cliStructure, testPatterns) // Step 4: Generate report const report = this.generateReport(cliStructure, testPatterns, coverage, analysisOptions) if (analysisOptions.verbose) { console.log('โœ… AST-based CLI coverage analysis complete') } return report } catch (error) { throw new Error(`AST CLI coverage analysis failed: ${error.message}`) } } /** * Discover CLI structure by parsing CLI definition files with AST * @param {Object} options - Analysis options * @returns {Promise<Object>} CLI structure */ async discoverCLIStructureAST(options) { const cliPath = options.cliPath || 'src/cli.mjs' if (options.verbose) { console.log(`๐Ÿ” Discovering CLI structure via AST: ${cliPath}`) } try { const content = readFileSync(cliPath, 'utf8') const structure = { commands: new Map(), globalOptions: new Map(), } // Parse the CLI file AST const ast = this.parseJavaScriptFile(content, cliPath) // Walk AST to find CLI definitions walk(ast, { CallExpression: (node) => { this.extractCLIDefinition(node, structure, options) }, ExportNamedDeclaration: (node) => { this.extractExportedCommands(node, structure, options) }, ExportDefaultDeclaration: (node) => { this.extractDefaultExport(node, structure, options) }, }) if (options.verbose) { console.log(`๐Ÿ“‹ Discovered ${structure.commands.size} commands via AST`) console.log(`๐Ÿ“‹ Found ${structure.globalOptions.size} global options via AST`) } return structure } catch (error) { throw new Error(`AST CLI structure discovery failed: ${error.message}`) } } /** * Parse JavaScript file into AST * @param {string} content - File content * @param {string} filePath - File path * @returns {Object} AST */ parseJavaScriptFile(content, filePath) { try { // Remove shebang if present let cleanContent = content if (content.startsWith('#!')) { const firstNewline = content.indexOf('\n') if (firstNewline !== -1) { cleanContent = content.substring(firstNewline + 1) } } return parse(cleanContent, { ecmaVersion: 2022, sourceType: 'module', allowReturnOutsideFunction: true, allowImportExportEverywhere: true, allowAwaitOutsideFunction: true, }) } catch (error) { throw new Error(`Failed to parse ${filePath}: ${error.message}`) } } /** * Extract CLI definition from CallExpression node * @param {Object} node - AST CallExpression node * @param {Object} structure - Structure to populate * @param {Object} options - Options */ extractCLIDefinition(node, structure, options) { const callee = node.callee // Look for defineCommand calls if (callee.type === 'Identifier' && callee.name === 'defineCommand') { this.extractDefineCommand(node, structure, options) } // Look for defineCLI calls if (callee.type === 'Identifier' && callee.name === 'defineCLI') { this.extractDefineCLI(node, structure, options) } } /** * Extract defineCommand definition * @param {Object} node - AST CallExpression node * @param {Object} structure - Structure to populate * @param {Object} options - Options */ extractDefineCommand(node, structure, options) { const args = node.arguments if (!args || args.length === 0) return const commandDef = args[0] if (commandDef.type !== 'ObjectExpression') return const command = this.parseCommandDefinition(commandDef) if (command) { structure.commands.set(command.name, command) if (options.verbose) { console.log(`๐Ÿ” Found command: ${command.name}`) } } } /** * Extract defineCLI definition * @param {Object} node - AST CallExpression node * @param {Object} structure - Structure to populate * @param {Object} options - Options */ extractDefineCLI(node, structure, options) { const args = node.arguments if (!args || args.length === 0) return const cliDef = args[0] if (cliDef.type !== 'ObjectExpression') return // Extract global options and subcommands this.parseCLIDefinition(cliDef, structure, options) } /** * Parse command definition object * @param {Object} objNode - AST ObjectExpression node * @returns {Object|null} Parsed command */ parseCommandDefinition(objNode) { const command = { name: null, description: '', subcommands: new Map(), arguments: new Map(), flags: new Map(), options: new Map(), tested: false, testFiles: [], } for (const prop of objNode.properties) { if (prop.type !== 'Property') continue const key = this.getPropertyKey(prop) const value = prop.value switch (key) { case 'meta': if (value.type === 'ObjectExpression') { this.parseMetaObject(value, command) } break case 'subCommands': if (value.type === 'ObjectExpression') { this.parseSubCommands(value, command) } break case 'args': if (value.type === 'ObjectExpression') { this.parseArgs(value, command) } break } } return command.name ? command : null } /** * Parse meta object for command name and description * @param {Object} metaNode - AST ObjectExpression node * @param {Object} command - Command to populate */ parseMetaObject(metaNode, command) { for (const prop of metaNode.properties) { if (prop.type !== 'Property') continue const key = this.getPropertyKey(prop) const value = prop.value if (key === 'name' && value.type === 'Literal') { command.name = value.value } else if (key === 'description' && value.type === 'Literal') { command.description = value.value } } } /** * Parse subcommands object * @param {Object} subCommandsNode - AST ObjectExpression node * @param {Object} command - Command to populate */ parseSubCommands(subCommandsNode, command) { for (const prop of subCommandsNode.properties) { if (prop.type !== 'Property') continue const subcommandName = this.getPropertyKey(prop) const subcommandDef = prop.value if ( subcommandDef.type === 'CallExpression' && subcommandDef.callee.type === 'Identifier' && subcommandDef.callee.name === 'defineCommand' ) { const subcommand = this.parseCommandDefinition(subcommandDef.arguments[0]) if (subcommand) { subcommand.name = subcommandName command.subcommands.set(subcommandName, subcommand) } } } } /** * Parse args object for flags and options * @param {Object} argsNode - AST ObjectExpression node * @param {Object} command - Command to populate */ parseArgs(argsNode, command) { for (const prop of argsNode.properties) { if (prop.type !== 'Property') continue const argName = this.getPropertyKey(prop) const argDef = prop.value if (argDef.type === 'ObjectExpression') { const arg = this.parseArgumentDefinition(argDef) if (arg) { if (arg.isFlag) { command.flags.set(argName, arg) } else { command.options.set(argName, arg) } } } } } /** * Parse argument definition * @param {Object} argDefNode - AST ObjectExpression node * @returns {Object|null} Parsed argument */ parseArgumentDefinition(argDefNode) { const arg = { name: '', description: '', type: 'string', isFlag: false, tested: false, testFiles: [], } for (const prop of argDefNode.properties) { if (prop.type !== 'Property') continue const key = this.getPropertyKey(prop) const value = prop.value switch (key) { case 'type': if (value.type === 'Literal') { arg.type = value.value arg.isFlag = value.value === 'boolean' } break case 'description': if (value.type === 'Literal') { arg.description = value.value } break case 'default': // Check if default is boolean if (value.type === 'Literal' && typeof value.value === 'boolean') { arg.isFlag = true } break } } return arg } /** * Parse CLI definition for global options * @param {Object} cliDefNode - AST ObjectExpression node * @param {Object} structure - Structure to populate * @param {Object} options - Options */ parseCLIDefinition(cliDefNode, structure, options) { for (const prop of cliDefNode.properties) { if (prop.type !== 'Property') continue const key = this.getPropertyKey(prop) const value = prop.value if (key === 'args' && value.type === 'ObjectExpression') { this.parseGlobalArgs(value, structure, options) } } } /** * Parse global arguments * @param {Object} argsNode - AST ObjectExpression node * @param {Object} structure - Structure to populate * @param {Object} options - Options */ parseGlobalArgs(argsNode, structure, options) { for (const prop of argsNode.properties) { if (prop.type !== 'Property') continue const argName = this.getPropertyKey(prop) const argDef = prop.value if (argDef.type === 'ObjectExpression') { const arg = this.parseArgumentDefinition(argDef) if (arg) { arg.name = argName structure.globalOptions.set(argName, arg) if (options.verbose) { console.log(`๐Ÿ” Found global option: ${argName}`) } } } } } /** * Extract exported commands * @param {Object} node - AST ExportNamedDeclaration node * @param {Object} structure - Structure to populate * @param {Object} options - Options */ extractExportedCommands(node, structure, options) { if (!node.declaration || node.declaration.type !== 'VariableDeclaration') return for (const declarator of node.declaration.declarations) { if (declarator.init && declarator.init.type === 'CallExpression') { this.extractCLIDefinition(declarator.init, structure, options) } } } /** * Extract default export * @param {Object} node - AST ExportDefaultDeclaration node * @param {Object} structure - Structure to populate * @param {Object} options - Options */ extractDefaultExport(node, structure, options) { if (node.declaration && node.declaration.type === 'CallExpression') { this.extractCLIDefinition(node.declaration, structure, options) } } /** * Get property key from AST Property node * @param {Object} prop - AST Property node * @returns {string} Property key */ getPropertyKey(prop) { if (prop.key.type === 'Identifier') { return prop.key.name } else if (prop.key.type === 'Literal') { return prop.key.value } return null } /** * Discover test patterns using AST parsing * @param {Object} options - Analysis options * @returns {Promise<Object>} Test patterns */ async discoverTestPatternsAST(options) { if (options.verbose) { console.log('๐Ÿงช Discovering test patterns via AST...') } const testFiles = this.findTestFiles(options.testDir, options) const patterns = { commands: new Map(), flags: new Map(), options: new Map(), } for (const testFile of testFiles) { try { const content = readFileSync(testFile, 'utf8') const ast = this.parseJavaScriptFile(content, testFile) walk(ast, { CallExpression: (node) => { this.extractTestCallPattern(node, patterns, testFile) }, }) } catch (error) { if (options.verbose) { console.log(`โš ๏ธ Could not analyze ${testFile}: ${error.message}`) } } } if (options.verbose) { console.log(`๐Ÿงช Analyzed ${testFiles.length} test files via AST`) console.log(`๐Ÿงช Found ${patterns.commands.size} command patterns`) console.log(`๐Ÿงช Found ${patterns.flags.size} flag patterns`) console.log(`๐Ÿงช Found ${patterns.options.size} option patterns`) } return patterns } /** * Extract test call patterns from AST * @param {Object} node - AST CallExpression node * @param {Object} patterns - Patterns to populate * @param {string} testFile - Test file path */ extractTestCallPattern(node, patterns, testFile) { const callee = node.callee if (!callee || callee.type !== 'Identifier') return const functionName = callee.name if (!['runLocalCitty', 'runCitty'].includes(functionName)) return const args = node.arguments if (!args || args.length === 0) return const firstArg = args[0] if (!firstArg || firstArg.type !== 'ArrayExpression') return const commandArgs = this.extractArrayElements(firstArg) if (commandArgs.length === 0) return if (this.options.verbose) { console.log(`๐Ÿ” Found ${functionName} call: [${commandArgs.join(', ')}] in ${testFile}`) } const command = commandArgs[0] const subcommand = commandArgs[1] if (command && !patterns.commands.has(command)) { patterns.commands.set(command, { name: command, tested: true, testFiles: [testFile], subcommands: new Map(), }) } if (subcommand && patterns.commands.has(command)) { const cmd = patterns.commands.get(command) if (!cmd.subcommands) cmd.subcommands = new Map() cmd.subcommands.set(subcommand, { name: subcommand, tested: true, testFiles: [testFile], }) } for (let i = 1; i < commandArgs.length; i++) { const arg = commandArgs[i] if (typeof arg === 'string' && arg.startsWith('--')) { const flagName = arg.replace('--', '') if (!patterns.flags.has(flagName)) { patterns.flags.set(flagName, { name: flagName, tested: true, testFiles: [testFile], }) } } } } /** * Extract elements from array expression * @param {Object} arrayNode - AST array expression node * @returns {Array} Array of string values */ extractArrayElements(arrayNode) { const elements = [] for (const element of arrayNode.elements) { if (element) { if (element.type === 'Literal') { elements.push(element.value) } else if (element.type === 'TemplateLiteral') { const staticParts = element.quasis.map((q) => q.value.raw).filter((p) => p.length > 0) if (staticParts.length > 0) { elements.push(staticParts[0]) } } } } return elements } /** * Find test files in directory * @param {string} dir - Directory to search * @param {Object} options - Search options * @returns {Array} Array of test file paths */ findTestFiles(dir, options) { const testFiles = [] if (!existsSync(dir)) { return testFiles } const items = readdirSync(dir) for (const item of items) { const fullPath = join(dir, item) const stat = statSync(fullPath) if (options.excludePatterns.some((pattern) => item.includes(pattern))) { continue } if (stat.isDirectory()) { testFiles.push(...this.findTestFiles(fullPath, options)) } else if (stat.isFile()) { if (options.includePatterns.some((pattern) => item.includes(pattern))) { testFiles.push(fullPath) } } } return testFiles } /** * Calculate coverage statistics * @param {Object} cliStructure - CLI structure * @param {Object} testPatterns - Test patterns * @returns {Object} Coverage statistics */ calculateCoverage(cliStructure, testPatterns) { let totalCommands = 0 let testedCommands = 0 let totalSubcommands = 0 let testedSubcommands = 0 let totalFlags = 0 let testedFlags = 0 let totalOptions = 0 let testedOptions = 0 // Calculate command coverage for (const [name, command] of cliStructure.commands) { totalCommands++ if (testPatterns.commands.has(name)) { testedCommands++ command.tested = true command.testFiles = testPatterns.commands.get(name).testFiles const testCmd = testPatterns.commands.get(name) if (testCmd && testCmd.subcommands) { for (const [subName, subcommand] of command.subcommands) { totalSubcommands++ if (testCmd.subcommands.has(subName)) { testedSubcommands++ subcommand.tested = true subcommand.testFiles = testCmd.subcommands.get(subName).testFiles } } } } else { for (const [subName, subcommand] of command.subcommands) { totalSubcommands++ } } } // Calculate global option coverage for (const [name, option] of cliStructure.globalOptions) { if (option.isFlag) { totalFlags++ if (testPatterns.flags.has(name)) { testedFlags++ option.tested = true option.testFiles = testPatterns.flags.get(name).testFiles } } else { totalOptions++ if (testPatterns.options.has(name)) { testedOptions++ option.tested = true option.testFiles = testPatterns.options.get(name).testFiles } } } // Calculate command-specific option coverage for (const [name, command] of cliStructure.commands) { for (const [flagName, flag] of command.flags) { totalFlags++ if (testPatterns.flags.has(flagName)) { testedFlags++ flag.tested = true flag.testFiles = testPatterns.flags.get(flagName).testFiles } } for (const [optionName, option] of command.options) { totalOptions++ if (testPatterns.options.has(optionName)) { testedOptions++ option.tested = true option.testFiles = testPatterns.options.get(optionName).testFiles } } } const totalItems = totalCommands + totalSubcommands + totalFlags + totalOptions const totalTested = testedCommands + testedSubcommands + testedFlags + testedOptions const overallPercentage = totalItems > 0 ? (totalTested / totalItems) * 100 : 0 return { commands: { tested: testedCommands, total: totalCommands, percentage: totalCommands > 0 ? (testedCommands / totalCommands) * 100 : 0, }, subcommands: { tested: testedSubcommands, total: totalSubcommands, percentage: totalSubcommands > 0 ? (testedSubcommands / totalSubcommands) * 100 : 0, }, flags: { tested: testedFlags, total: totalFlags, percentage: totalFlags > 0 ? (testedFlags / totalFlags) * 100 : 0, }, options: { tested: testedOptions, total: totalOptions, percentage: totalOptions > 0 ? (testedOptions / totalOptions) * 100 : 0, }, overall: { tested: totalTested, total: totalItems, percentage: overallPercentage, }, } } /** * Generate comprehensive report * @param {Object} cliStructure - CLI structure * @param {Object} testPatterns - Test patterns * @param {Object} coverage - Coverage statistics * @param {Object} options - Report options * @returns {Object} Generated report */ generateReport(cliStructure, testPatterns, coverage, options) { const timestamp = new Date().toISOString() return { metadata: { analyzedAt: timestamp, cliPath: options.cliPath || 'src/cli.mjs', testDir: options.testDir || 'test', analysisMethod: 'AST-based', totalTestFiles: this.findTestFiles(options.testDir, options).length, totalCommands: cliStructure.commands.size, totalSubcommands: coverage.subcommands ? coverage.subcommands.total : 0, totalFlags: coverage.flags.total, totalOptions: coverage.options.total, }, commands: this.convertCommandsToObject(cliStructure.commands), globalOptions: Object.fromEntries(cliStructure.globalOptions), coverage: { summary: coverage, details: this.generateCoverageDetails(cliStructure, coverage), }, recommendations: this.generateRecommendations(coverage, cliStructure), } } /** * Convert commands Map to object * @param {Map} commands - Commands Map * @returns {Object} Converted commands object */ convertCommandsToObject(commands) { const result = {} for (const [name, command] of commands) { result[name] = { ...command, subcommands: Object.fromEntries(command.subcommands || new Map()), arguments: Object.fromEntries(command.arguments || new Map()), flags: Object.fromEntries(command.flags || new Map()), options: Object.fromEntries(command.options || new Map()), } } return result } /** * Generate detailed coverage information * @param {Object} cliStructure - CLI structure * @param {Object} coverage - Coverage statistics * @returns {Object} Detailed coverage information */ generateCoverageDetails(cliStructure, coverage) { const details = { untestedCommands: [], untestedSubcommands: [], untestedFlags: [], untestedOptions: [], } for (const [name, command] of cliStructure.commands) { if (!command.tested) { details.untestedCommands.push({ name, description: command.description, }) } for (const [subName, subcommand] of command.subcommands) { if (!subcommand.tested) { details.untestedSubcommands.push({ command: name, subcommand: subName, description: subcommand.description, }) } } } for (const [name, option] of cliStructure.globalOptions) { if (!option.tested) { if (option.isFlag) { details.untestedFlags.push({ name, description: option.description, global: true, }) } else { details.untestedOptions.push({ name, description: option.description, global: true, }) } } } return details } /** * Generate recommendations for improving coverage * @param {Object} coverage - Coverage statistics * @param {Object} cliStructure - CLI structure * @returns {Array} Array of recommendations */ generateRecommendations(coverage, cliStructure) { const recommendations = [] if (coverage.commands.percentage < 100) { recommendations.push({ type: 'command', priority: 'high', message: `Add tests for ${ coverage.commands.total - coverage.commands.tested } untested commands`, count: coverage.commands.total - coverage.commands.tested, }) } if (coverage.subcommands.percentage < 100) { recommendations.push({ type: 'subcommand', priority: 'high', message: `Add tests for ${ coverage.subcommands.total - coverage.subcommands.tested } untested subcommands`, count: coverage.subcommands.total - coverage.subcommands.tested, }) } if (coverage.flags.percentage < 100) { recommendations.push({ type: 'flag', priority: 'medium', message: `Add tests for ${coverage.flags.total - coverage.flags.tested} untested flags`, count: coverage.flags.total - coverage.flags.tested, }) } if (coverage.options.percentage < 100) { recommendations.push({ type: 'option', priority: 'medium', message: `Add tests for ${ coverage.options.total - coverage.options.tested } untested options`, count: coverage.options.total - coverage.options.tested, }) } return recommendations } /** * Format report in the specified format * @param {Object} report - Report data * @param {Object} options - Formatting options * @returns {Promise<string>} Formatted report */ async formatReport(report, options = {}) { const { format = 'text' } = options switch (format.toLowerCase()) { case 'text': return this.generateTextReport(report, options) case 'json': return JSON.stringify(report, null, 2) default: throw new Error(`Unsupported format: ${format}`) } } /** * Generate text format report * @param {Object} report - Report data * @param {Object} options - Formatting options * @returns {string} Text report */ generateTextReport(report, options) { const lines = [] lines.push('๐Ÿš€ AST-Based CLI Test Coverage Analysis') lines.push('='.repeat(50)) lines.push('') // Summary lines.push('๐Ÿ“ˆ Summary:') lines.push( ` Commands: ${report.coverage.summary.commands.tested}/${ report.coverage.summary.commands.total } (${report.coverage.summary.commands.percentage.toFixed(1)}%)` ) if (report.coverage.summary.subcommands) { lines.push( ` Subcommands: ${report.coverage.summary.subcommands.tested}/${ report.coverage.summary.subcommands.total } (${report.coverage.summary.subcommands.percentage.toFixed(1)}%)` ) } lines.push( ` Flags: ${report.coverage.summary.flags.tested}/${ report.coverage.summary.flags.total } (${report.coverage.summary.flags.percentage.toFixed(1)}%)` ) lines.push( ` Options: ${report.coverage.summary.options.tested}/${ report.coverage.summary.options.total } (${report.coverage.summary.options.percentage.toFixed(1)}%)` ) lines.push( ` Overall: ${report.coverage.summary.overall.tested}/${ report.coverage.summary.overall.total } (${report.coverage.summary.overall.percentage.toFixed(1)}%)` ) lines.push('') // Analysis info lines.push('โ„น๏ธ Analysis Info:') lines.push(` Method: ${report.metadata.analysisMethod}`) lines.push(` Analyzed: ${new Date(report.metadata.analyzedAt).toLocaleString()}`) lines.push(` CLI Path: ${report.metadata.cliPath}`) lines.push(` Test Directory: ${report.metadata.testDir}`) lines.push(` Test Files: ${report.metadata.totalTestFiles}`) lines.push(` Commands: ${report.metadata.totalCommands}`) if (report.metadata.totalSubcommands > 0) { lines.push(` Subcommands: ${report.metadata.totalSubcommands}`) } lines.push(` Flags: ${report.metadata.totalFlags}`) lines.push(` Options: ${report.metadata.totalOptions}`) lines.push('') // Recommendations if (report.recommendations && report.recommendations.length > 0) { lines.push('๐Ÿ’ก Recommendations:') report.recommendations.forEach((rec, index) => { lines.push(` ${index + 1}. [${rec.priority.toUpperCase()}] ${rec.message}`) }) lines.push('') } return lines.join('\n') } }