UNPKG

@five-vm/cli

Version:

High-performance CLI for Five VM development with WebAssembly integration

819 lines 31.4 kB
/** * Five CLI Test Command * * Run test suites for Five VM bytecode with comprehensive validation, * performance benchmarking, and compliance verification. */ import { readFile, readdir, stat } from 'fs/promises'; import { join, basename } from 'path'; import chalk from 'chalk'; import ora from 'ora'; import { FiveSDK } from '../sdk/FiveSDK.js'; import { FiveTestRunner } from '../sdk/testing/TestRunner.js'; import { ConfigManager } from '../config/ConfigManager.js'; import { Connection, Keypair } from '@solana/web3.js'; import { FiveFileManager } from '../utils/FiveFileManager.js'; /** * Five test command implementation */ export const testCommand = { name: 'test', description: 'Run test suites', aliases: ['t'], options: [ { flags: '-p, --pattern <pattern>', description: 'Test file pattern (default: **/*.test.json)', defaultValue: '**/*.test.json' }, { flags: '-f, --filter <filter>', description: 'Run tests matching filter pattern', required: false }, { flags: '--timeout <ms>', description: 'Test timeout in milliseconds', defaultValue: 30000 }, { flags: '--max-cu <units>', description: 'Maximum compute units per test', defaultValue: 1000000 }, { flags: '--parallel <count>', description: 'Number of parallel test workers (0 = CPU count)', defaultValue: 0 }, { flags: '--benchmark', description: 'Run performance benchmarks', defaultValue: false }, { flags: '--coverage', description: 'Generate test coverage report', defaultValue: false }, { flags: '--watch', description: 'Watch for file changes and re-run tests', defaultValue: false }, { flags: '--format <format>', description: 'Output format', choices: ['text', 'json', 'junit'], defaultValue: 'text' }, { flags: '--verbose', description: 'Show detailed test output', defaultValue: false }, { flags: '--sdk-runner', description: 'Use modern SDK-based test runner (recommended)', defaultValue: false }, { flags: '--on-chain', description: 'Execute tests on-chain (deploy + execute)', defaultValue: false }, { flags: '--batch', description: 'Run all .bin files in batch mode', defaultValue: false }, { flags: '-t, --target <target>', description: 'Override target network (devnet, testnet, mainnet, local)', required: false }, { flags: '-n, --network <url>', description: 'Override network RPC URL', required: false }, { flags: '-k, --keypair <file>', description: 'Override keypair file path', required: false }, { flags: '--retry-failed', description: 'Retry only previously failed tests', defaultValue: false }, { flags: '--analyze-costs', description: 'Include detailed cost analysis in results', defaultValue: false } ], arguments: [ { name: 'path', description: 'Test directory or file (default: ./tests)', required: false } ], examples: [ { command: 'five test', description: 'Run all tests in ./tests directory' }, { command: 'five test --filter "token*" --verbose', description: 'Run token tests with verbose output' }, { command: 'five test ./my-tests --benchmark --format json', description: 'Run benchmarks with JSON output' }, { command: 'five test --watch --parallel 4', description: 'Watch mode with 4 parallel workers' }, { command: 'five test test-scripts/ --on-chain --target devnet', description: 'Run on-chain tests on devnet' }, { command: 'five test test-scripts/ --on-chain --batch --analyze-costs', description: 'Batch test all .bin files with cost analysis' } ], handler: async (args, options, context) => { const { logger } = context; try { const testPath = args[0] || './tests'; // Handle on-chain testing mode if (options.onChain) { await runOnChainTests(testPath, options, context); return; } // Use modern SDK-based test runner if requested if (options.sdkRunner) { await runWithSdkRunner(testPath, options, context); return; } // Legacy approach with SDK integration // Initialize SDK for testing const spinner = ora('Initializing Five SDK for testing...').start(); // No initialization needed for SDK - it's stateless const sdk = FiveSDK.create({ debug: options.verbose }); spinner.succeed('Five SDK initialized'); // Discover test files const testSuites = await discoverTestSuites(testPath, options, logger); if (testSuites.length === 0) { logger.warn('No test files found'); return; } logger.info(`Found ${testSuites.length} test suite(s) with ${getTotalTestCount(testSuites)} test(s)`); // Run tests const results = await runTestSuites(testSuites, sdk, options, context); // Display results displayTestResults(results, options, logger); // Handle watch mode if (options.watch) { await watchAndRerun(testPath, options, context); } // Exit with appropriate code const failed = results.some(suite => suite.results.some(test => !test.passed)); if (failed) { process.exit(1); } } catch (error) { logger.error('Test execution failed:', error); throw error; } } }; /** * Discover test suites from files */ async function discoverTestSuites(testPath, options, logger) { const testSuites = []; try { const stats = await stat(testPath); if (stats.isFile()) { // Single test file const suite = await loadTestSuite(testPath); if (suite) { testSuites.push(suite); } } else if (stats.isDirectory()) { // Directory of test files const files = await readdir(testPath, { recursive: true }); for (const file of files) { if (typeof file === 'string' && file.endsWith('.test.json')) { const filePath = join(testPath, file); const suite = await loadTestSuite(filePath); if (suite) { testSuites.push(suite); } } } } } catch (error) { logger.warn(`Failed to discover tests at ${testPath}: ${error}`); } // Apply filter if specified if (options.filter) { return testSuites.filter(suite => suite.name.includes(options.filter) || suite.testCases.some(test => test.name.includes(options.filter))); } return testSuites; } /** * Load test suite from JSON file */ async function loadTestSuite(filePath) { try { const content = await readFile(filePath, 'utf8'); const data = JSON.parse(content); return { name: data.name || basename(filePath, '.test.json'), description: data.description, testCases: data.tests || data.testCases || [] }; } catch (error) { console.warn(`Failed to load test suite ${filePath}: ${error}`); return null; } } /** * Run all test suites */ async function runTestSuites(testSuites, sdk, options, context) { const { logger } = context; const results = []; for (const suite of testSuites) { logger.info(`Running test suite: ${suite.name}`); const suiteResults = []; for (const testCase of suite.testCases) { const result = await runSingleTest(testCase, sdk, options, context); suiteResults.push(result); if (options.verbose) { displaySingleTestResult(result, logger); } } results.push({ suite, results: suiteResults }); } return results; } /** * Run a single test case */ async function runSingleTest(testCase, sdk, options, context) { const { logger } = context; const startTime = Date.now(); try { // Load bytecode using centralized manager const fileManager = FiveFileManager.getInstance(); const loadedFile = await fileManager.loadFile(testCase.bytecode, { validateFormat: true }); const bytecode = loadedFile.bytecode; // Validation already done by file manager with validateFormat: true const validation = { valid: true }; // Skip redundant validation // Validation handled by centralized file manager // Parse input parameters if specified let parameters = []; if (testCase.input) { const inputData = await readFile(testCase.input, 'utf8'); try { parameters = JSON.parse(inputData); } catch { // If not JSON, treat as raw string parameter parameters = [inputData]; } } // Execute with timeout using Five SDK const executionPromise = FiveSDK.executeLocally(bytecode, 0, // Default to first function parameters, { debug: options.verbose, trace: options.verbose, computeUnitLimit: options.maxCu, abi: loadedFile.abi // Pass ABI for function name resolution }); const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Test timeout')), options.timeout)); const result = await Promise.race([executionPromise, timeoutPromise]); const duration = Date.now() - startTime; // Validate result against expected const passed = validateTestResult(result, testCase.expected); return { name: testCase.name, passed, duration, computeUnits: result.computeUnitsUsed || 0, details: options.verbose ? result : undefined }; } catch (error) { const duration = Date.now() - startTime; // Check if error was expected const passed = testCase.expected.success === false && testCase.expected.error !== undefined && error instanceof Error && error.message.includes(testCase.expected.error); return { name: testCase.name, passed, duration, error: error instanceof Error ? error.message : 'Unknown error' }; } } /** * Validate test result against expected outcome */ function validateTestResult(result, expected) { // Check success/failure if (result.success !== expected.success) { return false; } // If expecting success, check result value if (expected.success && expected.result !== undefined) { if (JSON.stringify(result.result) !== JSON.stringify(expected.result)) { return false; } } // Check compute units limit if (expected.maxComputeUnits && result.computeUnitsUsed > expected.maxComputeUnits) { return false; } return true; } /** * Display single test result */ function displaySingleTestResult(result, logger) { const status = result.passed ? chalk.green('✓') : chalk.red('✗'); const duration = chalk.gray(`(${result.duration}ms)`); const cu = result.computeUnits ? chalk.gray(`[${result.computeUnits} CU]`) : ''; console.log(` ${status} ${result.name} ${duration} ${cu}`); if (!result.passed && result.error) { console.log(chalk.red(` Error: ${result.error}`)); } } /** * Display comprehensive test results */ function displayTestResults(results, options, logger) { if (options.format === 'json') { console.log(JSON.stringify(results, null, 2)); return; } console.log('\n' + chalk.bold('Test Results:')); let totalTests = 0; let totalPassed = 0; let totalDuration = 0; for (const { suite, results: suiteResults } of results) { const passed = suiteResults.filter(r => r.passed).length; const total = suiteResults.length; const suiteDuration = suiteResults.reduce((sum, r) => sum + r.duration, 0); totalTests += total; totalPassed += passed; totalDuration += suiteDuration; const status = passed === total ? chalk.green('✓') : chalk.red('✗'); console.log(`\n${status} ${suite.name}: ${passed}/${total} passed (${suiteDuration}ms)`); if (options.verbose || passed !== total) { suiteResults.forEach(result => displaySingleTestResult(result, logger)); } } // Summary console.log('\n' + chalk.bold('Summary:')); console.log(` Total: ${totalTests} tests`); console.log(` ${chalk.green(`Passed: ${totalPassed}`)}`); const failed = totalTests - totalPassed; if (failed > 0) { console.log(` ${chalk.red(`Failed: ${failed}`)}`); } console.log(` Duration: ${totalDuration}ms`); if (failed === 0) { console.log(chalk.green('\n✓ All tests passed!')); } else { console.log(chalk.red(`\n✗ ${failed} test(s) failed`)); } } /** * Watch for file changes and re-run tests */ async function watchAndRerun(testPath, options, context) { const { logger } = context; // Dynamic import for file watching const chokidar = await import('chokidar'); logger.info('Watching for file changes...'); const watcher = chokidar.watch([testPath, '**/*.bin'], { persistent: true, ignoreInitial: true }); watcher.on('change', async (filePath) => { logger.info(`File changed: ${filePath}`); logger.info('Re-running tests...'); try { // Re-run the test command const testSuites = await discoverTestSuites(testPath, options, logger); const sdk = FiveSDK.create({ debug: options.verbose }); const results = await runTestSuites(testSuites, sdk, options, context); displayTestResults(results, options, logger); } catch (error) { logger.error(`Test re-run failed: ${error}`); } }); // Handle graceful shutdown process.on('SIGINT', () => { logger.info('Stopping test watcher...'); watcher.close(); process.exit(0); }); } /** * Run on-chain tests with deploy + execute pipeline */ async function runOnChainTests(testPath, options, context) { const { logger } = context; logger.info('🚀 Starting On-Chain Test Pipeline'); try { // Apply configuration with CLI overrides const configManager = ConfigManager.getInstance(); const overrides = { target: options.target, network: options.network, keypair: options.keypair }; const config = await configManager.applyOverrides(overrides); const targetPrefix = ConfigManager.getTargetPrefix(config.target); logger.info(`${targetPrefix} Testing on ${config.target}`); logger.info(`Network: ${config.networks[config.target].rpcUrl}`); logger.info(`Keypair: ${config.keypairPath}`); // Discover .bin test files const testFiles = await discoverBinFiles(testPath, options); if (testFiles.length === 0) { logger.warn('No .bin test files found'); return; } logger.info(`Found ${testFiles.length} test script(s)`); // Setup Solana connection and keypair const connection = new Connection(config.networks[config.target].rpcUrl, 'confirmed'); const signerKeypair = await loadKeypair(config.keypairPath); logger.info(`Deployer: ${signerKeypair.publicKey.toString()}`); // Run batch testing const results = await runBatchOnChainTests(testFiles, connection, signerKeypair, options, config); // Display comprehensive results displayOnChainTestResults(results, options, logger); // Exit with appropriate code if (results.failed > 0) { logger.error(`❌ ${results.failed}/${results.totalScripts} tests failed`); process.exit(1); } else { logger.info(`✅ All ${results.passed}/${results.totalScripts} tests passed!`); } } catch (error) { logger.error('On-chain testing failed:', error); throw error; } } /** * Run tests using modern SDK-based test runner */ async function runWithSdkRunner(testPath, options, context) { const { logger } = context; logger.info('🚀 Using modern Five SDK Test Runner'); // Create test runner with options const runner = new FiveTestRunner({ timeout: options.timeout, maxComputeUnits: options.maxCu, parallel: options.parallel || 0, verbose: options.verbose, debug: options.verbose, trace: options.verbose, pattern: options.filter || '*', failFast: false }); try { // Discover test suites const testSuites = await runner.discoverTestSuites(testPath); if (testSuites.length === 0) { logger.warn('No test files found'); return; } logger.info(`Found ${testSuites.length} test suite(s)`); // Run test suites const results = await runner.runTestSuites(testSuites); // Display results in requested format if (options.format === 'json') { console.log(JSON.stringify(results, null, 2)); } else { displaySdkTestResults(results, logger); } // Check for failures const totalFailed = results.reduce((sum, r) => sum + r.failed, 0); if (totalFailed > 0) { process.exit(1); } } catch (error) { logger.error('SDK Test Runner failed:', error); process.exit(1); } } /** * Display SDK test results */ function displaySdkTestResults(results, logger) { logger.info('\n📊 Test Results Summary:'); let totalPassed = 0; let totalFailed = 0; let totalSkipped = 0; let totalDuration = 0; for (const result of results) { totalPassed += result.passed; totalFailed += result.failed; totalSkipped += result.skipped; totalDuration += result.duration; const status = result.failed === 0 ? '✅' : '❌'; logger.info(`${status} ${result.suite.name}: ${result.passed}/${result.passed + result.failed + result.skipped} passed (${result.duration}ms)`); if (result.failed > 0) { const failedTests = result.results.filter((r) => !r.passed); for (const test of failedTests) { logger.error(` ❌ ${test.name}: ${test.error || 'Test failed'}`); } } } logger.info(`\n🎯 Overall: ${totalPassed} passed, ${totalFailed} failed, ${totalSkipped} skipped (${totalDuration}ms)`); if (totalFailed === 0) { logger.info(chalk.green('🎉 All tests passed!')); } else { logger.error(chalk.red(`💥 ${totalFailed} test(s) failed`)); } } /** * Get total test count across all suites */ function getTotalTestCount(testSuites) { return testSuites.reduce((total, suite) => total + suite.testCases.length, 0); } /** * Discover .bin files for on-chain testing */ async function discoverBinFiles(testPath, options) { const binFiles = []; try { const stats = await stat(testPath); if (stats.isFile()) { // Single file - check if it's a .bin file if (testPath.endsWith('.bin')) { binFiles.push(testPath); } } else if (stats.isDirectory()) { // Directory - recursively find all .bin files const files = await readdir(testPath, { recursive: true }); for (const file of files) { if (typeof file === 'string' && file.endsWith('.bin')) { const fullPath = join(testPath, file); // Skip node_modules directories if (fullPath.includes('node_modules')) { continue; } try { // Verify it's actually a file, not a directory const fileStats = await stat(fullPath); if (fileStats.isFile()) { binFiles.push(fullPath); } } catch (error) { // Skip files that can't be accessed continue; } } } } } catch (error) { console.warn(`Failed to discover .bin files at ${testPath}: ${error}`); } // Apply filter if specified if (options.filter) { return binFiles.filter(file => basename(file).includes(options.filter)); } return binFiles.sort(); // Sort for consistent ordering } /** * Load keypair from file path */ async function loadKeypair(keypairPath) { try { const keypairData = await readFile(keypairPath, 'utf8'); const secretKey = JSON.parse(keypairData); return Keypair.fromSecretKey(new Uint8Array(secretKey)); } catch (error) { throw new Error(`Failed to load keypair from ${keypairPath}: ${error}`); } } /** * Run batch on-chain tests with deploy → execute → verify pipeline */ async function runBatchOnChainTests(testFiles, connection, signerKeypair, options, config) { const results = []; const startTime = Date.now(); let totalCost = 0; console.log(chalk.bold(`\n🚀 Running ${testFiles.length} On-Chain Tests\n`)); for (let i = 0; i < testFiles.length; i++) { const scriptFile = testFiles[i]; const scriptName = basename(scriptFile, '.bin'); const testStartTime = Date.now(); const spinner = ora(`[${i + 1}/${testFiles.length}] Testing ${scriptName}...`).start(); try { // Load bytecode using centralized manager const fileManager = FiveFileManager.getInstance(); const loadedFile = await fileManager.loadFile(scriptFile, { validateFormat: true }); const bytecode = loadedFile.bytecode; if (options.verbose || options.debug) { spinner.text = `[${i + 1}/${testFiles.length}] Deploying ${scriptName} (${bytecode.length} bytes)...`; } // Deploy script const deployResult = await FiveSDK.deployToSolana(bytecode, connection, signerKeypair, { debug: options.verbose || options.debug || false, network: config.target, computeBudget: 1000000, maxRetries: 3 }); if (!deployResult.success) { spinner.fail(`[${i + 1}/${testFiles.length}] ${scriptName} deployment failed`); results.push({ scriptFile, passed: false, deployResult: { success: false, error: deployResult.error, cost: deployResult.deploymentCost || 0 }, totalDuration: Date.now() - testStartTime, totalCost: deployResult.deploymentCost || 0, error: `Deployment failed: ${deployResult.error}` }); totalCost += deployResult.deploymentCost || 0; continue; } if (options.verbose || options.debug) { spinner.text = `[${i + 1}/${testFiles.length}] Executing ${scriptName}...`; } // Execute script (function 0 with no parameters) const executeResult = await FiveSDK.executeOnSolana(deployResult.programId, connection, signerKeypair, 0, // Function index 0 [], // No parameters [], // No additional accounts { debug: options.verbose || options.debug || false, network: config.target, computeUnitLimit: 1000000, maxRetries: 3 }); const testDuration = Date.now() - testStartTime; const testCost = (deployResult.deploymentCost || 0) + (executeResult.cost || 0); totalCost += testCost; const passed = deployResult.success && executeResult.success; if (passed) { spinner.succeed(`[${i + 1}/${testFiles.length}] ${scriptName} ✅ (${testDuration}ms, ${(testCost / 1e9).toFixed(4)} SOL)`); } else { spinner.fail(`[${i + 1}/${testFiles.length}] ${scriptName} ❌ (${testDuration}ms)`); } results.push({ scriptFile, passed, deployResult: { success: deployResult.success, scriptAccount: deployResult.programId, transactionId: deployResult.transactionId, cost: deployResult.deploymentCost || 0, error: deployResult.error }, executeResult: { success: executeResult.success, transactionId: executeResult.transactionId, computeUnitsUsed: executeResult.computeUnitsUsed, result: executeResult.result, error: executeResult.error }, totalDuration: testDuration, totalCost: testCost }); } catch (error) { const testDuration = Date.now() - testStartTime; spinner.fail(`[${i + 1}/${testFiles.length}] ${scriptName} ❌ (error)`); results.push({ scriptFile, passed: false, totalDuration: testDuration, totalCost: 0, error: error instanceof Error ? error.message : 'Unknown error' }); } } const totalDuration = Date.now() - startTime; const passed = results.filter(r => r.passed).length; const failed = results.length - passed; return { totalScripts: testFiles.length, passed, failed, totalCost, totalDuration, results }; } /** * Display comprehensive on-chain test results */ function displayOnChainTestResults(summary, options, logger) { console.log('\n' + chalk.bold('📊 On-Chain Test Results Summary:\n')); // Overall statistics const successRate = ((summary.passed / summary.totalScripts) * 100).toFixed(1); const avgCostPerScript = summary.totalCost / summary.totalScripts; const totalCostSOL = summary.totalCost / 1e9; console.log(`📈 ${chalk.green(`${summary.passed}/${summary.totalScripts}`)} tests passed (${successRate}%)`); console.log(`⏱️ Total duration: ${summary.totalDuration}ms`); console.log(`💰 Total cost: ${totalCostSOL.toFixed(6)} SOL`); console.log(`📊 Average cost per script: ${(avgCostPerScript / 1e9).toFixed(6)} SOL`); if (options.analyzeCosts) { console.log('\n' + chalk.bold('💰 Cost Analysis:\n')); let deploymentCost = 0; let executionCost = 0; for (const result of summary.results) { if (result.deployResult?.cost) { deploymentCost += result.deployResult.cost; } if (result.executeResult?.cost) { executionCost += result.executeResult.cost; } } console.log(`🚀 Total deployment cost: ${(deploymentCost / 1e9).toFixed(6)} SOL`); console.log(`⚡ Total execution cost: ${(executionCost / 1e9).toFixed(6)} SOL`); console.log(`📊 Deployment vs Execution: ${((deploymentCost / summary.totalCost) * 100).toFixed(1)}% : ${((executionCost / summary.totalCost) * 100).toFixed(1)}%`); } // Failed tests details if (summary.failed > 0) { console.log('\n' + chalk.red.bold('❌ Failed Tests:\n')); const failedResults = summary.results.filter(r => !r.passed); for (const result of failedResults) { const scriptName = basename(result.scriptFile, '.bin'); console.log(` ❌ ${scriptName}:`); if (result.error) { console.log(` Error: ${chalk.red(result.error)}`); } if (result.deployResult && !result.deployResult.success) { console.log(` Deployment: ${chalk.red('Failed')} - ${result.deployResult.error}`); } if (result.executeResult && !result.executeResult.success) { console.log(` Execution: ${chalk.red('Failed')} - ${result.executeResult.error}`); } console.log(` Duration: ${result.totalDuration}ms`); console.log(` Cost: ${(result.totalCost / 1e9).toFixed(6)} SOL\n`); } } // Successful tests (if verbose) if (options.verbose && summary.passed > 0) { console.log('\n' + chalk.green.bold('✅ Successful Tests:\n')); const passedResults = summary.results.filter(r => r.passed); for (const result of passedResults) { const scriptName = basename(result.scriptFile, '.bin'); const deployTx = result.deployResult?.transactionId?.substring(0, 8) || 'N/A'; const executeTx = result.executeResult?.transactionId?.substring(0, 8) || 'N/A'; const computeUnits = result.executeResult?.computeUnitsUsed || 0; console.log(` ✅ ${scriptName}:`); console.log(` Deploy: ${deployTx}... | Execute: ${executeTx}...`); console.log(` Compute Units: ${computeUnits.toLocaleString()}`); console.log(` Duration: ${result.totalDuration}ms | Cost: ${(result.totalCost / 1e9).toFixed(6)} SOL\n`); } } // Summary message if (summary.failed === 0) { console.log(chalk.green.bold('🎉 All on-chain tests passed successfully!\n')); } else { console.log(chalk.red.bold(`💥 ${summary.failed} test(s) failed. Check logs above for details.\n`)); } } //# sourceMappingURL=test.js.map