erosolar-cli
Version:
Unified AI agent framework for the command line - Multi-provider support with schema-driven tools, code intelligence, and transparent reasoning
549 lines • 17.8 kB
JavaScript
/**
* CLI Test Harness - PTY-based interactive CLI testing
*
* This module provides real runtime verification of CLI behavior by:
* 1. Spawning the CLI in a pseudo-terminal (PTY)
* 2. Sending simulated user input (including paste sequences)
* 3. Capturing and analyzing output
* 4. Verifying expected behaviors
*
* @license MIT
*/
import { spawn } from 'node:child_process';
import { join } from 'node:path';
import { EventEmitter } from 'node:events';
// ============================================================================
// CONSTANTS
// ============================================================================
const BRACKETED_PASTE_START = '\x1b[200~';
const BRACKETED_PASTE_END = '\x1b[201~';
const SPECIAL_KEYS = {
'enter': '\r',
'tab': '\t',
'escape': '\x1b',
'ctrl-c': '\x03',
'ctrl-d': '\x04',
};
// ============================================================================
// CLI TEST HARNESS
// ============================================================================
export class CLITestHarness extends EventEmitter {
config;
process = null;
output = '';
errors = '';
exitCode = null;
ptyModule = null;
constructor(config) {
super();
this.config = {
cwd: config.cwd,
env: config.env ?? {},
timeout: config.timeout ?? 30000,
usePty: config.usePty ?? false,
cliPath: config.cliPath ?? join(config.cwd, 'dist/bin/erosolar-optimized.js'),
};
}
/**
* Try to load node-pty for true PTY support
*/
async loadPtyModule() {
if (this.ptyModule)
return true;
try {
// Dynamic import to avoid hard dependency
// @ts-expect-error - node-pty is optional
this.ptyModule = await import('node-pty');
return true;
}
catch {
return false;
}
}
/**
* Start the CLI process
*/
async start() {
this.output = '';
this.errors = '';
this.exitCode = null;
const env = {
...process.env,
...this.config.env,
// Disable color output for easier parsing
NO_COLOR: '1',
FORCE_COLOR: '0',
// Set non-interactive mode hints
CI: '1',
};
if (this.config.usePty && await this.loadPtyModule()) {
await this.startWithPty(env);
}
else {
await this.startWithStdio(env);
}
}
/**
* Start CLI with PTY (for interactive features like bracketed paste)
*/
async startWithPty(env) {
const pty = this.ptyModule;
this.process = pty.spawn('node', [this.config.cliPath], {
name: 'xterm-256color',
cols: 120,
rows: 30,
cwd: this.config.cwd,
env,
});
this.process.onData((data) => {
this.output += data;
this.emit('output', data);
});
this.process.onExit(({ exitCode }) => {
this.exitCode = exitCode;
this.emit('exit', exitCode);
});
}
/**
* Start CLI with standard stdio (fallback, limited interactive support)
*/
async startWithStdio(env) {
this.process = spawn('node', [this.config.cliPath], {
cwd: this.config.cwd,
env,
stdio: ['pipe', 'pipe', 'pipe'],
});
this.process.stdout?.on('data', (data) => {
const str = data.toString();
this.output += str;
this.emit('output', str);
});
this.process.stderr?.on('data', (data) => {
const str = data.toString();
this.errors += str;
this.emit('error', str);
});
this.process.on('exit', (code) => {
this.exitCode = code ?? 0;
this.emit('exit', code);
});
}
/**
* Send input to the CLI
*/
write(input) {
if (!this.process) {
throw new Error('CLI process not started');
}
if (this.config.usePty && this.ptyModule) {
this.process.write(input);
}
else {
this.process.stdin?.write(input);
}
}
/**
* Send a bracketed paste sequence
*/
paste(content) {
this.write(BRACKETED_PASTE_START + content + BRACKETED_PASTE_END);
}
/**
* Send a special key
*/
sendKey(key) {
const sequence = SPECIAL_KEYS[key];
if (sequence) {
this.write(sequence);
}
}
/**
* Wait for output matching a pattern
*/
async waitForOutput(pattern, timeout = 5000) {
const startTime = Date.now();
const regex = typeof pattern === 'string' ? new RegExp(pattern, 'i') : pattern;
return new Promise((resolve, reject) => {
const check = () => {
if (regex.test(this.output)) {
resolve(this.output);
return;
}
if (Date.now() - startTime > timeout) {
reject(new Error(`Timeout waiting for pattern: ${pattern}`));
return;
}
setTimeout(check, 100);
};
check();
});
}
/**
* Wait for a specified duration
*/
async wait(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Get current output
*/
getOutput() {
return this.output;
}
/**
* Get current errors
*/
getErrors() {
return this.errors;
}
/**
* Stop the CLI process
*/
async stop() {
if (!this.process) {
return this.exitCode ?? 0;
}
return new Promise((resolve) => {
const timeout = setTimeout(() => {
this.process?.kill('SIGKILL');
}, 5000);
this.process.on('exit', (code) => {
clearTimeout(timeout);
resolve(code ?? 0);
});
// Try graceful shutdown first
this.sendKey('ctrl-c');
setTimeout(() => {
this.sendKey('ctrl-d');
}, 500);
});
}
/**
* Run a complete test scenario
*/
async runScenario(scenario) {
const startTime = Date.now();
const result = {
scenario,
passed: true,
duration: 0,
output: '',
errors: [],
expectations: [],
};
try {
// Start the CLI
await this.start();
// Wait for initial startup
await this.wait(1000);
// Execute input sequence
for (const input of scenario.inputs) {
await this.executeInput(input);
}
// Wait for processing
await this.wait(500);
// Check expectations
for (const expectation of scenario.expectations) {
const expResult = await this.checkExpectation(expectation);
result.expectations.push(expResult);
if (!expResult.passed) {
result.passed = false;
result.errors.push(expResult.reason || 'Expectation failed');
}
}
}
catch (error) {
result.passed = false;
result.errors.push(error instanceof Error ? error.message : String(error));
}
finally {
await this.stop();
result.output = this.output;
result.duration = Date.now() - startTime;
}
return result;
}
/**
* Execute a single input action
*/
async executeInput(input) {
switch (input.type) {
case 'text':
if (input.content) {
this.write(input.content);
}
break;
case 'paste':
if (input.content) {
this.paste(input.content);
}
break;
case 'key':
if (input.key) {
this.sendKey(input.key);
}
break;
case 'wait':
await this.wait(input.delay ?? 100);
break;
}
}
/**
* Check a single expectation
*/
async checkExpectation(expectation) {
const timeout = expectation.timeout ?? 5000;
try {
switch (expectation.type) {
case 'output_contains': {
const pattern = expectation.value;
try {
await this.waitForOutput(pattern, timeout);
return { expectation, passed: true, actual: this.output };
}
catch {
return {
expectation,
passed: false,
actual: this.output.slice(-500),
reason: `Output does not contain: "${pattern}"`,
};
}
}
case 'output_matches': {
const regex = expectation.value instanceof RegExp
? expectation.value
: new RegExp(expectation.value);
if (regex.test(this.output)) {
return { expectation, passed: true, actual: this.output };
}
return {
expectation,
passed: false,
actual: this.output.slice(-500),
reason: `Output does not match: ${regex}`,
};
}
case 'output_not_contains': {
const pattern = expectation.value;
if (!this.output.includes(pattern)) {
return { expectation, passed: true };
}
return {
expectation,
passed: false,
actual: this.output.slice(-500),
reason: `Output unexpectedly contains: "${pattern}"`,
};
}
case 'exit_code': {
const expected = expectation.value;
if (this.exitCode === expected) {
return { expectation, passed: true, actual: String(this.exitCode) };
}
return {
expectation,
passed: false,
actual: String(this.exitCode),
reason: `Exit code ${this.exitCode} !== expected ${expected}`,
};
}
default:
return {
expectation,
passed: false,
reason: `Unknown expectation type: ${expectation.type}`,
};
}
}
catch (error) {
return {
expectation,
passed: false,
reason: error instanceof Error ? error.message : String(error),
};
}
}
}
// ============================================================================
// PREDEFINED TEST SCENARIOS
// ============================================================================
/**
* Create a paste handling test scenario
*/
export function createPasteTestScenario(content, expectedLineCount) {
return {
id: `paste-${expectedLineCount}-lines`,
description: `Test pasting ${expectedLineCount} lines of content`,
category: 'paste',
inputs: [
{ type: 'wait', delay: 2000 }, // Wait for CLI to initialize
{ type: 'paste', content },
{ type: 'wait', delay: 500 },
{ type: 'key', key: 'enter' },
{ type: 'wait', delay: 1000 },
],
expectations: [
{
type: 'output_contains',
value: `${expectedLineCount} line`,
description: 'Should show line count in preview',
},
],
};
}
/**
* Create a multi-line input test scenario
*/
export function createMultiLineInputScenario() {
const multiLineContent = 'function test() {\n console.log("hello");\n return true;\n}';
return {
id: 'multi-line-input',
description: 'Test multi-line code input handling',
category: 'input',
inputs: [
{ type: 'wait', delay: 2000 },
{ type: 'paste', content: multiLineContent },
{ type: 'wait', delay: 500 },
{ type: 'key', key: 'enter' },
{ type: 'wait', delay: 2000 },
],
expectations: [
{
type: 'output_contains',
value: '4 line',
description: 'Should show 4 lines in preview',
},
{
type: 'output_not_contains',
value: 'error',
description: 'Should not show errors',
},
],
};
}
/**
* Create a slash command test scenario
*/
export function createSlashCommandScenario(command) {
return {
id: `slash-${command.replace('/', '')}`,
description: `Test /${command} slash command`,
category: 'command',
inputs: [
{ type: 'wait', delay: 2000 },
{ type: 'text', content: `/${command}` },
{ type: 'key', key: 'enter' },
{ type: 'wait', delay: 1000 },
],
expectations: [
{
type: 'output_not_contains',
value: 'Unknown command',
description: 'Command should be recognized',
},
],
};
}
// ============================================================================
// VERIFICATION INTEGRATION
// ============================================================================
/**
* Run verification tests for a specific claim type
*/
export async function runVerificationTests(claimType, workingDir) {
const harness = new CLITestHarness({
cwd: workingDir,
timeout: 60000,
usePty: true, // Try PTY first
});
const scenarios = [];
// Select scenarios based on claim type
switch (claimType) {
case 'paste_handling':
scenarios.push(createPasteTestScenario('line1\nline2\nline3', 3), createPasteTestScenario('a\nb\nc\nd\ne\nf\ng\nh\ni\nj', 10), createMultiLineInputScenario());
break;
case 'slash_commands':
scenarios.push(createSlashCommandScenario('help'), createSlashCommandScenario('model'), createSlashCommandScenario('clear'));
break;
case 'build_success':
// For build claims, we don't need PTY - just run npm build
scenarios.push({
id: 'build-check',
description: 'Verify project builds successfully',
category: 'command',
inputs: [],
expectations: [],
});
break;
default:
// Generic behavior test
scenarios.push({
id: 'startup-check',
description: 'Verify CLI starts without errors',
category: 'behavior',
inputs: [
{ type: 'wait', delay: 3000 },
{ type: 'key', key: 'ctrl-c' },
],
expectations: [
{
type: 'output_not_contains',
value: 'Error:',
description: 'Should not show errors on startup',
},
],
});
}
const results = [];
let allPassed = true;
for (const scenario of scenarios) {
const result = await harness.runScenario(scenario);
results.push(result);
if (!result.passed) {
allPassed = false;
}
}
const passed = results.filter(r => r.passed).length;
const failed = results.filter(r => !r.passed).length;
const summary = `${passed}/${results.length} tests passed${failed > 0 ? `, ${failed} failed` : ''}`;
return {
passed: allPassed,
results,
summary,
};
}
/**
* Verify a single claim using the appropriate method (PTY-based)
*/
export async function verifyClaim(claim, workingDir) {
const startTime = Date.now();
// Route ALL claims through PTY-based verification
const verification = await runVerificationTests(claim.context['feature'] || claim.category, workingDir);
return {
claim,
passed: verification.passed,
method: 'pty',
output: verification.results.map(r => r.output).join('\n'),
error: verification.passed ? undefined : verification.summary,
duration: Date.now() - startTime,
};
}
/**
* Verify all claims using unified PTY harness
*/
export async function verifyAllClaims(claims, workingDir) {
const results = [];
for (const claim of claims) {
const result = await verifyClaim(claim, workingDir);
results.push(result);
}
const passed = results.filter(r => r.passed).length;
const failed = results.filter(r => !r.passed).length;
return {
results,
summary: { total: claims.length, passed, failed },
};
}
export default CLITestHarness;
//# sourceMappingURL=cliTestHarness.js.map