mnemos-coder/dist/tools/E2EValidator.d.ts

CLI-based coding agent with graph-based execution loop and terminal UI

/**
 * End-to-end validation system for testing agent functionality
 */
import { EventEmitter } from 'events';
export interface TestScenario {
    id: string;
    name: string;
    description: string;
    category: 'feature_creation' | 'bug_fixing' | 'refactoring' | 'analysis' | 'optimization';
    difficulty: 'easy' | 'medium' | 'hard';
    command: string;
    setup?: {
        files: Record<string, string>;
        dependencies?: string[];
        environment?: Record<string, string>;
    };
    expectations: {
        success: boolean;
        duration_max_ms?: number;
        files_created?: string[];
        files_modified?: string[];
        files_deleted?: string[];
        content_contains?: Array<{
            file: string;
            text: string | RegExp;
        }>;
        content_not_contains?: Array<{
            file: string;
            text: string | RegExp;
        }>;
        exit_code?: number;
        output_contains?: string | RegExp;
        output_not_contains?: string | RegExp;
        memory_limit_mb?: number;
        custom_validators?: string[];
    };
    cleanup?: {
        preserve_files?: string[];
        custom_cleanup?: string;
    };
}
export interface ValidationResult {
    scenario_id: string;
    success: boolean;
    duration_ms: number;
    errors: string[];
    warnings: string[];
    details: {
        setup_success: boolean;
        execution_success: boolean;
        validation_success: boolean;
        cleanup_success: boolean;
    };
    metrics: {
        files_created: number;
        files_modified: number;
        files_deleted: number;
        peak_memory_mb: number;
        output_length: number;
    };
    artifacts: {
        sandbox_path: string;
        logs_path: string;
        output: string;
        error_output: string;
    };
}
export interface E2ETestSuite {
    name: string;
    description: string;
    scenarios: TestScenario[];
    global_setup?: () => Promise<void>;
    global_cleanup?: () => Promise<void>;
}
export declare class E2EValidator extends EventEmitter {
    private sandboxManager;
    private customValidators;
    private testResults;
    constructor();
    /**
     * Register a custom validator function
     */
    registerValidator(id: string, validator: (result: any, scenario: TestScenario) => Promise<string[]>): void;
    /**
     * Run a single test scenario
     */
    runScenario(scenario: TestScenario): Promise<ValidationResult>;
    /**
     * Run a complete test suite
     */
    runSuite(suite: E2ETestSuite): Promise<{
        name: string;
        results: ValidationResult[];
        summary: {
            total: number;
            passed: number;
            failed: number;
            duration_ms: number;
            success_rate: number;
        };
    }>;
    /**
     * Generate comprehensive test report
     */
    generateReport(results: ValidationResult[], outputPath: string, format?: 'html' | 'json' | 'markdown'): Promise<void>;
    /**
     * Get test statistics
     */
    getStatistics(results: ValidationResult[]): {
        success_rate: number;
        average_duration: number;
        common_errors: Array<{
            error: string;
            count: number;
        }>;
        performance_metrics: {
            fastest_test: number;
            slowest_test: number;
            average_memory: number;
            peak_memory: number;
        };
        category_breakdown: Record<string, {
            passed: number;
            failed: number;
        }>;
    };
    private setupScenario;
    private executeScenario;
    private validateResults;
    private collectMetrics;
    private registerBuiltinValidators;
    private generateJSONReport;
    private generateMarkdownReport;
    private generateHTMLReport;
}
export declare function createE2EValidator(): E2EValidator;
//# sourceMappingURL=E2EValidator.d.ts.map