@craftapit/tester/dist/core/TestRunner.d.ts

A focused, LLM-powered testing framework for natural language test scenarios

import { CapabilityRegistry } from './CapabilityRegistry';
import { BaseAdapter } from '../adapters/BaseAdapter';
import { LLMAdapter } from '../adapters/LLMAdapter';
import { TestResult } from '../types/results';
/**
 * LLM adapter type options
 */
export type LLMAdapterType = 'craftacoder' | 'anthropic' | 'openai' | 'ollama' | 'custom';
/**
 * Base configuration for TestRunner
 */
export interface TestRunnerConfig {
    /**
     * LLM adapter type to use
     * @default 'craftacoder'
     */
    llmAdapter?: LLMAdapterType;
    /**
     * LLM adapter configuration
     * The shape depends on the adapter type
     */
    llmAdapterConfig?: Record<string, any>;
    /**
     * Custom LLM adapter instance (if type is 'custom')
     */
    customLLMAdapter?: LLMAdapter;
    /**
     * Additional adapters to register
     */
    adapters?: Record<string, BaseAdapter>;
    /**
     * Addons to register
     * The addons should be instances of classes that have a registerCapabilities method
     */
    addons?: Array<{
        registerCapabilities: (registry: CapabilityRegistry) => void;
    }>;
    /**
     * Whether to enable caching
     * @default true
     */
    caching?: boolean;
    /**
     * Path to the cache file
     * If not provided, a default path will be used
     */
    cachePath?: string;
    /**
     * Timeout for test execution in milliseconds
     * @default 60000 (1 minute)
     */
    timeout?: number;
    /**
     * Whether to print verbose output
     * @default false
     */
    verbose?: boolean;
}
/**
 * A test file descriptor for batch operations
 */
export interface TestFile {
    /**
     * Path to the test file
     */
    path: string;
    /**
     * Content of the test file (if available)
     */
    content?: string;
    /**
     * Test metadata (if available)
     */
    metadata?: Record<string, any>;
}
/**
 * Test runner for executing test scenarios
 */
export declare class TestRunner {
    private registry;
    private executor;
    private parser;
    private llmAdapter;
    private config;
    /**
     * Create a new test runner
     * @param config Test runner configuration
     */
    constructor(config?: TestRunnerConfig);
    /**
     * Create an LLM adapter based on the configuration
     */
    private createLLMAdapter;
    /**
     * Initialize the test runner
     * This will initialize all adapters
     */
    initialize(): Promise<void>;
    /**
     * Run a test from a file path
     * @param filePath Path to the test file
     * @returns Test results
     */
    runTestFile(filePath: string): Promise<TestResult>;
    /**
     * Run a test from a string
     * @param test Test content as a string
     * @returns Test results
     */
    runTest(test: string): Promise<TestResult>;
    /**
     * Run all tests in a directory
     * @param directory Directory containing test files
     * @param pattern File pattern to match (default: '**\*.md')
     * @returns Test results by file path
     */
    runTestDirectory(directory: string, pattern?: string): Promise<Record<string, TestResult>>;
    /**
     * Discover test files in a directory
     * @param directory Directory to search
     * @param pattern File pattern to match
     */
    private discoverTestFiles;
    /**
     * Log test results
     * @param results Test results
     */
    private logTestResults;
    /**
     * Log test results summary
     * @param results Test results by file path
     */
    private logTestResultsSummary;
    /**
     * Clean up the test runner
     * This will clean up all adapters
     */
    cleanup(): Promise<void>;
}
/**
 * Create a test runner with the given configuration
 * @param config Test runner configuration
 * @returns A test runner instance
 */
export declare function createTestRunner(config: TestRunnerConfig): TestRunner;