@vfarcic/dot-ai/dist/evaluation/dataset-analyzer.d.ts

AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance

/**
 * Dataset Analyzer for Multi-Model Comparison
 *
 * Analyzes evaluation datasets to group them by scenario and extract
 * model responses for comparative evaluation.
 */
export interface DatasetSample {
    input: {
        issue: string;
        [key: string]: unknown;
    };
    output: string;
    performance: {
        duration_ms: number;
        input_tokens: number;
        output_tokens: number;
        total_tokens: number;
        model_version: string;
        sdk: string;
        iterations?: number;
        tool_calls_executed?: number;
        cache_read_tokens?: number;
        cache_creation_tokens?: number;
    };
    metadata: {
        timestamp: string;
        complexity: string;
        tags: string[];
        source: string;
        tool: string;
        test_scenario: string;
        failure_analysis: string;
    };
}
export interface ModelResponse {
    model: string;
    response: string;
    performance: {
        duration_ms: number;
        input_tokens: number;
        output_tokens: number;
        total_tokens: number;
        iterations?: number;
        tool_calls_executed?: number;
        cache_read_tokens?: number;
        cache_creation_tokens?: number;
    };
    metadata: {
        timestamp: string;
        complexity: string;
        test_scenario: string;
        issue?: string;
        interaction_count?: number;
        failure_analysis?: {
            failure_type?: string;
            failure_reason?: string;
            time_to_failure?: number;
            interaction_number?: number;
            issue?: string;
            [key: string]: unknown;
        };
        all_failures?: Array<{
            failure_type?: string;
            failure_reason?: string;
            time_to_failure?: number;
            interaction_number?: number;
            issue?: string;
            [key: string]: unknown;
        }>;
    };
}
export interface ComparisonScenario {
    issue: string;
    interaction_id: string;
    tool: string;
    models: ModelResponse[];
}
export declare class DatasetAnalyzer {
    private datasetDir;
    constructor(datasetDir?: string);
    /**
     * Find all available datasets for a specific tool
     */
    findDatasets(tool: string): string[];
    /**
     * Parse dataset filename to extract components
     * Format: {tool}_{interaction_id}_{sdk}_{model}_{timestamp}.jsonl
     */
    parseDatasetFilename(filename: string): {
        tool: string;
        interaction_id: string;
        sdk: string;
        model: string;
        timestamp: string;
    } | null;
    /**
     * Load and parse a dataset file
     */
    loadDataset(filepath: string): DatasetSample | null;
    /**
     * Group datasets by scenario for comparative evaluation
     * Returns scenarios that have data from multiple models
     * Groups by both tool and interaction_id to create separate evaluations for each phase
     */
    groupByScenario(tool: string): ComparisonScenario[];
    /**
     * Combine multiple interactions per model into a single response for evaluation
     */
    private combineModelInteractions;
    /**
     * Get summary of available models across all scenarios for a tool
     */
    getAvailableModels(tool: string): string[];
    /**
     * Get statistics about dataset availability
     */
    getDatasetStats(tool: string): {
        totalDatasets: number;
        availableModels: string[];
        scenariosWithMultipleModels: number;
        interactionTypes: string[];
    };
}
//# sourceMappingURL=dataset-analyzer.d.ts.map