@vfarcic/dot-ai
Version:
AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance
235 lines (234 loc) • 9.94 kB
JavaScript
;
/**
* Dataset Analyzer for Multi-Model Comparison
*
* Analyzes evaluation datasets to group them by scenario and extract
* model responses for comparative evaluation.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.DatasetAnalyzer = void 0;
const fs_1 = require("fs");
const path_1 = require("path");
class DatasetAnalyzer {
datasetDir;
constructor(datasetDir = './eval/datasets') {
this.datasetDir = datasetDir;
}
/**
* Find all available datasets for a specific tool
*/
findDatasets(tool) {
const files = (0, fs_1.readdirSync)(this.datasetDir);
return files
.filter(file => file.startsWith(`${tool}_`) && file.endsWith('.jsonl'))
.map(file => (0, path_1.join)(this.datasetDir, file));
}
/**
* Parse dataset filename to extract components
* Format: {tool}_{interaction_id}_{sdk}_{model}_{timestamp}.jsonl
*/
parseDatasetFilename(filename) {
const basename = filename.replace(/^.*\//, '').replace(/\.jsonl$/, '');
const parts = basename.split('_');
if (parts.length < 5)
return null;
// For remediate datasets: remediate_{phase}_{action}_vercel_{model}_{timestamp}
// e.g., remediate_manual_analyze_vercel_gpt_timestamp
const tool = parts[0];
const timestamp = parts[parts.length - 1];
// Find 'vercel' SDK position to split correctly
const sdkIndex = parts.indexOf('vercel');
if (sdkIndex === -1)
return null;
// interaction_id is everything between tool and sdk
const interaction_id = parts.slice(1, sdkIndex).join('_');
const sdk = parts[sdkIndex];
const model = parts.slice(sdkIndex + 1, -1).join('_');
return { tool, interaction_id, sdk, model, timestamp };
}
/**
* Load and parse a dataset file
*/
loadDataset(filepath) {
try {
const content = (0, fs_1.readFileSync)(filepath, 'utf8').trim();
if (!content)
return null;
return JSON.parse(content);
}
catch (error) {
console.warn(`Failed to load dataset ${filepath}:`, error);
return null;
}
}
/**
* Group datasets by scenario for comparative evaluation
* Returns scenarios that have data from multiple models
* Groups by both tool and interaction_id to create separate evaluations for each phase
*/
groupByScenario(tool) {
const datasets = this.findDatasets(tool);
const scenarioGroups = new Map();
// Group datasets by filename pattern up to provider, then by model
for (const filepath of datasets) {
const sample = this.loadDataset(filepath);
if (!sample)
continue;
// Extract scenario key from filename pattern (up to provider)
const filename = filepath.replace(/^.*\//, ''); // Remove directory path
const filenameParts = filename.split('_');
const beforeProvider = [];
for (const part of filenameParts) {
if (part === 'vercel')
break; // Stop at SDK name
beforeProvider.push(part);
}
const scenarioKey = beforeProvider.join('_');
// Group by model within each scenario
const modelKey = `${sample.performance.sdk}_${sample.performance.model_version}`;
if (!scenarioGroups.has(scenarioKey)) {
scenarioGroups.set(scenarioKey, new Map());
}
const modelGroups = scenarioGroups.get(scenarioKey);
if (!modelGroups.has(modelKey)) {
modelGroups.set(modelKey, []);
}
// Parse failure_analysis if it exists
let failure_analysis = undefined;
if (sample.metadata.failure_analysis && sample.metadata.failure_analysis !== "") {
try {
if (typeof sample.metadata.failure_analysis === 'string') {
failure_analysis = JSON.parse(sample.metadata.failure_analysis);
}
else {
failure_analysis = sample.metadata.failure_analysis;
}
}
catch {
// If parsing fails, treat as no failure analysis
failure_analysis = undefined;
}
}
modelGroups.get(modelKey).push({
model: modelKey,
response: sample.output,
performance: sample.performance,
metadata: {
timestamp: sample.metadata.timestamp,
complexity: sample.metadata.complexity,
test_scenario: sample.metadata.test_scenario,
issue: sample.input.issue,
failure_analysis
}
});
}
// Convert to comparison scenarios - include ALL scenarios (remove multi-model filter)
const scenarios = [];
for (const [scenarioKey, modelGroups] of scenarioGroups) {
// Flatten model groups: each model may have multiple interactions
const allModelResponses = [];
for (const [modelKey, interactions] of modelGroups) {
// Combine multiple interactions per model into a single response
if (interactions.length === 1) {
allModelResponses.push(interactions[0]);
}
else {
// Multiple interactions per model - combine them
const combinedResponse = this.combineModelInteractions(modelKey, interactions);
allModelResponses.push(combinedResponse);
}
}
// Get representative issue from first model's first interaction
const firstModel = Array.from(modelGroups.values())[0]?.[0];
const issue = firstModel?.metadata?.issue || scenarioKey;
scenarios.push({
issue,
interaction_id: scenarioKey,
tool,
models: allModelResponses
});
}
return scenarios;
}
/**
* Combine multiple interactions per model into a single response for evaluation
*/
combineModelInteractions(modelKey, interactions) {
// Sort interactions by timestamp
const sorted = interactions.sort((a, b) => new Date(a.metadata.timestamp).getTime() - new Date(b.metadata.timestamp).getTime());
// Create combined response showing all interactions
const combinedResponse = sorted.map((interaction, index) => `**Interaction ${index + 1}:**\n` +
`Issue: ${interaction.metadata.issue}\n` +
`Response: ${interaction.response}\n`).join('\n---\n');
// Aggregate performance metrics
const totalDuration = sorted.reduce((sum, i) => sum + i.performance.duration_ms, 0);
const totalInputTokens = sorted.reduce((sum, i) => sum + i.performance.input_tokens, 0);
const totalOutputTokens = sorted.reduce((sum, i) => sum + i.performance.output_tokens, 0);
// Collect all failure analyses from all interactions that have them
const allFailures = [];
sorted.forEach((interaction, index) => {
if (interaction.metadata.failure_analysis) {
allFailures.push({
interaction_number: index + 1,
issue: interaction.metadata.issue,
...interaction.metadata.failure_analysis
});
}
});
// Use the first failure as the primary failure_analysis, but preserve all failures
const primaryFailureAnalysis = allFailures.length > 0 ? allFailures[0] : undefined;
return {
model: modelKey,
response: combinedResponse,
performance: {
...sorted[0].performance,
duration_ms: totalDuration,
input_tokens: totalInputTokens,
output_tokens: totalOutputTokens,
total_tokens: totalInputTokens + totalOutputTokens
},
metadata: {
...sorted[0].metadata,
issue: sorted[0].metadata.issue, // Use first interaction's issue as primary
interaction_count: interactions.length,
failure_analysis: primaryFailureAnalysis,
all_failures: allFailures.length > 0 ? allFailures : undefined
}
};
}
/**
* Get summary of available models across all scenarios for a tool
*/
getAvailableModels(tool) {
const datasets = this.findDatasets(tool);
const models = new Set();
for (const filepath of datasets) {
const parsed = this.parseDatasetFilename(filepath);
if (parsed) {
models.add(`${parsed.sdk}_${parsed.model}`);
}
}
return Array.from(models).sort();
}
/**
* Get statistics about dataset availability
*/
getDatasetStats(tool) {
const scenarios = this.groupByScenario(tool);
const datasets = this.findDatasets(tool);
const interactionTypes = new Set();
for (const filepath of datasets) {
const parsed = this.parseDatasetFilename(filepath);
if (parsed) {
interactionTypes.add(parsed.interaction_id);
}
}
return {
totalDatasets: datasets.length,
availableModels: this.getAvailableModels(tool),
scenariosWithMultipleModels: scenarios.length,
interactionTypes: Array.from(interactionTypes).sort()
};
}
}
exports.DatasetAnalyzer = DatasetAnalyzer;