@vfarcic/dot-ai/dist/evaluation/metadata-loader.js

AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance

"use strict";
/**
 * Shared Metadata Loader
 *
 * Provides consistent access to model and tool metadata across all evaluators
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.loadEvaluationMetadata = loadEvaluationMetadata;
exports.buildModelPricingContext = buildModelPricingContext;
exports.buildToolContext = buildToolContext;
const fs_1 = require("fs");
const path_1 = require("path");
/**
 * Load model and tool metadata from model-metadata.json
 */
function loadEvaluationMetadata() {
    try {
        const metadataPath = (0, path_1.join)(process.cwd(), 'src', 'evaluation', 'model-metadata.json');
        const metadata = JSON.parse((0, fs_1.readFileSync)(metadataPath, 'utf8'));
        console.log(`✅ Loaded metadata for ${Object.keys(metadata.models || {}).length} models and ${Object.keys(metadata.tools || {}).length} tools`);
        return {
            models: metadata.models || {},
            tools: metadata.tools || {}
        };
    }
    catch (error) {
        console.warn('⚠️  Failed to load evaluation metadata:', error);
        return { models: {}, tools: {} };
    }
}
/**
 * Build model pricing context for evaluation prompts
 */
function buildModelPricingContext(models) {
    const modelIds = Object.keys(models);
    if (modelIds.length === 0) {
        return 'No pricing information available.';
    }
    const pricingLines = modelIds.map(modelId => {
        const model = models[modelId];
        const inputCost = model.pricing?.input_cost_per_million_tokens?.toFixed(2) || 'N/A';
        const outputCost = model.pricing?.output_cost_per_million_tokens?.toFixed(2) || 'N/A';
        const avgCost = model.pricing
            ? ((model.pricing.input_cost_per_million_tokens + model.pricing.output_cost_per_million_tokens) / 2).toFixed(2)
            : 'N/A';
        const contextWindow = model.context_window ? `${(model.context_window / 1000).toFixed(0)}K` : 'N/A';
        return `- **${modelId}** (${model.provider}): $${avgCost}/1M tokens ($${inputCost} input, $${outputCost} output) | Context: ${contextWindow} tokens`;
    });
    return `## Model Pricing Information\n\n${pricingLines.join('\n')}`;
}
/**
 * Build tool context for evaluation prompts (tool-specific description and constraints)
 */
function buildToolContext(toolName, tools) {
    const tool = tools[toolName];
    if (!tool) {
        return `No metadata available for tool: ${toolName}`;
    }
    return `## Tool Being Evaluated: ${tool.name}

**Description**: ${tool.description}

**Primary Function**: ${tool.primaryFunction}

**Test Timeout Constraint**: ${tool.testTimeout}

**Success Criteria**:
${tool.successCriteria.map((c) => `- ${c}`).join('\n')}

**Model Requirements**:
${Object.entries(tool.modelRequirements).map(([key, value]) => `- **${key}**: ${value}`).join('\n')}

**IMPORTANT**: When analyzing model failures, consider whether the model exceeded the timeout constraint. Models that timeout should be noted as failing due to timeout constraints rather than quality issues.`;
}