@dooor-ai/toolkit/dist/evals/cost.js

Guards, Evals & Observability for AI applications - works seamlessly with LangChain/LangGraph

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.CostEval = void 0;
const base_1 = require("./base");
/**
 * Eval that tracks cost per request
 *
 * @example
 * ```typescript
 * const eval = new CostEval({
 *   budgetLimitUsd: 0.10, // Alert if request costs > $0.10
 *   alertOnExceed: true
 * });
 * ```
 */
class CostEval extends base_1.Eval {
    constructor(config = {}) {
        super({
            threshold: config.budgetLimitUsd || 1.0,
            ...config,
        });
        // Pricing per 1M tokens (USD) - Updated 2025-01
        this.pricing = {
            "gpt-4o": { prompt: 5.0, completion: 15.0 },
            "gpt-4o-mini": { prompt: 0.15, completion: 0.6 },
            "gpt-4-turbo": { prompt: 10.0, completion: 30.0 },
            "gpt-3.5-turbo": { prompt: 0.5, completion: 1.5 },
            "claude-3-5-sonnet": { prompt: 3.0, completion: 15.0 },
            "claude-3-5-haiku": { prompt: 1.0, completion: 5.0 },
            "claude-3-opus": { prompt: 15.0, completion: 75.0 },
            "gemini-2.0-flash": { prompt: 0.0, completion: 0.0 }, // Free tier
            "gemini-1.5-pro": { prompt: 1.25, completion: 5.0 },
        };
        this.budgetLimitUsd = config.budgetLimitUsd || 1.0;
        this.alertOnExceed = config.alertOnExceed ?? true;
    }
    get name() {
        return "CostEval";
    }
    async evaluate(input, output, metadata) {
        const tokens = metadata.tokens;
        const model = metadata.model || "gpt-4o";
        if (!tokens || !tokens.prompt || !tokens.completion) {
            return {
                name: this.name,
                score: 0,
                passed: true,
                details: "No token data available",
                metadata: {
                    cost: 0,
                    currency: "USD",
                    model,
                },
                timestamp: new Date(),
            };
        }
        const cost = this.calculateCost(tokens, model);
        const passed = cost <= this.budgetLimitUsd;
        if (!passed && this.alertOnExceed) {
            console.warn(`⚠️ [CostEval] Budget exceeded: $${cost.toFixed(4)} > $${this.budgetLimitUsd.toFixed(4)}`);
        }
        return {
            name: this.name,
            score: cost,
            passed,
            details: passed
                ? `Cost within budget: $${cost.toFixed(4)}`
                : `Cost exceeded budget: $${cost.toFixed(4)} > $${this.budgetLimitUsd.toFixed(4)}`,
            metadata: {
                cost,
                currency: "USD",
                model,
                tokens: {
                    prompt: tokens.prompt,
                    completion: tokens.completion,
                    total: tokens.total,
                },
                budgetLimit: this.budgetLimitUsd,
            },
            timestamp: new Date(),
        };
    }
    /**
     * Calculate cost based on tokens and model
     */
    calculateCost(tokens, model) {
        const modelPricing = this.pricing[model] || this.pricing["gpt-4o"]; // Default to gpt-4o pricing
        const promptCost = (tokens.prompt / 1000000) * modelPricing.prompt;
        const completionCost = (tokens.completion / 1000000) * modelPricing.completion;
        return promptCost + completionCost;
    }
}
exports.CostEval = CostEval;