recoder-analytics
Version:
Comprehensive analytics and monitoring for the Recoder.xyz ecosystem
552 lines • 22.5 kB
JavaScript
"use strict";
/**
* Real-time Cost Monitoring & Budget Management
*
* Tracks AI model costs in real-time, monitors pricing changes, enforces budgets,
* and provides cost optimization recommendations.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.costTracker = exports.CostTracker = void 0;
const shared_1 = require("@recoder/shared");
const events_1 = require("events");
class CostTracker extends events_1.EventEmitter {
constructor() {
super();
this.costEvents = new Map();
this.budgetConfigs = new Map();
this.pricingData = new Map();
this.activeAlerts = new Map();
this.config = {
maxEventHistory: 50000, // Maximum cost events to store
budgetCheckInterval: 60000, // Check budgets every minute
pricingUpdateInterval: 3600000, // Update pricing every hour
costAggregationWindow: 86400000, // 24 hours for daily metrics
};
this.budgetCheckTimer = null;
this.pricingUpdateTimer = null;
this.isRunning = false;
this.initializePricing();
this.initializeDefaultBudgets();
}
initializePricing() {
// Initialize with current known pricing (as of implementation)
const initialPricing = [
{
modelName: 'claude-sonnet-4',
provider: 'anthropic',
inputTokenPrice: 0.000003, // $3 per million tokens
outputTokenPrice: 0.000015, // $15 per million tokens
effectiveDate: new Date(),
source: 'manual'
},
{
modelName: 'claude-haiku-3',
provider: 'anthropic',
inputTokenPrice: 0.00000025, // $0.25 per million tokens
outputTokenPrice: 0.00000125, // $1.25 per million tokens
effectiveDate: new Date(),
source: 'manual'
},
{
modelName: 'gpt-4-turbo',
provider: 'openai',
inputTokenPrice: 0.00001, // $10 per million tokens
outputTokenPrice: 0.00003, // $30 per million tokens
effectiveDate: new Date(),
source: 'manual'
},
{
modelName: 'gpt-4o',
provider: 'openai',
inputTokenPrice: 0.000005, // $5 per million tokens
outputTokenPrice: 0.000015, // $15 per million tokens
effectiveDate: new Date(),
source: 'manual'
},
{
modelName: 'gemini-2.5-pro',
provider: 'google',
inputTokenPrice: 0.00000125, // $1.25 per million tokens
outputTokenPrice: 0.000005, // $5 per million tokens
effectiveDate: new Date(),
source: 'manual'
},
{
modelName: 'deepseek-v3',
provider: 'deepseek',
inputTokenPrice: 0.00000027, // $0.27 per million tokens
outputTokenPrice: 0.0000011, // $1.1 per million tokens
effectiveDate: new Date(),
source: 'manual'
}
];
initialPricing.forEach(pricing => {
this.pricingData.set(pricing.modelName, pricing);
});
shared_1.Logger.info(`Initialized pricing data for ${initialPricing.length} models`);
}
initializeDefaultBudgets() {
// Set default budgets
const defaultBudgets = [
{
budgetType: 'daily',
amount: 100, // $100 per day total
warningThreshold: 0.8,
hardLimit: false,
rollover: true,
notifications: [],
active: true
},
{
budgetType: 'monthly',
amount: 2000, // $2000 per month total
warningThreshold: 0.9,
hardLimit: true,
rollover: false,
notifications: [],
active: true
}
];
defaultBudgets.forEach((budget, index) => {
this.budgetConfigs.set(`default_${index}`, budget);
});
shared_1.Logger.info(`Initialized ${defaultBudgets.length} default budget configurations`);
}
/**
* Start cost tracking and monitoring
*/
async start() {
if (this.isRunning) {
shared_1.Logger.warn('Cost tracker is already running');
return;
}
shared_1.Logger.info('Starting cost tracking...');
// Start budget monitoring
this.budgetCheckTimer = setInterval(() => {
this.checkBudgets();
}, this.config.budgetCheckInterval);
// Start pricing updates
this.pricingUpdateTimer = setInterval(() => {
this.updatePricing();
}, this.config.pricingUpdateInterval);
this.isRunning = true;
this.emit('trackingStarted');
// Perform initial checks
await this.checkBudgets();
await this.updatePricing();
shared_1.Logger.info('Cost tracking started successfully');
}
/**
* Stop cost tracking
*/
async stop() {
if (!this.isRunning) {
return;
}
shared_1.Logger.info('Stopping cost tracking...');
if (this.budgetCheckTimer) {
clearInterval(this.budgetCheckTimer);
this.budgetCheckTimer = null;
}
if (this.pricingUpdateTimer) {
clearInterval(this.pricingUpdateTimer);
this.pricingUpdateTimer = null;
}
this.isRunning = false;
this.emit('trackingStopped');
shared_1.Logger.info('Cost tracking stopped');
}
/**
* Track a cost event from a model request
*/
async trackCost(modelName, requestId, inputTokens, outputTokens, options) {
const pricing = this.pricingData.get(modelName);
if (!pricing) {
shared_1.Logger.warn(`No pricing data for model: ${modelName}`);
return 0;
}
const inputCost = inputTokens * pricing.inputTokenPrice;
const outputCost = outputTokens * pricing.outputTokenPrice;
const totalCost = inputCost + outputCost;
const costEvent = {
modelName,
provider: pricing.provider,
requestId,
timestamp: new Date(),
inputTokens,
outputTokens,
totalTokens: inputTokens + outputTokens,
cost: totalCost,
rateUsed: (inputCost + outputCost) / (inputTokens + outputTokens),
...options
};
// Store the event
const modelEvents = this.costEvents.get(modelName) || [];
modelEvents.push(costEvent);
// Maintain history limit
if (modelEvents.length > this.config.maxEventHistory) {
modelEvents.splice(0, modelEvents.length - this.config.maxEventHistory);
}
this.costEvents.set(modelName, modelEvents);
// Check for budget alerts
await this.checkCostThresholds(modelName, totalCost);
shared_1.Logger.debug(`Tracked cost for ${modelName}: $${totalCost.toFixed(4)}`);
return totalCost;
}
/**
* Get cost metrics for a model over a time period
*/
async getCostMetrics(modelName, timeframe = '24h') {
const events = this.getRecentCostEvents(modelName, timeframe);
if (events.length === 0) {
const pricing = this.pricingData.get(modelName);
return {
modelName,
provider: pricing?.provider || 'unknown',
currentRate: pricing?.inputTokenPrice || 0,
totalCost: 0,
requestCount: 0,
tokenCount: 0,
averageCostPerRequest: 0,
costTrend: 'stable',
timeframe,
lastUpdated: new Date()
};
}
const totalCost = events.reduce((sum, event) => sum + event.cost, 0);
const tokenCount = events.reduce((sum, event) => sum + event.totalTokens, 0);
const requestCount = events.length;
const averageCostPerRequest = totalCost / requestCount;
const currentRate = events[events.length - 1]?.rateUsed || 0;
// Analyze cost trend
const costTrend = await this.analyzeCostTrend(events);
return {
modelName,
provider: events[0].provider,
currentRate,
totalCost,
requestCount,
tokenCount,
averageCostPerRequest,
costTrend,
timeframe,
lastUpdated: new Date()
};
}
/**
* Get comprehensive cost report across all models
*/
async getCostReport(timeframe = '24h') {
const modelNames = Array.from(this.costEvents.keys());
const modelMetrics = await Promise.all(modelNames.map(name => this.getCostMetrics(name, timeframe)));
const totalCost = modelMetrics.reduce((sum, metrics) => sum + metrics.totalCost, 0);
// Sort by cost descending
const topCostModels = modelMetrics
.filter(m => m.totalCost > 0)
.sort((a, b) => b.totalCost - a.totalCost)
.slice(0, 10)
.map(metrics => ({
modelName: metrics.modelName,
cost: metrics.totalCost,
percentage: totalCost > 0 ? (metrics.totalCost / totalCost) * 100 : 0
}));
// Budget status
const budgetStatus = await this.getBudgetStatus();
return {
totalCost,
modelBreakdown: modelMetrics,
topCostModels,
budgetStatus
};
}
/**
* Set or update a budget configuration
*/
async setBudget(budgetId, config) {
this.budgetConfigs.set(budgetId, { ...config, active: true });
shared_1.Logger.info(`Set budget ${budgetId}: ${config.budgetType} $${config.amount}`);
// Immediate budget check
await this.checkBudgets();
}
/**
* Get current budget status
*/
async getBudgetStatus() {
const statuses = [];
for (const [budgetId, config] of this.budgetConfigs) {
if (!config.active)
continue;
const used = await this.calculateBudgetUsage(config);
const remaining = Math.max(0, config.amount - used);
const percentage = (used / config.amount) * 100;
let status = 'ok';
if (percentage >= 100)
status = 'exceeded';
else if (percentage >= config.warningThreshold * 100)
status = 'warning';
statuses.push({
budgetId,
config,
used,
remaining,
percentage,
status
});
}
return statuses;
}
/**
* Get cost optimization recommendations
*/
async getOptimizationRecommendations() {
const recommendations = [];
const modelMetrics = await Promise.all(Array.from(this.costEvents.keys()).map(name => this.getCostMetrics(name, '24h')));
// Sort by daily cost to focus on highest impact
const highCostModels = modelMetrics
.filter(m => m.totalCost > 5) // Focus on models costing >$5/day
.sort((a, b) => b.totalCost - a.totalCost);
for (const metrics of highCostModels) {
const alternatives = await this.findCostEffectiveAlternatives(metrics.modelName);
for (const alternative of alternatives) {
const currentDailyCost = metrics.totalCost;
const alternativeCost = await this.estimateAlternativeCost(metrics.modelName, alternative.modelName, metrics.tokenCount);
if (alternativeCost < currentDailyCost * 0.8) { // 20% savings threshold
const savings = currentDailyCost - alternativeCost;
const savingsPercentage = (savings / currentDailyCost) * 100;
recommendations.push({
currentModel: metrics.modelName,
recommendation: `Switch to ${alternative.modelName} for similar performance at lower cost`,
alternativeModel: alternative.modelName,
estimatedSavings: savings,
savingsPercentage,
tradeoffs: alternative.tradeoffs,
confidence: alternative.confidence
});
}
}
// Check for usage optimization
if (metrics.averageCostPerRequest > 0.10) { // $0.10 per request threshold
recommendations.push({
currentModel: metrics.modelName,
recommendation: 'Optimize request complexity to reduce token usage',
estimatedSavings: metrics.totalCost * 0.3, // Estimated 30% reduction
savingsPercentage: 30,
tradeoffs: ['May require prompt engineering', 'Possible slight quality reduction'],
confidence: 0.7
});
}
}
return recommendations.sort((a, b) => b.estimatedSavings - a.estimatedSavings);
}
/**
* Update pricing data from external sources
*/
async updatePricing() {
try {
// In a real implementation, this would fetch from provider APIs
shared_1.Logger.debug('Updating pricing data from external sources...');
// For now, just log that we checked
this.emit('pricingUpdated', new Date());
}
catch (error) {
shared_1.Logger.error('Failed to update pricing data:', error);
}
}
// Private helper methods
getRecentCostEvents(modelName, timeframe) {
const events = this.costEvents.get(modelName) || [];
const cutoff = this.getTimeframeCutoff(timeframe);
return events.filter(event => event.timestamp >= cutoff);
}
getTimeframeCutoff(timeframe) {
const now = new Date();
const match = timeframe.match(/^(\d+)([mhd])$/);
if (!match)
return new Date(now.getTime() - 24 * 60 * 60 * 1000); // Default 24h
const value = parseInt(match[1]);
const unit = match[2];
let milliseconds = 0;
switch (unit) {
case 'm':
milliseconds = value * 60 * 1000;
break;
case 'h':
milliseconds = value * 60 * 60 * 1000;
break;
case 'd':
milliseconds = value * 24 * 60 * 60 * 1000;
break;
}
return new Date(now.getTime() - milliseconds);
}
async analyzeCostTrend(events) {
if (events.length < 10)
return 'stable';
// Simple trend analysis based on cost per token over time
const first10 = events.slice(0, 10).map(e => e.rateUsed);
const last10 = events.slice(-10).map(e => e.rateUsed);
const firstAvg = first10.reduce((sum, rate) => sum + rate, 0) / first10.length;
const lastAvg = last10.reduce((sum, rate) => sum + rate, 0) / last10.length;
const change = (lastAvg - firstAvg) / firstAvg;
if (change > 0.05)
return 'increasing';
if (change < -0.05)
return 'decreasing';
return 'stable';
}
async calculateBudgetUsage(config) {
const now = new Date();
let cutoff;
switch (config.budgetType) {
case 'daily':
cutoff = new Date(now.getFullYear(), now.getMonth(), now.getDate());
break;
case 'weekly':
const dayOfWeek = now.getDay();
cutoff = new Date(now.getTime() - dayOfWeek * 24 * 60 * 60 * 1000);
cutoff.setHours(0, 0, 0, 0);
break;
case 'monthly':
cutoff = new Date(now.getFullYear(), now.getMonth(), 1);
break;
default:
cutoff = new Date(now.getTime() - 24 * 60 * 60 * 1000);
}
let totalCost = 0;
for (const [modelName, events] of this.costEvents) {
// Filter by model/provider if specified
if (config.modelName && modelName !== config.modelName)
continue;
if (config.provider) {
const pricing = this.pricingData.get(modelName);
if (pricing?.provider !== config.provider)
continue;
}
const relevantEvents = events.filter(event => event.timestamp >= cutoff);
totalCost += relevantEvents.reduce((sum, event) => sum + event.cost, 0);
}
return totalCost;
}
async checkBudgets() {
const budgetStatuses = await this.getBudgetStatus();
for (const status of budgetStatuses) {
const alertKey = `budget_${status.budgetId}`;
if (status.status === 'exceeded' && !this.activeAlerts.has(alertKey)) {
const alert = {
type: 'budget_exceeded',
severity: 'critical',
threshold: status.config.amount,
current: status.used,
timeframe: status.config.budgetType,
timestamp: new Date(),
message: `Budget exceeded: $${status.used.toFixed(2)} / $${status.config.amount}`,
recommendedActions: [
'Review recent high-cost requests',
'Consider switching to more cost-effective models',
'Implement request rate limiting'
]
};
this.activeAlerts.set(alertKey, alert);
this.emit('budgetAlert', alert);
}
else if (status.status === 'warning' && !this.activeAlerts.has(alertKey)) {
const alert = {
type: 'budget_warning',
severity: 'medium',
threshold: status.config.amount * status.config.warningThreshold,
current: status.used,
timeframe: status.config.budgetType,
timestamp: new Date(),
message: `Budget warning: ${status.percentage.toFixed(1)}% of ${status.config.budgetType} budget used`,
recommendedActions: [
'Monitor usage closely',
'Consider cost optimization strategies'
]
};
this.activeAlerts.set(alertKey, alert);
this.emit('budgetAlert', alert);
}
}
}
async checkCostThresholds(modelName, cost) {
// Check for unusually high single request cost
if (cost > 1.0) { // $1 per request threshold
const alert = {
type: 'usage_spike',
severity: 'high',
modelName,
threshold: 1.0,
current: cost,
timeframe: 'request',
timestamp: new Date(),
message: `High cost request detected: $${cost.toFixed(4)} for ${modelName}`,
recommendedActions: [
'Review request complexity',
'Check for excessive token usage',
'Consider breaking down large requests'
]
};
this.emit('costAlert', modelName, cost);
this.emit('budgetAlert', alert);
}
}
async findCostEffectiveAlternatives(modelName) {
const alternatives = [];
const currentPricing = this.pricingData.get(modelName);
if (!currentPricing)
return alternatives;
// Find models with lower cost per token
for (const [altModelName, altPricing] of this.pricingData) {
if (altModelName === modelName)
continue;
const currentAvgCost = (currentPricing.inputTokenPrice + currentPricing.outputTokenPrice) / 2;
const altAvgCost = (altPricing.inputTokenPrice + altPricing.outputTokenPrice) / 2;
if (altAvgCost < currentAvgCost * 0.8) { // At least 20% cheaper
alternatives.push({
modelName: altModelName,
confidence: this.calculateAlternativeConfidence(modelName, altModelName),
tradeoffs: this.getModelTradeoffs(modelName, altModelName)
});
}
}
return alternatives.sort((a, b) => b.confidence - a.confidence);
}
calculateAlternativeConfidence(current, alternative) {
// Simple confidence calculation based on model characteristics
// This would be more sophisticated in production
const confidenceMap = {
'claude-sonnet-4': {
'claude-haiku-3': 0.8,
'deepseek-v3': 0.7,
'gemini-2.5-pro': 0.75
},
'gpt-4-turbo': {
'gpt-4o': 0.9,
'deepseek-v3': 0.7,
'claude-haiku-3': 0.6
}
};
return confidenceMap[current]?.[alternative] || 0.5;
}
getModelTradeoffs(current, alternative) {
// Define known tradeoffs between models
const tradeoffMap = {
'claude-haiku-3': ['Faster but potentially lower quality for complex tasks'],
'deepseek-v3': ['Excellent for coding, may be less capable for general tasks'],
'gemini-2.5-pro': ['Good general performance, different reasoning style']
};
return tradeoffMap[alternative] || ['May have different performance characteristics'];
}
async estimateAlternativeCost(currentModel, alternativeModel, tokenCount) {
const altPricing = this.pricingData.get(alternativeModel);
if (!altPricing)
return 0;
// Estimate cost assuming similar token distribution
const avgCost = (altPricing.inputTokenPrice + altPricing.outputTokenPrice) / 2;
return tokenCount * avgCost;
}
}
exports.CostTracker = CostTracker;
// Export singleton instance
exports.costTracker = new CostTracker();
//# sourceMappingURL=cost-tracker.js.map