@callmedayz/ai-prompt-toolkit
Version:
Professional AI prompt engineering toolkit with advanced template features, real-time dashboards, conditional logic, template inheritance, live monitoring, OpenRouter integration, and 310+ model support
353 lines • 14 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.PromptAnalytics = void 0;
/**
* Prompt Performance Analytics System
*/
class PromptAnalytics {
constructor(config, client) {
this.dataPoints = [];
this.aggregations = new Map();
this.config = {
enableRealTimeMonitoring: true,
aggregationIntervals: ['hour', 'day', 'week'],
alertThresholds: {
successRate: { warning: 85, critical: 70 },
responseTime: { warning: 5000, critical: 10000 },
errorRate: { warning: 10, critical: 20 },
cost: { warning: 0.01, critical: 0.05 }
},
retentionPeriod: 90,
enableTrendAnalysis: true,
enableAnomalyDetection: true,
...config
};
this.client = client;
}
/**
* Record a test execution for analytics
*/
recordExecution(execution, model) {
const timestamp = execution.timestamp;
// Record individual metrics as data points
this.addDataPoint({
timestamp,
promptVersionId: execution.promptVersionId,
model,
metric: 'response_time',
value: execution.responseTime
});
this.addDataPoint({
timestamp,
promptVersionId: execution.promptVersionId,
model,
metric: 'token_usage',
value: execution.tokenUsage
});
this.addDataPoint({
timestamp,
promptVersionId: execution.promptVersionId,
model,
metric: 'cost',
value: execution.cost
});
this.addDataPoint({
timestamp,
promptVersionId: execution.promptVersionId,
model,
metric: 'success',
value: execution.success ? 1 : 0
});
// Trigger real-time analysis if enabled
if (this.config.enableRealTimeMonitoring) {
this.performRealTimeAnalysis(execution.promptVersionId);
}
}
/**
* Add a custom analytics data point
*/
addDataPoint(dataPoint) {
this.dataPoints.push(dataPoint);
// Clean up old data points based on retention period
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionPeriod);
this.dataPoints = this.dataPoints.filter(dp => dp.timestamp >= cutoffDate);
}
/**
* Generate aggregated analytics for a prompt version
*/
generateAggregation(promptVersionId, period, startTime, endTime) {
const now = new Date();
const defaultEndTime = endTime || now;
const defaultStartTime = startTime || this.getStartTimeForPeriod(period, defaultEndTime);
const relevantDataPoints = this.dataPoints.filter(dp => dp.promptVersionId === promptVersionId &&
dp.timestamp >= defaultStartTime &&
dp.timestamp <= defaultEndTime);
const executions = this.groupExecutions(relevantDataPoints);
const totalExecutions = executions.length;
if (totalExecutions === 0) {
return this.createEmptyAggregation(promptVersionId, period, defaultStartTime, defaultEndTime);
}
const successfulExecutions = executions.filter(e => e.success).length;
const successRate = (successfulExecutions / totalExecutions) * 100;
const errorRate = ((totalExecutions - successfulExecutions) / totalExecutions) * 100;
const responseTimeSum = executions.reduce((sum, e) => sum + e.responseTime, 0);
const tokenUsageSum = executions.reduce((sum, e) => sum + e.tokenUsage, 0);
const costSum = executions.reduce((sum, e) => sum + e.cost, 0);
const averageResponseTime = responseTimeSum / totalExecutions;
const averageTokenUsage = tokenUsageSum / totalExecutions;
const averageCost = costSum / totalExecutions;
const periodHours = this.getPeriodHours(period);
const throughput = totalExecutions / periodHours;
// Calculate trends
const trends = this.calculateTrends(promptVersionId, period, defaultStartTime);
const aggregation = {
period,
startTime: defaultStartTime,
endTime: defaultEndTime,
promptVersionId,
metrics: {
totalExecutions,
successRate,
averageResponseTime,
averageTokenUsage,
averageCost,
errorRate,
throughput
},
trends
};
// Store aggregation
const key = `${promptVersionId}_${period}`;
const existing = this.aggregations.get(key) || [];
existing.push(aggregation);
this.aggregations.set(key, existing);
return aggregation;
}
/**
* Compare performance between multiple prompt versions
*/
compareVersions(baselineVersionId, comparisonVersionIds, timeframe) {
// This would be implemented with actual version data
// For now, returning a placeholder structure
throw new Error('compareVersions method needs to be implemented with actual version data');
}
/**
* Generate performance insights and recommendations
*/
generateInsights(promptVersionId) {
const recentAggregations = this.getRecentAggregations(promptVersionId, 'day', 7);
const insights = [];
const trends = [];
const alerts = [];
if (recentAggregations.length === 0) {
return {
promptVersionId,
insights: [{
type: 'performance',
severity: 'low',
title: 'Insufficient Data',
description: 'Not enough data to generate meaningful insights',
recommendation: 'Run more tests to gather performance data',
impact: 'Cannot optimize without sufficient data',
confidence: 0
}],
trends: [],
alerts: []
};
}
// Analyze performance trends
const latestAggregation = recentAggregations[recentAggregations.length - 1];
// Check for performance issues
if (latestAggregation.metrics.successRate < this.config.alertThresholds.successRate.critical) {
alerts.push({
type: 'threshold',
severity: 'critical',
message: `Success rate (${latestAggregation.metrics.successRate.toFixed(1)}%) is below critical threshold`,
triggeredAt: new Date(),
metric: 'success_rate',
value: latestAggregation.metrics.successRate,
threshold: this.config.alertThresholds.successRate.critical
});
}
if (latestAggregation.metrics.averageResponseTime > this.config.alertThresholds.responseTime.warning) {
insights.push({
type: 'performance',
severity: latestAggregation.metrics.averageResponseTime > this.config.alertThresholds.responseTime.critical ? 'high' : 'medium',
title: 'High Response Time',
description: `Average response time is ${latestAggregation.metrics.averageResponseTime.toFixed(0)}ms`,
recommendation: 'Consider optimizing prompt length or switching to a faster model',
impact: 'Slower response times affect user experience',
confidence: 0.8
});
}
return {
promptVersionId,
insights,
trends,
alerts
};
}
/**
* Get analytics data for a specific time range
*/
getAnalyticsData(promptVersionId, startTime, endTime, metrics) {
return this.dataPoints.filter(dp => dp.promptVersionId === promptVersionId &&
dp.timestamp >= startTime &&
dp.timestamp <= endTime &&
(!metrics || metrics.includes(dp.metric)));
}
/**
* Export analytics data to JSON
*/
exportAnalytics() {
const data = {
dataPoints: this.dataPoints,
aggregations: Object.fromEntries(this.aggregations),
config: this.config,
exportedAt: new Date().toISOString()
};
return JSON.stringify(data, null, 2);
}
/**
* Import analytics data from JSON
*/
importAnalytics(jsonData) {
try {
const data = JSON.parse(jsonData);
if (data.dataPoints && Array.isArray(data.dataPoints)) {
this.dataPoints = data.dataPoints.map((dp) => ({
...dp,
timestamp: new Date(dp.timestamp)
}));
}
if (data.aggregations) {
this.aggregations = new Map(Object.entries(data.aggregations));
}
if (data.config) {
this.config = { ...this.config, ...data.config };
}
}
catch (error) {
throw new Error(`Failed to import analytics: ${error instanceof Error ? error.message : String(error)}`);
}
}
groupExecutions(dataPoints) {
const executionMap = new Map();
dataPoints.forEach(dp => {
const key = `${dp.timestamp.getTime()}_${dp.promptVersionId}`;
if (!executionMap.has(key)) {
executionMap.set(key, {});
}
const execution = executionMap.get(key);
switch (dp.metric) {
case 'response_time':
execution.responseTime = dp.value;
break;
case 'token_usage':
execution.tokenUsage = dp.value;
break;
case 'cost':
execution.cost = dp.value;
break;
case 'success':
execution.success = dp.value === 1;
break;
}
});
return Array.from(executionMap.values()).filter(e => e.responseTime !== undefined &&
e.tokenUsage !== undefined &&
e.cost !== undefined &&
e.success !== undefined);
}
getStartTimeForPeriod(period, endTime) {
const start = new Date(endTime);
switch (period) {
case 'hour':
start.setHours(start.getHours() - 1);
break;
case 'day':
start.setDate(start.getDate() - 1);
break;
case 'week':
start.setDate(start.getDate() - 7);
break;
case 'month':
start.setMonth(start.getMonth() - 1);
break;
}
return start;
}
getPeriodHours(period) {
switch (period) {
case 'hour': return 1;
case 'day': return 24;
case 'week': return 168;
case 'month': return 720; // approximate
default: return 24;
}
}
createEmptyAggregation(promptVersionId, period, startTime, endTime) {
return {
period,
startTime,
endTime,
promptVersionId,
metrics: {
totalExecutions: 0,
successRate: 0,
averageResponseTime: 0,
averageTokenUsage: 0,
averageCost: 0,
errorRate: 0,
throughput: 0
},
trends: {
responseTimeTrend: 'stable',
successRateTrend: 'stable',
costTrend: 'stable'
}
};
}
calculateTrends(promptVersionId, period, currentStartTime) {
// Get previous period for comparison
const previousEndTime = new Date(currentStartTime);
const previousStartTime = this.getStartTimeForPeriod(period, previousEndTime);
const previousAggregation = this.generateAggregation(promptVersionId, period, previousStartTime, previousEndTime);
const currentAggregation = this.generateAggregation(promptVersionId, period, currentStartTime);
return {
responseTimeTrend: this.getTrend(previousAggregation.metrics.averageResponseTime, currentAggregation.metrics.averageResponseTime, 'lower_is_better'),
successRateTrend: this.getTrend(previousAggregation.metrics.successRate, currentAggregation.metrics.successRate, 'higher_is_better'),
costTrend: this.getTrend(previousAggregation.metrics.averageCost, currentAggregation.metrics.averageCost, 'lower_is_better')
};
}
getTrend(previousValue, currentValue, direction) {
const threshold = 0.05; // 5% change threshold
const change = (currentValue - previousValue) / previousValue;
if (Math.abs(change) < threshold) {
return 'stable';
}
if (direction === 'higher_is_better') {
return change > 0 ? 'improving' : 'degrading';
}
else {
return change < 0 ? 'improving' : 'degrading';
}
}
getRecentAggregations(promptVersionId, period, count) {
const key = `${promptVersionId}_${period}`;
const aggregations = this.aggregations.get(key) || [];
return aggregations.slice(-count);
}
performRealTimeAnalysis(promptVersionId) {
// Generate insights and check for alerts
const insights = this.generateInsights(promptVersionId);
// In a real implementation, this would trigger notifications
// or update dashboards in real-time
if (insights.alerts.length > 0) {
console.warn(`Analytics Alert for ${promptVersionId}:`, insights.alerts);
}
}
}
exports.PromptAnalytics = PromptAnalytics;
//# sourceMappingURL=prompt-analytics.js.map