promptforge
Version:
Adaptive Prompt Intelligence & Orchestration SDK - Manage, optimize, and serve prompts for LLMs with versioning, feedback loops, and multi-provider support
190 lines • 6.23 kB
JavaScript
import { v4 as uuidv4 } from 'uuid';
import { FeedbackType, FeedbackSchema, PromptScoreSchema, } from '../types.js';
export class FeedbackSystem {
feedbacks;
scores;
constructor() {
this.feedbacks = new Map();
this.scores = new Map();
}
/**
* Add feedback for an execution
*/
async addFeedback(feedback) {
const completeFeedback = {
...feedback,
id: uuidv4(),
timestamp: new Date(),
};
const validated = FeedbackSchema.parse(completeFeedback);
this.feedbacks.set(validated.id, validated);
// Update prompt score
await this.updatePromptScore(feedback.promptId);
return validated;
}
/**
* Get feedback by ID
*/
async getFeedback(feedbackId) {
return this.feedbacks.get(feedbackId);
}
/**
* Get all feedback for a prompt
*/
async getPromptFeedback(promptId) {
return Array.from(this.feedbacks.values()).filter(f => f.promptId === promptId);
}
/**
* Get all feedback for an execution
*/
async getExecutionFeedback(executionId) {
return Array.from(this.feedbacks.values()).filter(f => f.executionId === executionId);
}
/**
* Calculate and get prompt score
*/
async getPromptScore(promptId) {
// Check if we have a cached score
let score = this.scores.get(promptId);
if (!score) {
// Calculate initial score
score = await this.calculatePromptScore(promptId);
}
return score;
}
/**
* Update prompt score based on feedback
*/
async updatePromptScore(promptId) {
const score = await this.calculatePromptScore(promptId);
this.scores.set(promptId, score);
}
/**
* Calculate prompt score using scoring algorithm
* Score = α * UserFeedbackScore + β * PerformanceScore + γ * CostEfficiencyScore
*/
async calculatePromptScore(promptId) {
const feedbacks = await this.getPromptFeedback(promptId);
if (feedbacks.length === 0) {
return {
promptId,
version: 1,
userFeedbackScore: 0.5,
performanceScore: 0.5,
costEfficiencyScore: 0.5,
compositeScore: 0.5,
executionCount: 0,
lastUpdated: new Date(),
};
}
// Calculate user feedback score
const userFeedbackScore = this.calculateUserFeedbackScore(feedbacks);
// For now, use placeholder values for performance and cost
// These would be calculated from actual metrics in production
const performanceScore = 0.7;
const costEfficiencyScore = 0.8;
// Weight factors (α, β, γ)
const alpha = 0.5; // User feedback weight
const beta = 0.3; // Performance weight
const gamma = 0.2; // Cost efficiency weight
const compositeScore = alpha * userFeedbackScore +
beta * performanceScore +
gamma * costEfficiencyScore;
const score = {
promptId,
version: 1, // TODO: Get from prompt metadata
userFeedbackScore,
performanceScore,
costEfficiencyScore,
compositeScore,
executionCount: feedbacks.length,
lastUpdated: new Date(),
};
return PromptScoreSchema.parse(score);
}
/**
* Calculate user feedback score from feedback entries
*/
calculateUserFeedbackScore(feedbacks) {
if (feedbacks.length === 0) {
return 0.5;
}
let totalScore = 0;
for (const feedback of feedbacks) {
if (feedback.type === FeedbackType.THUMBS_UP) {
totalScore += 1;
}
else if (feedback.type === FeedbackType.THUMBS_DOWN) {
totalScore += 0;
}
else if (feedback.type === FeedbackType.SCORE && feedback.score !== undefined) {
totalScore += feedback.score;
}
else {
totalScore += 0.5; // Neutral
}
}
return totalScore / feedbacks.length;
}
/**
* Get feedback statistics
*/
async getFeedbackStats(promptId) {
let feedbacks;
if (promptId) {
feedbacks = await this.getPromptFeedback(promptId);
}
else {
feedbacks = Array.from(this.feedbacks.values());
}
let positiveCount = 0;
let negativeCount = 0;
let totalScore = 0;
let scoreCount = 0;
for (const feedback of feedbacks) {
if (feedback.type === FeedbackType.THUMBS_UP) {
positiveCount++;
totalScore += 1;
scoreCount++;
}
else if (feedback.type === FeedbackType.THUMBS_DOWN) {
negativeCount++;
totalScore += 0;
scoreCount++;
}
else if (feedback.type === FeedbackType.SCORE && feedback.score !== undefined) {
totalScore += feedback.score;
scoreCount++;
}
}
return {
totalFeedback: feedbacks.length,
positiveCount,
negativeCount,
averageScore: scoreCount > 0 ? totalScore / scoreCount : 0,
positiveRate: feedbacks.length > 0 ? positiveCount / feedbacks.length : 0,
};
}
/**
* Get top performing prompts by score
*/
async getTopPrompts(limit = 10) {
const scores = Array.from(this.scores.values());
scores.sort((a, b) => b.compositeScore - a.compositeScore);
return scores.slice(0, limit);
}
/**
* Export feedback data
*/
exportFeedback(promptId) {
let feedbacks;
if (promptId) {
feedbacks = Array.from(this.feedbacks.values()).filter(f => f.promptId === promptId);
}
else {
feedbacks = Array.from(this.feedbacks.values());
}
return JSON.stringify(feedbacks, null, 2);
}
}
//# sourceMappingURL=feedback.js.map