ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
528 lines (525 loc) • 21.7 kB
JavaScript
/**
* AI Feedback Collector
*
* Revolutionary system for collecting feedback from AI users about their experience
* with AI-Debug tools, enabling automatic tool improvement based on AI usage patterns.
*/
export class AIFeedbackCollector {
feedbackEntries = [];
config;
analytics = null;
analyticsCache = null;
constructor(config = {}) {
this.config = {
enableAutoCollection: true,
feedbackFrequency: 'always',
persistenceMode: 'memory',
analysisEnabled: true,
privacyMode: 'anonymous',
feedbackPrompts: {
postSuccess: "🎉 Great! Your debugging session was successful. How was your experience with the AI-Debug tools? Please rate your satisfaction (1-10) and share what worked well.",
postFailure: "🔍 We noticed some challenges in your debugging session. Your feedback helps us improve! Please rate your experience (1-10) and tell us what could be better.",
postSession: "📊 Session complete! Quick feedback: How satisfied were you with the AI-Debug tools today? (1-10) Any suggestions for improvement?"
},
...config
};
}
/**
* Collect feedback from an AI user after tool usage
*/
async collectFeedback(sessionId, agentType, feedback) {
const feedbackId = `feedback_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const feedbackEntry = {
id: feedbackId,
timestamp: Date.now(),
sessionId,
agentType,
toolsUsed: feedback.toolsUsed || [],
taskDescription: feedback.taskDescription || 'No description provided',
outcome: feedback.outcome || 'success',
userExperience: {
satisfaction: 8,
efficiency: 8,
clarity: 8,
usefulness: 8,
...feedback.userExperience
},
feedback: {
strengths: [],
weaknesses: [],
suggestions: [],
wouldUseAgain: true,
recommendToOthers: true,
...feedback.feedback
},
technicalMetrics: {
responseTimeMs: 0,
tokensSaved: 0,
errorsEncountered: [],
recoveryActions: [],
...feedback.technicalMetrics
},
contextualData: {
framework: 'unknown',
projectComplexity: 'moderate',
userType: 'ai_assistant',
sessionDuration: 0,
...feedback.contextualData
}
};
this.feedbackEntries.push(feedbackEntry);
// Invalidate analytics cache
this.analyticsCache = null;
// Persist feedback if configured
if (this.config.persistenceMode === 'file') {
await this.persistFeedbackToFile(feedbackEntry);
}
return feedbackId;
}
/**
* Generate feedback prompt for AI users
*/
generateFeedbackPrompt(sessionOutcome, context) {
const basePrompt = this.config.feedbackPrompts[sessionOutcome === 'success' ? 'postSuccess' :
sessionOutcome === 'failure' ? 'postFailure' : 'postSession'];
const contextualInfo = `
**Session Summary:**
- Agent: ${context.agentType}
- Tools used: ${context.toolsUsed.join(', ')}
- Duration: ${Math.round(context.sessionDuration / 1000)}s
${context.contextLinesPreserved ? `- Context lines preserved: ${context.contextLinesPreserved}` : ''}
**Quick Feedback (1-5 minutes):**
1. Overall satisfaction (1-10):
2. What worked best?
3. What could be improved?
4. Would you use this agent again? (Yes/No)
5. Any specific suggestions?
Your feedback directly improves AI-Debug tools for all AI users! 🤖✨`;
return basePrompt + contextualInfo;
}
/**
* Get comprehensive feedback analytics
*/
async getFeedbackAnalytics(forceRefresh = false) {
// Use cached analytics if available and not forced to refresh
if (!forceRefresh && this.analyticsCache &&
Date.now() - this.analyticsCache.timestamp < 300000) { // 5 minute cache
return this.analyticsCache.data;
}
if (!this.config.analysisEnabled) {
throw new Error('Analytics disabled in configuration');
}
const analytics = this.calculateAnalytics();
// Cache the results
this.analyticsCache = {
timestamp: Date.now(),
data: analytics
};
return analytics;
}
/**
* Get feedback summary for specific agent
*/
getAgentFeedbackSummary(agentType) {
const agentFeedback = this.feedbackEntries.filter(entry => entry.agentType === agentType);
if (agentFeedback.length === 0) {
return {
totalSessions: 0,
averageRatings: {},
topStrengths: [],
topWeaknesses: [],
recentTrends: []
};
}
// Calculate average ratings
const averageRatings = {
satisfaction: this.calculateAverage(agentFeedback, 'satisfaction'),
efficiency: this.calculateAverage(agentFeedback, 'efficiency'),
clarity: this.calculateAverage(agentFeedback, 'clarity'),
usefulness: this.calculateAverage(agentFeedback, 'usefulness')
};
// Extract top strengths and weaknesses
const allStrengths = agentFeedback.flatMap(entry => entry.feedback.strengths);
const allWeaknesses = agentFeedback.flatMap(entry => entry.feedback.weaknesses);
const topStrengths = this.getTopMentions(allStrengths, 5);
const topWeaknesses = this.getTopMentions(allWeaknesses, 5);
// Analyze recent trends (last 10 entries)
const recentFeedback = agentFeedback.slice(-10);
const recentTrends = this.analyzeTrends(recentFeedback);
return {
totalSessions: agentFeedback.length,
averageRatings,
topStrengths,
topWeaknesses,
recentTrends
};
}
/**
* Configure feedback collection settings
*/
updateConfiguration(newConfig) {
this.config = { ...this.config, ...newConfig };
// Invalidate cache when configuration changes
this.analyticsCache = null;
}
/**
* Export feedback data for external analysis
*/
exportFeedbackData(format = 'json') {
if (format === 'json') {
return JSON.stringify({
metadata: {
exportTimestamp: Date.now(),
totalEntries: this.feedbackEntries.length,
configuration: this.config
},
feedbackEntries: this.feedbackEntries
}, null, 2);
}
// CSV format
const headers = [
'id', 'timestamp', 'agentType', 'outcome', 'satisfaction', 'efficiency',
'clarity', 'usefulness', 'wouldUseAgain', 'framework', 'complexity',
'sessionDuration', 'tokensavage', 'toolsUsed'
];
const csvData = this.feedbackEntries.map(entry => [
entry.id,
new Date(entry.timestamp).toISOString(),
entry.agentType,
entry.outcome,
entry.userExperience.satisfaction,
entry.userExperience.efficiency,
entry.userExperience.clarity,
entry.userExperience.usefulness,
entry.feedback.wouldUseAgain,
entry.contextualData.framework,
entry.contextualData.projectComplexity,
entry.contextualData.sessionDuration,
entry.technicalMetrics.tokensSaved,
entry.toolsUsed.join(';')
]);
return [headers, ...csvData]
.map(row => row.map(cell => `"${cell}"`).join(','))
.join('\n');
}
/**
* Clear all feedback data
*/
clearFeedbackData() {
this.feedbackEntries = [];
this.analyticsCache = null;
}
/**
* Get current configuration
*/
getConfiguration() {
return { ...this.config };
}
/**
* Calculate comprehensive analytics
*/
calculateAnalytics() {
const entries = this.feedbackEntries;
// Calculate average ratings
const averageRatings = {
satisfaction: this.calculateAverage(entries, 'satisfaction'),
efficiency: this.calculateAverage(entries, 'efficiency'),
clarity: this.calculateAverage(entries, 'clarity'),
usefulness: this.calculateAverage(entries, 'usefulness')
};
// Agent performance analysis
const agentTypes = [...new Set(entries.map(e => e.agentType))];
const agentPerformance = {};
agentTypes.forEach(agentType => {
const agentEntries = entries.filter(e => e.agentType === agentType);
const avgRating = agentEntries.reduce((sum, e) => sum + (e.userExperience.satisfaction + e.userExperience.efficiency +
e.userExperience.clarity + e.userExperience.usefulness) / 4, 0) / agentEntries.length;
agentPerformance[agentType] = {
usageCount: agentEntries.length,
averageRating: avgRating,
topStrengths: this.getTopMentions(agentEntries.flatMap(e => e.feedback.strengths), 3),
topWeaknesses: this.getTopMentions(agentEntries.flatMap(e => e.feedback.weaknesses), 3),
improvementTrends: this.calculateImprovementTrends(agentEntries)
};
});
// Tool effectiveness analysis
const allTools = [...new Set(entries.flatMap(e => e.toolsUsed))];
const toolEffectiveness = {};
allTools.forEach(tool => {
const toolEntries = entries.filter(e => e.toolsUsed.includes(tool));
const successRate = toolEntries.filter(e => e.outcome === 'success').length / toolEntries.length;
const avgResponseTime = toolEntries.reduce((sum, e) => sum + e.technicalMetrics.responseTimeMs, 0) / toolEntries.length;
const avgSatisfaction = toolEntries.reduce((sum, e) => sum + e.userExperience.satisfaction, 0) / toolEntries.length;
toolEffectiveness[tool] = {
usageCount: toolEntries.length,
successRate,
averageResponseTime: avgResponseTime,
userSatisfaction: avgSatisfaction
};
});
// Framework insights
const frameworks = [...new Set(entries.map(e => e.contextualData.framework))];
const frameworkInsights = {};
frameworks.forEach(framework => {
const frameworkEntries = entries.filter(e => e.contextualData.framework === framework);
frameworkInsights[framework] = {
sessionCount: frameworkEntries.length,
averageComplexity: this.getMostCommon(frameworkEntries.map(e => e.contextualData.projectComplexity)),
preferredAgents: this.getTopMentions(frameworkEntries.map(e => e.agentType), 3),
commonIssues: this.getTopMentions(frameworkEntries.flatMap(e => e.feedback.weaknesses), 3)
};
});
// Identify improvement opportunities
const improvementOpportunities = this.identifyImprovementOpportunities(entries);
return {
totalFeedbackEntries: entries.length,
averageRatings,
agentPerformance,
toolEffectiveness,
frameworkInsights,
improvementOpportunities
};
}
/**
* Helper method to calculate average ratings
*/
calculateAverage(entries, metric) {
if (entries.length === 0)
return 0;
return entries.reduce((sum, entry) => sum + entry.userExperience[metric], 0) / entries.length;
}
/**
* Get top mentioned items from array
*/
getTopMentions(items, limit) {
const counts = {};
items.forEach(item => {
counts[item] = (counts[item] || 0) + 1;
});
return Object.entries(counts)
.sort(([, a], [, b]) => b - a)
.slice(0, limit)
.map(([item]) => item);
}
/**
* Get most common value from array
*/
getMostCommon(items) {
return this.getTopMentions(items, 1)[0] || 'unknown';
}
/**
* Calculate improvement trends for agent
*/
calculateImprovementTrends(entries) {
// Sort by timestamp and calculate rolling average satisfaction
const sortedEntries = entries.sort((a, b) => a.timestamp - b.timestamp);
const windowSize = Math.min(5, entries.length);
const trends = [];
for (let i = windowSize - 1; i < sortedEntries.length; i++) {
const window = sortedEntries.slice(i - windowSize + 1, i + 1);
const avgSatisfaction = window.reduce((sum, e) => sum + e.userExperience.satisfaction, 0) / window.length;
trends.push(avgSatisfaction);
}
return trends;
}
/**
* Analyze recent trends
*/
analyzeTrends(recentEntries) {
const trends = [];
if (recentEntries.length < 3) {
trends.push('Not enough data for trend analysis');
return trends;
}
// Check satisfaction trend
const recentSatisfaction = recentEntries.slice(-3).map(e => e.userExperience.satisfaction);
const satisfactionTrend = recentSatisfaction[2] - recentSatisfaction[0];
if (satisfactionTrend > 1) {
trends.push('Satisfaction improving significantly');
}
else if (satisfactionTrend < -1) {
trends.push('Satisfaction declining, needs attention');
}
else {
trends.push('Satisfaction stable');
}
// Check error frequency
const recentErrors = recentEntries.slice(-5).filter(e => e.outcome === 'failure').length;
if (recentErrors === 0) {
trends.push('No recent failures - excellent reliability');
}
else if (recentErrors > 2) {
trends.push('High failure rate in recent sessions');
}
return trends;
}
/**
* Identify improvement opportunities based on feedback data
*/
identifyImprovementOpportunities(entries) {
const opportunities = [];
// Analyze low satisfaction ratings
const lowSatisfactionEntries = entries.filter(e => e.userExperience.satisfaction < 6);
if (lowSatisfactionEntries.length > entries.length * 0.2) {
opportunities.push({
area: 'User Satisfaction',
priority: 'high',
impact: lowSatisfactionEntries.length / entries.length,
description: 'High number of users reporting low satisfaction',
suggestedActions: [
'Review and improve low-performing agents',
'Enhance user experience design',
'Provide better error messages and guidance'
]
});
}
// Analyze tool performance issues
const errorEntries = entries.filter(e => e.technicalMetrics.errorsEncountered.length > 0);
if (errorEntries.length > entries.length * 0.15) {
opportunities.push({
area: 'Technical Reliability',
priority: 'high',
impact: errorEntries.length / entries.length,
description: 'High error rate affecting user experience',
suggestedActions: [
'Improve error handling and recovery',
'Add better input validation',
'Enhance system stability monitoring'
]
});
}
// Analyze efficiency concerns
const slowResponseEntries = entries.filter(e => e.technicalMetrics.responseTimeMs > 5000);
if (slowResponseEntries.length > entries.length * 0.25) {
opportunities.push({
area: 'Performance Optimization',
priority: 'medium',
impact: slowResponseEntries.length / entries.length,
description: 'Slow response times impacting user efficiency',
suggestedActions: [
'Optimize tool performance',
'Implement better caching strategies',
'Consider parallel processing for complex tasks'
]
});
}
return opportunities;
}
/**
* Persist feedback to file system
*/
async persistFeedbackToFile(feedbackEntry) {
try {
const fs = await import('fs');
const path = await import('path');
// Create feedback directory if it doesn't exist
const feedbackDir = path.join(process.cwd(), 'feedback-data');
if (!fs.existsSync(feedbackDir)) {
fs.mkdirSync(feedbackDir, { recursive: true });
}
// Create timestamped filename
const timestamp = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
const filename = `ai-feedback-${timestamp}.jsonl`;
const filepath = path.join(feedbackDir, filename);
// Append as JSONL (JSON Lines) for easy parsing
const feedbackLine = JSON.stringify({
...feedbackEntry,
persistedAt: new Date().toISOString()
}) + '\n';
fs.appendFileSync(filepath, feedbackLine, 'utf8');
console.log(`[AIFeedbackCollector] ✅ Persisted feedback ${feedbackEntry.id} to ${filename}`);
}
catch (error) {
console.error(`[AIFeedbackCollector] ❌ Failed to persist feedback: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
/**
* Load persisted feedback from files on startup
*/
async loadPersistedFeedback() {
if (this.config.persistenceMode !== 'file')
return;
try {
const fs = await import('fs');
const path = await import('path');
const feedbackDir = path.join(process.cwd(), 'feedback-data');
if (!fs.existsSync(feedbackDir))
return;
const files = fs.readdirSync(feedbackDir)
.filter(file => file.startsWith('ai-feedback-') && file.endsWith('.jsonl'))
.sort(); // Load in chronological order
let loadedCount = 0;
for (const file of files) {
const filepath = path.join(feedbackDir, file);
const content = fs.readFileSync(filepath, 'utf8');
const lines = content.split('\n').filter(line => line.trim());
for (const line of lines) {
try {
const entry = JSON.parse(line);
// Remove persistedAt before adding to memory
delete entry.persistedAt;
this.feedbackEntries.push(entry);
loadedCount++;
}
catch (parseError) {
console.warn(`[AIFeedbackCollector] Failed to parse feedback line in ${file}: ${parseError}`);
}
}
}
if (loadedCount > 0) {
console.log(`[AIFeedbackCollector] 📂 Loaded ${loadedCount} feedback entries from ${files.length} files`);
// Invalidate analytics cache since we loaded new data
this.analyticsCache = null;
}
}
catch (error) {
console.error(`[AIFeedbackCollector] ❌ Failed to load persisted feedback: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
/**
* Batch upload feedback entries (for cloud collection)
*/
async batchUploadFeedback(feedbackEntries) {
const result = {
successful: 0,
failed: 0,
errors: []
};
for (let i = 0; i < feedbackEntries.length; i++) {
try {
const entry = feedbackEntries[i];
if (!entry.sessionId || !entry.agentType) {
result.errors.push(`Entry ${i}: Missing required fields sessionId or agentType`);
result.failed++;
continue;
}
await this.collectFeedback(entry.sessionId, entry.agentType, entry);
result.successful++;
}
catch (error) {
result.errors.push(`Entry ${i}: ${error instanceof Error ? error.message : 'Unknown error'}`);
result.failed++;
}
}
return result;
}
/**
* Get all feedback data for analytics
*/
getAllFeedback() {
return [...this.feedbackEntries]; // Return a copy to prevent external modification
}
getStorageInfo() {
const memorySize = JSON.stringify(this.feedbackEntries).length;
const estimatedSizeKB = Math.round(memorySize / 1024);
return {
persistenceMode: this.config.persistenceMode,
memoryEntries: this.feedbackEntries.length,
storageLocation: this.config.persistenceMode === 'file'
? './feedback-data/'
: 'memory-only',
estimatedSize: `${estimatedSizeKB}KB`
};
}
}
//# sourceMappingURL=ai-feedback-collector.js.map