ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
229 lines • 10.2 kB
JavaScript
import { BaseToolHandler } from '../base-handler.js';
import { LocalDebugEngine } from '../../local-debug-engine.js';
import { AIExtractionUtils } from './ai-utils.js';
export class AILLMHandler extends BaseToolHandler {
tools = [
{
name: 'trace_llm_calls',
description: 'Monitor all LLM API calls with token usage, costs, and latency',
inputSchema: {
type: 'object',
properties: {
sessionId: { type: 'string', description: 'Debug session ID' },
provider: { type: 'string', description: 'Filter by provider (e.g., OpenAI, Anthropic)' },
includeDetails: { type: 'boolean', description: 'Include request/response details', default: false }
},
required: ['sessionId']
}
},
{
name: 'analyze_token_usage',
description: 'Analyze token usage patterns across LLM calls with cost breakdown',
inputSchema: {
type: 'object',
properties: {
sessionId: { type: 'string', description: 'Debug session ID' },
groupBy: {
type: 'string',
description: 'Group analysis by model, endpoint, or time',
enum: ['model', 'endpoint', 'time'],
default: 'model'
}
},
required: ['sessionId']
}
}
];
async handle(toolName, args, sessions) {
const session = sessions.get(args.sessionId);
if (!session) {
return {
content: [{ type: 'text', text: `Session not found: ${args.sessionId}` }]
};
}
try {
switch (toolName) {
case 'trace_llm_calls':
return await this.traceLLMCalls(args, session);
case 'analyze_token_usage':
return await this.analyzeTokenUsage(args, session);
default:
throw new Error(`Unknown tool: ${toolName}`);
}
}
catch (error) {
return {
content: [{
type: 'text',
text: `Error in ${toolName}: ${error instanceof Error ? error.message : String(error)}`
}]
};
}
}
async traceLLMCalls(args, session) {
const engine = session.engine || new LocalDebugEngine();
const networkRequests = engine.getNetworkRequests();
const llmCalls = AIExtractionUtils.extractLLMCalls(networkRequests);
const filteredCalls = args.provider
? llmCalls.filter(call => call.provider === args.provider)
: llmCalls;
if (filteredCalls.length === 0) {
return {
content: [{
type: 'text',
text: '## LLM API Calls Traced\n\nNo LLM API calls detected.'
}]
};
}
let report = '## LLM API Calls Traced\n\n';
// Summary statistics
const totalTokens = filteredCalls.reduce((sum, call) => sum + (call.tokens?.total || 0), 0);
const totalCost = filteredCalls.reduce((sum, call) => sum + (call.cost || 0), 0);
const avgLatency = filteredCalls.reduce((sum, call) => sum + (call.duration || 0), 0) / filteredCalls.length;
report += '### Summary\n';
report += `- **Total Calls:** ${filteredCalls.length}\n`;
report += `- **Total Tokens:** ${totalTokens.toLocaleString()}\n`;
report += `- **Total Cost:** $${totalCost.toFixed(4)}\n`;
report += `- **Average Latency:** ${avgLatency.toFixed(0)}ms\n\n`;
// Individual calls
report += '### Call Details\n\n';
for (const [index, call] of filteredCalls.entries()) {
report += `#### Call ${index + 1}\n`;
report += `- **Provider:** ${call.provider}\n`;
report += `- **Model:** ${call.model}\n`;
report += `- **Timestamp:** ${call.timestamp.toISOString()}\n`;
if (call.tokens) {
report += `- **Tokens:** ${call.tokens.total} `;
if (call.tokens.prompt && call.tokens.completion) {
report += `(prompt: ${call.tokens.prompt}, completion: ${call.tokens.completion})`;
}
report += '\n';
}
if (call.cost !== undefined) {
report += `- **Cost:** $${call.cost.toFixed(4)}\n`;
}
if (call.duration) {
report += `- **Duration:** ${call.duration}ms\n`;
}
if (call.error) {
report += `- **Error:** ${call.error}\n`;
}
if (args.includeDetails) {
report += '\n**Request:**\n```json\n' + JSON.stringify(call.request, null, 2) + '\n```\n';
report += '\n**Response:**\n```json\n' + JSON.stringify(call.response, null, 2) + '\n```\n';
}
report += '\n';
}
return {
content: [{ type: 'text', text: report }]
};
}
async analyzeTokenUsage(args, session) {
const engine = session.engine || new LocalDebugEngine();
const networkRequests = engine.getNetworkRequests();
const groupBy = args.groupBy || 'model';
const llmCalls = AIExtractionUtils.extractLLMCalls(networkRequests);
if (llmCalls.length === 0) {
return {
content: [{
type: 'text',
text: '## Token Usage Analysis\n\nNo LLM API calls found to analyze.'
}]
};
}
let report = '## Token Usage Analysis\n\n';
// Calculate totals
let totalTokens = 0;
let totalPromptTokens = 0;
let totalCompletionTokens = 0;
let totalCost = 0;
llmCalls.forEach(call => {
if (call.tokens) {
totalTokens += call.tokens.total;
totalPromptTokens += call.tokens.prompt || 0;
totalCompletionTokens += call.tokens.completion || 0;
}
totalCost += call.cost || 0;
});
report += `### Summary\n`;
report += `- **Total Tokens Used:** ${totalTokens.toLocaleString()}\n`;
report += `- **Prompt Tokens:** ${totalPromptTokens.toLocaleString()}\n`;
report += `- **Completion Tokens:** ${totalCompletionTokens.toLocaleString()}\n`;
report += `- **Total Cost:** $${totalCost.toFixed(4)}\n\n`;
// Group analysis
const groups = new Map();
llmCalls.forEach(call => {
let key;
switch (groupBy) {
case 'model':
key = call.model;
break;
case 'endpoint':
key = call.provider;
break;
case 'time':
key = new Date(call.timestamp).toISOString().split('T')[0]; // Group by day
break;
default:
key = call.model;
}
if (!groups.has(key)) {
groups.set(key, {
calls: 0,
tokens: 0,
promptTokens: 0,
completionTokens: 0,
cost: 0
});
}
const group = groups.get(key);
group.calls++;
if (call.tokens) {
group.tokens += call.tokens.total;
group.promptTokens += call.tokens.prompt || 0;
group.completionTokens += call.tokens.completion || 0;
}
group.cost += call.cost || 0;
});
report += `### Usage by ${groupBy.charAt(0).toUpperCase() + groupBy.slice(1)}\n\n`;
const sortedGroups = Array.from(groups.entries()).sort((a, b) => b[1].tokens - a[1].tokens);
sortedGroups.forEach(([key, stats]) => {
report += `#### ${key}\n`;
report += `- **Calls:** ${stats.calls}\n`;
report += `- **Tokens:** ${stats.tokens.toLocaleString()} (${Math.round(stats.tokens / totalTokens * 100)}%)\n`;
report += `- **Cost:** $${stats.cost.toFixed(4)}\n`;
report += `- **Avg Tokens/Call:** ${Math.round(stats.tokens / stats.calls)}\n\n`;
});
// Token efficiency analysis
report += `### Token Efficiency\n\n`;
if (totalPromptTokens > 0 && totalCompletionTokens > 0) {
const ratio = totalCompletionTokens / totalPromptTokens;
report += `- **Prompt/Completion Ratio:** 1:${ratio.toFixed(2)}\n`;
if (ratio < 0.5) {
report += `- ⚠️ **Low Output**: Generating less than half the input tokens\n`;
}
else if (ratio > 3) {
report += `- ⚠️ **High Output**: Consider if responses are too verbose\n`;
}
}
// Cost per token
const costPerToken = totalCost / totalTokens;
report += `- **Average Cost per Token:** $${(costPerToken * 1000).toFixed(6)}/1K tokens\n\n`;
// Recommendations
report += `### Optimization Recommendations\n\n`;
if (totalTokens > 100000) {
report += `1. **High Usage**: Consider caching frequent queries\n`;
}
const expensiveModels = sortedGroups.filter(([key, stats]) => stats.cost / stats.tokens > costPerToken * 1.5);
if (expensiveModels.length > 0) {
report += `2. **Cost Optimization**: Consider cheaper models for ${expensiveModels.map(e => e[0]).join(', ')}\n`;
}
if (totalPromptTokens > totalCompletionTokens * 2) {
report += `3. **Prompt Efficiency**: Your prompts are much longer than responses - consider optimization\n`;
}
return {
content: [{ type: 'text', text: report.trim() }]
};
}
}
//# sourceMappingURL=ai-llm-handler.js.map