ai-debug-local-mcp
Version:
šÆ ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
350 lines ā¢ 16.7 kB
JavaScript
import { BaseToolHandler } from '../base-handler.js';
import { LocalDebugEngine } from '../../local-debug-engine.js';
import { AIExtractionUtils } from './ai-utils.js';
export class AIPromptHandler extends BaseToolHandler {
tools = [
{
name: 'debug_prompt_templates',
description: 'Debug prompt template variable injection and monitor template usage',
inputSchema: {
type: 'object',
properties: {
sessionId: { type: 'string', description: 'Debug session ID' }
},
required: ['sessionId']
}
},
{
name: 'compare_prompt_versions',
description: 'A/B test different prompt versions for performance and quality',
inputSchema: {
type: 'object',
properties: {
sessionId: { type: 'string', description: 'Debug session ID' },
versions: {
type: 'array',
items: { type: 'string' },
description: 'Version identifiers to compare'
}
},
required: ['sessionId']
}
},
{
name: 'analyze_prompt_tokens',
description: 'Analyze token usage by prompt section to optimize costs',
inputSchema: {
type: 'object',
properties: {
sessionId: { type: 'string', description: 'Debug session ID' },
prompt: { type: 'string', description: 'Prompt text to analyze' }
},
required: ['sessionId', 'prompt']
}
},
{
name: 'monitor_prompt_injection',
description: 'Monitor and detect potential prompt injection security threats',
inputSchema: {
type: 'object',
properties: {
sessionId: { type: 'string', description: 'Debug session ID' }
},
required: ['sessionId']
}
}
];
async handle(toolName, args, sessions) {
const session = sessions.get(args.sessionId);
if (!session) {
return {
content: [{ type: 'text', text: `Session not found: ${args.sessionId}` }]
};
}
try {
switch (toolName) {
case 'debug_prompt_templates':
return await this.debugPromptTemplates(args, session);
case 'compare_prompt_versions':
return await this.comparePromptVersions(args, session);
case 'analyze_prompt_tokens':
return await this.analyzePromptTokens(args, session);
case 'monitor_prompt_injection':
return await this.monitorPromptInjection(args, session);
default:
throw new Error(`Unknown tool: ${toolName}`);
}
}
catch (error) {
return {
content: [{
type: 'text',
text: `Error in ${toolName}: ${error instanceof Error ? error.message : String(error)}`
}]
};
}
}
async debugPromptTemplates(args, session) {
// In a real implementation, this would monitor actual prompt template usage
// For now, we'll execute page evaluation to detect templates
const templateData = await session.page.evaluate(() => {
// This would hook into actual template engines like Handlebars, Mustache, etc.
// For demo purposes, returning mock data structure
return {
templates: []
};
});
let report = '## Prompt Template Debugging\n\n';
if (!templateData.templates || templateData.templates.length === 0) {
report += 'No prompt templates detected. This tool monitors:\n';
report += '- Template variable injection\n';
report += '- Variable sanitization\n';
report += '- Template usage patterns\n';
report += '- Potential security issues\n\n';
report += 'Ensure your application uses detectable template patterns.';
return {
content: [{ type: 'text', text: report }]
};
}
report += `**Templates Found:** ${templateData.templates.length}\n\n`;
for (const template of templateData.templates) {
report += `### Template: ${template.id}\n`;
report += `**Template:** ${template.template}\n`;
report += `**Variables:** ${template.variables.join(', ')}\n`;
if (template.warnings && template.warnings.length > 0) {
report += '\nā ļø **Warnings:**\n';
template.warnings.forEach((warning) => {
report += `- ${warning}\n`;
});
}
if (template.injections.length > 0) {
report += '\n### Recent Injections\n';
template.injections.slice(-5).forEach((injection, index) => {
report += `\n**Injection ${index + 1}:**\n`;
report += '```json\n' + JSON.stringify(injection.values, null, 2) + '\n```\n';
report += `**Result:** ${injection.finalPrompt.substring(0, 200)}...\n`;
});
}
report += '\n---\n\n';
}
return {
content: [{ type: 'text', text: report }]
};
}
async comparePromptVersions(args, session) {
const engine = session.engine || new LocalDebugEngine();
const networkRequests = engine.getNetworkRequests();
const versions = args.versions || ['A', 'B'];
// Extract LLM calls and group by version
const llmCalls = AIExtractionUtils.extractLLMCalls(networkRequests);
const versionData = {};
for (const version of versions) {
versionData[version] = llmCalls.filter(call => {
try {
const req = call.request;
return JSON.stringify(req).includes(`Version ${version}`) ||
(req.metadata && req.metadata.promptVersion === version);
}
catch {
return false;
}
});
}
let report = '## Prompt Version Comparison\n\n';
// Compare versions pairwise
if (versions.length >= 2) {
report += `### Version ${versions[0]} vs Version ${versions[1]}\n\n`;
const v1Data = versionData[versions[0]] || [];
const v2Data = versionData[versions[1]] || [];
if (v1Data.length === 0 && v2Data.length === 0) {
report += 'No version-specific LLM calls found. ';
report += 'Ensure requests include version metadata or version identifiers in prompts.\n';
}
else {
// Calculate metrics
const v1Metrics = this.calculateVersionMetrics(v1Data);
const v2Metrics = this.calculateVersionMetrics(v2Data);
report += '| Metric | Version ' + versions[0] + ' | Version ' + versions[1] + ' | Difference |\n';
report += '|--------|------------|------------|------------|\n';
report += `| Calls | ${v1Metrics.calls} | ${v2Metrics.calls} | ${v2Metrics.calls - v1Metrics.calls} |\n`;
report += `| Avg Token Usage | ${v1Metrics.avgTokens} | ${v2Metrics.avgTokens} | ${v2Metrics.avgTokens - v1Metrics.avgTokens} |\n`;
report += `| Avg Response Time | ${v1Metrics.avgDuration}ms | ${v2Metrics.avgDuration}ms | ${v2Metrics.avgDuration - v1Metrics.avgDuration}ms |\n`;
report += `| Avg Response Length | ${v1Metrics.avgResponseLength} | ${v2Metrics.avgResponseLength} | ${v2Metrics.avgResponseLength - v1Metrics.avgResponseLength} |\n`;
report += `| Total Cost | $${v1Metrics.totalCost.toFixed(4)} | $${v2Metrics.totalCost.toFixed(4)} | $${(v2Metrics.totalCost - v1Metrics.totalCost).toFixed(4)} |\n`;
report += '\n### Analysis\n';
if (v2Metrics.avgTokens < v1Metrics.avgTokens) {
report += `- ā
Version ${versions[1]} uses ${Math.round((1 - v2Metrics.avgTokens / v1Metrics.avgTokens) * 100)}% fewer tokens\n`;
}
if (v2Metrics.avgDuration < v1Metrics.avgDuration) {
report += `- ā
Version ${versions[1]} is ${Math.round((1 - v2Metrics.avgDuration / v1Metrics.avgDuration) * 100)}% faster\n`;
}
if (v2Metrics.totalCost < v1Metrics.totalCost) {
report += `- ā
Version ${versions[1]} is ${Math.round((1 - v2Metrics.totalCost / v1Metrics.totalCost) * 100)}% cheaper\n`;
}
}
}
return {
content: [{ type: 'text', text: report }]
};
}
async analyzePromptTokens(args, session) {
const prompt = args.prompt || '';
if (!prompt) {
return {
content: [{
type: 'text',
text: '## Prompt Token Analysis\n\nNo prompt provided. Please provide a prompt to analyze.'
}]
};
}
let report = '## Prompt Token Analysis\n\n';
// Split prompt into sections
const sections = this.splitPromptIntoSections(prompt);
report += '### Token Distribution by Section\n\n';
let totalTokens = 0;
const sectionData = [];
// Estimate tokens for each section (rough approximation: 1 token ā 4 chars)
for (const [sectionName, content] of Object.entries(sections)) {
const tokens = Math.ceil(content.length / 4);
totalTokens += tokens;
sectionData.push({ name: sectionName, tokens, percentage: 0 });
}
// Calculate percentages
sectionData.forEach(section => {
section.percentage = totalTokens > 0 ? Math.round((section.tokens / totalTokens) * 100) : 0;
});
// Display token distribution
sectionData.forEach(section => {
report += `**${section.name}:** ${section.tokens} tokens (${section.percentage}%)\n`;
});
report += `\n**Total Tokens:** ${totalTokens}\n`;
report += `**Estimated Cost:** $${(totalTokens * 0.00002).toFixed(6)} (GPT-3.5 pricing)\n\n`;
// Optimization suggestions
report += '### Optimization Suggestions\n\n';
const largestSection = sectionData.reduce((max, section) => section.tokens > max.tokens ? section : max);
if (largestSection.percentage > 50) {
report += `- ā ļø The "${largestSection.name}" section uses ${largestSection.percentage}% of tokens\n`;
report += ` Consider reducing verbosity or moving some content to context\n`;
}
if (totalTokens > 2000) {
report += '- ā ļø Prompt exceeds 2000 tokens - may hit model limits\n';
report += ' Consider using a more concise format\n';
}
if (sections.System && sections.System.length > 500) {
report += '- š” System prompt is lengthy - ensure all instructions are necessary\n';
}
return {
content: [{ type: 'text', text: report }]
};
}
async monitorPromptInjection(args, session) {
// Monitor for prompt injection attempts
const injectionData = await session.page.evaluate(() => {
// This would integrate with actual security monitoring
// For demo purposes, returning mock data
return {
injectionAttempts: [],
statistics: {
total: 0,
blocked: 0,
patterns: {}
}
};
});
let report = '## Prompt Injection Security Monitor\n\n';
report += '### Statistics\n';
report += `- **Total Attempts:** ${injectionData.statistics.total}\n`;
report += `- **Blocked:** ${injectionData.statistics.blocked}\n`;
report += `- **Success Rate:** ${injectionData.statistics.total > 0 ?
Math.round((injectionData.statistics.blocked / injectionData.statistics.total) * 100) : 100}%\n\n`;
if (injectionData.injectionAttempts.length > 0) {
report += '### Recent Injection Attempts\n\n';
injectionData.injectionAttempts.slice(-10).forEach((attempt, index) => {
const threatIcon = attempt.threat === 'high' ? 'š“' :
attempt.threat === 'medium' ? 'š”' : 'š¢';
report += `#### Attempt ${index + 1} ${threatIcon}\n`;
report += `- **Time:** ${new Date(attempt.timestamp).toLocaleString()}\n`;
report += `- **Threat:** ${attempt.threat}\n`;
report += `- **Pattern:** ${attempt.pattern}\n`;
report += `- **Blocked:** ${attempt.blocked ? 'ā
' : 'ā'}\n`;
report += `- **Input:** \`${attempt.input.substring(0, 100)}...\`\n\n`;
});
// Pattern analysis
report += '### Attack Patterns\n\n';
for (const [pattern, count] of Object.entries(injectionData.statistics.patterns)) {
report += `- **${pattern}:** ${count} attempts\n`;
}
}
else {
report += 'No injection attempts detected. The monitor watches for:\n';
report += '- Instruction override attempts\n';
report += '- System prompt manipulation\n';
report += '- Tag/delimiter injection\n';
report += '- Role confusion attacks\n';
report += '- Encoded/obfuscated payloads\n';
}
report += '\n### Security Recommendations\n';
report += '- Always validate and sanitize user inputs\n';
report += '- Use prompt guards and output validation\n';
report += '- Implement rate limiting for suspicious patterns\n';
report += '- Log and monitor all injection attempts\n';
return {
content: [{ type: 'text', text: report }]
};
}
calculateVersionMetrics(calls) {
if (calls.length === 0) {
return {
calls: 0,
avgTokens: 0,
avgDuration: 0,
avgResponseLength: 0,
totalCost: 0
};
}
let totalTokens = 0;
let totalDuration = 0;
let totalResponseLength = 0;
let totalCost = 0;
calls.forEach(call => {
totalTokens += call.tokens?.total || 0;
totalDuration += call.duration || 0;
totalCost += call.cost || 0;
if (call.response?.choices?.[0]?.message?.content) {
totalResponseLength += call.response.choices[0].message.content.length;
}
});
return {
calls: calls.length,
avgTokens: Math.round(totalTokens / calls.length),
avgDuration: Math.round(totalDuration / calls.length),
avgResponseLength: Math.round(totalResponseLength / calls.length),
totalCost
};
}
splitPromptIntoSections(prompt) {
const sections = {};
// Common patterns for section detection
const patterns = [
{ regex: /System:\s*(.+?)(?=\n\n|User:|Context:|$)/si, name: 'System' },
{ regex: /Context:\s*(.+?)(?=\n\n|User:|System:|$)/si, name: 'Context' },
{ regex: /User:\s*(.+?)(?=\n\n|System:|Context:|$)/si, name: 'User' },
{ regex: /Assistant:\s*(.+?)(?=\n\n|User:|System:|$)/si, name: 'Assistant' },
{ regex: /Instructions:\s*(.+?)(?=\n\n|User:|Context:|$)/si, name: 'Instructions' }
];
let remainingPrompt = prompt;
for (const pattern of patterns) {
const match = remainingPrompt.match(pattern.regex);
if (match) {
sections[pattern.name] = match[1].trim();
remainingPrompt = remainingPrompt.replace(match[0], '');
}
}
// If no sections found or remaining content, treat as single section
if (Object.keys(sections).length === 0 || remainingPrompt.trim()) {
sections['Main'] = remainingPrompt.trim() || prompt;
}
return sections;
}
}
//# sourceMappingURL=ai-prompt-handler.js.map