ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
378 lines • 10.6 kB
TypeScript
/**
* System Automation Handler - Native system mouse and keyboard control
* Enables AI models to control system cursor for Vim, terminal apps, and desktop automation
*
* Uses nut.js (modern, fast) with robotjs fallback for maximum compatibility
*/
import { Tool } from '@modelcontextprotocol/sdk/types.js';
import { BaseToolHandler } from './base-handler.js';
interface SystemMouseAction {
action: 'move' | 'click' | 'drag' | 'scroll' | 'doubleclick' | 'rightclick';
x?: number;
y?: number;
startX?: number;
startY?: number;
endX?: number;
endY?: number;
button?: 'left' | 'right' | 'middle';
scrollDirection?: 'up' | 'down' | 'left' | 'right';
scrollAmount?: number;
delay?: number;
}
interface ScreenAction {
action: 'capture' | 'find_text' | 'find_color' | 'get_pixel';
x?: number;
y?: number;
width?: number;
height?: number;
text?: string;
color?: string;
}
export declare class SystemAutomationHandler extends BaseToolHandler {
private robotjsFallback;
private macroSystem;
private backgroundAutomation;
tools: Tool[];
constructor();
private initializeRobotjsFallback;
getTools(): Tool[];
handle(name: string, args: any): Promise<any>;
/**
* 📝 List action macros with filtering and sorting
*/
private listActionMacros;
/**
* Control system mouse - the core functionality for AI models WITH VISUAL FEEDBACK
*/
controlSystemMouse(args: SystemMouseAction & {
captureVisualFeedback?: boolean;
analyzeRegion?: {
width: number;
height: number;
};
}): Promise<any>;
/**
* Screen capture and analysis capabilities
*/
controlSystemScreen(args: ScreenAction): Promise<any>;
swiftScreenshotCapture(args: {
action: 'full_screen' | 'region' | 'window' | 'application' | 'visible_windows';
x?: number;
y?: number;
width?: number;
height?: number;
windowId?: number;
bundleId?: string;
appName?: string;
options?: {
format?: 'png' | 'jpeg' | 'jpg';
quality?: number;
scale?: number;
};
outputPath?: string;
}): Promise<any>;
/**
* Specialized Vim automation - combines mouse and keyboard for terminal Vim control
*/
controlVimTerminal(args: {
action: 'click_line' | 'select_text' | 'scroll_page' | 'vim_command' | 'focus_window';
terminalBounds?: {
x: number;
y: number;
width: number;
height: number;
};
lineNumber?: number;
command?: string;
text?: string;
direction?: 'up' | 'down';
}): Promise<any>;
/**
* Get system information for automation context
*/
getSystemInfo(): Promise<any>;
private convertButtonToNut;
private convertKeyToNut;
/**
* 👁️ Visual AI Analysis - Screenshot analysis and screen understanding
*/
/**
* 🔍 Discover all application windows
*/
discoverApplicationWindows(args: {
filterByApplication?: string;
includeMinimized?: boolean;
}): Promise<any>;
/**
* 🎯 Create background application context
*/
createBackgroundAppContext(args: {
contextName: string;
applicationName: string;
windowTitle?: string;
preferredMethod?: string;
}): Promise<any>;
/**
* 🖱️ Control background application mouse
*/
controlBackgroundAppMouse(args: {
contextName: string;
action: string;
x: number;
y: number;
button?: string;
relative?: boolean;
}): Promise<any>;
/**
* ⌨️ Control background application keyboard with enhanced validation
*/
controlBackgroundAppKeyboard(args: {
contextName: string;
action: string;
text?: string;
key?: string;
keys?: string[];
}): Promise<any>;
/**
* 📋 List all background contexts
*/
listBackgroundContexts(args: {
includeWindowInfo?: boolean;
}): Promise<any>;
/**
* 🔄 Refresh background context window info
*/
refreshBackgroundContext(args: {
contextName: string;
}): Promise<any>;
/**
* 🗑️ Remove background context
*/
removeBackgroundContext(args: {
contextName: string;
}): Promise<any>;
/**
* 📸 Take background window screenshot
*/
takeBackgroundWindowScreenshot(args: {
contextName: string;
format?: 'png' | 'jpg';
quality?: number;
outputPath?: string;
}): Promise<any>;
/**
* Check NSAccessibility permissions
*/
private accessibilityCheckPermissions;
/**
* Get applications available for NSAccessibility interaction
*/
private accessibilityGetApplications;
/**
* 🚀 REVOLUTIONARY: Write text to background application WITHOUT focus switching!
*/
private accessibilityBackgroundTextInput;
/**
* Get UI elements from background application using NSAccessibility
*/
private accessibilityGetUIElements;
/**
* 🎯 Verify which window currently has focus
*/
private verifyWindowFocus;
/**
* 👁️ Analyze screen content visually
*/
private analyzeScreenVisually;
/**
* ⚡ Execute batch automation for dramatic performance improvement
*/
private executeBatchAutomation;
/**
* 🔬 Profile automation performance to identify bottlenecks
*/
private profileAutomationPerformance;
/**
* 🚀 Execute native CGEvent automation - 10x faster
*/
private executeNativeAutomation;
/**
* 🧠 Manage advanced UI cache
*/
private manageUICache;
/**
* ⚡ Control WebSocket automation server
*/
private controlWebSocketAutomation;
}
/**
* 🎬 INTELLIGENT ACTION MACRO SYSTEM
*
* Revolutionary feature that allows users to:
* 1. Describe actions in natural language
* 2. Record and intelligently parse action sequences
* 3. Save macros with visual verification
* 4. Replay with intelligent adaptation
* 5. Share and import macro libraries
*/
interface ActionMacro {
id: string;
name: string;
description: string;
actions: ParsedAction[];
visualCheckpoints: VisualCheckpoint[];
metadata: {
created: string;
lastUsed?: string;
useCount: number;
successRate: number;
tags: string[];
framework?: string;
environment?: string;
};
}
interface ParsedAction {
type: 'mouse' | 'keyboard' | 'screen' | 'vim' | 'wait' | 'verify';
description: string;
parameters: any;
visualExpectation?: string;
fallbackActions?: ParsedAction[];
}
interface VisualCheckpoint {
actionIndex: number;
description: string;
expectedChange: string;
screenshot?: string;
verificationRegion?: {
x: number;
y: number;
width: number;
height: number;
};
}
interface MacroExecution {
macroId: string;
startTime: string;
endTime?: string;
success: boolean;
steps: {
actionIndex: number;
action: ParsedAction;
result: any;
visualVerification?: {
passed: boolean;
actualChange: string;
screenshot?: string;
};
}[];
adaptations: string[];
failureReason?: string;
}
export declare class IntelligentActionMacroSystem {
private macros;
private executions;
private systemAutomation;
constructor(systemAutomationHandler: SystemAutomationHandler);
/**
* 🎯 CORE FEATURE: Parse natural language into intelligent actions
*/
parseNaturalLanguageActions(description: string, context?: {
framework?: string;
currentScreen?: string;
environment?: string;
}): Promise<ParsedAction[]>;
/**
* 🧠 INTELLIGENT: Single action parsing with smart defaults
*/
private parseSingleAction;
/**
* 🖱️ Smart click action parsing with intelligent coordinate detection
*/
private parseClickAction;
/**
* ⌨️ Smart keyboard action parsing
*/
private parseKeyboardAction;
/**
* 📷 Smart screen action parsing
*/
private parseScreenAction;
/**
* ✏️ Smart Vim action parsing
*/
private parseVimAction;
/**
* ⏰ Smart wait/delay action parsing
*/
private parseWaitAction;
/**
* 🎬 CORE FEATURE: Create and save action macro
*/
createMacro(request: {
name: string;
description: string;
naturalLanguageActions: string;
tags?: string[];
context?: any;
}): Promise<ActionMacro>;
/**
* 🚀 CORE FEATURE: Execute macro with intelligent adaptation
*/
executeMacro(macroId: string, options?: {
dryRun?: boolean;
skipVerification?: boolean;
adaptToChanges?: boolean;
}): Promise<MacroExecution>;
/**
* 📚 CORE FEATURE: Export/import macro libraries
*/
exportMacros(macroIds?: string[]): Promise<string>;
importMacros(macroLibrary: string): Promise<{
imported: number;
skipped: number;
errors: string[];
}>;
/**
* 📊 Get detailed analytics for action macros
*/
getMacroAnalytics(macroId?: string): any;
/**
* 🔍 HELPER: Extract coordinates from natural language
*/
private extractCoordinates;
/**
* 🔍 HELPER: Extract quoted text or text after keywords
*/
private extractQuotedText;
private extractAfterKeyword;
/**
* 🔍 HELPER: Extract key combinations
*/
private extractKeySequence;
/**
* 🔍 HELPER: Extract element descriptions for better targeting
*/
private extractElementDescription;
/**
* 📸 Generate visual checkpoints for macro verification
*/
private generateVisualCheckpoints;
/**
* 🎯 Execute individual parsed action
*/
private executeAction;
/**
* 👁️ Verify visual expectations after action execution
*/
private verifyVisualExpectation;
/**
* 🧠 Intelligent action adaptation when verification fails
*/
private adaptAction;
/**
* ⚡ Optimize action sequence for better performance
*/
private optimizeActionSequence;
}
export {};
//# sourceMappingURL=system-automation-handler.d.ts.map