ai-debug-local-mcp
Version:
🎯 ENHANCED AI GUIDANCE v4.1.2: Dramatically improved tool descriptions help AI users choose the right tools instead of 'close enough' options. Ultra-fast keyboard automation (10x speed), universal recording, multi-ecosystem debugging support, and compreh
496 lines (495 loc) • 21.4 kB
JavaScript
/**
* Background Window Automation - Cross-platform window messaging
* Enables AI to control applications without interrupting user's current window focus
*
* REVOLUTIONARY: NSAccessibility integration for true non-intrusive automation
*/
import { exec } from 'child_process';
import { promisify } from 'util';
import { accessibilityBridge } from './accessibility-bridge.js';
const execAsync = promisify(exec);
export class BackgroundWindowAutomation {
contexts = new Map();
platform;
constructor() {
this.platform = process.platform;
console.log(`🪟 Background Window Automation initialized for ${this.platform}`);
// Initialize accessibility bridge for revolutionary background interaction
if (this.platform === 'darwin') {
this.initializeAccessibility();
}
}
/**
* Initialize NSAccessibility bridge for true background interaction
*/
async initializeAccessibility() {
try {
await accessibilityBridge.initialize();
console.log('🚀 NSAccessibility bridge initialized for revolutionary background interaction!');
}
catch (error) {
console.warn('⚠️ NSAccessibility bridge failed to initialize:', error);
}
}
/**
* Discover all visible application windows
*/
async discoverWindows() {
switch (this.platform) {
case 'darwin':
return this.discoverMacOSWindows();
case 'win32':
return this.discoverWindowsWindows();
case 'linux':
return this.discoverLinuxWindows();
default:
throw new Error(`Platform ${this.platform} not supported`);
}
}
/**
* macOS window discovery using AppleScript
*/
async discoverMacOSWindows() {
try {
// First get all processes with windows
const processScript = `
tell application "System Events"
set processList to {}
repeat with proc in (every process whose background only is false)
try
set windowCount to count of windows of proc
if windowCount > 0 then
set processList to processList & {(name of proc) & "|" & (unix id of proc) & "|" & windowCount}
end if
end try
end repeat
return processList
end tell
`;
const { stdout: processOutput } = await execAsync(`osascript -e '${processScript.replace(/'/g, "\\'")}'`);
const processLines = processOutput.trim().split(',').map(line => line.trim()).filter(line => line.includes('|'));
console.log('🔍 Found processes with windows:', processLines.length);
const windows = [];
// For each process, get its windows
for (const processLine of processLines) {
const [processName, processId] = processLine.split('|');
try {
const windowScript = `
tell application "System Events"
set targetProcess to first process whose name is "${processName.replace(/"/g, '\\"')}"
set windowTitles to title of windows of targetProcess
set windowPositions to position of windows of targetProcess
set windowSizes to size of windows of targetProcess
set windowList to {}
repeat with i from 1 to count of windowTitles
try
set winTitle to item i of windowTitles
set winPos to item i of windowPositions
set winSize to item i of windowSizes
-- Default values for problematic properties
set windowList to windowList & {winTitle & "|" & (item 1 of winPos) & "|" & (item 2 of winPos) & "|" & (item 1 of winSize) & "|" & (item 2 of winSize) & "|true|false"}
on error
-- Skip problematic windows
end try
end repeat
return windowList
end tell
`;
const { stdout: windowOutput } = await execAsync(`osascript -e '${windowScript.replace(/'/g, "\\'")}'`);
const windowLines = windowOutput.trim().split(',').map(line => line.trim()).filter(line => line.includes('|'));
for (const windowLine of windowLines) {
const parts = windowLine.split('|');
if (parts.length >= 7) {
windows.push({
windowId: `${processId}-${parts[0]}`,
processId: parseInt(processId),
applicationName: processName,
windowTitle: parts[0],
bounds: {
x: parseInt(parts[1]),
y: parseInt(parts[2]),
width: parseInt(parts[3]),
height: parseInt(parts[4])
},
isVisible: parts[5] === 'true',
isMinimized: parts[6] === 'true',
ownerName: processName
});
}
}
}
catch (error) {
console.log(`⚠️ Failed to get windows for ${processName}:`, error instanceof Error ? error.message : String(error));
continue;
}
}
console.log(`🔍 Discovered ${windows.length} windows on macOS`);
return windows;
}
catch (error) {
console.error('❌ Failed to discover macOS windows:', error);
return [];
}
}
/**
* Windows window discovery (placeholder)
*/
async discoverWindowsWindows() {
console.log('🚧 Windows window discovery not yet implemented');
return [];
}
/**
* Linux window discovery (placeholder)
*/
async discoverLinuxWindows() {
console.log('🚧 Linux window discovery not yet implemented');
return [];
}
/**
* Create an automation context for a specific application
*/
async createContext(name, applicationName, windowTitle) {
console.log(`🎯 Creating automation context: ${name} for ${applicationName}`);
// Find the target window
const windows = await this.discoverWindows();
const targetWindow = windows.find(w => {
const appMatch = w.applicationName.toLowerCase().includes(applicationName.toLowerCase());
const titleMatch = !windowTitle || w.windowTitle.toLowerCase().includes(windowTitle.toLowerCase());
return appMatch && titleMatch && w.isVisible && !w.isMinimized;
});
if (!targetWindow) {
throw new Error(`Could not find window for application: ${applicationName}`);
}
// Check if accessibility is available for this app
let accessibilityEnabled = false;
if (this.platform === 'darwin') {
try {
const hasPermissions = await accessibilityBridge.checkPermissions();
if (hasPermissions) {
const apps = await accessibilityBridge.getApplications();
accessibilityEnabled = apps[targetWindow.applicationName] !== undefined;
}
}
catch (error) {
console.warn('Failed to check accessibility for', targetWindow.applicationName);
}
}
const context = {
name,
applicationName: targetWindow.applicationName,
processId: targetWindow.processId,
windowId: targetWindow.windowId,
preferredMethod: accessibilityEnabled ? 'accessibility' : (this.platform === 'darwin' ? 'direct_messaging' : 'focus_switching'),
windowBounds: targetWindow.bounds,
lastActive: new Date(),
accessibilityEnabled
};
this.contexts.set(name, context);
console.log(`✅ Context created: ${name} (PID: ${context.processId}, Method: ${context.preferredMethod})`);
return context;
}
/**
* Send mouse action to background application
*/
async sendMouseAction(contextName, action) {
const context = this.contexts.get(contextName);
if (!context) {
throw new Error(`Context not found: ${contextName}`);
}
console.log(`🖱️ Sending mouse action to ${contextName}: ${action.action} at (${action.x}, ${action.y})`);
// Transform coordinates if relative to window
let screenX = action.x;
let screenY = action.y;
if (action.relative && context.windowBounds) {
screenX = context.windowBounds.x + action.x;
screenY = context.windowBounds.y + action.y;
console.log(`📐 Transformed relative coordinates (${action.x}, ${action.y}) to screen coordinates (${screenX}, ${screenY})`);
}
switch (this.platform) {
case 'darwin':
return this.sendMacOSMouseAction(context, action, screenX, screenY);
default:
throw new Error(`Mouse actions not implemented for platform: ${this.platform}`);
}
}
/**
* macOS mouse action using AppleScript (will upgrade to CGEventPostToPid later)
*/
async sendMacOSMouseAction(context, action, screenX, screenY) {
try {
let script = '';
switch (action.action) {
case 'click':
// Use AppleScript to click without bringing window to front
script = `
tell application "System Events"
tell process "${context.applicationName}"
try
set frontmost to true
delay 0.1
click at {${screenX}, ${screenY}}
delay 0.1
on error
-- Fallback: direct coordinate click
do shell script "osascript -e 'tell application \\"System Events\\" to click at {${screenX}, ${screenY}}'"
end try
end tell
end tell
`;
break;
case 'move':
script = `
tell application "System Events"
tell process "${context.applicationName}"
set frontmost to true
delay 0.1
set mouseLocation to {${screenX}, ${screenY}}
end tell
end tell
`;
break;
default:
throw new Error(`Action ${action.action} not implemented for macOS`);
}
await execAsync(`osascript -e '${script.replace(/'/g, "\\'")}'`);
console.log(`✅ macOS mouse action completed: ${action.action}`);
}
catch (error) {
console.error(`❌ macOS mouse action failed:`, error);
throw error;
}
}
/**
* Send keyboard action to background application
* REVOLUTIONARY: Uses NSAccessibility when available for true background interaction!
*/
async sendKeyboardAction(contextName, action) {
const context = this.contexts.get(contextName);
if (!context) {
throw new Error(`Context not found: ${contextName}`);
}
console.log(`⌨️ Sending keyboard action to ${contextName}: ${action.action} (Method: ${context.preferredMethod})`);
// 🚀 REVOLUTIONARY: Use NSAccessibility for true background text input!
if (context.preferredMethod === 'accessibility' && action.action === 'type' && action.text) {
return this.sendAccessibilityTextInput(context, action.text);
}
switch (this.platform) {
case 'darwin':
return this.sendMacOSKeyboardAction(context, action);
default:
throw new Error(`Keyboard actions not implemented for platform: ${this.platform}`);
}
}
/**
* 🚀 REVOLUTIONARY: Send text using NSAccessibility - NO FOCUS SWITCHING!
*/
async sendAccessibilityTextInput(context, text) {
try {
console.log('🎯 Using NSAccessibility for revolutionary background text input...');
const result = await accessibilityBridge.writeTextToApplication(context.applicationName, text);
if (result.success) {
console.log(`✅ NSAccessibility text input successful! Found ${result.textFieldsFound} text fields.`);
}
else {
console.warn(`⚠️ NSAccessibility text input failed: ${result.error}`);
// Fallback to traditional method
throw new Error('Accessibility failed, falling back to AppleScript');
}
}
catch (error) {
console.log('📱 Falling back to AppleScript method...');
return this.sendMacOSKeyboardAction(context, { action: 'type', text });
}
}
/**
* macOS keyboard action using AppleScript
*/
async sendMacOSKeyboardAction(context, action) {
try {
let script = '';
switch (action.action) {
case 'type':
if (!action.text)
throw new Error('Text required for type action');
script = `
tell application "${context.applicationName}"
activate
delay 0.1
end tell
tell application "System Events"
keystroke "${action.text.replace(/"/g, '\\"')}"
end tell
`;
break;
case 'press':
if (!action.key)
throw new Error('Key required for press action');
script = `
tell application "${context.applicationName}"
activate
delay 0.1
end tell
tell application "System Events"
key code ${this.getKeyCode(action.key)}
end tell
`;
break;
default:
throw new Error(`Keyboard action ${action.action} not implemented for macOS`);
}
await execAsync(`osascript -e '${script.replace(/'/g, "\\'")}'`);
console.log(`✅ macOS keyboard action completed: ${action.action}`);
}
catch (error) {
console.error(`❌ macOS keyboard action failed:`, error);
throw error;
}
}
/**
* Get macOS key code for common keys
*/
getKeyCode(key) {
const keyCodes = {
'enter': 36,
'return': 36,
'escape': 53,
'tab': 48,
'space': 49,
'delete': 51,
'backspace': 51,
'up': 126,
'down': 125,
'left': 123,
'right': 124
};
return keyCodes[key.toLowerCase()] || 0;
}
/**
* List all active automation contexts
*/
listContexts() {
return Array.from(this.contexts.values());
}
/**
* Remove an automation context
*/
removeContext(name) {
return this.contexts.delete(name);
}
/**
* Take a screenshot of a background window without bringing it to focus
*/
async takeWindowScreenshot(contextName, options = {}) {
const context = this.contexts.get(contextName);
if (!context) {
return { success: false, error: `Context not found: ${contextName}` };
}
console.log(`📸 Taking screenshot of ${contextName} (${context.applicationName})`);
try {
// Refresh context to get latest window bounds
const windowInfo = await this.refreshContextWindow(contextName);
if (!windowInfo) {
return { success: false, error: 'Window not found or no longer available' };
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const format = options.format || 'png';
const outputPath = options.outputPath || `/tmp/bg-screenshot-${contextName}-${timestamp}.${format}`;
switch (this.platform) {
case 'darwin':
return this.takeScreenshotMacOS(windowInfo, outputPath, options);
default:
return { success: false, error: `Screenshot not implemented for platform: ${this.platform}` };
}
}
catch (error) {
console.error(`❌ Screenshot failed for ${contextName}:`, error);
return {
success: false,
error: error instanceof Error ? error.message : String(error)
};
}
}
/**
* macOS window screenshot using Swift ScreenCaptureKit with CGWindowListCreateImage fallback
*/
async takeScreenshotMacOS(windowInfo, outputPath, options) {
try {
// 🚀 REVOLUTIONARY: Use Swift screenshot service with CGWindowListCreateImage
// This captures windows even when they're behind other windows!
console.log(`🚀 Using Swift screenshot service for true background capture...`);
// Import Swift screenshot bridge
const { getSwiftScreenshotBridge } = await import('./swift-screenshot-bridge.js');
const bridge = await getSwiftScreenshotBridge();
// First, let's find the actual window ID from our windowInfo
let targetWindowId = null;
// Try to extract window ID from our windowInfo.windowId
if (windowInfo.windowId && windowInfo.windowId.includes('-')) {
// If windowId is in format "processId-title", we need to find the real CGWindowID
const windows = await this.discoverMacOSWindows();
const matchingWindow = windows.find(w => w.applicationName === windowInfo.applicationName &&
w.windowTitle === windowInfo.windowTitle &&
w.processId === windowInfo.processId);
if (matchingWindow) {
// Parse the actual window ID if available
targetWindowId = parseInt(matchingWindow.windowId.split('-')[0]) || null;
}
}
if (targetWindowId) {
try {
console.log(`📸 Attempting Swift window capture for window ID: ${targetWindowId}`);
// Use Swift service to capture specific window - THIS IS THE BREAKTHROUGH!
const screenshot = await bridge.captureWindow(targetWindowId, {
format: options.format || 'png',
quality: options.quality || 0.9
});
// Save the screenshot data to file
const fs = await import('fs');
const imageBuffer = Buffer.from(screenshot.data, 'base64');
fs.writeFileSync(outputPath, imageBuffer);
console.log(`✅ TRUE BACKGROUND CAPTURE SUCCESSFUL: ${screenshot.width}x${screenshot.height}`);
console.log(`📁 Saved to: ${outputPath}`);
return { success: true, path: outputPath };
}
catch (swiftError) {
console.log(`⚠️ Swift window capture failed: ${swiftError}. Falling back to region capture.`);
}
}
// Fallback to region capture if Swift capture fails
const { x, y, width, height } = windowInfo.bounds;
const regionArgs = [
'-R', `${x},${y},${width},${height}`,
'-x',
'-t', options.format || 'png',
outputPath
];
await execAsync(`screencapture ${regionArgs.join(' ')}`);
console.log(`✅ Fallback region capture: ${x},${y} ${width}x${height}`);
return { success: true, path: outputPath };
}
catch (error) {
console.error(`❌ macOS screenshot failed:`, error);
return {
success: false,
error: error instanceof Error ? error.message : String(error)
};
}
}
/**
* Get detailed window information for a context
*/
async refreshContextWindow(contextName) {
const context = this.contexts.get(contextName);
if (!context)
return null;
const windows = await this.discoverWindows();
const targetWindow = windows.find(w => w.processId === context.processId ||
w.windowId === context.windowId);
if (targetWindow && context) {
// Update context bounds
context.windowBounds = targetWindow.bounds;
context.lastActive = new Date();
}
return targetWindow || null;
}
}
//# sourceMappingURL=background-window-automation.js.map