xc-mcp/dist/utils/element-extraction.js

MCP server that wraps Xcode command-line tools for iOS/macOS development workflows

import { executeCommand } from './command.js';
/**
 * Extract interactive elements from an app's accessibility tree
 *
 * This queries the accessibility tree to find buttons, text fields, and other
 * interactive elements, returning their positions and properties.
 */
export async function extractAccessibilityElements(udid, bundleId) {
    try {
        // Query common interactive element types
        const elementTypes = [
            'XCUIElementTypeButton',
            'XCUIElementTypeTextField',
            'XCUIElementTypeSecureTextField',
            'XCUIElementTypeSwitch',
            'XCUIElementTypeSlider',
            'XCUIElementTypeStaticText',
            'XCUIElementTypeLink',
            'XCUIElementTypePickerWheel',
        ];
        const elements = [];
        // Query each element type with location capture
        for (const elementType of elementTypes) {
            try {
                const predicate = `type == "${elementType}"`;
                const command = `xcrun simctl query "${udid}" "${bundleId}" "${predicate}" --locations`;
                const result = await executeCommand(command, {
                    timeout: 10000,
                });
                if (result.code === 0 && result.stdout.trim()) {
                    // Parse the query output
                    const lines = result.stdout.trim().split('\n');
                    for (const line of lines) {
                        if (line.trim()) {
                            try {
                                // Try to parse as JSON or extract element info
                                const element = parseElementLine(line, elementType);
                                if (element) {
                                    elements.push(element);
                                }
                            }
                            catch {
                                // Skip lines that can't be parsed
                            }
                        }
                    }
                }
            }
            catch {
                // Continue with next element type if this one fails
            }
        }
        return elements;
    }
    catch (error) {
        console.error('[element-extraction] Error extracting elements:', error);
        return [];
    }
}
/**
 * Parse an element line from simctl query output
 *
 * The output format varies, but typically includes type, label, and location info
 */
function parseElementLine(line, elementType) {
    try {
        // Try to extract basic info from the line
        // Format typically: "Button, \"Label\", ... {x, y}"
        const element = {
            type: elementType,
        };
        // Extract label if present (between quotes)
        const labelMatch = line.match(/"([^"]+)"/);
        if (labelMatch) {
            element.label = labelMatch[1];
        }
        // Try to extract coordinates {x, y, width, height}
        const boundsMatch = line.match(/\{x ([\d.]+) y ([\d.]+) w ([\d.]+) h ([\d.]+)\}/);
        if (boundsMatch) {
            element.bounds = {
                x: Math.round(parseFloat(boundsMatch[1])),
                y: Math.round(parseFloat(boundsMatch[2])),
                width: Math.round(parseFloat(boundsMatch[3])),
                height: Math.round(parseFloat(boundsMatch[4])),
            };
        }
        // Extract state information
        if (line.includes('enabled: 0')) {
            element.enabled = false;
        }
        else if (line.includes('enabled: 1')) {
            element.enabled = true;
        }
        if (line.includes('hittable: 0')) {
            element.hittable = false;
        }
        else if (line.includes('hittable: 1')) {
            element.hittable = true;
        }
        // Only return if we got some meaningful data
        if (element.label || element.bounds) {
            return element;
        }
        return null;
    }
    catch {
        return null;
    }
}
/**
 * Get screen dimensions for the simulator
 */
export async function getScreenDimensions(udid) {
    try {
        // Use xcrun to get device info
        const command = `xcrun simctl list devices -j`;
        const result = await executeCommand(command, { timeout: 5000 });
        if (result.code === 0) {
            try {
                const data = JSON.parse(result.stdout);
                // Search for the device to get its screen size
                // For now, return common dimensions - can be enhanced with actual device info
                for (const runtime in data.devices) {
                    const devices = data.devices[runtime];
                    const device = devices.find((d) => d.udid === udid);
                    if (device) {
                        // Return common iPhone dimensions
                        // In reality, this would be retrieved from device capabilities
                        return {
                            width: 393, // iPhone 16 dimensions
                            height: 852,
                            scale: 3,
                        };
                    }
                }
            }
            catch {
                // Fallback to common dimensions
            }
        }
        return null;
    }
    catch {
        return null;
    }
}
//# sourceMappingURL=element-extraction.js.map