mcp-appium-visual/dist/lib/vision/imageProcessor.d.ts

MCP Server for Appium mobile automation with visual recovery

declare module "tesseract.js" {
    interface Page {
        words: Array<{
            text: string;
            confidence: number;
            bbox: {
                x0: number;
                y0: number;
                x1: number;
                y1: number;
            };
        }>;
    }
}
/**
 * Image processing utilities for enhanced visual recovery
 */
export declare class ImageProcessor {
    /**
     * Performs template matching to find a smaller image within a larger one
     * @param screenshotPath Path to the full screenshot
     * @param templatePath Path to the template image to find
     * @param threshold Matching threshold (0.0 to 1.0)
     * @returns Coordinates of the match or null if not found
     */
    static findTemplateInImage(screenshotPath: string, templatePath: string, threshold?: number): Promise<{
        x: number;
        y: number;
        width: number;
        height: number;
    } | null>;
    /**
     * Performs OCR on an image to extract text
     * @param imagePath Path to the image
     * @param options Configuration options
     * @returns Extracted text and bounding boxes
     */
    static performOCR(imagePath: string, options?: {
        lang?: string;
        rect?: {
            left: number;
            top: number;
            width: number;
            height: number;
        };
    }): Promise<{
        text: string;
        confidence: number;
        words: Array<{
            text: string;
            confidence: number;
            bbox: {
                x0: number;
                y0: number;
                x1: number;
                y1: number;
            };
        }>;
    }>;
    /**
     * Compares two images and returns the difference as a percentage
     * @param image1Path Path to first image
     * @param image2Path Path to second image
     * @param options Comparison options
     * @returns Difference as percentage and diff image path
     */
    static compareImages(image1Path: string, image2Path: string, options?: {
        threshold?: number;
        outputDiffPath?: string;
        ignoreRegions?: Array<{
            x: number;
            y: number;
            width: number;
            height: number;
        }>;
    }): Promise<{
        diffPercentage: number;
        diffImagePath: string | null;
    }>;
    /**
     * Finds elements in an image based on visual characteristics
     * @param imagePath Path to the image
     * @returns Information about detected UI elements
     */
    static detectUIElements(imagePath: string): Promise<Array<{
        type: string;
        confidence: number;
        bbox: {
            x: number;
            y: number;
            width: number;
            height: number;
        };
    }>>;
    /**
     * Calculate the difference between two colors
     * @param color1 First color
     * @param color2 Second color
     * @returns Difference value
     */
    private static colorDifference;
}
/**
 * Visual recovery utilities for UI elements
 */
export declare class VisualRecovery {
    /**
     * Find an element by appearance when traditional locators fail
     * @param baseScreenshotPath Path to previous screenshot containing element
     * @param currentScreenshotPath Path to current screenshot
     * @param elementRegion Region of element in base screenshot
     * @returns Coordinates of element in new screenshot or null if not found
     */
    static recoverElementByAppearance(baseScreenshotPath: string, currentScreenshotPath: string, elementRegion: {
        x: number;
        y: number;
        width: number;
        height: number;
    }): Promise<{
        x: number;
        y: number;
        width: number;
        height: number;
    } | null>;
    /**
     * Find text in an image and return its location
     * @param screenshotPath Path to the screenshot
     * @param text Text to find
     * @param options Optional parameters
     * @returns Coordinates of the text or null if not found
     */
    static findTextInImage(screenshotPath: string, text: string, options?: {
        minConfidence?: number;
        searchRegion?: {
            x: number;
            y: number;
            width: number;
            height: number;
        };
    }): Promise<{
        x: number;
        y: number;
        width: number;
        height: number;
        confidence: number;
    } | null>;
    /**
     * Find a UI element by visual characteristics
     * @param screenshotPath Path to the screenshot
     * @param options Search options
     * @returns Element coordinates or null if not found
     */
    static findElementByVisualCharacteristics(screenshotPath: string, options: {
        elementType?: "button" | "text" | "input" | "checkbox" | "toggle" | "any";
        nearText?: string;
        expectedText?: string;
        color?: {
            r: number;
            g: number;
            b: number;
            tolerance: number;
        };
        region?: {
            x: number;
            y: number;
            width: number;
            height: number;
        };
    }): Promise<{
        x: number;
        y: number;
        width: number;
        height: number;
        confidence: number;
        type: string;
    } | null>;
}
//# sourceMappingURL=imageProcessor.d.ts.map