donobu/dist/managers/PageInspector.d.ts

Create browser automations with an LLM agent and replay them as Playwright scripts.

import type { ElementHandle, Page } from 'playwright';
import type { InteractableElement } from '../models/InteractableElement';
/**
 * A class for identifying, attributing, and annotating interactable elements on web pages.
 *
 * The PageInspector provides functionality to:
 * - Find and attribute interactable elements with unique identifiers
 * - Retrieve information about attributed elements
 * - Add visual annotations (numbered indicators) to interactable elements
 * - Clean up both attributes and annotations
 *
 * Interactable elements are determined using comprehensive heuristics including:
 * - Standard HTML interactive elements (buttons, inputs, links, etc.)
 * - Elements with ARIA roles indicating interactivity
 * - Elements with event handlers or CSS classes suggesting interactivity
 * - Elements that are visible, enabled, and accessible at their coordinates
 *
 * This class is designed to work with Playwright's Page and Frame objects
 * and handles cross-frame navigation, shadow DOM, and various edge cases.
 *
 * WARNING: It is REQUIRED that the {@code installInteractiveElementsTracker} has been
 *          run in the browser context before calling this class's methods.
 *
 * @example
 * ```typescript
 * // Basic usage
 * await PlaywrightUtils.setupBasicBrowserContext(browserContext);
 * const inspector = new PageInspector();
 *
 * const page = await browserContext.newPage();
 * await page.goto("https://google.com");
 *
 * // Find and attribute interactable elements
 * await inspector.attributeInteractableElements(page);
 *
 * // Get information about interactable elements
 * const elements = await inspector.getAttributedInteractableElements(page);
 *
 * // Add visual annotations to the page
 * await inspector.annotateInteractableElements(page);
 *
 * // Clean up when done
 * await inspector.removeDonobuAnnotations(page);
 * await inspector.deattributeVisibleInteractableElements(page);
 * ```
 *
 * @remarks
 * This class uses custom HTML attributes (`data-donobu-interactable` by default)
 * to mark elements, and creates a shadow DOM container for annotations to avoid
 * style conflicts with the target page.
 *
 * All methods will throw a {@link PageClosedException} if the page is closed
 * during operation.
 */
export declare class PageInspector {
    readonly interactableElementAttribute: string;
    readonly interactableAnnotationAttribute: string;
    /**
     * WARNING: It is REQUIRED that the {@code installInteractiveElementsTracker} has been
     *          run in the browser context before calling this class's methods.
     */
    constructor(interactableElementAttribute?: string, interactableAnnotationAttribute?: string);
    /**
     * Assigns a globally unique attribute to all visible and interactable elements in the page.
     *
     * This method performs the following steps:
     * 1. Removes any pre-existing interactable element attributes from the page
     * 2. Assigns sequential numeric values as attributes to interactable elements in the main frame
     * 3. Processes child frames that are visible in the viewport and assigns attributes to their interactable elements
     *
     * The method identifies "interactable" elements based on tag names, ARIA roles, CSS classes, and other heuristics.
     * Only elements that are:
     * - Visible (non-zero dimensions and not hidden via CSS)
     * - More than 50% in the viewport
     * - Not disabled or inert
     * - Actually reachable at their coordinates (topmost in z-index)
     * will receive the attribute.
     *
     * @param page - The Playwright Page object to process
     * @throws {PageClosedException} If the page is closed during processing
     * @returns {Promise<void>} A promise that resolves when all elements have been attributed
     */
    attributeInteractableElements(page: Page): Promise<void>;
    /**
     * Retrieves all elements that have been previously attributed with the interactable element attribute.
     *
     * This method:
     * 1. Searches all frames in the page (including the main frame and child frames)
     * 2. Collects elements with the {@link interactableElementAttribute} attribute
     * 3. Creates an {@link InteractableElement} object for each attributed element
     *
     * For each interactable element, it extracts:
     * - The attribute value (serving as a unique identifier)
     * - A simplified HTML snippet representation of the element
     *   * For 'select' elements, the complete HTML (including options) is preserved
     *   * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
     *   * For all other elements, only the opening tag without children is captured
     *   * For the main scrolling element (document.scrollingElement), adds special decoration indicating it's the page's main scrolling element
     *
     * Note: This method only finds elements that have been previously attributed using
     * the {@link attributeInteractableElements} method.
     *
     * @param page - The Playwright Page object to process
     * @returns {Promise<InteractableElement[]>} A promise that resolves to an array of
     *         interactable elements with their attribute values and HTML snippets
     * @throws {PageClosedException} If the page is closed during processing
     *
     * @example
     * const inspector = new PageInspector();
     * await inspector.attributeInteractableElements(page);
     * const elements = await inspector.getAttributedInteractableElements(page);
     * // elements = [{ donobuAttributeValue: "0", htmlSnippet: "<button id=\"submit\">Submit</button>"}]
     */
    getAttributedInteractableElements(page: Page): Promise<InteractableElement[]>;
    /**
     * Visually annotates all interactable elements with numbered indicators on the page.
     *
     * This method:
     * 1. Processes all accessible frames in the page
     * 2. Creates (or reuses) a shadow DOM container to isolate annotation styling
     * 3. Places circular numbered indicators over each element that has the
     *    {@link interactableElementAttribute} attribute
     *
     * The annotations:
     * - Are positioned at the center of each interactable element
     * - Have the same numeric value as the element's attribute
     * - Are styled as black circles with red borders and white text
     * - Are placed in a shadow DOM to avoid style conflicts with the page
     * - Have the {@link interactableAnnotationAttribute} for identification
     * - Are non-interactive (pointer-events: none)
     *
     * Note: This method requires elements to be previously attributed using the
     * {@link attributeInteractableElements} method to find the elements to annotate.
     *
     * @param page - The Playwright Page object to process
     * @returns {Promise<void>} A promise that resolves when all elements have been annotated
     * @throws {PageClosedException} If the page is closed during processing
     *
     * @example
     * const inspector = new PageInspector();
     * await inspector.attributeInteractableElements(page);
     * await inspector.annotateInteractableElements(page);
     */
    annotateInteractableElements(page: Page): Promise<void>;
    /**
     * Removes all visual annotations from the page that were created by
     * the {@link annotateInteractableElements} method.
     *
     * This method:
     * 1. Processes all accessible frames in the page
     * 2. Finds and removes the shadow DOM container with ID 'annotation-shadow-container'
     *    that contains all the annotations
     *
     * This effectively removes all numbered indicators that were previously placed
     * over interactable elements, leaving the page in its original visual state.
     * Note that this only removes the visual annotations, not the
     * {@link interactableElementAttribute} attributes on the elements themselves.
     *
     * @param page - The Playwright Page object to process
     * @returns {Promise<void>} A promise that resolves when all annotations have been removed
     * @throws {PageClosedException} If the page is closed during processing
     *
     * @example
     * const inspector = new PageInspector();
     * await inspector.attributeInteractableElements(page);
     * await inspector.annotateInteractableElements(page);
     * // ... do some operations with the annotations visible ...
     * await inspector.removeDonobuAnnotations(page);
     * // All visual annotations are now removed from the page
     */
    removeDonobuAnnotations(page: Page): Promise<void>;
    /**
     * Removes all interactable element attributes that were previously added to elements in the page.
     *
     * This method:
     * 1. Processes all accessible frames in the page
     * 2. Finds all elements with the {@link interactableElementAttribute} attribute
     * 3. Removes this attribute from each element
     *
     * This effectively undoes the changes made by the {@link attributeInteractableElements} method,
     * returning the page's DOM to its original state without the custom attributes.
     * Note that this does not affect any visual annotations - to remove those, use
     * the {@link removeDonobuAnnotations} method separately.
     *
     * This method is automatically called at the beginning of {@link attributeInteractableElements}
     * to ensure a clean state before adding new attributes, but can also be called
     * independently to clean up the DOM.
     *
     * @param page - The Playwright Page object to process
     * @returns {Promise<void>} A promise that resolves when all attributes have been removed
     * @throws {PageClosedException} If the page is closed during processing
     *
     * @example
     * const inspector = new PageInspector();
     * await inspector.attributeInteractableElements(page);
     * // ... perform operations with attributed elements ...
     * await inspector.deattributeInteractableElements(page);
     * // All interactable element attributes are now removed from the page
     */
    deattributeInteractableElements(page: Page): Promise<void>;
    /**
     * Retrieves the HTML snippet for a single element.
     *
     * This method:
     * 1. Extracts a simplified HTML snippet representation of the element
     *   * For 'select' elements, the complete HTML (including options) is preserved
     *   * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
     *   * For all other elements, only the opening tag without children is captured
     * 2. Strips any Donobu-specific attributes from the snippet
     *
     * @example
     * const inspector = new PageInspector();
     * const submitButton = page.querySelector('button[type="submit"]');
     * const htmlSnippet = await inspector.getHtmlSnippet(submitButton);
     * // htmlSnippet = "<button type=\"submit\">Submit</button>"
     */
    getHtmlSnippet(elementHandle: ElementHandle<HTMLElement | SVGElement>): Promise<string>;
    /**
     * Converts an HTML attribute to a JavaScript attribute. For example,
     * "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
     * prefix, and the conversion from kebab-case to camelCase.
     */
    static convertToJsAttribute(htmlAttribute: string): string;
    /**
     * An internal method that is injected into page/frame contexts to find and attribute interactable elements.
     *
     * This method:
     * 1. Identifies potentially interactable elements using a comprehensive selector
     * 2. Filters elements based on visibility, position in viewport, and interactability
     * 3. Assigns unique sequential numeric values to the interactable attribute
     *
     * The method uses several criteria to determine if an element is truly interactable:
     * - Element must be visible (non-zero dimensions, not hidden via CSS)
     * - Element must have at least 50% of its area within the viewport
     * - Element must not be disabled, inert, or have pointer-events:none
     * - Element must be the topmost element at its coordinates (using point sampling)
     *
     * Special handling is provided for label elements, which will attribute their
     * associated form controls as well.
     *
     * This method can process both standard DOM elements and elements within shadow roots,
     * ensuring thorough coverage of modern web applications.
     *
     * @param arg - A tuple containing [offset: number, interactableAttribute: string]
     *              where offset is the starting value for sequential numbering and
     *              interactableAttribute is the attribute name to assign
     * @returns The updated offset after assigning attributes (for sequential numbering across frames)
     * @private
     *
     * @remarks
     * This method is designed to be injected into the page context using page.evaluate()
     * and should not be called directly from Node.js code.
     */
    private static attributeElementsInContext;
    private static frameFilter;
}
//# sourceMappingURL=PageInspector.d.ts.map