donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
991 lines • 47.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.PageInspector = void 0;
const PageClosedException_1 = require("../exceptions/PageClosedException");
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
/**
* A class for identifying, attributing, and annotating interactable elements on web pages.
*
* The PageInspector provides functionality to:
* - Find and attribute interactable elements with unique identifiers
* - Retrieve information about attributed elements
* - Add visual annotations (numbered indicators) to interactable elements
* - Clean up both attributes and annotations
*
* Interactable elements are determined using comprehensive heuristics including:
* - Standard HTML interactive elements (buttons, inputs, links, etc.)
* - Elements with ARIA roles indicating interactivity
* - Elements with event handlers or CSS classes suggesting interactivity
* - Elements that are visible, enabled, and accessible at their coordinates
*
* This class is designed to work with Playwright's Page and Frame objects
* and handles cross-frame navigation, shadow DOM, and various edge cases.
*
* WARNING: It is REQUIRED that the {@code installInteractiveElementsTracker} has been
* run in the browser context before calling this class's methods.
*
* @example
* ```typescript
* // Basic usage
* await PlaywrightUtils.setupBasicBrowserContext(browserContext);
* const inspector = new PageInspector();
*
* const page = await browserContext.newPage();
* await page.goto("https://google.com");
*
* // Find and attribute interactable elements
* await inspector.attributeInteractableElements(page);
*
* // Get information about interactable elements
* const elements = await inspector.getAttributedInteractableElements(page);
*
* // Add visual annotations to the page
* await inspector.annotateInteractableElements(page);
*
* // Clean up when done
* await inspector.removeDonobuAnnotations(page);
* await inspector.deattributeVisibleInteractableElements(page);
* ```
*
* @remarks
* This class uses custom HTML attributes (`data-donobu-interactable` by default)
* to mark elements, and creates a shadow DOM container for annotations to avoid
* style conflicts with the target page.
*
* All methods will throw a {@link PageClosedException} if the page is closed
* during operation.
*/
class PageInspector {
/**
* WARNING: It is REQUIRED that the {@code installInteractiveElementsTracker} has been
* run in the browser context before calling this class's methods.
*/
constructor(interactableElementAttribute = 'data-donobu-interactable', interactableAnnotationAttribute = 'data-donobu-annotation') {
this.interactableElementAttribute = interactableElementAttribute;
this.interactableAnnotationAttribute = interactableAnnotationAttribute;
}
/**
* Assigns a globally unique attribute to all visible and interactable elements in the page.
*
* This method performs the following steps:
* 1. Removes any pre-existing interactable element attributes from the page
* 2. Assigns sequential numeric values as attributes to interactable elements in the main frame
* 3. Processes child frames that are visible in the viewport and assigns attributes to their interactable elements
*
* The method identifies "interactable" elements based on tag names, ARIA roles, CSS classes, and other heuristics.
* Only elements that are:
* - Visible (non-zero dimensions and not hidden via CSS)
* - More than 50% in the viewport
* - Not disabled or inert
* - Actually reachable at their coordinates (topmost in z-index)
* will receive the attribute.
*
* @param page - The Playwright Page object to process
* @throws {PageClosedException} If the page is closed during processing
* @returns {Promise<void>} A promise that resolves when all elements have been attributed
*/
async attributeInteractableElements(page) {
try {
// Remove any preexisting attributes
await this.deattributeInteractableElements(page);
// Get viewport dimensions and scroll position properly
const viewportInfo = await page.evaluate(() => {
return {
viewportWidth: window.innerWidth,
viewportHeight: window.innerHeight,
scrollX: window.scrollX || window.pageXOffset,
scrollY: window.scrollY || window.pageYOffset,
};
});
// 1) Attribute elements in the main page
let annotationOffset = await page.evaluate(PageInspector.attributeElementsInContext, [0, this.interactableElementAttribute]);
// 2) Check child frames, attributing elements if the frame is (partially) in view
const frames = page
.frames()
.filter((frame) => PageInspector.frameFilter(frame) && frame !== page.mainFrame());
for (const frame of frames) {
const elementHandle = await frame.frameElement();
if (!elementHandle) {
continue;
}
const boundingBox = await elementHandle.boundingBox();
if (!boundingBox) {
continue;
}
// boundingBox coordinates are already in viewport space, so we need to account for scroll
const isInViewport = boundingBox.x + boundingBox.width > 0 &&
boundingBox.x < viewportInfo.viewportWidth &&
boundingBox.y + boundingBox.height > 0 &&
boundingBox.y < viewportInfo.viewportHeight;
if (isInViewport) {
annotationOffset = await frame.evaluate(PageInspector.attributeElementsInContext, [annotationOffset, this.interactableElementAttribute]);
}
}
}
catch (error) {
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Retrieves all elements that have been previously attributed with the interactable element attribute.
*
* This method:
* 1. Searches all frames in the page (including the main frame and child frames)
* 2. Collects elements with the {@link interactableElementAttribute} attribute
* 3. Creates an {@link InteractableElement} object for each attributed element
*
* For each interactable element, it extracts:
* - The attribute value (serving as a unique identifier)
* - A simplified HTML snippet representation of the element
* * For 'select' elements, the complete HTML (including options) is preserved
* * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
* * For all other elements, only the opening tag without children is captured
* * For the main scrolling element (document.scrollingElement), adds special decoration indicating it's the page's main scrolling element
*
* Note: This method only finds elements that have been previously attributed using
* the {@link attributeInteractableElements} method.
*
* @param page - The Playwright Page object to process
* @returns {Promise<InteractableElement[]>} A promise that resolves to an array of
* interactable elements with their attribute values and HTML snippets
* @throws {PageClosedException} If the page is closed during processing
*
* @example
* const inspector = new PageInspector();
* await inspector.attributeInteractableElements(page);
* const elements = await inspector.getAttributedInteractableElements(page);
* // elements = [{ donobuAttributeValue: "0", htmlSnippet: "<button id=\"submit\">Submit</button>"}]
*/
async getAttributedInteractableElements(page) {
try {
const frames = page.frames().filter(PageInspector.frameFilter);
const aggregate = {};
for (const frame of frames) {
const frameMap = await frame.evaluate((interactableAttr) => {
/* --- helpers running in the browser context --- */
function stripDonobuAttrs(el) {
const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
let node = el;
while (node) {
Array.from(node.attributes).forEach((attr) => {
// Strip out Donobu attributes since those are not a part of the
// original HTML.
if (attr.name.startsWith('data-donobu')) {
node.removeAttribute(attr.name);
}
});
node = walker.nextNode();
}
}
/** helper to compute live scroll directions for el */
function getScrollDirections(el) {
// Special case for when the document body is not the scrollingElement
// element. This may happen if the scrollingElement is the
// root <html> element. In this case, it makes no sense to report
// scrollability on <body> and on scrollingElement, since we should
// use the scrollingElement instead.
if (el === document.body &&
document.scrollingElement !== document.body) {
return [];
}
const dirs = [];
const isRoot = el === document.scrollingElement;
const style = getComputedStyle(el);
// Add a small margin so we do not waste time reporting scrollability
// for an element that is not materially scrollable.
const marginPx = 1;
const canY = el.scrollHeight > el.clientHeight + marginPx &&
(isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowY));
const canX = el.scrollWidth > el.clientWidth + marginPx &&
(isRoot || /(auto|scroll|overlay|visible)/.test(style.overflowX));
if (canY) {
if (el.scrollTop > 0) {
dirs.push('UP');
}
if (el.scrollTop < el.scrollHeight - el.clientHeight) {
dirs.push('DOWN');
}
}
if (canX) {
if (el.scrollLeft > 0) {
dirs.push('LEFT');
}
if (el.scrollLeft < el.scrollWidth - el.clientWidth) {
dirs.push('RIGHT');
}
}
return dirs;
}
function serialise(el) {
const deepClone = (el.tagName.toLowerCase() === 'select'
? el.cloneNode(true)
: el.cloneNode(false));
stripDonobuAttrs(deepClone);
if (el.tagName.toLowerCase() === 'select') {
const scrollComment = el === document.scrollingElement
? '<!-- This is the main page scrolling element -->'
: '';
return scrollComment + deepClone.outerHTML; // full markup incl. <option>s
}
// Get the text content of the original element
const textContent = el.textContent?.trim() || '';
if (textContent) {
// Truncate text if longer than 32 characters
const displayText = textContent.length > 32
? textContent.substring(0, 32) + '...'
: textContent;
// Return opening tag + text + closing tag
const fullTag = deepClone.outerHTML;
const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
const tagName = el.tagName.toLowerCase();
const scrollComment = el === document.scrollingElement
? '<!-- This is the main page scrolling element -->'
: '';
return `${scrollComment}${openingTag}${displayText}</${tagName}>`;
}
else {
// opening tag only
const html = deepClone.outerHTML;
const scrollComment = el === document.scrollingElement
? '<!-- This is the main page scrolling element -->'
: '';
return scrollComment + html.slice(0, html.indexOf('>') + 1);
}
}
const out = {};
// Recursively process document and all shadow roots
const processNode = (root) => {
// Find elements with the interactable attribute
root.querySelectorAll(`[${interactableAttr}]`).forEach((el) => {
const val = el.getAttribute(interactableAttr);
if (!val) {
return;
}
out[val] = {
htmlSnippet: serialise(el),
scrollable: getScrollDirections(el),
};
});
// Recursively process any child shadow roots
root.querySelectorAll('*').forEach((el) => {
if (el.shadowRoot) {
processNode(el.shadowRoot);
}
});
};
// Start processing from the document root
processNode(document);
return out;
}, this.interactableElementAttribute);
Object.assign(aggregate, frameMap);
}
return Object.keys(aggregate)
.sort((a, b) => Number(a) - Number(b))
.map((key) => ({
donobuAttributeValue: key,
htmlSnippet: aggregate[key].htmlSnippet,
scrollable: aggregate[key].scrollable,
}));
}
catch (error) {
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Visually annotates all interactable elements with numbered indicators on the page.
*
* This method:
* 1. Processes all accessible frames in the page
* 2. Creates (or reuses) a shadow DOM container to isolate annotation styling
* 3. Places circular numbered indicators over each element that has the
* {@link interactableElementAttribute} attribute
*
* The annotations:
* - Are positioned at the center of each interactable element
* - Have the same numeric value as the element's attribute
* - Are styled as black circles with red borders and white text
* - Are placed in a shadow DOM to avoid style conflicts with the page
* - Have the {@link interactableAnnotationAttribute} for identification
* - Are non-interactive (pointer-events: none)
*
* Note: This method requires elements to be previously attributed using the
* {@link attributeInteractableElements} method to find the elements to annotate.
*
* @param page - The Playwright Page object to process
* @returns {Promise<void>} A promise that resolves when all elements have been annotated
* @throws {PageClosedException} If the page is closed during processing
*
* @example
* const inspector = new PageInspector();
* await inspector.attributeInteractableElements(page);
* await inspector.annotateInteractableElements(page);
*/
async annotateInteractableElements(page) {
try {
// Filter frames as needed
const frames = page
.frames()
.filter((frame) => PageInspector.frameFilter(frame));
for (const frame of frames) {
await frame.evaluate(([interactableAttr, annotationAttr]) => {
// 1) Ensure we have a shadow container in the main document
let container = document.getElementById('annotation-shadow-container');
if (!container) {
container = document.createElement('div');
container.id = 'annotation-shadow-container';
// Position container so child elements can be absolutely placed
Object.assign(container.style, {
position: 'absolute',
top: '0',
left: '0',
width: '100%',
height: '100%',
pointerEvents: 'none', // Let clicks pass through
zIndex: '2147483647', // win every z-index fight
});
// Check if document.body exists before trying to append.
if (document.body) {
document.body.appendChild(container);
}
else if (document.documentElement) {
// Fall back to document.documentElement if body does not exist.
document.documentElement.appendChild(container);
}
else {
// If neither exists, we can't proceed with annotations in this frame.
console.warn(`Cannot create annotation container for ${window.location.href} since the document structure is incomplete`);
return;
}
// Attach a shadow root
const shadowRoot = container.attachShadow({ mode: 'open' });
// Add a <style> element inside the shadow root to reset and define annotation styles
const style = document.createElement('style');
style.textContent = `
:host {
all: initial; /* Reset styles in shadow root */
}
.annotation {
position: absolute;
z-index: 2147483647;
background-color: black;
color: white;
width: 40px;
height: 40px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-size: 14px;
font-weight: bold;
line-height: 20px;
text-align: center;
box-shadow: 0px 2px 4px rgba(0,0,0,0.2);
border: 4px solid #FF4136;
pointer-events: none;
}
`;
shadowRoot.appendChild(style);
}
// Retrieve the shadow root to place annotation elements
const containerEl = document.getElementById('annotation-shadow-container');
if (!containerEl?.shadowRoot) {
return;
}
const shadowRoot = containerEl.shadowRoot;
// 2) Factory to create a new annotation inside the shadow root
const createAnnotation = (value) => {
const annotation = document.createElement('div');
annotation.classList.add('annotation');
annotation.dataset[annotationAttr] = '1';
annotation.textContent = value;
return annotation;
};
// 3) Position annotation relative to an element
const positionAnnotation = (annotation, element) => {
const rect = element.getBoundingClientRect();
// Center the annotation on the element, adjusting for its size
// Since container is absolute, we need to account for scroll position
const x = rect.left + rect.width / 2 - 20 + window.scrollX;
const y = rect.top + rect.height / 2 - 20 + window.scrollY;
annotation.style.left = `${x}px`;
annotation.style.top = `${y}px`;
};
// 4) Traverse DOM (including any nested shadow roots) to find interactable elements
const processNode = (root) => {
// Find elements with the interactable attribute
const elements = root.querySelectorAll(`[${interactableAttr}]`);
elements.forEach((element) => {
const value = element.getAttribute(interactableAttr);
if (value) {
const annotation = createAnnotation(value);
shadowRoot.appendChild(annotation);
positionAnnotation(annotation, element);
}
});
// Recursively process any child shadow roots
root.querySelectorAll('*').forEach((el) => {
if (el.shadowRoot) {
processNode(el.shadowRoot);
}
});
};
// Start processing from the (frame) document root
processNode(document);
}, [
this.interactableElementAttribute,
PageInspector.convertToJsAttribute(this.interactableAnnotationAttribute),
]);
}
}
catch (error) {
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Removes all visual annotations from the page that were created by
* the {@link annotateInteractableElements} method.
*
* This method:
* 1. Processes all accessible frames in the page
* 2. Finds and removes the shadow DOM container with ID 'annotation-shadow-container'
* that contains all the annotations
*
* This effectively removes all numbered indicators that were previously placed
* over interactable elements, leaving the page in its original visual state.
* Note that this only removes the visual annotations, not the
* {@link interactableElementAttribute} attributes on the elements themselves.
*
* @param page - The Playwright Page object to process
* @returns {Promise<void>} A promise that resolves when all annotations have been removed
* @throws {PageClosedException} If the page is closed during processing
*
* @example
* const inspector = new PageInspector();
* await inspector.attributeInteractableElements(page);
* await inspector.annotateInteractableElements(page);
* // ... do some operations with the annotations visible ...
* await inspector.removeDonobuAnnotations(page);
* // All visual annotations are now removed from the page
*/
async removeDonobuAnnotations(page) {
try {
const frames = page
.frames()
.filter((frame) => PageInspector.frameFilter(frame));
for (const frame of frames) {
await frame.evaluate(() => {
const containerId = 'annotation-shadow-container';
const container = document.getElementById(containerId);
if (container) {
container.remove();
}
});
}
}
catch (error) {
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
throw error;
}
}
/**
* Removes all interactable element attributes that were previously added to elements in the page.
*
* This method:
* 1. Processes all accessible frames in the page
* 2. Finds all elements with the {@link interactableElementAttribute} attribute
* 3. Removes this attribute from each element
*
* This effectively undoes the changes made by the {@link attributeInteractableElements} method,
* returning the page's DOM to its original state without the custom attributes.
* Note that this does not affect any visual annotations - to remove those, use
* the {@link removeDonobuAnnotations} method separately.
*
* This method is automatically called at the beginning of {@link attributeInteractableElements}
* to ensure a clean state before adding new attributes, but can also be called
* independently to clean up the DOM.
*
* @param page - The Playwright Page object to process
* @returns {Promise<void>} A promise that resolves when all attributes have been removed
* @throws {PageClosedException} If the page is closed during processing
*
* @example
* const inspector = new PageInspector();
* await inspector.attributeInteractableElements(page);
* // ... perform operations with attributed elements ...
* await inspector.deattributeInteractableElements(page);
* // All interactable element attributes are now removed from the page
*/
async deattributeInteractableElements(page) {
try {
const frames = page.frames().filter(PageInspector.frameFilter);
const attr = this.interactableElementAttribute;
for (const frame of frames) {
await frame.evaluate(([a]) => {
/** Depth-first removal inside document & every shadow root */
const removeDeep = (root) => {
root
.querySelectorAll(`[${a}]`)
.forEach((el) => el.removeAttribute(a));
root.querySelectorAll('*').forEach((el) => {
const sr = el.shadowRoot;
if (sr) {
removeDeep(sr);
}
});
};
removeDeep(document);
}, [attr]);
}
}
catch (error) {
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Retrieves the HTML snippet for a single element.
*
* This method:
* 1. Extracts a simplified HTML snippet representation of the element
* * For 'select' elements, the complete HTML (including options) is preserved
* * For elements with text content, includes opening tag, truncated text (max 32 chars), and closing tag
* * For all other elements, only the opening tag without children is captured
* 2. Strips any Donobu-specific attributes from the snippet
*
* @example
* const inspector = new PageInspector();
* const submitButton = page.querySelector('button[type="submit"]');
* const htmlSnippet = await inspector.getHtmlSnippet(submitButton);
* // htmlSnippet = "<button type=\"submit\">Submit</button>"
*/
async getHtmlSnippet(elementHandle) {
try {
// Evaluate in the element's context to get the HTML snippet
const htmlSnippet = await elementHandle.evaluate((element) => {
// Helper function to strip Donobu attributes
function stripDonobuAttrs(el) {
const walker = document.createTreeWalker(el, NodeFilter.SHOW_ELEMENT);
let node = el;
while (node) {
Array.from(node.attributes).forEach((attr) => {
// Strip out Donobu attributes since those are not a part of the
// original HTML.
if (attr.name.startsWith('data-donobu')) {
node.removeAttribute(attr.name);
}
});
node = walker.nextNode();
}
}
// Helper function to serialize element
function serialise(el) {
const deepClone = el.tagName.toLowerCase() === 'select'
? el.cloneNode(true)
: el.cloneNode(false);
stripDonobuAttrs(deepClone);
if (el.tagName.toLowerCase() === 'select') {
return deepClone.outerHTML; // full markup incl. <option>s
}
// Get the text content of the original element
const textContent = el.textContent?.trim() || '';
if (textContent) {
// Truncate text if longer than 32 characters
const displayText = textContent.length > 32
? textContent.substring(0, 32) + '...'
: textContent;
// Return opening tag + text + closing tag
const fullTag = deepClone.outerHTML;
const openingTag = fullTag.slice(0, fullTag.indexOf('>') + 1);
const tagName = el.tagName.toLowerCase();
return `${openingTag}${displayText}</${tagName}>`;
}
else {
// opening tag only
const html = deepClone.outerHTML;
return html.slice(0, html.indexOf('>') + 1);
}
}
return serialise(element);
});
return htmlSnippet;
}
catch (error) {
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Converts an HTML attribute to a JavaScript attribute. For example,
* "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
* prefix, and the conversion from kebab-case to camelCase.
*/
static convertToJsAttribute(htmlAttribute) {
if (htmlAttribute.startsWith('data-')) {
htmlAttribute = htmlAttribute.substring(5);
}
const parts = htmlAttribute.split('-');
const jsAttribute = parts[0] +
parts
.slice(1)
.map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase())
.join('');
return jsAttribute;
}
/**
* An internal method that is injected into page/frame contexts to find and attribute interactable elements.
*
* This method:
* 1. Identifies potentially interactable elements using a comprehensive selector
* 2. Filters elements based on visibility, position in viewport, and interactability
* 3. Assigns unique sequential numeric values to the interactable attribute
*
* The method uses several criteria to determine if an element is truly interactable:
* - Element must be visible (non-zero dimensions, not hidden via CSS)
* - Element must have at least 50% of its area within the viewport
* - Element must not be disabled, inert, or have pointer-events:none
* - Element must be the topmost element at its coordinates (using point sampling)
*
* Special handling is provided for label elements, which will attribute their
* associated form controls as well.
*
* This method can process both standard DOM elements and elements within shadow roots,
* ensuring thorough coverage of modern web applications.
*
* @param arg - A tuple containing [offset: number, interactableAttribute: string]
* where offset is the starting value for sequential numbering and
* interactableAttribute is the attribute name to assign
* @returns The updated offset after assigning attributes (for sequential numbering across frames)
* @private
*
* @remarks
* This method is designed to be injected into the page context using page.evaluate()
* and should not be called directly from Node.js code.
*/
static attributeElementsInContext(arg) {
let offset = arg[0];
const interactableAttribute = arg[1];
// --- Utility Functions ---
function isElementVisible(rect, style) {
return (rect.width > 0 &&
rect.height > 0 &&
style.display !== 'none' &&
style.visibility !== 'hidden');
}
function isElementMoreThanHalfInViewport(rect) {
const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
const visibleWidth = Math.min(rect.right, viewportWidth) - Math.max(rect.left, 0);
const visibleHeight = Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0);
const visibleArea = Math.max(0, visibleWidth) * Math.max(0, visibleHeight);
const elementArea = rect.width * rect.height;
return visibleArea >= elementArea / 2;
}
function isElementEnabled(element, style) {
// Check standard disabled attribute (for form controls like button, input, etc.)
if (element.hasAttribute('disabled')) {
return false;
}
// Check for ARIA attributes that indicate non-interactivity
if (element.getAttribute('aria-hidden') === 'true') {
return false;
}
// Check for pointer-events: none which prevents interactions
if (style.pointerEvents === 'none') {
return false;
}
// Check for inert attribute which makes elements non-interactive
if (element.hasAttribute('inert')) {
return false;
}
// If the element is in a form and the fieldset is disabled, it might be disabled as well
let parent = element.parentElement;
while (parent) {
if (parent.tagName.toLowerCase() === 'fieldset' &&
parent.hasAttribute('disabled') &&
element.tagName.toLowerCase() !== 'legend') {
return false;
}
parent = parent.parentElement;
}
return true;
}
function isScrollable(el) {
const canY = el.scrollHeight > el.clientHeight;
const canX = el.scrollWidth > el.clientWidth;
// If nothing overflows, bail early
if (!canY && !canX) {
return false;
}
// The document’s scrolling element works even when overflow is “visible”
if (el === document.scrollingElement) {
return true;
}
const s = getComputedStyle(el);
const rect = el.getBoundingClientRect();
const visible = rect.width > 0 && rect.height > 0;
if (!visible) {
return false;
}
const yOK = canY && /(auto|scroll|overlay)/.test(s.overflowY);
const xOK = canX && /(auto|scroll|overlay)/.test(s.overflowX);
return yOK || xOK;
}
/**
* Generate a few test points on the element's bounding box. We only need
* a small offset (1px) from each corner, plus the center.
*/
function getPointsToCheck(rect) {
const cornerOffset = 1;
return [
{ x: rect.left + cornerOffset, y: rect.top + cornerOffset },
{ x: rect.right - cornerOffset, y: rect.top + cornerOffset },
{ x: rect.left + cornerOffset, y: rect.bottom - cornerOffset },
{ x: rect.right - cornerOffset, y: rect.bottom - cornerOffset },
{ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 },
];
}
/**
* Like `elementFromPoint` but continues walking shadow roots if found.
*/
function getDeepElementFromPoint(x, y) {
// 1. Use elementsFromPoint - returns full stack, top-first
const stack = document.elementsFromPoint(x, y);
// 2. Pick the first element
let hit = stack[0];
if (!hit) {
return null;
}
// 3. Walk down through open shadow roots, still honoring the ignore rule
// Use .elementsFromPoint on each shadow root if available
while (hit.shadowRoot) {
const innerStack = hit.shadowRoot.elementsFromPoint?.(x, y) ?? [];
const next = innerStack[0];
if (!next || next === hit) {
break;
}
hit = next;
}
return hit;
}
function getAllInteractableElements(root) {
const selector = [
// ----- STANDARD HTML INTERACTIVE ELEMENTS -----
// Basic form controls and interactive elements
'button',
'button svg', // SVG inside buttons are also clickable
'input',
'textarea',
'a',
'select',
'summary', // Disclosure (details/summary) elements
'details',
'audio[controls]', // Media controls
'video[controls]',
// ----- ARIA ROLE-BASED INTERACTIVE ELEMENTS -----
// Elements with explicit interactive ARIA roles
'[role="button"]',
'[role="option"]',
'[role="link"]',
'[role="checkbox"]',
'[role="radio"]',
'[role="tab"]',
'[role="menuitem"]',
'[role="menuitemcheckbox"]',
'[role="menuitemradio"]',
'[role="combobox"]',
'[role="listbox"]',
'[role="searchbox"]',
'[role="spinbutton"]',
'[role="slider"]',
'[role="switch"]',
'[role="menu"]',
'[role="menubar"]',
'[role="treeitem"]',
// ----- INTERACTIVE ARIA ATTRIBUTES -----
// Elements with popup behavior
'[aria-haspopup]',
'[aria-controls]',
'[aria-expanded]', // Elements that can expand/collapse
'[aria-pressed]', // Toggle buttons
'[aria-selected]', // Selectable items
// ----- EDITABLE AND FOCUSABLE ELEMENTS -----
// Elements that can receive input
'[contenteditable="true"]',
// Elements that can be dragged
'[draggable="true"]',
// Elements with explicit focus capability
'[tabindex]:not([tabindex="-1"])',
// ----- CSS FRAMEWORK-SPECIFIC CLASSES -----
// Bootstrap & similar frameworks
'.btn', // Standard button class
'.dropdown-toggle', // Dropdown triggers
'.nav-link', // Navigation links
'.page-link', // Pagination links
'.card-header[data-toggle="collapse"]', // Collapsible headers
'.accordion-button', // Accordion toggles
'.close', // Close buttons
'.modal-close', // Modal close buttons
// Material Design & Angular Material
'.mdc-button',
'.mat-button',
'.mat-icon-button',
'.mat-fab',
'.mat-mini-fab',
'.mat-menu-item',
'.mat-tab-label',
// Foundation
'.button',
'.menu-item',
'.accordion-title',
// Tailwind UI & Headless UI components
'.tw-button',
'[x-on\\:click]', // Alpine.js click handlers
// Vue-based frameworks
'.v-btn', // Vuetify
'.el-button', // Element UI
// React-based frameworks
'.ant-btn', // Ant Design
'.chakra-button', // Chakra UI
'.mui-button', // Material-UI
// ----- CUSTOM INTERACTIVE PATTERNS -----
// Common custom interactive classes
'.clickable',
'.selectable',
'.interactive',
'.toggle',
'.expandable',
'.switch',
'.slider',
// Common dropdown/select libraries
'.select2-selection',
'.chosen-single',
'.vs__dropdown-toggle', // Vue Select
// Tabs and accordion components
'.tab',
'.tab-header',
'.tab-title',
'.accordion-header',
// Mobile-friendly interactive elements
'.swipe-item',
'.touch-target',
// ----- COMMON COMPONENT PATTERNS -----
// Cards and tiles that are often clickable
'.card[onclick]',
'.tile[onclick]',
'.card a', // Links inside cards
// Social media & e-commerce patterns
'.share-button',
'.like-button',
'.add-to-cart',
'.product-card',
// Notification and alert controls
'.alert .close',
'.toast .close',
'.notification-action',
].join(', ');
let elements = Array.from(root.querySelectorAll(selector));
// Dive into shadow roots
root.querySelectorAll('*').forEach((el) => {
if (el.shadowRoot) {
// Recurse
elements = elements.concat(getAllInteractableElements(el.shadowRoot));
}
});
// Remove duplicates
return Array.from(new Set(elements));
}
// 1) Gather candidate elements
const interactableElements = getAllInteractableElements(document)
.concat(window.__donobu.getInteractiveElements())
.filter((el) => !el.id || (typeof el.id === 'string' && !el.id.startsWith('donobu-')));
const uniqueElements = new Set(interactableElements);
const maybeAddScrollable = (el) => {
if (isScrollable(el) && !uniqueElements.has(el)) {
uniqueElements.add(el);
}
};
document.querySelectorAll('*').forEach(maybeAddScrollable);
if (document.scrollingElement) {
maybeAddScrollable(document.scrollingElement);
}
// 2) Iterate and assign numbers
uniqueElements.forEach((element) => {
if (element === document.scrollingElement) {
// Special-case: always keep the root scrolling element
element.setAttribute(interactableAttribute, offset.toString());
offset++;
return; // skip the usual checks
}
else if (element.hasAttribute(interactableAttribute)) {
// Skip if this element already carries a value (e.g. assigned via <label>)
return;
}
const rect = element.getBoundingClientRect();
const style = window.getComputedStyle(element);
const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
const enabled = isElementEnabled(element, style);
if (!visible || !enabled) {
return;
}
// Check a few probe points to make sure the element is top-most
for (const pt of getPointsToCheck(rect)) {
let elToCheck = getDeepElementFromPoint(pt.x, pt.y);
while (elToCheck) {
if (elToCheck === element) {
element.setAttribute(interactableAttribute, offset.toString());
offset++;
return; // this element done
}
// Handle <label> -> control mapping
if (elToCheck.tagName.toLowerCase() === 'label' &&
elToCheck.htmlFor) {
const forId = elToCheck.htmlFor;
const control = document.getElementById(forId);
if (control &&
!control.hasAttribute(interactableAttribute) // prevent double number
) {
control.setAttribute(interactableAttribute, offset.toString());
offset++;
}
return;
}
elToCheck = elToCheck.parentElement;
}
}
});
return offset;
}
static frameFilter(frame) {
return (!frame.isDetached() &&
!frame.url().startsWith('about:') &&
!frame.url().startsWith('chrome:') &&
!frame.url().startsWith('edge:'));
}
}
exports.PageInspector = PageInspector;
//# sourceMappingURL=PageInspector.js.map