donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
1,071 lines • 50.3 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.PlaywrightUtils = void 0;
const Logger_1 = require("./Logger");
const MiscUtils_1 = require("./MiscUtils");
const path_1 = __importDefault(require("path"));
const PageClosedException_1 = require("../exceptions/PageClosedException");
const child_process_1 = require("child_process");
const envVars_1 = require("../envVars");
/**
* Miscellaneous utility functions for working with the Playwright SDK. If you are looking to
* instantiate a Playwright instance, see PlaywrightSetup instead.
*/
class PlaywrightUtils {
/**
* Attempts to take a screenshot of the given page in PNG format, returning
* the raw byte array. If the operation fails, an empty array is returned.
* Note that if the Donobu control panel is present in the page, it will be
* excluded from the screenshot.
*/
static async takePngScreenshot(page) {
try {
const style = `#${PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID} { display: none !important; }`;
return await page.screenshot({
style,
type: 'png',
});
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Generate valid selectors for the given element. The generated selectors are
* in a priority order based on their ability to identify the given element.
* For example, an unambiguous selector that matches the element exactly are
* first in the list, and weaker selectors that match multiple elements are
* last.
*/
static async generateSelectors(element) {
return element.evaluate((elem) => {
return window.donobuGenerateSmartSelectors(elem);
});
}
/**
* Returns a JavaScript code snippet intended to run as an initialization
* script for `DonobuFlow` flows. This script helps other scripts to know
* which elements have mouse-click related event listeners attached to them
* via inspecting the {@code window.donobuGetClickableElements} element array.
*/
static clickableElementsTrackerInitScript() {
return PlaywrightUtils.CLICKABLE_ELEMENTS_TRACKER_INIT_SCRIPT;
}
/**
* Returns a JavaScript code snippet intended to run as an initialization
* script for `DonobuFlow` flows. This script helps flows handle/track
* prompts/confirmations. This is done specially since these browser actions
* pause the Javascript main thread and also cause issues with running various
* Playwright actions. See `DonobuFlow.onDialog(Dialog)` for details.
*/
static dialogPromptTrackerInitScript() {
return PlaywrightUtils.DIALOG_PROMPT_TRACKER_INIT_SCRIPT;
}
/**
* Returns a JavaScript code snippet intended to run as an initialization
* script for `DonobuFlow` flows. This is an in-page version of the
* `SelectorGenerator` class so that smart selectors can be generated in
* the web client. This is done so the
* {@link PlaywrightUtils.pageInteractionsTrackerInitScript()} can generate
* smart selectors on the fly and pass them to the `PageInteractionTracker`
* so the page interactions can be converted to synthetic `ToolCallResult`s.
*/
static smartSelectorGeneratorInitScript() {
return PlaywrightUtils.SMART_SELECTOR_GENERATOR_INIT_SCRIPT;
}
/**
* This script is used by the `RunAccessibilityTestTool` to test the
* accessibility of a webpage. This script is injected on page load, rather
* than when running the `RunAccessibilityTestTool` because we need to bypass
* webpage Content Security Policy (CSP); this is done by injecting the
* script via `BrowserContext#addInitScript`.
*/
static accessibilityTestInitScript() {
return PlaywrightUtils.ACCESSIBILITY_TEST_INIT_SCRIPT;
}
static pageInteractionsTrackerInitScript() {
return PlaywrightUtils.PAGE_INTERACTIONS_TRACKER_INIT_SCRIPT;
}
static donobuControlPanelInitScript() {
return PlaywrightUtils.DONOBU_CONTROL_PANEL_INIT_SCRIPT;
}
/**
* Creates a visual sparkle effect at the specified browser-page coordinates,
* automatically detecting if the point lies inside a same-origin iframe (and nesting deeper
* if necessary). Injects the sparkle into that frame’s document at the correct local coords.
*
* @param page - The Playwright Page object
* @param x - The x-coordinate in the top-level page coordinate space (relative to the browser window)
* @param y - The y-coordinate in the top-level page coordinate space
* @param durationMs - How long the sparkle remains (ms, default 800)
* @param emoji - The emoji to display (default '✨')
* @param size - CSS font-size for the emoji (default '40px')
*/
static async createSparkleEffect(page, x, y, durationMs = 800, emoji = '✨', size = '40px') {
// 1) Find the deepest same-origin frame covering (x, y), recursing if iframes nest further.
// Also get the coordinate in that frame’s local coordinate system.
const { targetFrame, localX, localY } = await this.findDeepestFrameAtPoint(page, x, y);
// 2) Inject the sparkle container & style in that frame
await targetFrame.evaluate(({ x, y, emoji, size }) => {
const containerId = 'playwright-sparkle-shadow-container';
// Remove old container if needed
const old = document.getElementById(containerId);
if (old)
old.remove();
const container = document.createElement('div');
container.id = containerId;
const shadowRoot = container.attachShadow({ mode: 'open' });
const sparkle = document.createElement('span');
sparkle.id = 'playwright-sparkle-element';
sparkle.textContent = emoji;
// Position & style
sparkle.style.position = 'fixed';
sparkle.style.left = `${x}px`;
sparkle.style.top = `${y}px`;
sparkle.style.fontSize = size;
sparkle.style.transform = 'translate(-50%, -50%)';
sparkle.style.zIndex = '2147483647'; // Very large to float above most overlays
sparkle.style.pointerEvents = 'none';
sparkle.style.userSelect = 'none';
sparkle.style.filter = 'drop-shadow(0 0 2px rgba(255,255,255,0.7))';
sparkle.style.animation = 'sparkle-appear 0.3s ease-out forwards';
const styleEl = document.createElement('style');
styleEl.textContent = `
:host {
all: initial;
}
@keyframes sparkle-appear {
0% { opacity: 0; transform: translate(-50%, -50%) scale(0.5); }
50% { opacity: 1; transform: translate(-50%, -50%) scale(1.2); }
100% { opacity: 1; transform: translate(-50%, -50%) scale(1); }
}
@keyframes sparkle-disappear {
0% { opacity: 1; transform: translate(-50%, -50%) scale(1); }
100% { opacity: 0; transform: translate(-50%, -50%) scale(0.5); }
}
`;
shadowRoot.appendChild(styleEl);
shadowRoot.appendChild(sparkle);
document.body.appendChild(container);
}, { x: localX, y: localY, emoji, size });
// 3) Wait the duration
await page.waitForTimeout(durationMs);
// 4) Fade out and remove
await targetFrame.evaluate(() => {
const container = document.getElementById('playwright-sparkle-shadow-container');
if (container && container.shadowRoot) {
const sparkle = container.shadowRoot.getElementById('playwright-sparkle-element');
if (sparkle) {
sparkle.style.animation = 'sparkle-disappear 0.3s ease-in forwards';
}
setTimeout(() => {
container.remove();
}, 300);
}
});
// 5) Wait for removal
await page.waitForTimeout(300);
}
/**
* Recursively finds the deepest same-origin Frame that covers the point (top-level coordinates).
* Returns the Frame plus the local coordinates (relative to that frame's viewport).
*
* If no same-origin child frame covers (x, y), it returns the main frame and the original coords.
*/
static async findDeepestFrameAtPoint(page, x, y, frame = page.mainFrame()) {
// We'll attempt to see if any child <iframe> in this frame covers the coordinate
// Steps:
// (1) Convert the top-level coordinate (x, y) into this frame's local coordinate system.
// (2) If outside the frame's bounding client rect (in its parent), skip further checks.
// (3) Check each child <iframe> inside this frame's DOM. If the point is inside that <iframe>,
// convert to the child frame's coords and recurse. If the child frame is cross-origin,
// skip it.
//
// If none of the children contain the point, we are the "deepest" frame for that point.
// If we're the mainFrame, there's no parent boundingRect. So we start with (x, y) as is.
let localX = x;
let localY = y;
// If there's a parent frame, we need to see where the <iframe> itself is in parent's coords
// But we only do that if frame is NOT the main frame
const parentFrame = frame.parentFrame();
if (parentFrame) {
// Evaluate bounding rect of the <iframe> that hosts this child frame
// We'll do a small trick: find the <iframe> element in the parent that has frame.url()
// or frame.name(). There's no direct method, so we rely on the frame element handle:
const frameElement = await frame.frameElement();
const box = await frameElement.boundingBox();
if (!box) {
// If we can't get bounding box, fallback
return { targetFrame: page.mainFrame(), localX: x, localY: y };
}
// The bounding box's coords are relative to the parent's main page.
// Convert our (x, y) from the parent's coordinate system to this child's local system.
if (x < box.x ||
y < box.y ||
x > box.x + box.width ||
y > box.y + box.height) {
// The point is not inside this child frame's bounding box at all
// so it's not relevant. We'll skip it. The caller will continue searching siblings.
return { targetFrame: page.mainFrame(), localX: x, localY: y };
}
// So we are inside this child frame. Convert (x, y) to local coords
localX = x - box.x; // offset from left
localY = y - box.y; // offset from top
// Now we must also account for the child frame's own scrolling. Evaluate in the child:
const scrollOffsets = await frame.evaluate(() => {
return { sx: window.scrollX, sy: window.scrollY };
});
localX += scrollOffsets.sx;
localY += scrollOffsets.sy;
}
// Next, see if there's a deeper child frame (iframe) in this frame that covers (localX, localY)
const childFrames = frame.childFrames();
for (const child of childFrames) {
// We'll see if child is same-origin by trying a small evaluate
// If it's cross-origin, we'll catch the error & skip
try {
// Convert localX/Y into child's bounding box
const childElement = await child.frameElement();
const childBox = await childElement.boundingBox();
if (!childBox) {
continue;
}
// Check if local point is inside child's bounding box
if (localX >= childBox.x &&
localY >= childBox.y &&
localX <= childBox.x + childBox.width &&
localY <= childBox.y + childBox.height) {
// Convert to child's local coords
let childLocalX = localX - childBox.x;
let childLocalY = localY - childBox.y;
// Add child's scroll offset
const childScroll = await child.evaluate(() => {
return { sx: window.scrollX, sy: window.scrollY };
});
childLocalX += childScroll.sx;
childLocalY += childScroll.sy;
// Recurse deeper
return await this.findDeepestFrameAtPoint(page, childLocalX, childLocalY, child);
}
}
catch {
// Probably cross-origin or some other error, skip
}
}
// If we get here, none of the children contain the point (or are cross-origin),
// so this frame is the deepest coverage.
return { targetFrame: frame, localX, localY };
}
static async getLocatorOrItsLabel(element) {
const timeoutMilliseconds = 5000;
try {
const id = await element.getAttribute('id', {
timeout: timeoutMilliseconds,
});
if (id) {
// Get the underlying ElementHandle first
const handle = await element.elementHandle({
timeout: timeoutMilliseconds / 5,
});
if (handle) {
// From the handle, retrieve the frame that owns this element
const frame = await handle.ownerFrame();
if (frame) {
// Look for label with matching 'for' attribute
const labelByFor = frame.locator(`label[for="${id}"]`);
if ((await labelByFor.count()) > 0) {
await labelByFor.waitFor({
state: 'visible',
timeout: timeoutMilliseconds / 5,
});
return labelByFor;
}
}
}
}
// Fallback: look for a wrapping label
const wrappingLabel = element.locator('xpath=ancestor::label');
if ((await wrappingLabel.count()) > 0) {
await wrappingLabel.waitFor({
state: 'visible',
timeout: timeoutMilliseconds / 5,
});
return wrappingLabel;
}
// Neither "for" nor wrapping label found
return element;
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
throw error;
}
}
/**
* Reads and clears the desired next state as directed by the in-flow control
* panel. Note that the control panel does not have carte blanche control, as
* we only support the control panel signaling an intent to pause and resume a
* flow.
*/
static async popControlPanelNextDesiredState(page) {
try {
const desiredNextState = await page.evaluate(() => {
const tmpState = window.donobuNextState;
window.donobuNextState = null;
return tmpState;
});
if (desiredNextState) {
return desiredNextState;
}
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
return null;
}
/**
* Hides the control panel by setting its display to 'none'. This ensures the
* panel is not visible and does not intercept mouse events.
*/
static async hideControlPanel(focusedPage, flowMetadata) {
if (!flowMetadata.isControlPanelEnabled) {
return;
}
for (const page of focusedPage.context().pages()) {
try {
await page.evaluate((panelId) => {
const element = document.querySelector(`#${panelId}`);
if (element) {
element.style.display = 'none';
}
}, PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID);
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
// Pass; this can happen normally.
}
else {
Logger_1.appLogger.error(`Unexpected exception while attempting to show control panel for flow '${flowMetadata.id}'`, error);
}
}
}
}
/**
* Shows the control panel by resetting its display property. Assumes the
* original display was 'block'.
*/
static async showControlPanel(focusedPage, flowMetadata) {
if (!flowMetadata.isControlPanelEnabled) {
return;
}
for (const page of focusedPage.context().pages()) {
try {
await page.evaluate((panelId) => {
const element = document.querySelector(`#${panelId}`);
if (element) {
element.style.display = 'block';
}
}, PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID);
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
// Pass; this can happen normally.
}
else {
Logger_1.appLogger.error(`Unexpected exception while attempting to show control panel for flow '${flowMetadata.id}'`, error);
}
}
}
}
/**
* Updates all control panels in the given browser context. If the provided
* headline is non-null, the headline in the control panel will be updated.
*/
static async updateControlPanel(focusedPage, flowMetadata, headline) {
const controlPanelQuerySelector = `#${PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID}`;
const controlPanelHeadlineQuerySelector = `#${PlaywrightUtils.DONOBU_CONTROL_PANEL_HEADLINE_ELEMENT_ID}`;
for (const page of focusedPage.context().pages()) {
try {
const controlPanel = page.locator(controlPanelQuerySelector).first();
if ((await controlPanel.count()) === 1) {
await controlPanel.evaluate((element, state) => {
element.donobuFlowState = state;
}, flowMetadata.state.toString());
if (headline) {
const controlPanelHeadline = page
.locator(controlPanelHeadlineQuerySelector)
.first();
if ((await controlPanelHeadline.count()) === 1) {
const escapedMessage = headline.replace(/[\\"']/g, '\\$&');
await controlPanelHeadline.evaluate((element, message) => {
element.textContent = message;
}, escapedMessage);
}
}
}
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
// Pass; this can happen normally.
}
else {
Logger_1.appLogger.error(`Unexpected exception while attempting to show control panel for flow '${flowMetadata.id}'`, error);
}
}
}
}
/**
* Returns all elements that have the
* {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE} HTML attribute.
**/
static async getAttributedInteractableElements(page) {
try {
const interactableElements = [];
const frames = page
.frames()
.filter((frame) => PlaywrightUtils.frameFilter(frame));
for (const frame of frames) {
const attributeMap = await frame.evaluate((interactableAttribute) => {
const getOuterHTMLWithoutChildrenExceptForSelectElements = (element) => {
if (element.tagName.toLowerCase() === 'select') {
return element.outerHTML;
}
else {
const tempElement = document.createElement('div');
tempElement.appendChild(element.cloneNode(false));
return tempElement.innerHTML;
}
};
const attributeToHtmlMap = {};
const elements = document.querySelectorAll(`[${interactableAttribute}]`);
elements.forEach((element) => {
const attributeValue = element.getAttribute(interactableAttribute);
const outerHtml = getOuterHTMLWithoutChildrenExceptForSelectElements(element);
if (attributeValue) {
attributeToHtmlMap[attributeValue] = outerHtml;
}
});
return attributeToHtmlMap;
}, PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE);
Object.entries(attributeMap).forEach(([donobuAttributeValue, htmlSnippet]) => {
interactableElements.push({
donobuAttributeValue,
htmlSnippet,
});
});
}
return interactableElements;
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Assigns a globally unique {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE}
* attribute value to each visible, interactable, element in the given page.
* Any pre-existing {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE}
* attributes will be removed.
*/
static async attributeInteractableElements(page) {
try {
// Remove any preexisting attributes
await this.deattributeVisibleInteractableElements(page);
// Get the main page's viewport and scroll offsets
const [viewWidth, viewHeight, viewX, viewY] = await page.evaluate(() => {
return [
window.innerWidth,
window.innerHeight,
Math.round(window.scrollX),
Math.round(window.scrollY),
];
});
// 1) Attribute elements in the main page
let annotationOffset = await page.evaluate(this.attributeElementsInContext, [0, PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE]);
// 2) Check child frames, attributing elements if the frame is (partially) in view
const frames = page
.frames()
.filter((frame) => PlaywrightUtils.frameFilter(frame) && frame !== page.mainFrame());
for (const frame of frames) {
const elementHandle = await frame.frameElement();
if (!elementHandle) {
continue;
}
const boundingBox = await elementHandle.boundingBox();
if (!boundingBox) {
continue;
}
const { x: frameX, y: frameY, width: frameWidth, height: frameHeight, } = boundingBox;
// Check if the frame is visible in the main viewport
const intersectsViewport = frameX < viewX + viewWidth &&
frameX + frameWidth > viewX &&
frameY < viewY + viewHeight &&
frameY + frameHeight > viewY;
if (intersectsViewport) {
// Update annotation offset from this frame's results
annotationOffset = await frame.evaluate(this.attributeElementsInContext, [
annotationOffset,
PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE,
]);
}
}
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Annotate all elements in the given page that have a
* {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE} HTML attribute.
*
* The annotations are placed in a shadow root to avoid site-specific CSS,
* each having a {@link PlaywrightUtils.DONOBU_ANNOTATION_ATTRIBUTE} attribute.
*/
static async annotateInteractableElements(page) {
try {
// Filter frames as needed
const frames = page
.frames()
.filter((frame) => PlaywrightUtils.frameFilter(frame));
for (const frame of frames) {
await frame.evaluate(([interactableAttr, annotationAttr]) => {
// 1) Ensure we have a shadow container in the main document
let container = document.getElementById('annotation-shadow-container');
if (!container) {
container = document.createElement('div');
container.id = 'annotation-shadow-container';
// Position container so child elements can be absolutely placed
Object.assign(container.style, {
position: 'absolute',
top: '0',
left: '0',
width: '100%',
height: '100%',
pointerEvents: 'none', // Let clicks pass through
});
// Check if document.body exists before trying to append.
if (document.body) {
document.body.appendChild(container);
}
else if (document.documentElement) {
// Fall back to document.documentElement if body does not exist.
document.documentElement.appendChild(container);
}
else {
// If neither exists, we can't proceed with annotations in this frame.
console.warn(`Cannot create annotation container for ${window.location.href} since the document structure is incomplete`);
return;
}
// Attach a shadow root
const shadowRoot = container.attachShadow({ mode: 'open' });
// Add a <style> element inside the shadow root to reset and define annotation styles
const style = document.createElement('style');
style.textContent = `
:host {
all: initial; /* Reset styles in shadow root */
}
.annotation {
position: absolute;
z-index: 2147483647;
background-color: black;
color: white;
width: 40px;
height: 40px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-size: 14px;
font-weight: bold;
line-height: 20px;
text-align: center;
box-shadow: 0px 2px 4px rgba(0,0,0,0.2);
border: 4px solid #FF4136;
pointer-events: none;
}
`;
shadowRoot.appendChild(style);
}
// Retrieve the shadow root to place annotation elements
const containerEl = document.getElementById('annotation-shadow-container');
if (!containerEl?.shadowRoot) {
return;
}
const shadowRoot = containerEl.shadowRoot;
// 2) Factory to create a new annotation inside the shadow root
const createAnnotation = (value) => {
const annotation = document.createElement('div');
annotation.classList.add('annotation');
annotation.dataset[annotationAttr] = '1';
annotation.textContent = value;
return annotation;
};
// 3) Position annotation relative to an element
const positionAnnotation = (annotation, element) => {
const rect = element.getBoundingClientRect();
// Center the annotation on the element, adjusting for its size
const x = rect.left + rect.width / 2 - 20 + window.scrollX;
const y = rect.top + rect.height / 2 - 20 + window.scrollY;
annotation.style.left = `${x}px`;
annotation.style.top = `${y}px`;
};
// 4) Traverse DOM (including any nested shadow roots) to find interactable elements
const processNode = (root) => {
// Find elements with the interactable attribute
const elements = root.querySelectorAll(`[${interactableAttr}]`);
elements.forEach((element) => {
const value = element.getAttribute(interactableAttr);
if (value) {
const annotation = createAnnotation(value);
shadowRoot.appendChild(annotation);
positionAnnotation(annotation, element);
}
});
// Recursively process any child shadow roots
root.querySelectorAll('*').forEach((el) => {
if (el.shadowRoot) {
processNode(el.shadowRoot);
}
});
};
// Start processing from the (frame) document root
processNode(document);
}, [
PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE,
PlaywrightUtils.convertToJsAttribute(PlaywrightUtils.DONOBU_ANNOTATION_ATTRIBUTE),
]);
}
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Removes all annotations with a {@link PlaywrightUtils.DONOBU_ANNOTATION_ATTRIBUTE}
* HTML attribute in the given page, including any containers in shadow DOM.
*/
static async removeDonobuAnnotations(page) {
try {
const frames = page
.frames()
.filter((frame) => PlaywrightUtils.frameFilter(frame));
for (const frame of frames) {
await frame.evaluate(() => {
const containerId = 'annotation-shadow-container';
const container = document.getElementById(containerId);
if (container) {
container.remove();
}
});
}
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
throw error;
}
}
/**
* Removes the {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE} HTML
* attribute for all elements in the given page. This attribute is normally
* added by the {@link PlaywrightUtils.attributeInteractableElements(Page)}
* function.
*/
static async deattributeVisibleInteractableElements(page) {
try {
const frames = page
.frames()
.filter((frame) => PlaywrightUtils.frameFilter(frame));
for (const frame of frames) {
await frame.evaluate((attr) => {
document.querySelectorAll(`[${attr}]`).forEach((element) => {
element.removeAttribute(attr);
});
}, PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE);
}
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
static async parseUnambiguousSelector(elementHandle) {
try {
return await elementHandle.evaluate((element) => {
const escapeCssIdentifier = (ident) => {
return ident.replace(/([!"#$%&'()*+,./:;<=>?@[\\]^{|}~])/g, '\\$1');
};
const getPath = (el) => {
if (el.id)
return '#' + escapeCssIdentifier(el.id);
if (!el.parentNode || el.parentNode.nodeType === Node.DOCUMENT_NODE)
return '';
const siblings = Array.from(el.parentNode.children).filter((e) => e.tagName === el.tagName);
const index = siblings.indexOf(el) + 1;
const tag = el.tagName.toLowerCase();
const parentPath = getPath(el.parentNode);
return (parentPath +
(parentPath ? ' > ' : '') +
tag +
(siblings.length > 1 ? ':nth-child(' + index + ')' : ''));
};
return getPath(element);
});
}
catch (error) {
if (PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
/**
* Converts an HTML attribute to a JavaScript attribute. For example,
* "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
* prefix, and the conversion from kebab-case to camelCase.
*/
static convertToJsAttribute(htmlAttribute) {
if (htmlAttribute.startsWith('data-')) {
htmlAttribute = htmlAttribute.substring(5);
}
const parts = htmlAttribute.split('-');
const jsAttribute = parts[0] +
parts
.slice(1)
.map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase())
.join('');
return jsAttribute;
}
/**
* Returned true IFF the given error is a Playwright error regarding page closing,
* of if the given error is an instance of {@link PageClosedException}.
*/
static isPageClosedError(error) {
if (error instanceof PageClosedException_1.PageClosedException) {
return true;
}
else {
const exceptionMessage = error?.message?.toLowerCase();
if (!exceptionMessage) {
return false;
}
else {
return (exceptionMessage.includes('detached') ||
exceptionMessage.includes('context was destroyed') ||
exceptionMessage.includes('browser has been closed'));
}
}
}
static async ensurePlaywrightInstallation() {
return PlaywrightUtils.runPlaywrightCli(['install', '--with-deps']);
}
static async runPlaywrightCli(args) {
try {
// First, resolve the package root
const packagePath = require.resolve('playwright-core/package.json');
// Then construct the path to cli.js
const cliPath = path_1.default.join(path_1.default.dirname(packagePath), 'cli.js');
Logger_1.appLogger.debug(`Found Playwright CLI at: ${cliPath}`);
Logger_1.appLogger.info(`Running Playwright CLI with args: ${args.join(' ')}`);
const env = {
...process.env,
PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS: '1',
PLAYWRIGHT_SKIP_BROWSER_GC: '1',
PLAYWRIGHT_SKIP_BROWSER_VALIDATION: '1',
FORCE_COLOR: '0',
NODE_NO_WARNINGS: '1',
ELECTRON_RUN_AS_NODE: '1',
};
if (process.env[envVars_1.ENV_VAR_NAMES.USE_PACKAGED_PLAYWRIGHT_BROWSERS]) {
env['PLAYWRIGHT_BROWSERS_PATH'] = path_1.default.join(MiscUtils_1.MiscUtils.baseWorkingDirectory(), '.playwright');
}
const childProcess = (0, child_process_1.spawn)(process.execPath, [cliPath, ...args], {
stdio: 'inherit',
env: env,
windowsHide: true,
});
return new Promise((resolve, reject) => {
childProcess.on('exit', (code) => {
if (code === 0) {
Logger_1.appLogger.debug('Playwright CLI completed successfully');
resolve();
}
else {
reject(new Error(`Playwright CLI failed with code ${code}`));
}
});
childProcess.on('error', (error) => {
reject(new Error(`Failed to execute Playwright CLI: ${error.message}`));
});
});
}
catch (error) {
throw new Error(`Failed to initialize Playwright CLI: ${error instanceof Error ? error.message : String(error)}`);
}
}
/**
* Attempts to wait until the currently focused page is stable. If the page
* never stabilizes, it just returns after timing out. If any error occurs,
* it is logged and ignored. If page is null, this function has no effect.
*/
static async waitForPageStability(page) {
const waitTimeMilliseconds = 5000;
const minimumWaitTimeMilliseconds = 1000;
if (page) {
try {
await Promise.all([
page.waitForLoadState('load', {
timeout: waitTimeMilliseconds,
}),
page.waitForTimeout(minimumWaitTimeMilliseconds),
]);
}
catch (error) {
if (!PlaywrightUtils.isPageClosedError(error)) {
// Pass, just move on and hope for the best, we waited long enough.
Logger_1.appLogger.error(`Error while waiting for page to reach steady state for page ${page.url}`, error);
}
}
}
}
static frameFilter(frame) {
return (!frame.isDetached() &&
!frame.url().startsWith('about:') &&
!frame.url().startsWith('chrome:') &&
!frame.url().startsWith('edge:'));
}
/**
* This function is injected into the page (or frame) context. It finds
* "interactable" elements, checks visibility, and sets a unique attribute.
* It returns the updated offset after labeling.
*/
static attributeElementsInContext(arg) {
let offset = arg[0];
const interactableAttribute = arg[1];
// --- Utility Functions ---
function isElementVisible(rect, style) {
return (rect.width > 0 &&
rect.height > 0 &&
style.display !== 'none' &&
style.visibility !== 'hidden');
}
function isElementMoreThanHalfInViewport(rect) {
const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
const visibleWidth = Math.min(rect.right, viewportWidth) - Math.max(rect.left, 0);
const visibleHeight = Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0);
const visibleArea = Math.max(0, visibleWidth) * Math.max(0, visibleHeight);
const elementArea = rect.width * rect.height;
return visibleArea >= elementArea / 2;
}
function isElementEnabled(element, style) {
// Check standard disabled attribute (for form controls like button, input, etc.)
if (element.hasAttribute('disabled')) {
return false;
}
// Check for ARIA attributes that indicate non-interactivity
if (element.getAttribute('aria-disabled') === 'true' ||
element.getAttribute('aria-hidden') === 'true') {
return false;
}
// Check for pointer-events: none which prevents interactions
if (style.pointerEvents === 'none') {
return false;
}
// Check for ARIA attributes indicating disabled state
if (element.getAttribute('aria-disabled') === 'true') {
return false;
}
// Check for inert attribute which makes elements non-interactive
if (element.hasAttribute('inert')) {
return false;
}
// If the element is in a form and the fieldset is disabled, it might be disabled as well
let parent = element.parentElement;
while (parent) {
if (parent.tagName.toLowerCase() === 'fieldset' &&
parent.hasAttribute('disabled') &&
element.tagName.toLowerCase() !== 'legend') {
return false;
}
parent = parent.parentElement;
}
return true;
}
/**
* Generate a few test points on the element's bounding box. We only need
* a small offset (1px) from each corner, plus the center.
*/
function getPointsToCheck(rect) {
const cornerOffset = 1;
return [
{ x: rect.left + cornerOffset, y: rect.top + cornerOffset },
{ x: rect.right - cornerOffset, y: rect.top + cornerOffset },
{ x: rect.left + cornerOffset, y: rect.bottom - cornerOffset },
{ x: rect.right - cornerOffset, y: rect.bottom - cornerOffset },
{ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 },
];
}
/**
* Like `elementFromPoint` but continues walking shadow roots if found.
*/
function getDeepElementFromPoint(x, y) {
let el = document.elementFromPoint(x, y);
while (el && el.shadowRoot) {
const shadowEl = el.shadowRoot.elementFromPoint(x, y);
if (!shadowEl || shadowEl === el)
break;
el = shadowEl;
}
return el;
}
function getAllInteractableElements(root) {
const selector = [
// Basic interactive elements
'button',
'button svg',
'input',
'textarea',
'a',
'select',
// Common ARIA roles
'[role="button"]',
'[role="option"]',
'[role="link"]',
'[role="checkbox"]',
'[role="radio"]',
'[role="tab"]',
'[role="menuitem"]',
'[role="menuitemcheckbox"]',
'[role="menuitemradio"]',
'[role="combobox"]',
'[role="listbox"]',
'[role="searchbox"]',
'[role="spinbutton"]',
'[role="slider"]',
'[role="switch"]',
// Elements with popup behavior
'[aria-haspopup]',
'[aria-controls]',
// Editable elements
'[contenteditable="true"]',
// Draggable elements
'[draggable="true"]',
// Elements that can receive focus
'[tabindex]:not([tabindex="-1"])',
].join(', ');
let elements = Array.from(root.querySelectorAll(selector));
// Dive into shadow roots
root.querySelectorAll('*').forEach((el) => {
if (el.shadowRoot) {
// Recurse
elements = elements.concat(getAllInteractableElements(el.shadowRoot));
}
});
// Remove duplicates
return Array.from(new Set(elements));
}
// 1) Gather candidate elements (including any custom "clickable" from donobuGetClickableElements)
let interactableElements = getAllInteractableElements(document);
if (window.donobuGetClickableElements) {
interactableElements = interactableElements.concat(window
.donobuGetClickableElements()
// Exclude elements with a "donobu-" prefixed ID.
.filter((el) => !el.id || !el.id.startsWith('donobu-')));
}
// Use a Set to avoid duplicates
const uniqueElements = new Set(interactableElements);
// 2) Check each element for visibility + actual "interactability"
uniqueElements.forEach((element) => {
if (typeof element.getBoundingClientRect !== 'function') {
// Should be rare, but just in case
return;
}
const rect = element.getBoundingClientRect();
const style = window.getComputedStyle(element);
const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
// Check if the element is actually enabled/interactive
const enabled = isElementEnabled(element, style);
if (!visible || !enabled) {
return;
}
let annotated = false;
// 3) Test small set of points on the element to confirm it's truly topmost
for (const pt of getPointsToCheck(rect)) {
let elToCheck = getDeepElementFromPoint(pt.x, pt.y);
// Walk up the DOM tree from that point
while (elToCheck) {
if (elToCheck === element) {
// Found the actual element
element.setAttribute(interactableAttribute, offset.toString());
offset++;
annotated = true;
break;
}
else if (elToCheck.tagName?.toLowerCase() === 'label' &&
elToCheck.htmlFor) {
// If we found a label referencing an element, also mark that
const forId = elToCheck.htmlFor;
const associatedInput = document.getElementById(forId);
if (associatedInput) {
associatedInput.setAttribute(interactableAttribute, offset.toString());
offset++;
}
annotated = true;
break;
}
elToCheck = elToCheck.parentElement;
}
if (annotated) {
break;
}
}
});
return offset;
}
}
exports.PlaywrightUtils = PlaywrightUtils;
PlaywrightUtils.ACCESSIBILITY_TEST_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('axe.js'));
PlaywrightUtils.CLICKABLE_ELEMENTS_TRACKER_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('clickable-elements-tracker.js'));
PlaywrightUtils.DIALOG_PROMPT_TRACKER_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('dialog-prompt-tracker.js'));
PlaywrightUtils.SMART_SELECTOR_GENERATOR_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('smart-selector-generator.js'));
PlaywrightUtils.PAGE_INTERACTIONS_TRACKER_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('page-interactions-tracker.js'));
PlaywrightUtils.DONOBU_CONTROL_PANEL_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('control-panel.js'));
// WARNING: If the control panel ID is changed here, you must also change the
// control-panel.js control panel ID value.
PlaywrightUtils.DONOBU_CONTROL_PANEL