donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
297 lines • 13.6 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.InteractionVisualizer = void 0;
const Logger_1 = require("../utils/Logger");
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
class InteractionVisualizer {
constructor(defaultMessageDurationMillis) {
this.defaultMessageDurationMillis = defaultMessageDurationMillis;
this.cursorPos = { x: 150, y: 150 };
}
/* ------------------------------------------------------------------ */
/* Public API */
/* ------------------------------------------------------------------ */
/**
* Moves the virtual cursor to the center of the specified element and optionally displays a message.
*
* @param page - The Playwright page instance where the cursor will be displayed.
* @param locator - Optional target element to point at. If omitted, cursor remains at current position.
* @param message - Optional message to display near the cursor during the interaction.
* @param duration - Duration in milliseconds for the cursor animation and message display.
* Defaults to the instance's configured duration. If ≤ 0, no action is taken
*
* @returns Promise that resolves when the cursor movement and message display are complete
*
* @remarks
* - The target element will be scrolled into view if necessary
* - The cursor animates smoothly to the element's center point
* - Messages are positioned automatically to avoid viewport edges
* - The virtual cursor does not interfere with actual page interactions
*/
async pointAt(page, locator, message, duration = this.defaultMessageDurationMillis) {
if (!duration || duration <= 0) {
return;
}
try {
let target = { x: this.cursorPos.x, y: this.cursorPos.y };
if (locator) {
const box = await locator.boundingBox();
if (box) {
target = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
}
}
await this.ensureContainer(page);
await Promise.all([
this.moveCursor(page, target, duration / 2),
message?.trim()
? this.showMessage(page, target, message.trim(), duration)
: Promise.resolve(),
]);
}
catch (error) {
if (!PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
Logger_1.appLogger.warn('Failed to move virtual mouse', error);
}
}
}
/**
* Shows the virtual mouse cursor on the page.
*
* @param page - The Playwright page instance where the cursor will be displayed.
*
* @returns Promise that resolves when the cursor is shown.
*
* @remarks
* - If the cursor doesn't exist yet, it will be created.
* - The cursor will be made visible with a smooth opacity transition.
*/
async showMouse(page) {
try {
await this.ensureContainer(page);
await this.ensureCursor(page);
await page.evaluate(([containerId]) => {
const root = document.getElementById(containerId)
.shadowRoot;
const cursor = root.querySelector('.donobu-virtual-mouse');
if (cursor) {
cursor.classList.remove('hidden');
}
}, [InteractionVisualizer.CONTAINER_ID]);
}
catch (error) {
if (!PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
Logger_1.appLogger.warn('Failed to show virtual mouse', error);
}
}
}
/**
* Hides the virtual mouse cursor on the page.
*
* @param page - The Playwright page instance where the cursor is displayed.
*
* @returns Promise that resolves when the cursor is hidden
*
* @remarks
* - The cursor will be hidden with a smooth opacity transition
* - The cursor element remains in the DOM but becomes invisible
*/
async hideMouse(page) {
try {
const id = InteractionVisualizer.CONTAINER_ID;
await page.evaluate(([containerId]) => {
const container = document.getElementById(containerId);
if (!container?.shadowRoot) {
return;
}
const cursor = container.shadowRoot.querySelector('.donobu-virtual-mouse');
if (cursor) {
cursor.classList.add('hidden');
}
}, [id]);
}
catch (error) {
if (!PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
Logger_1.appLogger.warn('Failed to hide virtual mouse', error);
}
}
}
/* ------------------------------------------------------------------ */
/* Private helpers */
/* ------------------------------------------------------------------ */
async ensureContainer(page) {
const id = InteractionVisualizer.CONTAINER_ID;
await page.evaluate(([containerId, css]) => {
if (document.getElementById(containerId)) {
return;
}
const el = document.createElement('div');
el.id = containerId;
Object.assign(el.style, {
position: 'fixed',
top: '0',
left: '0',
width: '100vw',
height: '100vh',
pointerEvents: 'none',
zIndex: '2147483646', // just below the message itself
});
const shadow = el.attachShadow({ mode: 'open' });
const style = document.createElement('style');
style.textContent = css;
shadow.appendChild(style);
// Append to <html> so it is not clipped by overflow/transform on <body>
(document.documentElement || document.body).appendChild(el);
}, [id, InteractionVisualizer.CSS]);
}
async ensureCursor(page) {
const id = InteractionVisualizer.CONTAINER_ID;
await page.evaluate(([containerId, position, svg, svgAttrs, tipX, tipY]) => {
const root = document.getElementById(containerId).shadowRoot;
let cursor = root.querySelector('.donobu-virtual-mouse');
if (cursor) {
return;
}
cursor = document.createElement('div');
cursor.className = 'donobu-virtual-mouse hidden';
// Hybrid approach: try insertAdjacentHTML first, fallback to createElementNS
try {
cursor.insertAdjacentHTML('beforeend', svg);
}
catch (_error) {
// Fallback: create SVG programmatically using attributes object
const attrs = JSON.parse(svgAttrs);
const svgElement = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
// Set SVG attributes
Object.entries(attrs.svg).forEach(([key, value]) => {
svgElement.setAttribute(key, value);
});
// Create and add path element
const path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
Object.entries(attrs.path).forEach(([key, value]) => {
path.setAttribute(key, value);
});
svgElement.appendChild(path);
cursor.appendChild(svgElement);
}
const pos = position;
cursor.style.transitionDuration = '0s';
cursor.style.transform = `translate(${pos.x - tipX}px, ${pos.y - tipY}px)`;
root.appendChild(cursor);
}, [
id,
this.cursorPos,
InteractionVisualizer.SVG_MOUSE,
JSON.stringify(InteractionVisualizer.SVG_MOUSE_JSON),
InteractionVisualizer.TIP_X,
InteractionVisualizer.TIP_Y,
]);
}
async moveCursor(page, target, duration) {
await this.ensureCursor(page);
const id = InteractionVisualizer.CONTAINER_ID;
await page.evaluate(([containerId, end, duration, tipX, tipY]) => {
const root = document.getElementById(containerId).shadowRoot;
const cursor = root.querySelector('.donobu-virtual-mouse');
cursor.style.transitionDuration = `${duration}ms`;
const endPos = end;
cursor.style.transform = `translate(${endPos.x - tipX}px, ${endPos.y - tipY}px)`;
cursor.classList.add('rippling');
const done = new Promise((resolve) => {
const finish = () => {
cursor.removeEventListener('transitionend', finish);
cursor.classList.remove('rippling');
resolve();
};
cursor.addEventListener('transitionend', finish);
setTimeout(finish, duration + 50);
});
return done;
}, [
id,
target,
duration,
InteractionVisualizer.TIP_X,
InteractionVisualizer.TIP_Y,
]);
this.cursorPos = target;
}
async showMessage(page, target, text, duration) {
const id = InteractionVisualizer.CONTAINER_ID;
await page.evaluate(([containerId, target, text, duration]) => {
containerId = containerId;
target = target;
text = text;
duration = duration;
const root = document.getElementById(containerId)
.shadowRoot;
const msg = document.createElement('div');
msg.className = 'donobu-message';
msg.textContent = text;
root.appendChild(msg);
/* ── compute final coordinates ────────────────────────────────── */
const { width, height } = msg.getBoundingClientRect();
let x = target.x - width / 2;
let y = target.y + 24; // default - below cursor
/* clamp horizontally inside viewport */
x = Math.max(8, Math.min(window.innerWidth - width - 8, x));
/* if it overflows bottom, flip above the element */
if (y + height + 8 > window.innerHeight) {
y = target.y - height - 24;
}
/* if still off-screen at the top, clamp to 8 px margin */
if (y < 8) {
y = 8;
}
msg.style.left = `${x}px`;
msg.style.top = `${y}px`;
/* auto-remove after the requested duration */
setTimeout(() => msg.remove(), duration);
}, [id, target, text, duration]);
}
}
exports.InteractionVisualizer = InteractionVisualizer;
/* ------------------------------------------------------------------ */
/* Constants */
/* ------------------------------------------------------------------ */
InteractionVisualizer.RAW_MOUSE_D = 'M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z';
// Arrow tip within the 24×24 viewBox — matches the M command at the start of RAW_MOUSE_D.
InteractionVisualizer.TIP_X = 4.037;
InteractionVisualizer.TIP_Y = 4.688;
InteractionVisualizer.SVG_MOUSE = `
<svg xmlns="http://www.w3.org/2000/svg"
width="32"
height="32"
viewBox="0 0 24 24"
fill="oklch(13.09% 0.005 165.18)"
stroke="#FF781B"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round">
<path d="${InteractionVisualizer.RAW_MOUSE_D}" />
</svg>`;
InteractionVisualizer.SVG_MOUSE_JSON = {
svg: {
xmlns: 'http://www.w3.org/2000/svg',
width: '32',
height: '32',
viewBox: '0 0 24 24',
fill: 'oklch(13.09% 0.005 165.18)',
stroke: '#FF781B',
'stroke-width': '2',
'stroke-linecap': 'round',
'stroke-linejoin': 'round',
},
path: {
d: InteractionVisualizer.RAW_MOUSE_D,
},
};
InteractionVisualizer.CSS = `
.donobu-message{position:absolute;z-index:2147483647;background:#000;color:#fff;padding:8px 10px;border-radius:5px;font:12px/1 'Source Sans Pro','Helvetica Neue',Helvetica,Arial,sans-serif;max-width:300px;white-space:pre-wrap;box-shadow:2px 2px 10px rgba(0,0,0,.2);pointer-events:none}
.donobu-virtual-mouse{width:24px;height:24px;position:absolute;z-index:2147483646;filter:drop-shadow(1px 1px 1px rgba(0,0,0,.5));transition:transform .15s ease-in-out;pointer-events:none;opacity:1;visibility:visible}
.donobu-virtual-mouse.hidden{opacity:0;visibility:hidden}
.donobu-virtual-mouse svg{width:100%;height:100%;display:block}
.donobu-virtual-mouse.rippling::after{content:"";position:absolute;left:${InteractionVisualizer.TIP_X}px;top:${InteractionVisualizer.TIP_Y}px;width:24px;height:24px;border-radius:50%;background:#FF781B;transform:translate(-50%,-50%) scale(.5);opacity:.4;animation:ripple-pop .6s ease-out forwards}
@keyframes ripple-pop{40%{transform:translate(-50%,-50%) scale(1);opacity:.5}100%{transform:translate(-50%,-50%) scale(2);opacity:0}}
`;
InteractionVisualizer.CONTAINER_ID = 'donobu-iv-overlay';
//# sourceMappingURL=InteractionVisualizer.js.map