donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
208 lines (207 loc) • 9.47 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.WebTargetInspector = void 0;
const PageInteractionTracker_1 = require("../bindings/PageInteractionTracker");
const SetDonobuAnnotations_1 = require("../bindings/SetDonobuAnnotations");
const PageClosedException_1 = require("../exceptions/PageClosedException");
const BrowserUtils_1 = require("../utils/BrowserUtils");
const Logger_1 = require("../utils/Logger");
const PageLogListeners_1 = require("../utils/PageLogListeners");
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
const PageInspector_1 = require("./PageInspector");
/**
* Web implementation of {@link TargetInspectorBase}.
*
* Wraps the existing {@link PageInspector} (which operates on Playwright
* {@link Page} objects) behind the target-agnostic interface. Holds a
* reference to the mutable {@link WebTarget} so the same inspector instance
* tracks page/tab switches automatically.
*
* Also owns the {@link BrowserContext} and {@link InteractionVisualizer}
* so that all web-specific lifecycle (initialization, cursor, session
* persistence) is encapsulated here rather than in the flow engine.
*
* The underlying {@link pageInspector} is exposed publicly so that
* web-specific code (e.g. {@link ReplayableInteraction}) can still call
* Page-specific helpers like `getHtmlSnippet`.
*/
class WebTargetInspector {
constructor(_target, _browserContext, _interactionVisualizer) {
this._target = _target;
this._browserContext = _browserContext;
this._interactionVisualizer = _interactionVisualizer;
this.type = 'web';
this.pageInspector = new PageInspector_1.PageInspector();
}
/* ------------------------------------------------------------------ */
/* Target accessors */
/* ------------------------------------------------------------------ */
get target() {
return this._target;
}
get connected() {
return this._target.current !== null;
}
get interactableElementAttribute() {
return this.pageInspector.interactableElementAttribute;
}
requirePage() {
if (!this._target.current) {
throw new PageClosedException_1.PageClosedException();
}
return this._target.current;
}
/* ------------------------------------------------------------------ */
/* Element inspection */
/* ------------------------------------------------------------------ */
async attributeInteractableElements() {
await this.pageInspector.attributeInteractableElements(this.requirePage());
}
async getAttributedInteractableElements() {
return this.pageInspector.getAttributedInteractableElements(this.requirePage());
}
async annotateInteractableElements() {
await this.pageInspector.annotateInteractableElements(this.requirePage());
}
async removeAnnotations() {
await this.pageInspector.removeDonobuAnnotations(this.requirePage());
}
/* ------------------------------------------------------------------ */
/* Screenshots */
/* ------------------------------------------------------------------ */
async takeCleanScreenshot() {
return PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(this.requirePage());
}
async takeAnnotatedScreenshot() {
return PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(this.requirePage());
}
async captureScreenshot() {
return PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(this.requirePage());
}
/* ------------------------------------------------------------------ */
/* Connection lifecycle */
/* ------------------------------------------------------------------ */
checkConnectedOrThrow() {
if (!this._target.current) {
throw new PageClosedException_1.PageClosedException();
}
}
checkTargetAliveOrThrow() {
if (this._target.current?.isClosed()) {
throw new PageClosedException_1.PageClosedException();
}
}
isTargetClosedError(error) {
return PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error);
}
async handleTargetClosed() {
const allPages = this._browserContext.pages();
if (allPages.length === 0) {
this._target.current = null;
return {
recovered: false,
reason: 'Stopped flow due to the browser unexpectedly closing!',
};
}
this._target.current = allPages[0];
return { recovered: true };
}
/* ------------------------------------------------------------------ */
/* Interaction cursor */
/* ------------------------------------------------------------------ */
async showInteractionCursor() {
if (this._target.current) {
await this._interactionVisualizer.showMouse(this._target.current);
}
}
async hideInteractionCursor() {
if (this._target.current) {
await this._interactionVisualizer.hideMouse(this._target.current);
}
}
/* ------------------------------------------------------------------ */
/* Location & platform identity */
/* ------------------------------------------------------------------ */
getCurrentLocation() {
return this._target.current?.url() ?? 'about:blank';
}
getPlatformPromptInfo() {
return WebTargetInspector.PROMPT_INFO;
}
getContextDescription() {
if (!this._target.current) {
return 'The web browser has no open pages.';
}
return `The current web browser tabs are:
- ${this._target.current
.context()
.pages()
.map((page) => page.url())
.join('\n- ')}
The active (i.e. in focus) tab is ${this._target.current.url()}`;
}
/* ------------------------------------------------------------------ */
/* Initialization & session state */
/* ------------------------------------------------------------------ */
async initialize(callbacks) {
const browserContext = this._browserContext;
// Register page handler — assigns new tabs/popups as the focused page
// and optionally installs the virtual mouse on domcontentloaded.
browserContext.on('page', (page) => this.handleNewPage(page, callbacks));
await PlaywrightUtils_1.PlaywrightUtils.setupBasicBrowserContext(browserContext);
// Wire flow-engine handlers onto the browser context.
if (callbacks.dialogHandler) {
browserContext.on('dialog', callbacks.dialogHandler);
}
if (callbacks.interactionTrackingHost) {
await PageInteractionTracker_1.PageInteractionTracker.register(callbacks.interactionTrackingHost, browserContext);
}
await SetDonobuAnnotations_1.SetDonobuAnnotations.register(this.pageInspector, browserContext);
// Ensure a page is assigned.
if (!this._target.current) {
const existingPages = browserContext.pages();
for (let i = 0; i < existingPages.length; ++i) {
// Reload so that the init scripts registered above will run.
await existingPages[i].reload();
this.handleNewPage(existingPages[i], callbacks);
}
if (existingPages.length === 0) {
// The call to `newPage` will trigger the page handler above.
await browserContext.newPage();
}
}
}
async persistSessionState(persistence, flowId) {
try {
const browserState = await BrowserUtils_1.BrowserUtils.getBrowserStorageState(this._browserContext);
await persistence.setBrowserState(flowId, browserState);
}
catch (error) {
Logger_1.appLogger.error('Failed to persist browser state when completing flow', error);
}
}
/* ------------------------------------------------------------------ */
/* Internal helpers */
/* ------------------------------------------------------------------ */
handleNewPage(page, callbacks) {
this._target.current = page;
(0, PageLogListeners_1.registerPageLogListeners)(page);
if (callbacks.metadata.runMode !== 'INSTRUCT') {
page.on('domcontentloaded', async () => {
await this._interactionVisualizer.showMouse(page);
});
}
}
}
exports.WebTargetInspector = WebTargetInspector;
WebTargetInspector.PROMPT_INFO = {
systemPreamble: `You are a web browser automation agent to help people navigate webpages to
accomplish an OVERALL OBJECTIVE. These webpages are being rendered in a web
browser and are powered using the Microsoft Playwright framework.`,
screenshotSubject: 'a webpage',
currentViewDescription: 'web page (i.e. the current viewport)',
annotatedViewDescription: 'viewport of the web page',
interactionTarget: 'website',
targetNoun: 'webpage',
};
//# sourceMappingURL=WebTargetInspector.js.map