donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
100 lines • 4.37 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.PageInteractionTracker = void 0;
const MiscUtils_1 = require("../utils/MiscUtils");
const ClickTool_1 = require("../tools/ClickTool");
const PressKeyTool_1 = require("../tools/PressKeyTool");
const Logger_1 = require("../utils/Logger");
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
/**
* Tracks direct user actions that have occurred in the web browser.
* This differs from normal Donobu flows which use GPT to analyze pages and propose actions.
* This tracker records direct user actions for reliable replay and GPT visibility.
*/
class PageInteractionTracker {
constructor(donobuFlow) {
this.donobuFlow = donobuFlow;
}
name() {
return PageInteractionTracker.NAME;
}
/**
* Handles tracking of page interactions.
*
* Assumes:
* - args is a single-element array with first element being JSON convertible
* - Existence of 'data-donobu-marker' DOM attribute for events with target elements
* - donobuMarker field in payload contains the attribute value if applicable
*/
async call(source, ...args) {
const flowState = this.donobuFlow.metadata.state;
if (flowState === 'PAUSED' ||
flowState === 'WAITING_ON_USER_FOR_NEXT_ACTION') {
const timestamp = new Date().getTime();
const event = args[0];
Logger_1.appLogger.debug(`Event: ${JSON.stringify(event)}`);
const postCallImage = await PlaywrightUtils_1.PlaywrightUtils.takePngScreenshot(source.page);
const postCallImageId = await this.donobuFlow.persistence.savePngScreenShot(this.donobuFlow.metadata.id, postCallImage);
const frameSelector = source.frame.url() === source.page.url()
? null
: await PlaywrightUtils_1.PlaywrightUtils.parseUnambiguousSelector(await source.frame.frameElement());
const elementSelectors = event.selectors;
const externalToolCall = await this.createToolCall(event, elementSelectors, frameSelector, postCallImageId, source.page.url(), timestamp);
if (externalToolCall) {
this.donobuFlow.invokedToolCalls.push(externalToolCall);
await this.donobuFlow.persistence.saveToolCall(this.donobuFlow.metadata.id, externalToolCall);
}
}
return null;
}
async createToolCall(event, elementSelectors, frameSelector, postCallImageId, pageUrl, timestamp) {
const selectorForReplay = {
element: elementSelectors,
frame: frameSelector,
};
switch (event.type) {
case 'click': {
return {
id: MiscUtils_1.MiscUtils.createAdHocToolCallId(),
toolName: ClickTool_1.ClickTool.NAME,
parameters: {
selector: selectorForReplay,
},
outcome: {
isSuccessful: true,
forLlm: 'Clicked the element.',
metadata: selectorForReplay,
},
postCallImageId: postCallImageId,
page: pageUrl,
startedAt: timestamp,
completedAt: timestamp,
};
}
case 'keydown': {
return {
id: MiscUtils_1.MiscUtils.createAdHocToolCallId(),
toolName: PressKeyTool_1.PressKeyTool.NAME,
parameters: {
selector: selectorForReplay,
key: event.key,
},
outcome: {
isSuccessful: true,
forLlm: 'Pressed the key.',
metadata: selectorForReplay,
},
postCallImageId: postCallImageId,
page: pageUrl,
startedAt: timestamp,
completedAt: timestamp,
};
}
default:
return null;
}
}
}
exports.PageInteractionTracker = PageInteractionTracker;
PageInteractionTracker.NAME = 'donobuTrackInteraction';
//# sourceMappingURL=PageInteractionTracker.js.map