donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
170 lines • 8.38 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ReplayableInteraction = void 0;
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
const Tool_1 = require("./Tool");
const Logger_1 = require("../utils/Logger");
const PageClosedException_1 = require("../exceptions/PageClosedException");
/**
* Tools that specify a numbered Donobu annotation to indicate which element to
* interact with and support deterministic reruns should extend this interface.
*/
class ReplayableInteraction extends Tool_1.Tool {
constructor(name, description, parametersTypeNameForGptlessSchema, parametersTypeNameForGptSchema) {
super(name, description, parametersTypeNameForGptlessSchema, parametersTypeNameForGptSchema);
}
async call(context, parameters) {
const page = context.page;
const locators = await ReplayableInteraction.getLocatorsOrderedByMatchCount(page, parameters.selector.frame, parameters.selector.element);
if (!locators.length) {
return {
isSuccessful: false,
forLlm: 'FAILED! Unable to resolve HTML element to operate with.',
metadata: null,
};
}
return this.callCore(context, parameters, locators, parameters.selector);
}
async callFromGpt(context, parameters) {
const page = context.page;
const elementSelector = `[${PlaywrightUtils_1.PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE}="${parameters.annotation}"]`;
let locatorData = null;
for (const frame of page.frames()) {
if (frame.isDetached()) {
continue;
}
const locator = frame.locator(elementSelector);
if ((await locator.count()) > 0) {
const selectorCandidates = await PlaywrightUtils_1.PlaywrightUtils.generateSelectors(locator);
const frameSelector = frame.parentFrame() === null
? null
: await PlaywrightUtils_1.PlaywrightUtils.parseUnambiguousSelector(await frame.frameElement());
locatorData = {
locators: [locator],
selectorForReplay: {
element: selectorCandidates,
frame: frameSelector,
},
};
}
}
if (!locatorData || !locatorData.locators.length) {
return {
isSuccessful: false,
forLlm: 'FAILED! Unable to resolve HTML element to operate with.',
metadata: null,
};
}
return this.callCore(context, parameters, locatorData.locators, locatorData.selectorForReplay);
}
async callCore(context, parameters, locators, selectorForReplay) {
const timeoutMilliseconds = 1000;
const page = context.page;
for (const locator of locators) {
try {
const count = await locator.count();
for (let i = 0; i < count; ++i) {
try {
const element = (await PlaywrightUtils_1.PlaywrightUtils.getLocatorOrItsLabel(locator.nth(i))).first();
await element.scrollIntoViewIfNeeded({
timeout: timeoutMilliseconds,
});
const box = await element.boundingBox({
timeout: timeoutMilliseconds,
});
if (!box) {
throw new Error(`Failed to retrieve element bounding box for '${element}'; element may be offscreen or detached.`);
}
if (parameters.rationale) {
await context.toolTipper.blipToolTipAtElement(page, element, parameters.rationale);
}
// Hide the control panel so that it will not block an interaction if the
// target element is underneath the panel.
await PlaywrightUtils_1.PlaywrightUtils.hideControlPanel(context.page, context.metadata);
const forLlm = await this.invoke(context, parameters, element);
await PlaywrightUtils_1.PlaywrightUtils.showControlPanel(context.page, context.metadata);
await PlaywrightUtils_1.PlaywrightUtils.waitForPageStability(page);
return {
isSuccessful: true,
forLlm: forLlm,
metadata: selectorForReplay,
};
}
catch (elementError) {
Logger_1.appLogger.error(`Failed to interact with element '${locator.nth(i)}' due to exception, will fail over to remaining elements (if any)`, elementError);
}
}
}
catch (locatorError) {
Logger_1.appLogger.error(`Failed to interact with locator '${locator.toString()}' due to exception, will fail over to remaining locators (if any)`, locatorError);
}
}
return {
isSuccessful: false,
forLlm: 'FAILED! Unable to apply operatation.',
metadata: null,
};
}
/**
* Retrieves a list of {@link Locator} objects based on the provided selector
* candidates, ordered by their match count in ascending order. If the match
* count for a locator exceeds
* {@link ReplayableInteraction.MAX_LOCATOR_MATCH_COUNT}, then they are ignored, as
* it is considered too broad of a locator to be useful.
*
* This method iterates through a list of CSS selector candidates and creates
* a {@link Locator} for each candidate. It counts the number of elements that
* match each selector within a given page. If a frame selector is provided,
* it looks for the elements within the specified frame; otherwise, it
* searches within the entire page. Only locators with a positive match count
* are added to the result list.
*
* @param page - The the web page to search within.
* @param frameSelector - An optional CSS selector for a frame within the
* page. If {@code null}, the search is performed in the entire page.
* @param selectorCandidates A list of CSS selector strings to be used for
* locating elements.
* @return A list of {@link Locator} objects that have been found, ordered by
* their match count in ascending order.
*/
static async getLocatorsOrderedByMatchCount(page, frameSelector, selectorCandidates) {
try {
const locators = [];
for (const selectorCandidate of selectorCandidates) {
try {
const elementLocator = frameSelector
? page.frameLocator(frameSelector).locator(selectorCandidate)
: page.locator(selectorCandidate);
const count = await elementLocator.count();
if (count > 0 &&
ReplayableInteraction.MAX_LOCATOR_MATCH_COUNT >= count) {
locators.push(elementLocator);
}
}
catch (e) {
Logger_1.appLogger.warn(`Invalid selector: ${selectorCandidate}`, e);
}
}
// Create array of objects containing locator and its count
const locatorsWithCounts = await Promise.all(locators.map(async (locator) => ({
locator,
count: await locator.count(),
})));
// Sort by count and return just the locators
return locatorsWithCounts
.sort((a, b) => a.count - b.count)
.map((item) => item.locator);
}
catch (error) {
if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
throw new PageClosedException_1.PageClosedException();
}
else {
throw error;
}
}
}
}
exports.ReplayableInteraction = ReplayableInteraction;
ReplayableInteraction.MAX_LOCATOR_MATCH_COUNT = 5;
//# sourceMappingURL=ReplayableInteraction.js.map