donobu/dist/tools/ClickTool.js

Create browser automations with an LLM agent and replay them as Playwright scripts.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ClickTool = exports.ClickGptSchema = exports.ClickNonGptSchema = exports.ClickCoreSchema = void 0;
const v4_1 = require("zod/v4");
const Logger_1 = require("../utils/Logger");
const MiscUtils_1 = require("../utils/MiscUtils");
const TargetUtils_1 = require("../utils/TargetUtils");
const ReplayableInteraction_1 = require("./ReplayableInteraction");
exports.ClickCoreSchema = v4_1.z.object({
    button: v4_1.z
        .enum(['left', 'right', 'middle'])
        .optional()
        .describe('The mouse button to use for the click. Defaults to "left".'),
});
exports.ClickNonGptSchema = v4_1.z.object({
    ...ReplayableInteraction_1.SelectorBasedSchema.shape,
    ...exports.ClickCoreSchema.shape,
});
exports.ClickGptSchema = v4_1.z.object({
    ...ReplayableInteraction_1.AnnotationBasedSchema.shape,
    ...exports.ClickCoreSchema.shape,
});
class ClickTool extends ReplayableInteraction_1.ReplayableInteraction {
    constructor() {
        super(ClickTool.NAME, 'Click an element on a webpage.', exports.ClickCoreSchema, exports.ClickNonGptSchema, exports.ClickGptSchema);
    }
    async invoke(context, parameters, handles) {
        const element = handles.label ?? handles.target;
        const page = (0, TargetUtils_1.webPage)(context);
        const box = await element.boundingBox();
        const button = parameters.button ?? 'left';
        if (!box) {
            throw new Error(`Failed to retrieve element bounding box for '${element}'; element may be offscreen or detached.`);
        }
        const clickDuration = MiscUtils_1.MiscUtils.generateHumanLikeClickDurationInMs();
        try {
            try {
                await element.hover({ timeout: 1000 });
                await page.waitForTimeout(50); // Small delay to let the hover settle
            }
            catch {
                // This is best-effort, just move on.
            }
            await context.interactionVisualizer.pointAt(page, element);
            await element.click({
                button: button,
                delay: clickDuration,
                timeout: 1000,
            });
            return `${MiscUtils_1.MiscUtils.capitalize(button)}-clicked on: `;
        }
        catch (error) {
            (0, Logger_1.logErrorWithoutStack)(`Failed to directly ${button}-click element, failing over to position based click...`, error, 'warn');
            const x = box.x + box.width / 2;
            const y = box.y + box.height / 2;
            await page.mouse.move(x, y);
            await page.mouse.click(x, y, {
                delay: clickDuration,
            });
            return `${MiscUtils_1.MiscUtils.capitalize(button)}-clicked at ${x}px by ${y}px on: `;
        }
    }
}
exports.ClickTool = ClickTool;
ClickTool.NAME = 'click';
//# sourceMappingURL=ClickTool.js.map