donobu/dist/apis/SpecialFlowsApi.js

Create browser automations with an LLM agent and replay them as Playwright scripts.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SpecialFlowsApi = void 0;
const JsonUtils_1 = require("../utils/JsonUtils");
const Logger_1 = require("../utils/Logger");
const CreateBrowserCookieReportTool_1 = require("../tools/CreateBrowserCookieReportTool");
const GoToWebpageTool_1 = require("../tools/GoToWebpageTool");
const ClickTool_1 = require("../tools/ClickTool");
const MarkObjectiveNotCompletableTool_1 = require("../tools/MarkObjectiveNotCompletableTool");
const MarkObjectiveCompleteTool_1 = require("../tools/MarkObjectiveCompleteTool");
const WaitTool_1 = require("../tools/WaitTool");
const ExtractPublicFacebookEntityDataTool_1 = require("../tools/ExtractPublicFacebookEntityDataTool");
const ChooseSelectOptionTool_1 = require("../tools/ChooseSelectOptionTool");
const ExtractPaymentProviderKeyTool_1 = require("../tools/ExtractPaymentProviderKeyTool");
const GoForwardOrBackTool_1 = require("../tools/GoForwardOrBackTool");
const InputRandomizedEmailAddressTool_1 = require("../tools/InputRandomizedEmailAddressTool");
const InputTextTool_1 = require("../tools/InputTextTool");
const PressKeyTool_1 = require("../tools/PressKeyTool");
const ScrollPageTool_1 = require("../tools/ScrollPageTool");
const ExtractGoogleStreetviewEntityDataTool_1 = require("../tools/ExtractGoogleStreetviewEntityDataTool");
const AggregateExtractedStreetviewDataTool_1 = require("../tools/AggregateExtractedStreetviewDataTool");
const GoToGoogleMapsStreetViewTool_1 = require("../tools/GoToGoogleMapsStreetViewTool");
const NavigateWithinStreetView_1 = require("../tools/NavigateWithinStreetView");
const GetEntityDataFromGoogleMapResult_1 = require("../tools/GetEntityDataFromGoogleMapResult");
const DetectBrokenLinksTool_1 = require("../tools/DetectBrokenLinksTool");
const envVars_1 = require("../envVars");
/**
 * This class defines the API for special flows (i.e. flows that are essentially
 * pre-defined and have a specific purpose).
 */
class SpecialFlowsApi {
    constructor(donobuFlowsManager) {
        this.donobuFlowsManager = donobuFlowsManager;
    }
    async analyzeCookieConsent(req, res) {
        const httpBodyParams = req.body;
        const url = httpBodyParams.url;
        const overallObjective = `
You are a web-browsing QA agent verifying a website's cookie banner behavior and compliance. Follow the steps below carefully.
If any step (such as rejecting cookies) is not possible due to the absence of a cookie banner or a lack of “Reject”/“Customize” options,
document your observations but do not conclude “objective not completable.” Instead, continue with the process and produce a final report.

1. Initial Cookie Report:
    - After the webpage initially loads, but before interacting with any cookie banner, generate a comprehensive cookie report.
    - This report should capture all cookies present in the browser at this moment.

2. Check for Cookie Banner:
    - If a cookie banner is present:
      - Look for a “Reject” option.
      - If found, select “Reject” (reject all non-essential cookies).
      - If there is no explicit “Reject” option, select the most restrictive choice (e.g., “Customize” and deselect all possible categories).
      - Document the absence of a direct “Reject” button as a potential compliance concern.
      - Proceed to Step 3.
    - If no cookie banner is present:
      - Document the absence of a banner as a potential compliance concern.
      - For the relevant schema fields, set:
        - cookieBannerDisplayed = false
        - rejectOptionAvailable = false
        - retainedCookies = [] (empty list, assuming no cookies are added after rejecting).
      - Skip Step 3 and proceed directly to Step 4.

3. Post-Rejection Cookie Report:
    - Generate a second cookie report after you have selected “Reject” or the most restrictive settings in the cookie banner.
    - This second report helps determine if any non-essential cookies remain.

4. Analysis & Summary:
    - Compare the two cookie reports (initial vs. post-rejection).
    - If a second report was generated (i.e., a cookie banner existed):
      - Verify whether only strictly necessary cookies remain post-rejection.
      - Identify any non-essential cookies that were retained and flag them as potential compliance issues.
      - Document whether a “Reject” option was available and whether the site appears to respect the user’s preferences.
    - If no banner was present:
      - Note that no second report was generated.
      - Document the absence of a cookie banner as a compliance concern, since users could not explicitly reject non-essential cookies.

5. End the Flow Gracefully:
    - Regardless of whether a cookie banner was found, finalize your findings in a clear, structured format.
    - Do not mark the objective as “not completable” or abruptly end the flow just because the banner was missing.
    - Instead, complete all the documentation steps and conclude with a summary of potential compliance issues and overall observations.`;
        const resultJsonSchema = {
            type: 'object',
            required: [
                'cookieBannerDisplayed',
                'potentialCookieIssues',
                'rejectOptionAvailable',
                'initialCookies',
                'retainedCookies',
            ],
            additionalProperties: false,
            properties: {
                cookieBannerDisplayed: {
                    type: 'boolean',
                    description: 'Whether a cookie banner was displayed on the page.',
                },
                potentialCookieIssues: {
                    type: 'array',
                    items: {
                        type: 'string',
                    },
                    description: 'The list of potential cookie issues (if any).\nExamples:\n- A marketing/advertising cookies being retained.\n- Lack of a "Reject" option in the cookie banner.\n- No cookie banner present (if expected).\n- Other indications of non-compliance with regional cookie laws.',
                },
                rejectOptionAvailable: {
                    type: 'boolean',
                    description: 'Indicates whether the cookie banner had a "Reject" option.\n- true: Banner provided a "Reject" option.\n- false: Banner lacked a "Reject" option.',
                },
                initialCookies: {
                    type: 'array',
                    items: {
                        type: 'string',
                    },
                    description: 'Cookies discovered immediately after page load, before any user interaction.',
                },
                retainedCookies: {
                    type: 'array',
                    items: {
                        type: 'string',
                    },
                    description: 'Cookies that remain after attempting to reject or minimize non-essential cookies.',
                },
            },
        };
        const browserConfig = this.useDevice('Desktop Chromium');
        const flowParams = {
            targetWebsite: url,
            overallObjective: overallObjective,
            browser: browserConfig,
            maxIterations: 10,
            defaultToolTipDurationMilliseconds: 0,
            initialRunMode: 'AUTONOMOUS',
            isControlPanelEnabled: false,
            allowedTools: [
                CreateBrowserCookieReportTool_1.CreateBrowserCookieReportTool.NAME,
                GoToWebpageTool_1.GoToWebpageTool.NAME,
                ClickTool_1.ClickTool.NAME,
                MarkObjectiveNotCompletableTool_1.MarkObjectiveNotCompletableTool.NAME,
                MarkObjectiveCompleteTool_1.MarkObjectiveCompleteTool.NAME,
                WaitTool_1.WaitTool.NAME,
            ],
            toolCallsOnStart: [
                {
                    name: GoToWebpageTool_1.GoToWebpageTool.NAME,
                    parameters: {
                        url: url.toString(),
                    },
                },
                {
                    name: WaitTool_1.WaitTool.NAME,
                    parameters: {
                        seconds: 3,
                    },
                },
                {
                    name: CreateBrowserCookieReportTool_1.CreateBrowserCookieReportTool.NAME,
                    parameters: {},
                },
            ],
            resultJsonSchema: resultJsonSchema,
        };
        const flowHandle = await this.donobuFlowsManager.createFlow(flowParams);
        const isAsync = req.query.async === 'true';
        if (isAsync) {
            res.json({ id: flowHandle.donobuFlow.metadata.id });
        }
        else {
            await this.wrapUp(res, flowHandle);
        }
    }
    async detectBrokenLinks(req, res) {
        const httpBodyParams = req.body;
        const browserConfig = this.useDevice('Desktop Chromium');
        const flowParams = {
            targetWebsite: httpBodyParams.url,
            browser: browserConfig,
            defaultToolTipDurationMilliseconds: 0,
            initialRunMode: 'DETERMINISTIC',
            isControlPanelEnabled: false,
            toolCallsOnStart: [
                {
                    name: GoToWebpageTool_1.GoToWebpageTool.NAME,
                    parameters: {
                        url: httpBodyParams.url,
                    },
                },
                {
                    name: DetectBrokenLinksTool_1.DetectBrokenLinksTool.NAME,
                    parameters: {},
                },
            ],
        };
        const flowHandle = await this.donobuFlowsManager.createFlow(flowParams);
        res.json({ id: flowHandle.donobuFlow.metadata.id });
    }
    async extractPublicFacebookEntityData(req, res) {
        const httpBodyParams = req.body;
        const overallObjective = `Source Facebook transparency data for ${httpBodyParams.facebookEntityName || httpBodyParams.facebookEntityUrl}`;
        const browserConfig = this.maybeUseBrowserBase('Desktop Firefox');
        const flowParams = {
            targetWebsite: 'https://www.facebook.com',
            overallObjective: overallObjective,
            browser: browserConfig,
            maxIterations: 0,
            gptConfigNameOverride: 'claude-3-5-sonnet-latest',
            defaultToolTipDurationMilliseconds: 0,
            initialRunMode: 'AUTONOMOUS',
            isControlPanelEnabled: false,
            allowedTools: [ExtractPublicFacebookEntityDataTool_1.ExtractPublicFacebookEntityDataTool.NAME],
            toolCallsOnStart: [
                {
                    name: ExtractPublicFacebookEntityDataTool_1.ExtractPublicFacebookEntityDataTool.NAME,
                    parameters: httpBodyParams,
                },
            ],
        };
        const flowHandle = await this.donobuFlowsManager.createFlow(flowParams);
        await this.wrapUp(res, flowHandle);
    }
    async extractPaymentProviderData(req, res) {
        const httpBodyParams = req.body;
        const url = this.parseUrl(httpBodyParams.website);
        if (!url) {
            throw new Error(`Invalid website parameter: ${httpBodyParams.website}`);
        }
        const overallObjective = `
      Find the payments provider for ${url}

      Try to checkout some product. Make up real sounding data if needed.
      Add a paid product to cart. Sometimes you might need to scroll to see the option to add product to cart.
      Once a product is in cart, continue to checkout.
      Fill out the shopping cart page with made up shipping and other info and go to payments details page.
      Continue through the flow until you see an option to input credit card number on the page.
      Do not actually submit credit card details. At the final checkout page extract payment provider information.
    `;
        const browserConfig = this.maybeUseBrowserBase('Desktop Chromium');
        const flowParams = {
            targetWebsite: url,
            overallObjective: overallObjective,
            browser: browserConfig,
            maxIterations: 30,
            defaultToolTipDurationMilliseconds: 0,
            initialRunMode: 'AUTONOMOUS',
            isControlPanelEnabled: false,
            allowedTools: [
                ChooseSelectOptionTool_1.ChooseSelectOptionTool.NAME,
                ClickTool_1.ClickTool.NAME,
                ExtractPaymentProviderKeyTool_1.ExtractPaymentProviderKeyTool.NAME,
                GoForwardOrBackTool_1.GoForwardOrBackTool.NAME,
                InputRandomizedEmailAddressTool_1.InputRandomizedEmailAddressTool.NAME,
                InputTextTool_1.InputTextTool.NAME,
                GoToWebpageTool_1.GoToWebpageTool.NAME,
                MarkObjectiveNotCompletableTool_1.MarkObjectiveNotCompletableTool.NAME,
                PressKeyTool_1.PressKeyTool.NAME,
                ScrollPageTool_1.ScrollPageTool.NAME,
            ],
        };
        const flowHandle = await this.donobuFlowsManager.createFlow(flowParams);
        await this.wrapUp(res, flowHandle);
    }
    async extractGoogleStreetviewEntityData(req, res) {
        const httpBodyParams = req.body;
        if (!httpBodyParams.entityName && !httpBodyParams.googleMapsUrl) {
            throw new Error('entityName and googleMapsUrl cannot both be null.');
        }
        else if (httpBodyParams.entityName && !httpBodyParams.entityLocation) {
            throw new Error('entityLocation cannot be null if entityName is non-null.');
        }
        let entityName;
        let entityLocation;
        let inputTokensForTmpFlow;
        let completionTokensForTmpFlow;
        if (httpBodyParams.entityName) {
            entityName = httpBodyParams.entityName;
            entityLocation = httpBodyParams.entityLocation;
            inputTokensForTmpFlow = 0;
            completionTokensForTmpFlow = 0;
        }
        else {
            const tmpFlow = await this.divineTargetEntityDataFromGoogleMapUrl(httpBodyParams.googleMapsUrl);
            const entityData = JsonUtils_1.JsonUtils.objectToJson(tmpFlow.metadata.result);
            entityName = entityData.entityName;
            entityLocation = entityData.entityAddress;
            inputTokensForTmpFlow = tmpFlow.metadata.inputTokensUsed;
            completionTokensForTmpFlow = tmpFlow.metadata.completionTokensUsed;
        }
        const overallObjective = `
      Attempt to locate the business/entity, ${entityName}, at (or around!) the location ${entityLocation}, using Google Maps street view.
      Note that the initial street view will likely already been oriented towards the business/entity, so if
      business/entity signage is initially blurry, first try zooming in the view before rotating the view.

      Business/entity data can be extracted using the '${ExtractGoogleStreetviewEntityDataTool_1.ExtractGoogleStreetviewEntityDataTool.NAME}' tool.

      If the target business/entity is not found, DO NOT instantly give up! Here is what to do for various conditions:
      - If building is visible but business/entity signage wasn't a match, prefer zooming in first.
      - Business/entity signage is illegible/blurry/etc -> Try zooming in.
      - Business/entity is not present -> Try rotating, or, moving the current position of the street view.

      If the street view has been adjusted, be sure to follow that up with a call to the '${ExtractGoogleStreetviewEntityDataTool_1.ExtractGoogleStreetviewEntityDataTool.NAME}' tool.

      Once confident about the existence or non-existence of the target business/entity at the given location,
      using (perhaps multiple times) the '${ExtractGoogleStreetviewEntityDataTool_1.ExtractGoogleStreetviewEntityDataTool.NAME}' tool, call the '${AggregateExtractedStreetviewDataTool_1.AggregateExtractedStreetviewDataTool.NAME}' tool.
    `;
        const browserConfig = this.maybeUseBrowserBase('Desktop Chromium');
        const flowParams = {
            targetWebsite: 'https://maps.google.com',
            overallObjective: overallObjective,
            browser: browserConfig,
            maxIterations: 10,
            defaultToolTipDurationMilliseconds: 0,
            initialRunMode: 'AUTONOMOUS',
            isControlPanelEnabled: false,
            allowedTools: [
                ClickTool_1.ClickTool.NAME,
                GoToGoogleMapsStreetViewTool_1.GoToGoogleMapsStreetViewTool.NAME,
                NavigateWithinStreetView_1.NavigateWithinStreetViewTool.NAME,
                ExtractGoogleStreetviewEntityDataTool_1.ExtractGoogleStreetviewEntityDataTool.NAME,
                AggregateExtractedStreetviewDataTool_1.AggregateExtractedStreetviewDataTool.NAME,
            ],
            toolCallsOnStart: [
                {
                    name: GoToGoogleMapsStreetViewTool_1.GoToGoogleMapsStreetViewTool.NAME,
                    parameters: {
                        entityName,
                        entityLocation,
                    },
                },
                {
                    name: ExtractGoogleStreetviewEntityDataTool_1.ExtractGoogleStreetviewEntityDataTool.NAME,
                    parameters: {
                        entityName,
                    },
                },
            ],
        };
        const flowHandle = await this.donobuFlowsManager.createFlow(flowParams);
        flowHandle.donobuFlow.metadata.inputTokensUsed += inputTokensForTmpFlow;
        flowHandle.donobuFlow.metadata.completionTokensUsed +=
            completionTokensForTmpFlow;
        await this.wrapUpWithStreetDataRecovery(res, flowHandle);
    }
    async divineTargetEntityDataFromGoogleMapUrl(googleMapUrl) {
        const browserConfig = this.maybeUseBrowserBase('Desktop Chromium');
        const flowParams = {
            targetWebsite: googleMapUrl,
            overallObjective: 'Extract the business/entity data from the current Google Maps page.',
            browser: browserConfig,
            maxIterations: 1,
            defaultToolTipDurationMilliseconds: 0,
            initialRunMode: 'AUTONOMOUS',
            isControlPanelEnabled: false,
            allowedTools: [
                GetEntityDataFromGoogleMapResult_1.GetEntityDataFromGoogleMapResultTool.NAME,
                GoToWebpageTool_1.GoToWebpageTool.NAME,
            ],
            toolCallsOnStart: [],
        };
        const flowHandle = await this.donobuFlowsManager.createFlow(flowParams);
        await flowHandle.job;
        return flowHandle.donobuFlow;
    }
    // Helper methods
    async wrapUp(res, flowHandle) {
        const flowId = flowHandle.donobuFlow.metadata.id;
        const flow = flowHandle.donobuFlow;
        await flowHandle.job;
        if (!flow.metadata.result) {
            res.status(500);
        }
        else if (typeof flow.metadata.result.failed === 'string' &&
            flow.metadata.result.failed.toLowerCase().includes('internal error')) {
            res.status(500);
        }
        else if (flow.metadata.state !== 'SUCCESS') {
            res.status(400);
        }
        res.header('x-input-tokens-used', flow.metadata.inputTokensUsed.toString());
        res.header('x-completion-tokens-used', flow.metadata.completionTokensUsed.toString());
        res.header('x-gpt-config-name', flow.metadata.gptConfigName ?? '');
        res.header('x-flow-id', flow.metadata.id);
        res.json(flow.metadata.result ?? {
            error: `Unexpected null result for flow ${flowId}.`,
        });
    }
    async wrapUpWithStreetDataRecovery(res, flowHandle) {
        const flowId = flowHandle.donobuFlow.metadata.id;
        const flow = flowHandle.donobuFlow;
        await flowHandle.job;
        let flowResult = flow.metadata.result;
        if (!flowResult) {
            Logger_1.appLogger.error(`Unexpected null result for flow ${flowId}.`);
            res.status(500);
        }
        else if (flow.metadata.state !== 'SUCCESS') {
            flowResult = AggregateExtractedStreetviewDataTool_1.AggregateExtractedStreetviewDataTool.aggregateStreetviewData(flow.invokedToolCalls, flow.metadata);
        }
        else {
            res.status(200);
        }
        res.header('x-input-tokens-used', flow.metadata.inputTokensUsed.toString());
        res.header('x-completion-tokens-used', flow.metadata.completionTokensUsed.toString());
        res.header('x-gpt-config-name', flow.metadata.gptConfigName ?? '');
        res.header('x-flow-id', flowHandle.donobuFlow.metadata.id);
        res.json(flowResult || {});
    }
    parseUrl(url) {
        if (!url) {
            return null;
        }
        if (!url.startsWith('http')) {
            url = `https://${url}`;
        }
        return url;
    }
    useDevice(deviceName) {
        return {
            initialState: undefined,
            persistState: false,
            using: {
                type: 'device',
                deviceName: deviceName,
                headless: true,
            },
        };
    }
    /**
     * Returns a BrowserConfig for BrowserBase if the BROWSERBASE_PROJECT_ID and
     * BROWSERBASE_API_KEY environment variables are present, otherwise, returns a
     * BrowserConfig for the given device name.
     */
    maybeUseBrowserBase(deviceName) {
        if (process.env[envVars_1.ENV_VAR_NAMES.BROWSERBASE_PROJECT_ID] &&
            process.env[envVars_1.ENV_VAR_NAMES.BROWSERBASE_API_KEY]) {
            return {
                initialState: undefined,
                persistState: false,
                using: {
                    type: 'browserBase',
                    sessionArgs: {
                        projectId: process.env[envVars_1.ENV_VAR_NAMES.BROWSERBASE_PROJECT_ID] ?? '',
                        browserSettings: {
                            advancedStealth: false, // Advanced Stealth is only available on BrowserBase enterprise plans.
                        },
                        proxies: true,
                    },
                },
            };
        }
        else {
            return {
                initialState: undefined,
                persistState: false,
                using: {
                    type: 'device',
                    deviceName: deviceName,
                    headless: true,
                },
            };
        }
    }
}
exports.SpecialFlowsApi = SpecialFlowsApi;
//# sourceMappingURL=SpecialFlowsApi.js.map