magnitude-core/dist/agent/browserAgent.js

Magnitude e2e testing agent

import { Agent } from ".";
import { BrowserConnector } from "@/connectors/browserConnector";
import { buildDefaultBrowserAgentOptions } from "@/ai/util";
import { narrateBrowserAgent } from "./narrator";
import { partitionHtml, serializeToMarkdown } from 'magnitude-extract';
import EventEmitter from "eventemitter3";
// export interface StartAgentWithWebOptions {
//     agentBaseOptions?: Partial<AgentOptions>;
//     webConnectorOptions?: BrowserConnectorOptions;
// }
const DEFAULT_BROWSER_AGENT_TEMP = 0.2;
// Helper function to start a web agent
export async function startBrowserAgent(options //StartAgentWithWebOptions = {}
) {
    //console.log("sba options:", options);
    const { agentOptions, browserOptions } = buildDefaultBrowserAgentOptions({ agentOptions: options ?? {}, browserOptions: options ?? {} });
    const agent = new BrowserAgent({
        agentOptions: agentOptions,
        browserOptions: browserOptions,
    });
    if (options?.narrate || process.env.MAGNITUDE_NARRATE) {
        narrateBrowserAgent(agent);
        //agent.events.on('actionStarted', (action: any) => { console.log(action) })
    }
    //console.log('starting agent')
    await agent.start();
    //console.log('agent started');
    return agent;
}
async function getFullPageContent(page) {
    // 1. Get all iframe element handles
    const iframeHandles = await page.locator('iframe').elementHandles();
    // 2. Iterate through each iframe handle
    for (const iframeHandle of iframeHandles) {
        // 3. Get the Frame object for the iframe
        const frame = await iframeHandle.contentFrame();
        if (frame) {
            // 4. Get the HTML content of the iframe
            const iframeContent = await frame.content();
            // 5. Use evaluate to replace the iframe element with its content.
            // We pass the content as an argument to avoid issues with string escaping.
            await iframeHandle.evaluate((iframeNode, { content }) => {
                // Create a new div element to hold the iframe's content
                const div = document.createElement('div');
                div.innerHTML = content;
                // Add a data-attribute to mark that this was an expanded iframe
                div.dataset.expandedFromIframe = 'true';
                div.dataset.iframeSrc = iframeNode.getAttribute('src') || '';
                // Replace the iframeNode with the new div
                iframeNode.parentNode?.replaceChild(div, iframeNode);
            }, { content: iframeContent });
        }
    }
    // 6. Return the final, modified page content
    return page.content();
}
export class BrowserAgent extends Agent {
    browserAgentEvents = new EventEmitter();
    constructor({ agentOptions, browserOptions }) {
        //console.log("agent options:", agent);
        //console.log("browser options:", browserOptions);
        super({
            ...agentOptions,
            connectors: [new BrowserConnector(browserOptions || {}), ...(agentOptions?.connectors ?? [])]
        });
    }
    get page() {
        return this.require(BrowserConnector).getHarness().page;
    }
    get context() {
        return this.require(BrowserConnector).getHarness().context;
    }
    async nav(url) {
        this.browserAgentEvents.emit('nav', url);
        await this.require(BrowserConnector).getHarness().navigate(url);
    }
    async extract(instructions, schema) {
        this.browserAgentEvents.emit('extractStarted', instructions, schema);
        //const htmlContent = await this.page.content();
        const htmlContent = await getFullPageContent(this.page);
        // const accessibilityTree = await this.page.accessibility.snapshot({ interestingOnly: true });
        // const pageRepr = renderMinimalAccessibilityTree(accessibilityTree);
        const partitionOptions = {
            extractImages: true,
            extractForms: true,
            extractLinks: true,
            skipNavigation: false, // NAVIGATION SKIPPING IS BROKEN
            minTextLength: 3,
            includeOriginalHtml: false,
            includeMetadata: true
        };
        // Process HTML
        const result = partitionHtml(htmlContent, partitionOptions);
        // Configure markdown serializer options
        const markdownOptions = {
            includeMetadata: false,
            includePageNumbers: true,
            includeElementIds: false,
            includeCoordinates: false,
            preserveHierarchy: true,
            escapeSpecialChars: true,
            includeFormFields: true,
            includeImageMetadata: true
        };
        // Convert to markdown
        const markdown = serializeToMarkdown(result, markdownOptions);
        const screenshot = await this.require(BrowserConnector).getHarness().screenshot();
        const data = await this.models.extract(instructions, schema, screenshot, markdown);
        this.browserAgentEvents.emit('extractDone', instructions, data);
        return data;
    }
}