magnitude-core/dist/agent/browserAgent.js

Magnitude e2e testing agent

import { Agent } from ".";
import { BrowserConnector } from "@/connectors/browserConnector";
import { buildDefaultBrowserAgentOptions } from "@/ai/util";
import { renderMinimalAccessibilityTree } from "@/web/util";
import { narrateBrowserAgent } from "./narrator";
import EventEmitter from "eventemitter3";
// export interface StartAgentWithWebOptions {
//     agentBaseOptions?: Partial<AgentOptions>;
//     webConnectorOptions?: BrowserConnectorOptions;
// }
const DEFAULT_BROWSER_AGENT_TEMP = 0.2;
// Helper function to start a web agent
export async function startBrowserAgent(options //StartAgentWithWebOptions = {}
) {
    const { agentOptions, browserOptions } = buildDefaultBrowserAgentOptions({ agentOptions: options ?? {}, browserOptions: options ?? {} });
    const agent = new BrowserAgent({
        agentOptions: agentOptions,
        browserOptions: browserOptions,
    });
    if (options?.narrate || process.env.MAGNITUDE_NARRATE) {
        narrateBrowserAgent(agent);
        //agent.events.on('actionStarted', (action: any) => { console.log(action) })
    }
    await agent.start();
    return agent;
}
export class BrowserAgent extends Agent {
    browserAgentEvents = new EventEmitter();
    constructor({ agentOptions, browserOptions }) {
        //console.log("agent options:", agent);
        super({
            ...agentOptions,
            connectors: [new BrowserConnector(browserOptions || {}), ...(agentOptions?.connectors ?? [])]
        });
    }
    get page() {
        return this.require(BrowserConnector).getHarness().page;
    }
    get context() {
        return this.require(BrowserConnector).getHarness().context;
    }
    async nav(url) {
        this.browserAgentEvents.emit('nav', url);
        await this.require(BrowserConnector).getHarness().navigate(url);
    }
    async extract(instructions, schema) {
        this.browserAgentEvents.emit('extractStarted', instructions, schema);
        //const htmlContent = await this.page.content();
        const accessibilityTree = await this.page.accessibility.snapshot({ interestingOnly: true });
        const pageRepr = renderMinimalAccessibilityTree(accessibilityTree);
        const screenshot = await this.require(BrowserConnector).getHarness().screenshot();
        const data = await this.model.extract(instructions, schema, screenshot, pageRepr);
        this.browserAgentEvents.emit('extractDone', instructions, data);
        return data;
    }
}