magnitude-core/dist/actions/webActions.js

Magnitude e2e testing agent

import { createAction } from ".";
import { z } from "zod";
import { BrowserConnector } from "@/connectors/browserConnector"; // Changed from WebInteractionFacet
// For separate grounding
export const clickTargetAction = createAction({
    name: 'mouse:click',
    description: "Click something",
    schema: z.object({
        target: z.string().describe("Where exactly to click"),
    }),
    resolver: async ({ input: { target }, agent }) => {
        const web = agent.require(BrowserConnector);
        const harness = web.getHarness();
        const screenshot = await web.getLastScreenshot();
        const { x, y } = await web.requireGrounding().locateTarget(screenshot, target);
        await harness.click({ x, y });
    },
    //render: ({ x, y}) => `⊙ Clicked (${})`
});
// For separate grounding
// export const clickTargetAndType = createAction({
//     name: 'browser:type',
//     description: "Click something and type into it",
//     schema: z.object({
//         target: z.string().describe("Where exactly to click before typing"),
//         content: z.string().describe("Content to type, insert sequences <enter> or <tab> for those keypresses respectively."),
//     }),
//     resolver: async ({ input: { target, content }, agent }) => {
//         const web = agent.require(BrowserConnector);
//         const harness = web.getHarness();
//         const screenshot = await web.getLastScreenshot();
//         const { x, y } = await web.requireGrounding().locateTarget(screenshot, target);
//         await harness.clickAndType({ x, y, content });
//     }
// });
// For separate grounding
export const scrollTargetAction = createAction({
    name: 'mouse:scroll',
    description: "Hover mouse over target and scroll",
    schema: z.object({
        target: z.string().describe("Somewhere specific inside the container to scroll in"),
        deltaX: z.number().int().describe("Pixels to scroll horizontally"),
        deltaY: z.number().int().describe("Pixels to scroll vertically"),
    }),
    resolver: async ({ input: { target, deltaX, deltaY }, agent }) => {
        const web = agent.require(BrowserConnector);
        const harness = web.getHarness();
        const screenshot = await web.getLastScreenshot();
        const { x, y } = await web.requireGrounding().locateTarget(screenshot, target);
        await harness.scroll({ x, y, deltaX, deltaY });
    }
});
// For grounded planner
export const clickCoordAction = createAction({
    name: 'mouse:click',
    description: "Click something",
    schema: z.object({
        x: z.number().int(),
        y: z.number().int(),
    }),
    resolver: async ({ input: { x, y }, agent }) => {
        const web = agent.require(BrowserConnector);
        const harness = web.getHarness();
        await harness.click({ x, y });
    },
    render: ({ x, y }) => `⊙ click (${x}, ${y})`
});
export const mouseDoubleClickAction = createAction({
    name: 'mouse:double_click',
    schema: z.object({
        x: z.number().int(),
        y: z.number().int(),
    }),
    resolver: async ({ input: { x, y }, agent }) => {
        const web = agent.require(BrowserConnector);
        const harness = web.getHarness();
        await harness.doubleClick({ x, y });
    },
    render: ({ x, y }) => `⊙ double click (${x}, ${y})`
});
export const mouseRightClickAction = createAction({
    name: 'mouse:right_click',
    schema: z.object({
        x: z.number().int(),
        y: z.number().int(),
    }),
    resolver: async ({ input: { x, y }, agent }) => {
        await agent.require(BrowserConnector).getHarness().rightClick({ x, y });
    },
    render: ({ x, y }) => `⊙ right click (${x}, ${y})`
});
export const mouseDragAction = createAction({
    name: 'mouse:drag',
    description: "Click and hold mouse in one location and release in another",
    schema: z.object({
        from: z.object({ x: z.number().int(), y: z.number().int() }),
        to: z.object({ x: z.number().int(), y: z.number().int() })
    }),
    resolver: async ({ input: { from, to }, agent }) => {
        const web = agent.require(BrowserConnector);
        const harness = web.getHarness();
        await harness.drag({ x1: from.x, y1: from.y, x2: to.x, y2: to.y });
    },
    render: ({ from, to }) => `⤡ drag (${from.x}, ${from.y}) -> (${to.x}, ${to.y})`
});
export const typeAction = createAction({
    name: 'keyboard:type',
    description: "Make sure to click where you need to type first", // make sure you click into it first
    schema: z.object({
        content: z.string().describe("Content to type"),
    }),
    resolver: async ({ input: { content }, agent }) => {
        const webConnector = agent.require(BrowserConnector);
        const harness = webConnector.getHarness();
        await harness.type({ content });
    },
    render: ({ content }) => `⌨︎ type "${content}"`
});
export const keyboardEnterAction = createAction({
    name: 'keyboard:enter',
    resolver: async ({ agent }) => {
        await agent.require(BrowserConnector).getHarness().enter();
    },
    render: () => `⏎ press enter`
});
export const keyboardTabAction = createAction({
    name: 'keyboard:tab',
    resolver: async ({ agent }) => {
        await agent.require(BrowserConnector).getHarness().tab();
    },
    render: () => `⇥ press tab`
});
export const keyboardBackspaceAction = createAction({
    name: 'keyboard:backspace',
    resolver: async ({ agent }) => {
        await agent.require(BrowserConnector).getHarness().backspace();
    },
    render: () => `⌫ press backspace`
});
export const keyboardSelectAllAction = createAction({
    name: 'keyboard:select_all',
    description: "Select all content in the active text area (CTRL+A)",
    resolver: async ({ input: { content }, agent }) => {
        await agent.require(BrowserConnector).getHarness().selectAll();
    },
    render: () => `⬚ select all`
});
// For grounded planner
export const scrollCoordAction = createAction({
    name: 'mouse:scroll',
    description: "Hover mouse over target and scroll",
    schema: z.object({
        x: z.number().int(),
        y: z.number().int(),
        deltaX: z.number().int().describe("Pixels to scroll horizontally"),
        deltaY: z.number().int().describe("Pixels to scroll vertically"),
    }),
    resolver: async ({ input: { x, y, deltaX, deltaY }, agent }) => {
        const webConnector = agent.require(BrowserConnector);
        const harness = webConnector.getHarness();
        await harness.scroll({ x, y, deltaX, deltaY });
    },
    render: ({ x, y, deltaX, deltaY }) => `↕ scroll (${deltaX}px, ${deltaY}px)`
});
// Grounding agnostic
export const switchTabAction = createAction({
    name: 'browser:tab:switch',
    description: "Switch to a tab that is already open",
    schema: z.object({
        index: z.number().int().describe("Index of tab to switch to"),
    }),
    resolver: async ({ input: { index }, agent }) => {
        const webConnector = agent.require(BrowserConnector);
        const harness = webConnector.getHarness();
        await harness.switchTab({ index });
    },
    render: ({ index }) => `⧉ switch to tab ${index}`
});
export const newTabAction = createAction({
    name: 'browser:tab:new',
    description: "Open and switch to a new tab",
    schema: z.object({}),
    resolver: async ({ agent }) => {
        const webConnector = agent.require(BrowserConnector);
        const harness = webConnector.getHarness();
        await harness.newTab();
    },
    render: () => `⊞ open new tab`
});
export const navigateAction = createAction({
    name: 'browser:nav',
    description: "Navigate to a URL directly",
    schema: z.object({
        url: z.string().describe('URL to navigate to'),
    }),
    resolver: async ({ input: { url }, agent }) => {
        const webConnector = agent.require(BrowserConnector);
        const harness = webConnector.getHarness();
        await harness.navigate(url);
    },
    render: ({ url }) => `⛓︎ navigate to ${url}`
});
export const goBackAction = createAction({
    name: 'browser:nav:back',
    description: "Go back",
    schema: z.object({}),
    resolver: async ({ agent }) => {
        const webConnector = agent.require(BrowserConnector);
        const harness = webConnector.getHarness();
        await harness.goBack();
    },
    render: () => `← navigate back`
});
// gets overused currently if we include this
export const waitAction = createAction({
    name: 'wait',
    description: "Wait for some time",
    schema: z.object({
        seconds: z.number()
    }),
    resolver: async ({ input: { seconds }, agent }) => {
        await new Promise((resolve) => setTimeout(resolve, seconds * 1000));
    },
    render: ({ seconds }) => `◴ wait for ${seconds}s`
});
// export const webActions = [
//     clickTargetAction,
//     clickTargetAndType,
//     scrollTargetAction,
//     switchTabAction,
// ] as const;
export const agnosticWebActions = [
    newTabAction,
    switchTabAction,
    navigateAction,
    typeAction,
    keyboardEnterAction,
    keyboardTabAction,
    keyboardBackspaceAction,
    keyboardSelectAllAction,
    //waitAction,
];
export const coordWebActions = [
    clickCoordAction,
    mouseDoubleClickAction,
    mouseRightClickAction,
    scrollCoordAction,
    mouseDragAction,
    //typeAction
];
export const targetWebActions = [
    clickTargetAction,
    //typeAction,
    //clickTargetAndType,
    scrollTargetAction
];