magnitude-core
Version:
Magnitude e2e testing agent
157 lines (156 loc) • 6.63 kB
JavaScript
//import { Observation, BamlRenderable } from "@/memory";
import { WebHarness } from "@/web/harness";
import { agnosticWebActions, coordWebActions, targetWebActions } from '@/actions/webActions';
import { BrowserProvider } from "@/web/browserProvider";
import logger from "@/logger";
import { Observation } from "@/memory/observation";
import { GroundingService, moondreamTargetingInstructions } from "@/ai/grounding";
export class BrowserConnector {
id = "web";
harness;
options;
browser;
context;
logger;
grounding;
constructor(options = {}) {
this.options = options;
this.logger = logger.child({
name: `connectors.${this.id}`
});
if (this.options.grounding) {
this.grounding = new GroundingService({ client: this.options.grounding });
}
}
requireGrounding() {
if (!this.grounding)
throw new Error("Grounding not configured on web connector");
return this.grounding;
}
async onStart() {
this.logger.info("Starting...");
const dpr = process.env.DEVICE_PIXEL_RATIO ?
parseInt(process.env.DEVICE_PIXEL_RATIO) :
process.platform === 'darwin' ? 2 : 1;
const browserContextOptions = {
viewport: { width: 1024, height: 768 },
deviceScaleFactor: dpr,
...this.options.browser?.contextOptions
};
const browserOptions = this.options.browser;
this.logger.info("Creating new browser context.");
this.context = await BrowserProvider.getInstance().newContext({
...browserOptions,
contextOptions: browserContextOptions
});
// if (browserOptions) {
// //let browserInstance: Browser;
// if ('instance' in browserOptions) {
// //browserInstance = browserOptions.instance;
// this.context = await browserOptions.instance.newContext(browserContextOptions);
// } else {
// // todo: launch browser via browserprovider with these launch options
// }
// } else {
// //browserInstance = await BrowserProvider.getInstance();
// this.context = await BrowserProvider.getInstance().newContext(browserContextOptions);
// this.logger.info("Using singleton browser provider.");
// }
//browserOptions.instance
//let browserInstance = this.options.browser?.instance;
// if (browserInstance) {
// this.context = await browserInstance.newContext(browserContextOptions)
// } else {
// this.context = await BrowserProvider.getInstance().newContext(browserContextOptions);
// this.logger.info("Using singleton browser provider.");
// }
this.harness = new WebHarness(this.context, {
virtualScreenDimensions: this.options.virtualScreenDimensions
});
await this.harness.start();
this.logger.info("WebHarness started.");
if (this.options.url) {
this.logger.info(`Navigating to initial URL: ${this.options.url}`);
await this.harness.navigate(this.options.url);
//await this.harness.waitForStability();
}
this.logger.info("Started successfully.");
}
async onStop() {
this.logger.info("Stopping...");
if (this.context) {
await this.context.close();
this.logger.info("Browser context closed.");
}
// Note: We don't close this.browser here if obtained from BrowserProvider,
// as BrowserProvider manages the singleton browser lifecycle.
// If this.options.browser was provided, its lifecycle is managed externally.
this.logger.info("Stopped successfully.");
}
getActionSpace() {
if (this.grounding) {
// Separate grounding
return [...targetWebActions, ...agnosticWebActions];
}
else {
// Planner is grounded
return [...coordWebActions, ...agnosticWebActions];
}
}
// public get page(): Page {
// if (!this.harness || !this.harness.page) {
// throw new Error("WebInteractionConnector: Harness or Page is not available. Ensure onStart has completed.");
// }
// return this.harness.page;
// }
getHarness() {
if (!this.harness) {
throw new Error("WebInteractionConnector: Harness is not available. Ensure onStart has completed.");
}
return this.harness;
}
async captureCurrentState() {
if (!this.harness || !this.harness.page) {
throw new Error("WebInteractionConnector: Harness or Page is not available for capturing state.");
}
const [screenshot, tabs] = await Promise.all([
this.harness.screenshot(),
this.harness.retrieveTabState()
]);
//const resizedScreenshot = await screenshot.resize()
// if (this.options.autoResize) {
// return { screenshot: await screenshot.resize(this.options.autoResize.width, this.options.autoResize.height), tabs: tabs };
// }
return { screenshot: await this.transformScreenshot(screenshot), tabs: tabs };
}
async transformScreenshot(screenshot) {
if (this.options.virtualScreenDimensions) {
return await screenshot.resize(this.options.virtualScreenDimensions.width, this.options.virtualScreenDimensions.height);
}
else {
return screenshot;
}
}
async getLastScreenshot() {
//return { image: "", dimensions: { width: 0, height: 0 } };
// TODO: better to use last
return (await this.captureCurrentState()).screenshot;
}
async collectObservations() {
const currentState = await this.captureCurrentState();
const observations = [];
const currentTabs = currentState.tabs;
let tabInfo = "Open Tabs:\n";
currentTabs.tabs.forEach((tab, index) => {
tabInfo += `${index === currentTabs.activeTab ? '[ACTIVE] ' : ''}${tab.title} (${tab.url})`;
});
observations.push(Observation.fromConnector(this.id, await this.transformScreenshot(currentState.screenshot), { type: 'screenshot', limit: 3, dedupe: true }));
observations.push(Observation.fromConnector(this.id, tabInfo, { type: 'tabinfo', limit: 1 }));
return observations;
}
async getInstructions() {
if (this.grounding) {
return moondreamTargetingInstructions;
}
}
}