@auto-browse/auto-browse
Version:
AI-powered browser automation
387 lines (386 loc) • 14.7 kB
JavaScript
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import debug from 'debug';
import { callOnPageNoTrace, waitForCompletion } from './tools/utils.js';
import { ManualPromise } from './manualPromise.js';
import { Tab } from './tab.js';
import { outputFile, defaultConfig } from './config.js';
import { sessionManager } from './session-manager.js';
import { createSessionManagerContextFactory } from './browserContextFactory.js';
const testDebug = debug('pw:mcp:test');
export class Context {
tools;
config;
_browserContextPromise;
_browserContextFactory;
_tabs = [];
_currentTab;
_modalStates = [];
_pendingAction;
_downloads = [];
clientVersion;
// Singleton pattern for compatibility with existing auto-browse-ts code
static instance;
constructor(tools = [], config = defaultConfig, browserContextFactory) {
this.tools = tools;
this.config = config;
this._browserContextFactory = browserContextFactory || createSessionManagerContextFactory();
testDebug('create context');
}
static getInstance(tools, config) {
if (!Context.instance)
Context.instance = new Context(tools, config);
return Context.instance;
}
clientSupportsImages() {
if (this.config.imageResponses === 'allow')
return true;
if (this.config.imageResponses === 'omit')
return false;
return !this.clientVersion?.name.includes('cursor');
}
modalStates() {
return this._modalStates;
}
setModalState(modalState, inTab) {
this._modalStates.push({ ...modalState, tab: inTab });
}
clearModalState(modalState) {
this._modalStates = this._modalStates.filter(state => state !== modalState);
}
modalStatesMarkdown() {
const result = ['### Modal state'];
if (this._modalStates.length === 0)
result.push('- There is no modal state present');
for (const state of this._modalStates) {
const tool = this.tools.find(tool => tool.clearsModalState === state.type);
result.push(`- [${state.description}]: can be handled by the "${tool?.schema.name}" tool`);
}
return result;
}
tabs() {
return this._tabs;
}
currentTabOrDie() {
if (!this._currentTab)
throw new Error('No current snapshot available. Capture a snapshot or navigate to a new location first.');
return this._currentTab;
}
async newTab() {
const { browserContext } = await this._ensureBrowserContext();
const page = await browserContext.newPage();
this._currentTab = this._tabs.find(t => t.page === page);
return this._currentTab;
}
async selectTab(index) {
this._currentTab = this._tabs[index - 1];
await this._currentTab.page.bringToFront();
}
async ensureTab() {
// First try to get existing page from session manager
try {
const existingPage = sessionManager.getPage();
if (existingPage && !existingPage.isClosed()) {
// Find or create tab for existing page
let existingTab = this._tabs.find(t => t.page === existingPage);
if (!existingTab) {
existingTab = new Tab(this, existingPage, tab => this._onPageClosed(tab));
this._tabs.push(existingTab);
}
this._currentTab = existingTab;
return existingTab;
}
}
catch (error) {
// Session manager doesn't have a page, continue with normal flow
}
const { browserContext } = await this._ensureBrowserContext();
if (!this._currentTab)
await browserContext.newPage();
return this._currentTab;
}
async listTabsMarkdown() {
if (!this._tabs.length)
return '### No tabs open';
const lines = ['### Open tabs'];
for (let i = 0; i < this._tabs.length; i++) {
const tab = this._tabs[i];
const title = await tab.title();
const url = tab.page.url();
const current = tab === this._currentTab ? ' (current)' : '';
lines.push(`- ${i + 1}:${current} [${title}] (${url})`);
}
return lines.join('\n');
}
async closeTab(index) {
const tab = index === undefined ? this._currentTab : this._tabs[index - 1];
await tab?.page.close();
return await this.listTabsMarkdown();
}
async run(tool, params) {
// Tab management is done outside of the action() call.
const toolResult = await tool.handle(this, tool.schema.inputSchema.parse(params || {}));
const { code, action, waitForNetwork, captureSnapshot, resultOverride } = toolResult;
const racingAction = action ? () => this._raceAgainstModalDialogs(action) : undefined;
if (resultOverride)
return resultOverride;
if (!this._currentTab) {
return {
content: [{
type: 'text',
text: 'No open pages available. Use the "browser_navigate" tool to navigate to a page first.',
}],
};
}
const tab = this.currentTabOrDie();
// TODO: race against modal dialogs to resolve clicks.
let actionResult;
try {
if (waitForNetwork)
actionResult = await waitForCompletion(this, tab, async () => racingAction?.()) ?? undefined;
else
actionResult = await racingAction?.() ?? undefined;
}
finally {
if (captureSnapshot && !this._javaScriptBlocked())
await tab.captureSnapshot();
}
const result = [];
result.push(`- Ran Playwright code:
\`\`\`js
${code.join('\n')}
\`\`\`
`);
if (this.modalStates().length) {
result.push(...this.modalStatesMarkdown());
return {
content: [{
type: 'text',
text: result.join('\n'),
}],
};
}
if (this._downloads.length) {
result.push('', '### Downloads');
for (const entry of this._downloads) {
if (entry.finished)
result.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`);
else
result.push(`- Downloading file ${entry.download.suggestedFilename()} ...`);
}
result.push('');
}
if (this.tabs().length > 1)
result.push(await this.listTabsMarkdown(), '');
if (this.tabs().length > 1)
result.push('### Current tab');
result.push(`- Page URL: ${tab.page.url()}`, `- Page Title: ${await tab.title()}`);
if (captureSnapshot && tab.hasSnapshot())
result.push(tab.snapshotOrDie().text());
const content = actionResult?.content ?? [];
return {
content: [
...content,
{
type: 'text',
text: result.join('\n'),
}
],
};
}
async waitForTimeout(time) {
if (!this._currentTab || this._javaScriptBlocked()) {
await new Promise(f => setTimeout(f, time));
return;
}
await callOnPageNoTrace(this._currentTab.page, page => {
return page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
});
}
async _raceAgainstModalDialogs(action) {
this._pendingAction = {
dialogShown: new ManualPromise(),
};
let result;
try {
await Promise.race([
action().then(r => result = r),
this._pendingAction.dialogShown,
]);
}
finally {
this._pendingAction = undefined;
}
return result;
}
_javaScriptBlocked() {
return this._modalStates.some(state => state.type === 'dialog');
}
dialogShown(tab, dialog) {
this.setModalState({
type: 'dialog',
description: `"${dialog.type()}" dialog with message "${dialog.message()}"`,
dialog,
}, tab);
this._pendingAction?.dialogShown.resolve();
}
async downloadStarted(tab, download) {
const entry = {
download,
finished: false,
outputFile: await outputFile(this.config, download.suggestedFilename())
};
this._downloads.push(entry);
await download.saveAs(entry.outputFile);
entry.finished = true;
}
_onPageCreated(page) {
const tab = new Tab(this, page, tab => this._onPageClosed(tab));
this._tabs.push(tab);
if (!this._currentTab)
this._currentTab = tab;
}
_onPageClosed(tab) {
this._modalStates = this._modalStates.filter(state => state.tab !== tab);
const index = this._tabs.indexOf(tab);
if (index === -1)
return;
this._tabs.splice(index, 1);
if (this._currentTab === tab)
this._currentTab = this._tabs[Math.min(index, this._tabs.length - 1)];
if (!this._tabs.length)
void this.close();
}
async close() {
if (!this._browserContextPromise)
return;
testDebug('close context');
const promise = this._browserContextPromise;
this._browserContextPromise = undefined;
await promise.then(async ({ browserContext, close }) => {
if (this.config.saveTrace)
await browserContext.tracing.stop();
await close();
});
}
async _setupRequestInterception(context) {
if (this.config.network?.allowedOrigins?.length) {
await context.route('**', route => route.abort('blockedbyclient'));
for (const origin of this.config.network.allowedOrigins)
await context.route(`*://${origin}/**`, route => route.continue());
}
if (this.config.network?.blockedOrigins?.length) {
for (const origin of this.config.network.blockedOrigins)
await context.route(`*://${origin}/**`, route => route.abort('blockedbyclient'));
}
}
_ensureBrowserContext() {
if (!this._browserContextPromise) {
this._browserContextPromise = this._setupBrowserContext();
this._browserContextPromise.catch(() => {
this._browserContextPromise = undefined;
});
}
return this._browserContextPromise;
}
async _setupBrowserContext() {
const result = await this._browserContextFactory.createContext();
const { browserContext } = result;
await this._setupRequestInterception(browserContext);
// Handle existing pages from session manager
for (const page of browserContext.pages())
this._onPageCreated(page);
browserContext.on('page', page => this._onPageCreated(page));
if (this.config.saveTrace) {
await browserContext.tracing.start({
name: 'trace',
screenshots: false,
snapshots: true,
sources: false,
});
}
return result;
}
// Legacy compatibility methods for existing auto-browse-ts code
async allFramesSnapshot() {
const tab = this.currentTabOrDie();
const page = tab.page;
const visibleFrames = await page
.locator('iframe')
.filter({ visible: true })
.all();
const lastSnapshotFrames = visibleFrames.map(frame => frame.contentFrame());
const snapshots = await Promise.all([
page.locator('html').ariaSnapshot({ ref: true }),
...lastSnapshotFrames.map(async (frame, index) => {
const snapshot = await frame.locator('html').ariaSnapshot({
ref: true
});
const args = [];
const src = await frame.owner().getAttribute('src');
if (src)
args.push(`src=${src}`);
const name = await frame.owner().getAttribute('name');
if (name)
args.push(`name=${name}`);
return (`\n# iframe ${args.join(' ')}\n` +
snapshot.replaceAll('[ref=', `[ref=f${index}`));
})
]);
return snapshots.join('\n');
}
refLocator(ref) {
const tab = this.currentTabOrDie();
const page = tab.page;
let frame = page.mainFrame();
const match = ref.match(/^f(\d+)(.*)/);
if (match) {
const frameIndex = parseInt(match[1], 10);
const visibleFrames = page.locator('iframe').filter({ visible: true });
frame = visibleFrames.nth(frameIndex).contentFrame();
ref = match[2];
}
return frame.locator(`aria-ref=${ref}`);
}
async createPage() {
const tab = await this.ensureTab();
return tab.page;
}
existingPage() {
try {
return sessionManager.getPage();
}
catch {
if (this._currentTab)
return this._currentTab.page;
throw new Error('No page available');
}
}
async submitFileChooser(paths) {
const fileChooserState = this._modalStates.find(state => state.type === 'fileChooser');
if (!fileChooserState)
throw new Error('No file chooser visible');
await fileChooserState.fileChooser.setFiles(paths);
this.clearModalState(fileChooserState);
}
hasFileChooser() {
return this._modalStates.some(state => state.type === 'fileChooser');
}
clearFileChooser() {
this._modalStates = this._modalStates.filter(state => state.type !== 'fileChooser');
}
}
export const context = Context.getInstance();