donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
856 lines • 41.5 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.extendPage = extendPage;
const crypto_1 = require("crypto");
const v4_1 = require("zod/v4");
const GptClient_1 = require("../../clients/GptClient");
const VercelAiGptClient_1 = require("../../clients/VercelAiGptClient");
const envVars_1 = require("../../envVars");
const GptApiKeysNotSetupException_1 = require("../../exceptions/GptApiKeysNotSetupException");
const TestNotFoundException_1 = require("../../exceptions/TestNotFoundException");
const ToolCallFailedException_1 = require("../../exceptions/ToolCallFailedException");
const ToolRequiresGptException_1 = require("../../exceptions/ToolRequiresGptException");
const DonobuFlowsManager_1 = require("../../managers/DonobuFlowsManager");
const InteractionVisualizer_1 = require("../../managers/InteractionVisualizer");
const ToolManager_1 = require("../../managers/ToolManager");
const WebTargetInspector_1 = require("../../managers/WebTargetInspector");
const ControlPanel_1 = require("../../models/ControlPanel");
const AnalyzePageTextTool_1 = require("../../tools/AnalyzePageTextTool");
const AssertTool_1 = require("../../tools/AssertTool");
const AuditTool_1 = require("../../tools/AuditTool");
const ChangeWebBrowserTabTool_1 = require("../../tools/ChangeWebBrowserTabTool");
const CreateBrowserCookieReportTool_1 = require("../../tools/CreateBrowserCookieReportTool");
const GoToWebpageTool_1 = require("../../tools/GoToWebpageTool");
const RunAccessibilityTestTool_1 = require("../../tools/RunAccessibilityTestTool");
const ansi_1 = require("../../utils/ansi");
const BrowserUtils_1 = require("../../utils/BrowserUtils");
const buildProvenance_1 = require("../../utils/buildProvenance");
const Logger_1 = require("../../utils/Logger");
const MiscUtils_1 = require("../../utils/MiscUtils");
const PlaywrightUtils_1 = require("../../utils/PlaywrightUtils");
const cache_1 = require("../ai/cache/cache");
const cacheLocator_1 = require("../ai/cache/cacheLocator");
const locateElement_1 = require("../ai/locate/locateElement");
const PageAi_1 = require("../ai/PageAi");
const gptClients_1 = require("../test/fixtures/gptClients");
const donobuTestStack_1 = require("../test/utils/donobuTestStack");
const originalGotoRegistry_1 = require("./originalGotoRegistry");
const SmartSelector_1 = require("./SmartSelector");
const tbd_1 = require("./tbd");
/**
* Resolve a URL against the browser context's baseURL (if set), mirroring
* what Playwright does internally before issuing the navigation request.
* This gives us the *intended* URL (before any server-side redirects).
*/
function resolveBaseUrl(page, url) {
const baseURL = page.context()._options?.baseURL;
// Only resolve against baseURL if the URL is relative (no scheme like https:).
if (baseURL && !/^[a-zA-Z][a-zA-Z\d+\-.]*:/.test(url)) {
try {
return new URL(url, baseURL).href;
}
catch {
return url;
}
}
return url;
}
// Donobu page extension helpers: decorate Playwright pages with Donobu behaviors and keep one
// coherent flow (and persistence record) per browser context so new tabs share state safely.
const PLACEHOLDER_FLOW_URL = 'https://example.com';
// Cache the shared Donobu state per browser context so every tab in that context reuses the same
// flow metadata, persistence, GPT client, and visualizer. WeakMap ensures cleanup when contexts die.
const contextSharedState = new WeakMap();
/**
* Decorate a Playwright page with Donobu capabilities.
* - If the browser context already has an extended page, reuse its shared state (flowId,
* persistence, GPT client, cache settings); passing a different flowId will throw.
* - Otherwise, initialize a fresh flow and register it for future tabs in this context.
*/
async function extendPage(page, options) {
await PlaywrightUtils_1.PlaywrightUtils.setupBasicBrowserContext(page.context());
const browserContext = page.context();
let contextEntry = contextSharedState.get(browserContext);
if (contextEntry) {
const existingFlowId = contextEntry.sharedState.donobuFlowMetadata.id;
if (options?.flowId && options.flowId !== existingFlowId) {
throw new Error(`extendPage called with flowId ${options.flowId}, but this browser context is already bound to flow ${existingFlowId}. New tabs share the existing flow; omit flowId or reuse the same one.`);
}
}
else {
const donobuStack = options?.donobuStack ?? (await (0, donobuTestStack_1.getOrCreateDonobuStack)());
const persistence = await donobuStack.flowsPersistenceRegistry.get();
const testsPersistence = await donobuStack.testsPersistenceRegistry.get();
const initialBrowserState = await BrowserUtils_1.BrowserUtils.getBrowserStorageState(browserContext);
const resolvedGptClient = await buttonUpGptClient(options?.gptClient);
const defaultMessageDuration = options?.visualCueDurationMs ?? 0;
const interactionVisualizer = new InteractionVisualizer_1.InteractionVisualizer(defaultMessageDuration);
const sharedState = {
donobuFlowMetadata: {
id: options?.flowId ?? (0, crypto_1.randomUUID)(),
metadataVersion: 1,
name: null,
createdWithDonobuVersion: MiscUtils_1.MiscUtils.DONOBU_VERSION,
target: 'web',
web: {
browser: {
persistState: true,
using: {
type: 'device',
deviceName: 'Desktop Chromium',
headless: options?.headless,
},
},
targetWebsite: PLACEHOLDER_FLOW_URL,
},
envVars: options?.envVars ?? null,
gptConfigName: null,
hasGptConfigNameOverride: false,
customTools: null,
defaultMessageDuration: defaultMessageDuration,
runMode: 'DETERMINISTIC',
isControlPanelEnabled: false,
callbackUrl: null,
overallObjective: null,
allowedTools: [],
resultJsonSchema: null,
result: null,
inputTokensUsed: 0,
completionTokensUsed: 0,
maxToolCalls: null,
startedAt: new Date().getTime(),
completedAt: null,
state: 'RUNNING_ACTION',
nextState: null,
provenance: (0, buildProvenance_1.buildProvenance)('CODE'),
},
interactionVisualizer: interactionVisualizer,
donobuStack: donobuStack,
pageAi: undefined,
pageAiCache: undefined,
persistence: persistence,
testsPersistence: testsPersistence,
initialBrowserState: initialBrowserState,
gptClient: resolvedGptClient,
controlPanelFactory: options?.controlPanelFactory,
runtimeDirectives: {
clearPageAiCache: MiscUtils_1.MiscUtils.yn(envVars_1.env.data.DONOBU_PAGE_AI_CLEAR_CACHE),
},
tbdSessions: [],
aiInvocations: [],
};
const showMouse = async (p) => {
if (interactionVisualizer.defaultMessageDurationMillis > 0) {
await interactionVisualizer.showMouse(p);
}
};
contextEntry = {
sharedState,
cacheFilepath: options?.cacheFilepath,
showMouse,
pageListenerRegistered: false,
};
contextSharedState.set(browserContext, contextEntry);
}
const { sharedState, cacheFilepath, showMouse } = contextEntry;
const extendedPage = applyDonobuExtensions(page, sharedState, cacheFilepath);
if (!contextEntry.pageListenerRegistered) {
browserContext.on('page', (newPage) => {
newPage.on('domcontentloaded', showMouse);
});
contextEntry.pageListenerRegistered = true;
}
page.on('domcontentloaded', showMouse);
return extendedPage;
}
/**
* Apply Donobu behavior to a page, reusing the provided shared state.
* Attaches AI, assertions, tool runner, tab switching, and other helpers so every tab
* records into the same flow.
*/
function applyDonobuExtensions(target, sharedState, cacheFilepath) {
if (!(0, originalGotoRegistry_1.hasOriginalGoto)(target)) {
(0, originalGotoRegistry_1.registerOriginalGoto)(target, target.goto);
}
/**
* Lazily initialises the shared PageAiCache, used by both page.ai flow
* caching and assertion caching (stored in the same file).
*/
function getOrInitPageAiCache() {
if (!sharedState.pageAiCache) {
sharedState.pageAiCache = cacheFilepath
? new cache_1.FilePageAiCache(cacheFilepath)
: new cache_1.InMemoryPageAiCache([]);
}
return sharedState.pageAiCache;
}
const page = target;
page._dnb = sharedState;
async function act(instruction, options) {
if (!sharedState.pageAi) {
const gptClient = getGptClient(page);
if (!gptClient) {
throw new GptApiKeysNotSetupException_1.GptApiKeysNotSetupException(`No AI connection available. Establish a connection by setting an API key via an environment variable.
Valid options:
- ${envVars_1.env.keys.ANTHROPIC_API_KEY}
- ${envVars_1.env.keys.GOOGLE_GENERATIVE_AI_API_KEY}
- ${envVars_1.env.keys.OPENAI_API_KEY}`);
}
const cache = getOrInitPageAiCache();
sharedState.pageAi = new PageAi_1.PageAi(sharedState.donobuStack, gptClient, cache);
}
const pageAiInstance = sharedState.pageAi;
if (!pageAiInstance) {
throw new Error('Failed to initialize Page AI.');
}
const clearAiCacheForRun = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
const revisedOptions = {
envVars: sharedState.donobuFlowMetadata.envVars ?? [],
gptClient: getGptClient(page, options?.gptClient),
...(options ? options : {}),
};
if (clearAiCacheForRun) {
await pageAiInstance.invalidate(page, instruction, revisedOptions);
Logger_1.appLogger.debug(`Page.AI cache invalidated for this run via ${envVars_1.env.keys.DONOBU_PAGE_AI_CLEAR_CACHE}`);
}
return pageAiInstance.ai(page, instruction, revisedOptions);
}
const pageAi = Object.assign(act, {
act,
assert: async (assertion, options) => {
const aiInvocationStartedAt = Date.now();
let aiInvocationCacheHit = false;
let aiInvocationCacheStored = false;
let aiInvocationError = undefined;
let aiInvocationAssertSteps;
let aiInvocationVerification;
try {
const useCache = options?.cache !== false;
const clearCache = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
const retries = options?.retries ?? 0;
const retryDelaySeconds = options?.retryDelaySeconds ?? 3;
// Distill env var names from `{{$.env.*}}` interpolations in the
// assertion plus any explicitly provided names/overrides. Cached
// Playwright steps may carry the same `{{$.env.X}}` placeholders in
// their `value`/`attributeValue` fields, so we resolve env data at
// replay time and let the executor interpolate before applying.
const envVarNames = (0, DonobuFlowsManager_1.distillAllowedEnvVariableNames)(assertion, [
...(options?.envVars ?? []),
...Object.keys(options?.envVals ?? {}),
]);
const hasEnvRefs = envVarNames.length > 0;
const resolveEnvData = async () => {
if (!hasEnvRefs) {
return undefined;
}
const envData = await sharedState.donobuStack.envDataManager.getByNames(envVarNames);
if (options?.envVals) {
for (const [k, v] of Object.entries(options.envVals)) {
if (v === undefined) {
delete envData[k];
}
else {
envData[k] = v;
}
}
}
return envData;
};
// --- Cache lookup (when enabled and not clearing) ---
if (useCache && !clearCache) {
const cache = getOrInitPageAiCache();
const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
const cached = await cache.getAssert({ pageUrl, assertion });
if (cached) {
aiInvocationCacheHit = true;
aiInvocationAssertSteps = cached.steps;
Logger_1.appLogger.debug(`Assert cache HIT for: "${assertion}" - running cached Playwright assertion`);
const envData = await resolveEnvData();
let lastError = null;
for (let attempt = 0; attempt <= retries; attempt++) {
if (attempt > 0) {
Logger_1.appLogger.info(`Retry ${attempt} of ${retries} for cached assert`);
await page.waitForTimeout(retryDelaySeconds * 1000);
}
try {
await cached.run({ page, envData });
return; // Assertion passed
}
catch (error) {
lastError = error;
}
}
// All retry attempts exhausted
throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, {
isSuccessful: false,
// Strip ANSI: Playwright matchers style their messages for the
// terminal, but this string flows into JSON-stringified exception
// messages, the LLM, and HTML/markdown reports — places where the
// codes never render and just become visible junk.
forLlm: `Assertion FAILED (cached) for: ${assertion}\nPlaywright Error: ${(0, ansi_1.stripAnsi)(lastError?.message ?? '')}`,
metadata: {
cached: true,
steps: cached.steps,
},
});
}
}
// --- Cache invalidation (when clearing) ---
if (useCache && clearCache) {
const cache = getOrInitPageAiCache();
const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
await cache.deleteAssert({ pageUrl, assertion });
Logger_1.appLogger.debug(`Assert cache invalidated for: "${assertion}"`);
}
// Make env vars available to runTool's envData for `{{$.env.*}}`
// interpolation inside `assertionToTestFor` and so AssertTool can
// instruct the AI to emit placeholders in cached step values. Mirrors
// PageAi.ai for `act`: metadata.envVars is set (overwriting), envVals
// is restored.
if (hasEnvRefs) {
sharedState.donobuFlowMetadata.envVars = envVarNames;
}
const previousEnvVals = sharedState.envVals;
sharedState.envVals = options?.envVals;
let result;
try {
// --- Cache miss or cache disabled: run AI assertion ---
result = await runTool(page, AssertTool_1.AssertTool.NAME, {
assertionToTestFor: assertion,
retries: options?.retries,
retryWaitSeconds: options?.retryDelaySeconds,
}, options?.gptClient);
}
finally {
sharedState.envVals = previousEnvVals;
}
aiInvocationVerification = result.outcome.metadata?.verification;
if (!result.outcome.isSuccessful) {
throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, result.outcome);
}
// --- Cache the Playwright assertion for future runs ---
if (useCache) {
const steps = result.outcome.metadata?.playwrightAssertionSteps;
if (Array.isArray(steps) && steps.length > 0) {
try {
const cache = getOrInitPageAiCache();
const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
await cache.putAssert({ pageUrl, assertion, steps });
aiInvocationCacheStored = true;
Logger_1.appLogger.debug(`Assert cache STORED for: "${assertion}"`);
}
catch (error) {
Logger_1.appLogger.debug(`Skipping assert cache for: "${assertion}" - failed to persist: ${error.message}`);
}
}
}
}
catch (e) {
aiInvocationError = e;
throw e;
}
finally {
sharedState.aiInvocations.push({
kind: 'assert',
description: assertion,
startedAt: aiInvocationStartedAt,
endedAt: Date.now(),
cacheHit: aiInvocationCacheHit,
cacheStored: aiInvocationCacheStored,
passed: aiInvocationError === undefined,
error: aiInvocationError !== undefined
? { message: aiInvocationError?.message }
: undefined,
assertSteps: aiInvocationAssertSteps,
verification: aiInvocationVerification,
});
}
},
extract: async (schema, options) => {
const gptClient = getGptClient(page, options?.gptClient);
if (!gptClient) {
throw new ToolRequiresGptException_1.ToolRequiresGptException('extract');
}
const webpageRawText = await page.locator('body').innerText({
timeout: 5000,
});
const systemMsg = {
type: 'system',
text: `The current date and time in ISO 8601 format is ${new Date().toISOString()}.
The current webpage URL is: ${page.url()}
The current webpage title is: ${await page.title()}
You will also be given:
- A viewport screenshot of the webpage.
- The raw textual content of the webpage. Note that since this is raw textual content,
the text may be a bit jumbled, have its styling lost, careful positioning lost, etc.
${options?.instruction ? '- A user instruction.\n' : ''}
Use this information to return an appropriate JSON object.`,
};
const screenShotItem = {
type: 'jpeg',
bytes: await PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(page),
};
const webpageTextItem = {
type: 'text',
text: webpageRawText,
};
const userInstructionItem = options?.instruction
? {
type: 'text',
text: `USER INSTRUCTION: ${options.instruction}`,
}
: undefined;
const userMsg = {
type: 'user',
items: [
screenShotItem,
webpageTextItem,
...(userInstructionItem ? [userInstructionItem] : []),
],
};
// Set up timeout to prevent indefinite hangs
const timeoutMillis = options?.timeout ?? 60000;
const abortController = new AbortController();
const timeoutId = setTimeout(() => {
abortController.abort(`Extract operation timed out after ${timeoutMillis} milliseconds`);
}, timeoutMillis);
try {
const resp = await gptClient.getStructuredOutput([systemMsg, userMsg], schema, { signal: abortController.signal });
sharedState.donobuFlowMetadata.resultJsonSchema =
v4_1.z.toJSONSchema(schema);
sharedState.donobuFlowMetadata.result = JSON.parse(JSON.stringify(resp.output));
await sharedState.persistence.setFlowMetadata(sharedState.donobuFlowMetadata);
return schema.parse(resp.output);
}
finally {
clearTimeout(timeoutId);
}
},
analyzePageText: async (analysisToRun, options) => {
const result = await page.run(AnalyzePageTextTool_1.AnalyzePageTextTool.NAME, {
analysisToRun: analysisToRun,
additionalRelevantContext: options?.additionalContext ?? '',
}, options);
return result.forLlm;
},
createCookieReport: async (options) => {
const result = await page.run(CreateBrowserCookieReportTool_1.CreateBrowserCookieReportTool.NAME, {}, options);
return result.metadata;
},
locate: async (description, options) => {
const aiInvocationStartedAt = Date.now();
let aiInvocationCacheHit = false;
let aiInvocationCacheStored = false;
let aiInvocationError = undefined;
const useCache = options?.cache !== false;
const clearCache = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
// Distill env var names referenced by the description plus any
// explicitly provided names/overrides. Resolve env data locally — locate
// does not flow through `runTool`, so we don't mutate sharedState here.
const envVarNames = (0, DonobuFlowsManager_1.distillAllowedEnvVariableNames)(description, [
...(options?.envVars ?? []),
...Object.keys(options?.envVals ?? {}),
]);
const hasEnvRefs = envVarNames.length > 0;
const resolveEnvData = async () => {
if (!hasEnvRefs) {
return undefined;
}
const envData = await sharedState.donobuStack.envDataManager.getByNames(envVarNames);
if (options?.envVals) {
for (const [k, v] of Object.entries(options.envVals)) {
if (v === undefined) {
delete envData[k];
}
else {
envData[k] = v;
}
}
}
return envData;
};
// The user-supplied `timeout` (default 30s) is the budget for the
// ENTIRE locate operation — cache-hit hydration wait + AI fallback.
// We start the abort timer here so the cache path's `waitFor` and the
// AI path share one bounded clock.
const timeoutMillis = options?.timeout ?? 30_000;
const startedAt = Date.now();
const abortController = new AbortController();
const timeoutId = setTimeout(() => {
abortController.abort(`Locate operation timed out after ${timeoutMillis} milliseconds`);
}, timeoutMillis);
try {
// --- Cache lookup (when enabled and not clearing) ---
if (useCache && !clearCache) {
const cache = getOrInitPageAiCache();
const cached = await cache.getLocate({ pageUrl, description });
if (cached) {
const envData = await resolveEnvData();
const candidate = cached.run({ page, envData });
// Cache replay can outrun page hydration — the no-cache path
// gets an implicit hydration window from the AI round-trip
// latency, but a cache hit fires immediately and may see a
// partially-mounted DOM. Wait (within the operation's overall
// budget) for the locator to attach before validating.
const remaining = Math.max(timeoutMillis - (Date.now() - startedAt), 100);
try {
await candidate.first().waitFor({
state: 'attached',
timeout: remaining,
});
Logger_1.appLogger.debug(`Locate cache HIT for: "${description}" — rebuilt locator from cache`);
Logger_1.appLogger.info(`Located: ${candidate}`);
aiInvocationCacheHit = true;
return candidate;
}
catch {
// Locator did not attach within the patience window. Either
// the page has drifted or the cache is genuinely stale.
// Invalidate and fall through to the AI path; the AI call
// gets whatever budget remains on the abort timer.
Logger_1.appLogger.debug(`Locate cache STALE for "${description}" (no match within ${remaining}ms) — re-running AI`);
await cache.deleteLocate({ pageUrl, description });
}
}
}
// --- Cache invalidation (when clearing) ---
if (useCache && clearCache) {
const cache = getOrInitPageAiCache();
await cache.deleteLocate({ pageUrl, description });
Logger_1.appLogger.debug(`Locate cache invalidated for: "${description}"`);
}
// --- Cache miss / cache disabled / stale-cache fallthrough: run AI ---
const gptClient = getGptClient(page, options?.gptClient);
if (!gptClient) {
throw new ToolRequiresGptException_1.ToolRequiresGptException('locate');
}
const envData = await resolveEnvData();
const { locator, result } = await (0, locateElement_1.locateElement)(page, description, gptClient, { signal: abortController.signal, envData });
// --- Cache the result for future runs ---
if (useCache) {
try {
const cache = getOrInitPageAiCache();
await cache.putLocate({ pageUrl, description, result });
aiInvocationCacheStored = true;
Logger_1.appLogger.debug(`Locate cache STORED for: "${description}"`);
}
catch (error) {
Logger_1.appLogger.debug(`Skipping locate cache for: "${description}" — failed to persist: ${error.message}`);
}
}
Logger_1.appLogger.info(`Located: ${locator}`);
return locator;
}
catch (e) {
aiInvocationError = e;
throw e;
}
finally {
sharedState.aiInvocations.push({
kind: 'locate',
description,
startedAt: aiInvocationStartedAt,
endedAt: Date.now(),
cacheHit: aiInvocationCacheHit,
cacheStored: aiInvocationCacheStored,
passed: aiInvocationError === undefined,
error: aiInvocationError !== undefined
? { message: aiInvocationError?.message }
: undefined,
});
clearTimeout(timeoutId);
}
},
});
page.ai = pageAi;
page.find = (selector, options) => {
return new SmartSelector_1.SmartSelectorImpl(page, {
element: [selector, ...(options?.failover ?? [])],
frame: options?.frame,
});
};
page.goto = async (url, options) => {
const startedAt = new Date().getTime();
const flowId = sharedState.donobuFlowMetadata.id;
const effectiveUrl = resolveBaseUrl(page, url);
// First navigation sets the target website and records the tool call with screenshots.
if (sharedState.donobuFlowMetadata.web?.targetWebsite === PLACEHOLDER_FLOW_URL) {
sharedState.donobuFlowMetadata = {
...sharedState.donobuFlowMetadata,
web: {
...sharedState.donobuFlowMetadata.web,
targetWebsite: effectiveUrl,
},
};
await sharedState.persistence.setFlowMetadata(sharedState.donobuFlowMetadata);
// Keep the test record in sync with the resolved target URL.
const testId = sharedState.donobuFlowMetadata.testId;
if (testId) {
try {
const test = await sharedState.testsPersistence.getTestById(testId);
test.web = sharedState.donobuFlowMetadata.web;
await sharedState.testsPersistence.updateTest(test);
}
catch (error) {
if (error instanceof TestNotFoundException_1.TestNotFoundException) {
Logger_1.appLogger.warn(`Flow ${flowId} metadata is referencing a test that does not exist. Test ID: ${testId}`);
}
else {
Logger_1.appLogger.error(`Failed to update test with new target URL. Test ID: ${testId}`, error);
}
}
}
}
try {
const originalGoto = (0, originalGotoRegistry_1.getOriginalGoto)(page);
const resp = await originalGoto.call(page, url, options);
const pageTitle = await page.title();
const postCallImage = await PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(page);
const postCallImageId = await sharedState.persistence.saveScreenShot(flowId, postCallImage);
const completedAt = new Date().getTime();
await sharedState.persistence.setToolCall(flowId, {
id: MiscUtils_1.MiscUtils.createAdHocToolCallId(),
toolName: GoToWebpageTool_1.GoToWebpageTool.NAME,
parameters: {
url: effectiveUrl,
},
outcome: {
isSuccessful: true,
forLlm: `Successfully navigated to ${effectiveUrl}`,
metadata: {
pageTitle: pageTitle,
resolvedUrl: page.url(),
},
},
postCallImageId: postCallImageId,
page: effectiveUrl,
startedAt: startedAt,
completedAt: completedAt,
});
return resp;
}
catch (error) {
// Best-effort screenshot and tool-call recording - if the page is gone
// these will fail, and we must not let that mask the original error.
try {
const postCallImage = await PlaywrightUtils_1.PlaywrightUtils.takeViewportScreenshot(page);
const postCallImageId = await sharedState.persistence.saveScreenShot(flowId, postCallImage);
const completedAt = new Date().getTime();
await sharedState.persistence.setToolCall(flowId, {
id: MiscUtils_1.MiscUtils.createAdHocToolCallId(),
toolName: GoToWebpageTool_1.GoToWebpageTool.NAME,
parameters: {
url: effectiveUrl,
},
outcome: {
isSuccessful: false,
forLlm: `FAILED! ${typeof error}: ${error.message}`,
metadata: null,
},
postCallImageId: postCallImageId,
page: effectiveUrl,
startedAt: startedAt,
completedAt: completedAt,
});
}
catch (recordingError) {
Logger_1.appLogger.warn('Failed to record goto failure (original error will still be thrown):', recordingError);
}
throw error;
}
};
page.run = async (toolName, toolParams, options) => {
// Thin wrapper to run a Donobu tool and throw on failure so tests can await it directly.
const result = await runTool(page, toolName, toolParams, options?.gptClient);
if (!result.outcome.isSuccessful) {
throw new ToolCallFailedException_1.ToolCallFailedException(toolName, result.outcome);
}
return result.outcome;
};
// Switch tabs by URL and return the extended page so it continues logging to the same flow.
page.changeTab = async (url) => {
const result = await page.run(ChangeWebBrowserTabTool_1.ChangeWebBrowserTabTool.NAME, {
tabUrl: url,
});
if (result.isSuccessful) {
const matchingPage = page
.context()
.pages()
.find((tab) => tab.url() === url);
if (!matchingPage) {
// This is entirely unexpected since the tool already said the active
// tab successfully switched.
throw new Error(`Unable to locate tab with URL ${url} after switching.`);
}
return applyDonobuExtensions(matchingPage, sharedState, cacheFilepath);
}
throw new ToolCallFailedException_1.ToolCallFailedException(ChangeWebBrowserTabTool_1.ChangeWebBrowserTabTool.NAME, result);
};
// Convenience wrapper to run the accessibility tool and return its metadata.
page.runAccessibilityTest = async () => {
const result = await page.run(RunAccessibilityTestTool_1.RunAccessibilityTestTool.NAME);
return result.metadata;
};
page.audit = async (options) => {
// Convert the user-facing AuditOptions (false to disable, RegExp patterns)
// into the tool's serializable format (enabled: false, string patterns).
const toEnabled = (opt) => {
if (opt === undefined) {
return undefined;
}
if (opt === false) {
return { enabled: false };
}
return { ...opt, enabled: true };
};
const { gptClient, ...opts } = options ?? {};
const toolParams = {
url: opts.url,
pageLoad: toEnabled(opts.pageLoad),
accessibility: toEnabled(opts.accessibility),
uniqueIds: toEnabled(opts.uniqueIds),
uniqueTestIds: toEnabled(opts.uniqueTestIds),
};
// Convert RegExp patterns to source strings.
const ce = toEnabled(opts.consoleErrors);
if (ce && 'ignore' in ce && ce.ignore) {
toolParams.consoleErrors = {
...ce,
ignore: ce.ignore.map((r) => r.source),
};
}
else {
toolParams.consoleErrors = ce;
}
const ne = toEnabled(opts.networkErrors);
if (ne && 'ignore' in ne && ne.ignore) {
toolParams.networkErrors = {
...ne,
ignore: ne.ignore.map((r) => r.source),
};
}
else {
toolParams.networkErrors = ne;
}
const result = await page.run(AuditTool_1.AuditTool.NAME, toolParams, { gptClient });
return result.metadata;
};
// Interactive pause: opens a control panel and optionally mirrors the
// headless browser via CDP screencast so the user can explore and then
// resume with optional instructions.
page.tbd = () => (0, tbd_1.tbd)(page);
page.once('close', () => {
const otherOpenPages = page
.context()
.pages()
.filter((p) => p !== page && !p.isClosed());
if (otherOpenPages.length > 0) {
return;
}
// Only mark the flow complete when the last page in the context closes.
void finalizeFlowOnPageClose(sharedState);
});
return page;
}
/**
* Run a Donobu tool with full context and persistence wiring.
*/
async function runTool(page, toolName, toolParams, gptClient) {
const clonedParams = toolParams ? JSON.parse(JSON.stringify(toolParams)) : {};
clonedParams.rationale ||= '';
const webTarget = { type: 'web', current: page };
const envData = await page._dnb.donobuStack.envDataManager.getByNames(page._dnb.donobuFlowMetadata.envVars ?? []);
if (page._dnb.envVals) {
for (const [k, v] of Object.entries(page._dnb.envVals)) {
if (v === undefined) {
delete envData[k];
}
else {
envData[k] = v;
}
}
}
const toolCallContext = {
flowsManager: page._dnb.donobuStack.flowsManager,
envData,
targetInspector: new WebTargetInspector_1.WebTargetInspector(webTarget, page.context(), page._dnb.interactionVisualizer),
controlPanel: new ControlPanel_1.NoOpControlPanel(),
persistence: page._dnb.persistence,
gptClient: getGptClient(page, gptClient),
interactionVisualizer: page._dnb.interactionVisualizer,
proposedToolCalls: [],
invokedToolCalls: [],
metadata: page._dnb.donobuFlowMetadata,
toolCallId: clonedParams.toolCallId ?? MiscUtils_1.MiscUtils.createAdHocToolCallId(),
};
const tools = page._dnb.donobuStack.toolRegistry.allTools();
const result = await new ToolManager_1.ToolManager(tools).invokeTool(toolCallContext, toolName, clonedParams, false);
return result;
}
/**
* Resolve the GPT client to use, wrapping non-Donobu clients when needed.
*/
function getGptClient(page, gptClient) {
if (gptClient) {
if (!(gptClient instanceof GptClient_1.GptClient)) {
return new VercelAiGptClient_1.VercelAiGptClient(gptClient);
}
else {
return gptClient;
}
}
else if (page._dnb.gptClient) {
return page._dnb.gptClient;
}
else {
return null;
}
}
/**
* Ensure we have a GPT client, logging non-configuration failures but allowing flows that do
* not need AI to proceed.
*/
async function buttonUpGptClient(gptClient) {
try {
if (!gptClient) {
gptClient = await (0, gptClients_1.getOrCreateDefaultGptClient)();
}
return gptClient;
}
catch (error) {
if (!(error instanceof GptApiKeysNotSetupException_1.GptApiKeysNotSetupException)) {
const err = error;
Logger_1.appLogger.error(`Failed to set up GPT client! ${err.name}: ${err.message}`);
}
// Do nothing since we may not need a GPT client at all anyway.
// If something later on requires a GPT client, it will throw an error.
return gptClient;
}
finally {
if (gptClient) {
const modelName = 'modelName' in gptClient.config
? gptClient.config.modelName
: undefined;
Logger_1.appLogger.info(`Set up ${gptClient.config.type} client${modelName ? ` for model: ${modelName}` : ''}`);
}
}
}
/**
* Finalize flow metadata once the flow ends, persisting completion time and state.
*/
async function finalizeFlowOnPageClose(sharedState) {
if (sharedState.donobuFlowMetadata.state === 'SUCCESS' ||
sharedState.donobuFlowMetadata.state === 'FAILED') {
return;
}
sharedState.donobuFlowMetadata.state = 'SUCCESS';
sharedState.donobuFlowMetadata.completedAt = new Date().getTime();
try {
await sharedState.persistence.setFlowMetadata(sharedState.donobuFlowMetadata);
}
catch (error) {
Logger_1.appLogger.error('Failed to persist Donobu flow metadata when page closed.', error);
}
}
//# sourceMappingURL=extendPage.js.map