UNPKG

@gguf/claw

Version:

Multi-channel AI gateway with extensible messaging integrations

1,631 lines (1,620 loc) 56.9 kB
import { t as parseBooleanValue } from "./boolean-BsqeuxE6.js"; import { i as loadConfig, l as writeConfigFile } from "./config-PQiujvsf.js"; import { i as getImageMetadata, n as buildImageResizeSideGrid, s as resizeToJpeg, t as IMAGE_REDUCE_QUALITY_STEPS } from "./image-ops-CI1VknD1.js"; import { C as DEFAULT_AI_SNAPSHOT_EFFICIENT_DEPTH, E as DEFAULT_BROWSER_DEFAULT_PROFILE_NAME, T as DEFAULT_AI_SNAPSHOT_MAX_CHARS, a as resolveOpenClawUserDataDir, c as captureScreenshot, f as snapshotAria, h as withBrowserNavigationPolicy, s as resolveBrowserExecutableForPlatform, w as DEFAULT_AI_SNAPSHOT_EFFICIENT_MAX_CHARS } from "./chrome-Dd5zBIFu.js"; import { a as resolveProfile, c as getUsedColors, d as deriveDefaultBrowserCdpPortRange, f as getPwAiModule$1, l as getUsedPorts, o as allocateCdpPort, r as parseHttpUrl, s as allocateColor, u as isValidProfileName } from "./server-context-DAWsUNUs.js"; import { a as resolvePathWithinRoot, i as resolveExistingPathsWithinRoot, n as DEFAULT_TRACE_DIR, r as DEFAULT_UPLOAD_DIR, t as DEFAULT_DOWNLOAD_DIR } from "./paths-D1WZUYry.js"; import { r as saveMediaBuffer, t as ensureMediaDir } from "./store-DSGYY-H9.js"; import { t as movePathToTrash } from "./trash-DLzIf__E.js"; import fs from "node:fs"; import path from "node:path"; import fs$1 from "node:fs/promises"; import crypto from "node:crypto"; //#region src/browser/routes/agent.act.shared.ts const ACT_KINDS = [ "click", "close", "drag", "evaluate", "fill", "hover", "scrollIntoView", "press", "resize", "select", "type", "wait" ]; function isActKind(value) { if (typeof value !== "string") return false; return ACT_KINDS.includes(value); } const ALLOWED_CLICK_MODIFIERS = new Set([ "Alt", "Control", "ControlOrMeta", "Meta", "Shift" ]); function parseClickButton(raw) { if (raw === "left" || raw === "right" || raw === "middle") return raw; } function parseClickModifiers(raw) { if (raw.filter((m) => !ALLOWED_CLICK_MODIFIERS.has(m)).length) return { error: "modifiers must be Alt|Control|ControlOrMeta|Meta|Shift" }; return { modifiers: raw.length ? raw : void 0 }; } //#endregion //#region src/browser/routes/utils.ts /** * Extract profile name from query string or body and get profile context. * Query string takes precedence over body for consistency with GET routes. */ function getProfileContext(req, ctx) { let profileName; if (typeof req.query.profile === "string") profileName = req.query.profile.trim() || void 0; if (!profileName && req.body && typeof req.body === "object") { const body = req.body; if (typeof body.profile === "string") profileName = body.profile.trim() || void 0; } try { return ctx.forProfile(profileName); } catch (err) { return { error: String(err), status: 404 }; } } function jsonError(res, status, message) { res.status(status).json({ error: message }); } function toStringOrEmpty(value) { if (typeof value === "string") return value.trim(); if (typeof value === "number" || typeof value === "boolean") return String(value).trim(); return ""; } function toNumber(value) { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim()) { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : void 0; } } function toBoolean(value) { return parseBooleanValue(value, { truthy: [ "true", "1", "yes" ], falsy: [ "false", "0", "no" ] }); } function toStringArray(value) { if (!Array.isArray(value)) return; const strings = value.map((v) => toStringOrEmpty(v)).filter(Boolean); return strings.length ? strings : void 0; } //#endregion //#region src/browser/routes/agent.shared.ts const SELECTOR_UNSUPPORTED_MESSAGE = [ "Error: 'selector' is not supported. Use 'ref' from snapshot instead.", "", "Example workflow:", "1. snapshot action to get page state with refs", "2. act with ref: \"e123\" to interact with element", "", "This is more reliable for modern SPAs." ].join("\n"); function readBody(req) { const body = req.body; if (!body || typeof body !== "object" || Array.isArray(body)) return {}; return body; } function resolveTargetIdFromBody(body) { return (typeof body.targetId === "string" ? body.targetId.trim() : "") || void 0; } function resolveTargetIdFromQuery(query) { return (typeof query.targetId === "string" ? query.targetId.trim() : "") || void 0; } function handleRouteError(ctx, res, err) { const mapped = ctx.mapTabError(err); if (mapped) return jsonError(res, mapped.status, mapped.message); jsonError(res, 500, String(err)); } function resolveProfileContext(req, res, ctx) { const profileCtx = getProfileContext(req, ctx); if ("error" in profileCtx) { jsonError(res, profileCtx.status, profileCtx.error); return null; } return profileCtx; } async function getPwAiModule() { return await getPwAiModule$1({ mode: "soft" }); } async function requirePwAi(res, feature) { const mod = await getPwAiModule(); if (mod) return mod; jsonError(res, 501, [ `Playwright is not available in this gateway build; '${feature}' is unsupported.`, "Install the full Playwright package (not playwright-core) and restart the gateway, or reinstall with browser support.", "Docs: /tools/browser#playwright-requirement" ].join("\n")); return null; } async function withRouteTabContext(params) { const profileCtx = resolveProfileContext(params.req, params.res, params.ctx); if (!profileCtx) return; try { const tab = await profileCtx.ensureTabAvailable(params.targetId); return await params.run({ profileCtx, tab, cdpUrl: profileCtx.profile.cdpUrl }); } catch (err) { handleRouteError(params.ctx, params.res, err); return; } } async function withPlaywrightRouteContext(params) { return await withRouteTabContext({ req: params.req, res: params.res, ctx: params.ctx, targetId: params.targetId, run: async ({ profileCtx, tab, cdpUrl }) => { const pw = await requirePwAi(params.res, params.feature); if (!pw) return; return await params.run({ profileCtx, tab, cdpUrl, pw }); } }); } //#endregion //#region src/browser/routes/agent.act.ts function resolveDownloadPathOrRespond(res, requestedPath) { const downloadPathResult = resolvePathWithinRoot({ rootDir: DEFAULT_DOWNLOAD_DIR, requestedPath, scopeLabel: "downloads directory" }); if (!downloadPathResult.ok) { res.status(400).json({ error: downloadPathResult.error }); return null; } return downloadPathResult.path; } function buildDownloadRequestBase(cdpUrl, targetId, timeoutMs) { return { cdpUrl, targetId, timeoutMs: timeoutMs ?? void 0 }; } function respondWithDownloadResult(res, targetId, result) { res.json({ ok: true, targetId, download: result }); } function registerBrowserAgentActRoutes(app, ctx) { app.post("/act", async (req, res) => { const body = readBody(req); const kindRaw = toStringOrEmpty(body.kind); if (!isActKind(kindRaw)) return jsonError(res, 400, "kind is required"); const kind = kindRaw; const targetId = resolveTargetIdFromBody(body); if (Object.hasOwn(body, "selector") && kind !== "wait") return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: `act:${kind}`, run: async ({ cdpUrl, tab, pw }) => { const evaluateEnabled = ctx.state().resolved.evaluateEnabled; switch (kind) { case "click": { const ref = toStringOrEmpty(body.ref); if (!ref) return jsonError(res, 400, "ref is required"); const doubleClick = toBoolean(body.doubleClick) ?? false; const timeoutMs = toNumber(body.timeoutMs); const buttonRaw = toStringOrEmpty(body.button) || ""; const button = buttonRaw ? parseClickButton(buttonRaw) : void 0; if (buttonRaw && !button) return jsonError(res, 400, "button must be left|right|middle"); const parsedModifiers = parseClickModifiers(toStringArray(body.modifiers) ?? []); if (parsedModifiers.error) return jsonError(res, 400, parsedModifiers.error); const modifiers = parsedModifiers.modifiers; const clickRequest = { cdpUrl, targetId: tab.targetId, ref, doubleClick }; if (button) clickRequest.button = button; if (modifiers) clickRequest.modifiers = modifiers; if (timeoutMs) clickRequest.timeoutMs = timeoutMs; await pw.clickViaPlaywright(clickRequest); return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); } case "type": { const ref = toStringOrEmpty(body.ref); if (!ref) return jsonError(res, 400, "ref is required"); if (typeof body.text !== "string") return jsonError(res, 400, "text is required"); const text = body.text; const submit = toBoolean(body.submit) ?? false; const slowly = toBoolean(body.slowly) ?? false; const timeoutMs = toNumber(body.timeoutMs); const typeRequest = { cdpUrl, targetId: tab.targetId, ref, text, submit, slowly }; if (timeoutMs) typeRequest.timeoutMs = timeoutMs; await pw.typeViaPlaywright(typeRequest); return res.json({ ok: true, targetId: tab.targetId }); } case "press": { const key = toStringOrEmpty(body.key); if (!key) return jsonError(res, 400, "key is required"); const delayMs = toNumber(body.delayMs); await pw.pressKeyViaPlaywright({ cdpUrl, targetId: tab.targetId, key, delayMs: delayMs ?? void 0 }); return res.json({ ok: true, targetId: tab.targetId }); } case "hover": { const ref = toStringOrEmpty(body.ref); if (!ref) return jsonError(res, 400, "ref is required"); const timeoutMs = toNumber(body.timeoutMs); await pw.hoverViaPlaywright({ cdpUrl, targetId: tab.targetId, ref, timeoutMs: timeoutMs ?? void 0 }); return res.json({ ok: true, targetId: tab.targetId }); } case "scrollIntoView": { const ref = toStringOrEmpty(body.ref); if (!ref) return jsonError(res, 400, "ref is required"); const timeoutMs = toNumber(body.timeoutMs); const scrollRequest = { cdpUrl, targetId: tab.targetId, ref }; if (timeoutMs) scrollRequest.timeoutMs = timeoutMs; await pw.scrollIntoViewViaPlaywright(scrollRequest); return res.json({ ok: true, targetId: tab.targetId }); } case "drag": { const startRef = toStringOrEmpty(body.startRef); const endRef = toStringOrEmpty(body.endRef); if (!startRef || !endRef) return jsonError(res, 400, "startRef and endRef are required"); const timeoutMs = toNumber(body.timeoutMs); await pw.dragViaPlaywright({ cdpUrl, targetId: tab.targetId, startRef, endRef, timeoutMs: timeoutMs ?? void 0 }); return res.json({ ok: true, targetId: tab.targetId }); } case "select": { const ref = toStringOrEmpty(body.ref); const values = toStringArray(body.values); if (!ref || !values?.length) return jsonError(res, 400, "ref and values are required"); const timeoutMs = toNumber(body.timeoutMs); await pw.selectOptionViaPlaywright({ cdpUrl, targetId: tab.targetId, ref, values, timeoutMs: timeoutMs ?? void 0 }); return res.json({ ok: true, targetId: tab.targetId }); } case "fill": { const fields = (Array.isArray(body.fields) ? body.fields : []).map((field) => { if (!field || typeof field !== "object") return null; const rec = field; const ref = toStringOrEmpty(rec.ref); const type = toStringOrEmpty(rec.type); if (!ref || !type) return null; const value = typeof rec.value === "string" || typeof rec.value === "number" || typeof rec.value === "boolean" ? rec.value : void 0; return value === void 0 ? { ref, type } : { ref, type, value }; }).filter((field) => field !== null); if (!fields.length) return jsonError(res, 400, "fields are required"); const timeoutMs = toNumber(body.timeoutMs); await pw.fillFormViaPlaywright({ cdpUrl, targetId: tab.targetId, fields, timeoutMs: timeoutMs ?? void 0 }); return res.json({ ok: true, targetId: tab.targetId }); } case "resize": { const width = toNumber(body.width); const height = toNumber(body.height); if (!width || !height) return jsonError(res, 400, "width and height are required"); await pw.resizeViewportViaPlaywright({ cdpUrl, targetId: tab.targetId, width, height }); return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); } case "wait": { const timeMs = toNumber(body.timeMs); const text = toStringOrEmpty(body.text) || void 0; const textGone = toStringOrEmpty(body.textGone) || void 0; const selector = toStringOrEmpty(body.selector) || void 0; const url = toStringOrEmpty(body.url) || void 0; const loadStateRaw = toStringOrEmpty(body.loadState); const loadState = loadStateRaw === "load" || loadStateRaw === "domcontentloaded" || loadStateRaw === "networkidle" ? loadStateRaw : void 0; const fn = toStringOrEmpty(body.fn) || void 0; const timeoutMs = toNumber(body.timeoutMs) ?? void 0; if (fn && !evaluateEnabled) return jsonError(res, 403, ["wait --fn is disabled by config (browser.evaluateEnabled=false).", "Docs: /gateway/configuration#browser-openclaw-managed-browser"].join("\n")); if (timeMs === void 0 && !text && !textGone && !selector && !url && !loadState && !fn) return jsonError(res, 400, "wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn"); await pw.waitForViaPlaywright({ cdpUrl, targetId: tab.targetId, timeMs, text, textGone, selector, url, loadState, fn, timeoutMs }); return res.json({ ok: true, targetId: tab.targetId }); } case "evaluate": { if (!evaluateEnabled) return jsonError(res, 403, ["act:evaluate is disabled by config (browser.evaluateEnabled=false).", "Docs: /gateway/configuration#browser-openclaw-managed-browser"].join("\n")); const fn = toStringOrEmpty(body.fn); if (!fn) return jsonError(res, 400, "fn is required"); const ref = toStringOrEmpty(body.ref) || void 0; const evalTimeoutMs = toNumber(body.timeoutMs); const evalRequest = { cdpUrl, targetId: tab.targetId, fn, ref, signal: req.signal }; if (evalTimeoutMs !== void 0) evalRequest.timeoutMs = evalTimeoutMs; const result = await pw.evaluateViaPlaywright(evalRequest); return res.json({ ok: true, targetId: tab.targetId, url: tab.url, result }); } case "close": await pw.closePageViaPlaywright({ cdpUrl, targetId: tab.targetId }); return res.json({ ok: true, targetId: tab.targetId }); default: return jsonError(res, 400, "unsupported kind"); } } }); }); app.post("/hooks/file-chooser", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const ref = toStringOrEmpty(body.ref) || void 0; const inputRef = toStringOrEmpty(body.inputRef) || void 0; const element = toStringOrEmpty(body.element) || void 0; const paths = toStringArray(body.paths) ?? []; const timeoutMs = toNumber(body.timeoutMs); if (!paths.length) return jsonError(res, 400, "paths are required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "file chooser hook", run: async ({ cdpUrl, tab, pw }) => { const uploadPathsResult = await resolveExistingPathsWithinRoot({ rootDir: DEFAULT_UPLOAD_DIR, requestedPaths: paths, scopeLabel: `uploads directory (${DEFAULT_UPLOAD_DIR})` }); if (!uploadPathsResult.ok) { res.status(400).json({ error: uploadPathsResult.error }); return; } const resolvedPaths = uploadPathsResult.paths; if (inputRef || element) { if (ref) return jsonError(res, 400, "ref cannot be combined with inputRef/element"); await pw.setInputFilesViaPlaywright({ cdpUrl, targetId: tab.targetId, inputRef, element, paths: resolvedPaths }); } else { await pw.armFileUploadViaPlaywright({ cdpUrl, targetId: tab.targetId, paths: resolvedPaths, timeoutMs: timeoutMs ?? void 0 }); if (ref) await pw.clickViaPlaywright({ cdpUrl, targetId: tab.targetId, ref }); } res.json({ ok: true }); } }); }); app.post("/hooks/dialog", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const accept = toBoolean(body.accept); const promptText = toStringOrEmpty(body.promptText) || void 0; const timeoutMs = toNumber(body.timeoutMs); if (accept === void 0) return jsonError(res, 400, "accept is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "dialog hook", run: async ({ cdpUrl, tab, pw }) => { await pw.armDialogViaPlaywright({ cdpUrl, targetId: tab.targetId, accept, promptText, timeoutMs: timeoutMs ?? void 0 }); res.json({ ok: true }); } }); }); app.post("/wait/download", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const out = toStringOrEmpty(body.path) || ""; const timeoutMs = toNumber(body.timeoutMs); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "wait for download", run: async ({ cdpUrl, tab, pw }) => { let downloadPath; if (out.trim()) { const resolvedDownloadPath = resolveDownloadPathOrRespond(res, out); if (!resolvedDownloadPath) return; downloadPath = resolvedDownloadPath; } const requestBase = buildDownloadRequestBase(cdpUrl, tab.targetId, timeoutMs); const result = await pw.waitForDownloadViaPlaywright({ ...requestBase, path: downloadPath }); respondWithDownloadResult(res, tab.targetId, result); } }); }); app.post("/download", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const ref = toStringOrEmpty(body.ref); const out = toStringOrEmpty(body.path); const timeoutMs = toNumber(body.timeoutMs); if (!ref) return jsonError(res, 400, "ref is required"); if (!out) return jsonError(res, 400, "path is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "download", run: async ({ cdpUrl, tab, pw }) => { const downloadPath = resolveDownloadPathOrRespond(res, out); if (!downloadPath) return; const requestBase = buildDownloadRequestBase(cdpUrl, tab.targetId, timeoutMs); const result = await pw.downloadViaPlaywright({ ...requestBase, ref, path: downloadPath }); respondWithDownloadResult(res, tab.targetId, result); } }); }); app.post("/response/body", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const url = toStringOrEmpty(body.url); const timeoutMs = toNumber(body.timeoutMs); const maxChars = toNumber(body.maxChars); if (!url) return jsonError(res, 400, "url is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "response body", run: async ({ cdpUrl, tab, pw }) => { const result = await pw.responseBodyViaPlaywright({ cdpUrl, targetId: tab.targetId, url, timeoutMs: timeoutMs ?? void 0, maxChars: maxChars ?? void 0 }); res.json({ ok: true, targetId: tab.targetId, response: result }); } }); }); app.post("/highlight", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const ref = toStringOrEmpty(body.ref); if (!ref) return jsonError(res, 400, "ref is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "highlight", run: async ({ cdpUrl, tab, pw }) => { await pw.highlightViaPlaywright({ cdpUrl, targetId: tab.targetId, ref }); res.json({ ok: true, targetId: tab.targetId }); } }); }); } //#endregion //#region src/browser/routes/agent.debug.ts function registerBrowserAgentDebugRoutes(app, ctx) { app.get("/console", async (req, res) => { const targetId = resolveTargetIdFromQuery(req.query); const level = typeof req.query.level === "string" ? req.query.level : ""; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "console messages", run: async ({ cdpUrl, tab, pw }) => { const messages = await pw.getConsoleMessagesViaPlaywright({ cdpUrl, targetId: tab.targetId, level: level.trim() || void 0 }); res.json({ ok: true, messages, targetId: tab.targetId }); } }); }); app.get("/errors", async (req, res) => { const targetId = resolveTargetIdFromQuery(req.query); const clear = toBoolean(req.query.clear) ?? false; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "page errors", run: async ({ cdpUrl, tab, pw }) => { const result = await pw.getPageErrorsViaPlaywright({ cdpUrl, targetId: tab.targetId, clear }); res.json({ ok: true, targetId: tab.targetId, ...result }); } }); }); app.get("/requests", async (req, res) => { const targetId = resolveTargetIdFromQuery(req.query); const filter = typeof req.query.filter === "string" ? req.query.filter : ""; const clear = toBoolean(req.query.clear) ?? false; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "network requests", run: async ({ cdpUrl, tab, pw }) => { const result = await pw.getNetworkRequestsViaPlaywright({ cdpUrl, targetId: tab.targetId, filter: filter.trim() || void 0, clear }); res.json({ ok: true, targetId: tab.targetId, ...result }); } }); }); app.post("/trace/start", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const screenshots = toBoolean(body.screenshots) ?? void 0; const snapshots = toBoolean(body.snapshots) ?? void 0; const sources = toBoolean(body.sources) ?? void 0; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "trace start", run: async ({ cdpUrl, tab, pw }) => { await pw.traceStartViaPlaywright({ cdpUrl, targetId: tab.targetId, screenshots, snapshots, sources }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/trace/stop", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const out = toStringOrEmpty(body.path) || ""; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "trace stop", run: async ({ cdpUrl, tab, pw }) => { const id = crypto.randomUUID(); const dir = DEFAULT_TRACE_DIR; await fs$1.mkdir(dir, { recursive: true }); const tracePathResult = resolvePathWithinRoot({ rootDir: dir, requestedPath: out, scopeLabel: "trace directory", defaultFileName: `browser-trace-${id}.zip` }); if (!tracePathResult.ok) { res.status(400).json({ error: tracePathResult.error }); return; } const tracePath = tracePathResult.path; await pw.traceStopViaPlaywright({ cdpUrl, targetId: tab.targetId, path: tracePath }); res.json({ ok: true, targetId: tab.targetId, path: path.resolve(tracePath) }); } }); }); } //#endregion //#region src/browser/screenshot.ts const DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE = 2e3; const DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES = 5 * 1024 * 1024; async function normalizeBrowserScreenshot(buffer, opts) { const maxSide = Math.max(1, Math.round(opts?.maxSide ?? DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE)); const maxBytes = Math.max(1, Math.round(opts?.maxBytes ?? DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES)); const meta = await getImageMetadata(buffer); const width = Number(meta?.width ?? 0); const height = Number(meta?.height ?? 0); const maxDim = Math.max(width, height); if (buffer.byteLength <= maxBytes && (maxDim === 0 || width <= maxSide && height <= maxSide)) return { buffer }; const sideGrid = buildImageResizeSideGrid(maxSide, maxDim > 0 ? Math.min(maxSide, maxDim) : maxSide); let smallest = null; for (const side of sideGrid) for (const quality of IMAGE_REDUCE_QUALITY_STEPS) { const out = await resizeToJpeg({ buffer, maxSide: side, quality, withoutEnlargement: true }); if (!smallest || out.byteLength < smallest.size) smallest = { buffer: out, size: out.byteLength }; if (out.byteLength <= maxBytes) return { buffer: out, contentType: "image/jpeg" }; } const best = smallest?.buffer ?? buffer; throw new Error(`Browser screenshot could not be reduced below ${(maxBytes / (1024 * 1024)).toFixed(0)}MB (got ${(best.byteLength / (1024 * 1024)).toFixed(2)}MB)`); } //#endregion //#region src/browser/routes/agent.snapshot.ts async function saveBrowserMediaResponse(params) { await ensureMediaDir(); const saved = await saveMediaBuffer(params.buffer, params.contentType, "browser", params.maxBytes); params.res.json({ ok: true, path: path.resolve(saved.path), targetId: params.targetId, url: params.url }); } function registerBrowserAgentSnapshotRoutes(app, ctx) { app.post("/navigate", async (req, res) => { const body = readBody(req); const url = toStringOrEmpty(body.url); const targetId = toStringOrEmpty(body.targetId) || void 0; if (!url) return jsonError(res, 400, "url is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "navigate", run: async ({ cdpUrl, tab, pw }) => { const result = await pw.navigateViaPlaywright({ cdpUrl, targetId: tab.targetId, url, ...withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy) }); res.json({ ok: true, targetId: tab.targetId, ...result }); } }); }); app.post("/pdf", async (req, res) => { await withPlaywrightRouteContext({ req, res, ctx, targetId: toStringOrEmpty(readBody(req).targetId) || void 0, feature: "pdf", run: async ({ cdpUrl, tab, pw }) => { const pdf = await pw.pdfViaPlaywright({ cdpUrl, targetId: tab.targetId }); await saveBrowserMediaResponse({ res, buffer: pdf.buffer, contentType: "application/pdf", maxBytes: pdf.buffer.byteLength, targetId: tab.targetId, url: tab.url }); } }); }); app.post("/screenshot", async (req, res) => { const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || void 0; const fullPage = toBoolean(body.fullPage) ?? false; const ref = toStringOrEmpty(body.ref) || void 0; const element = toStringOrEmpty(body.element) || void 0; const type = body.type === "jpeg" ? "jpeg" : "png"; if (fullPage && (ref || element)) return jsonError(res, 400, "fullPage is not supported for element screenshots"); await withRouteTabContext({ req, res, ctx, targetId, run: async ({ profileCtx, tab, cdpUrl }) => { let buffer; if (profileCtx.profile.driver === "extension" || !tab.wsUrl || Boolean(ref) || Boolean(element)) { const pw = await requirePwAi(res, "screenshot"); if (!pw) return; buffer = (await pw.takeScreenshotViaPlaywright({ cdpUrl, targetId: tab.targetId, ref, element, fullPage, type })).buffer; } else buffer = await captureScreenshot({ wsUrl: tab.wsUrl ?? "", fullPage, format: type, quality: type === "jpeg" ? 85 : void 0 }); const normalized = await normalizeBrowserScreenshot(buffer, { maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE, maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES }); await saveBrowserMediaResponse({ res, buffer: normalized.buffer, contentType: normalized.contentType ?? `image/${type}`, maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, targetId: tab.targetId, url: tab.url }); } }); }); app.get("/snapshot", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) return; const targetId = typeof req.query.targetId === "string" ? req.query.targetId.trim() : ""; const mode = req.query.mode === "efficient" ? "efficient" : void 0; const labels = toBoolean(req.query.labels) ?? void 0; const format = (req.query.format === "aria" ? "aria" : req.query.format === "ai" ? "ai" : void 0) ?? (mode ? "ai" : await getPwAiModule() ? "ai" : "aria"); const limitRaw = typeof req.query.limit === "string" ? Number(req.query.limit) : void 0; const hasMaxChars = Object.hasOwn(req.query, "maxChars"); const maxCharsRaw = typeof req.query.maxChars === "string" ? Number(req.query.maxChars) : void 0; const limit = Number.isFinite(limitRaw) ? limitRaw : void 0; const resolvedMaxChars = format === "ai" ? hasMaxChars ? typeof maxCharsRaw === "number" && Number.isFinite(maxCharsRaw) && maxCharsRaw > 0 ? Math.floor(maxCharsRaw) : void 0 : mode === "efficient" ? DEFAULT_AI_SNAPSHOT_EFFICIENT_MAX_CHARS : DEFAULT_AI_SNAPSHOT_MAX_CHARS : void 0; const interactiveRaw = toBoolean(req.query.interactive); const compactRaw = toBoolean(req.query.compact); const depthRaw = toNumber(req.query.depth); const refsModeRaw = toStringOrEmpty(req.query.refs).trim(); const refsMode = refsModeRaw === "aria" ? "aria" : refsModeRaw === "role" ? "role" : void 0; const interactive = interactiveRaw ?? (mode === "efficient" ? true : void 0); const compact = compactRaw ?? (mode === "efficient" ? true : void 0); const depth = depthRaw ?? (mode === "efficient" ? DEFAULT_AI_SNAPSHOT_EFFICIENT_DEPTH : void 0); const selector = toStringOrEmpty(req.query.selector); const frameSelector = toStringOrEmpty(req.query.frame); const selectorValue = selector.trim() || void 0; const frameSelectorValue = frameSelector.trim() || void 0; try { const tab = await profileCtx.ensureTabAvailable(targetId || void 0); if ((labels || mode === "efficient") && format === "aria") return jsonError(res, 400, "labels/mode=efficient require format=ai"); if (format === "ai") { const pw = await requirePwAi(res, "ai snapshot"); if (!pw) return; const wantsRoleSnapshot = labels === true || mode === "efficient" || interactive === true || compact === true || depth !== void 0 || Boolean(selectorValue) || Boolean(frameSelectorValue); const roleSnapshotArgs = { cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, selector: selectorValue, frameSelector: frameSelectorValue, refsMode, options: { interactive: interactive ?? void 0, compact: compact ?? void 0, maxDepth: depth ?? void 0 } }; const snap = wantsRoleSnapshot ? await pw.snapshotRoleViaPlaywright(roleSnapshotArgs) : await pw.snapshotAiViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, ...typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {} }).catch(async (err) => { if (String(err).toLowerCase().includes("_snapshotforai")) return await pw.snapshotRoleViaPlaywright(roleSnapshotArgs); throw err; }); if (labels) { const labeled = await pw.screenshotWithLabelsViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, refs: "refs" in snap ? snap.refs : {}, type: "png" }); const normalized = await normalizeBrowserScreenshot(labeled.buffer, { maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE, maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES }); await ensureMediaDir(); const saved = await saveMediaBuffer(normalized.buffer, normalized.contentType ?? "image/png", "browser", DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES); const imageType = normalized.contentType?.includes("jpeg") ? "jpeg" : "png"; return res.json({ ok: true, format, targetId: tab.targetId, url: tab.url, labels: true, labelsCount: labeled.labels, labelsSkipped: labeled.skipped, imagePath: path.resolve(saved.path), imageType, ...snap }); } return res.json({ ok: true, format, targetId: tab.targetId, url: tab.url, ...snap }); } const snap = profileCtx.profile.driver === "extension" || !tab.wsUrl ? requirePwAi(res, "aria snapshot").then(async (pw) => { if (!pw) return null; return await pw.snapshotAriaViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, limit }); }) : snapshotAria({ wsUrl: tab.wsUrl ?? "", limit }); const resolved = await Promise.resolve(snap); if (!resolved) return; return res.json({ ok: true, format, targetId: tab.targetId, url: tab.url, ...resolved }); } catch (err) { handleRouteError(ctx, res, err); } }); } //#endregion //#region src/browser/routes/agent.storage.ts function parseStorageKind(raw) { if (raw === "local" || raw === "session") return raw; return null; } function parseStorageMutationRequest(kindParam, body) { return { kind: parseStorageKind(toStringOrEmpty(kindParam)), targetId: resolveTargetIdFromBody(body) }; } function parseRequiredStorageMutationRequest(kindParam, body) { const parsed = parseStorageMutationRequest(kindParam, body); if (!parsed.kind) return null; return { kind: parsed.kind, targetId: parsed.targetId }; } function parseStorageMutationOrRespond(res, kindParam, body) { const parsed = parseRequiredStorageMutationRequest(kindParam, body); if (!parsed) { jsonError(res, 400, "kind must be local|session"); return null; } return parsed; } function parseStorageMutationFromRequest(req, res) { const body = readBody(req); const parsed = parseStorageMutationOrRespond(res, req.params.kind, body); if (!parsed) return null; return { body, parsed }; } function registerBrowserAgentStorageRoutes(app, ctx) { app.get("/cookies", async (req, res) => { await withPlaywrightRouteContext({ req, res, ctx, targetId: resolveTargetIdFromQuery(req.query), feature: "cookies", run: async ({ cdpUrl, tab, pw }) => { const result = await pw.cookiesGetViaPlaywright({ cdpUrl, targetId: tab.targetId }); res.json({ ok: true, targetId: tab.targetId, ...result }); } }); }); app.post("/cookies/set", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const cookie = body.cookie && typeof body.cookie === "object" && !Array.isArray(body.cookie) ? body.cookie : null; if (!cookie) return jsonError(res, 400, "cookie is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "cookies set", run: async ({ cdpUrl, tab, pw }) => { await pw.cookiesSetViaPlaywright({ cdpUrl, targetId: tab.targetId, cookie: { name: toStringOrEmpty(cookie.name), value: toStringOrEmpty(cookie.value), url: toStringOrEmpty(cookie.url) || void 0, domain: toStringOrEmpty(cookie.domain) || void 0, path: toStringOrEmpty(cookie.path) || void 0, expires: toNumber(cookie.expires) ?? void 0, httpOnly: toBoolean(cookie.httpOnly) ?? void 0, secure: toBoolean(cookie.secure) ?? void 0, sameSite: cookie.sameSite === "Lax" || cookie.sameSite === "None" || cookie.sameSite === "Strict" ? cookie.sameSite : void 0 } }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/cookies/clear", async (req, res) => { await withPlaywrightRouteContext({ req, res, ctx, targetId: resolveTargetIdFromBody(readBody(req)), feature: "cookies clear", run: async ({ cdpUrl, tab, pw }) => { await pw.cookiesClearViaPlaywright({ cdpUrl, targetId: tab.targetId }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.get("/storage/:kind", async (req, res) => { const kind = parseStorageKind(toStringOrEmpty(req.params.kind)); if (!kind) return jsonError(res, 400, "kind must be local|session"); const targetId = resolveTargetIdFromQuery(req.query); const key = toStringOrEmpty(req.query.key); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "storage get", run: async ({ cdpUrl, tab, pw }) => { const result = await pw.storageGetViaPlaywright({ cdpUrl, targetId: tab.targetId, kind, key: key.trim() || void 0 }); res.json({ ok: true, targetId: tab.targetId, ...result }); } }); }); app.post("/storage/:kind/set", async (req, res) => { const mutation = parseStorageMutationFromRequest(req, res); if (!mutation) return; const key = toStringOrEmpty(mutation.body.key); if (!key) return jsonError(res, 400, "key is required"); const value = typeof mutation.body.value === "string" ? mutation.body.value : ""; await withPlaywrightRouteContext({ req, res, ctx, targetId: mutation.parsed.targetId, feature: "storage set", run: async ({ cdpUrl, tab, pw }) => { await pw.storageSetViaPlaywright({ cdpUrl, targetId: tab.targetId, kind: mutation.parsed.kind, key, value }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/storage/:kind/clear", async (req, res) => { const mutation = parseStorageMutationFromRequest(req, res); if (!mutation) return; await withPlaywrightRouteContext({ req, res, ctx, targetId: mutation.parsed.targetId, feature: "storage clear", run: async ({ cdpUrl, tab, pw }) => { await pw.storageClearViaPlaywright({ cdpUrl, targetId: tab.targetId, kind: mutation.parsed.kind }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/offline", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const offline = toBoolean(body.offline); if (offline === void 0) return jsonError(res, 400, "offline is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "offline", run: async ({ cdpUrl, tab, pw }) => { await pw.setOfflineViaPlaywright({ cdpUrl, targetId: tab.targetId, offline }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/headers", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const headers = body.headers && typeof body.headers === "object" && !Array.isArray(body.headers) ? body.headers : null; if (!headers) return jsonError(res, 400, "headers is required"); const parsed = {}; for (const [k, v] of Object.entries(headers)) if (typeof v === "string") parsed[k] = v; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "headers", run: async ({ cdpUrl, tab, pw }) => { await pw.setExtraHTTPHeadersViaPlaywright({ cdpUrl, targetId: tab.targetId, headers: parsed }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/credentials", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const clear = toBoolean(body.clear) ?? false; const username = toStringOrEmpty(body.username) || void 0; const password = typeof body.password === "string" ? body.password : void 0; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "http credentials", run: async ({ cdpUrl, tab, pw }) => { await pw.setHttpCredentialsViaPlaywright({ cdpUrl, targetId: tab.targetId, username, password, clear }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/geolocation", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const clear = toBoolean(body.clear) ?? false; const latitude = toNumber(body.latitude); const longitude = toNumber(body.longitude); const accuracy = toNumber(body.accuracy) ?? void 0; const origin = toStringOrEmpty(body.origin) || void 0; await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "geolocation", run: async ({ cdpUrl, tab, pw }) => { await pw.setGeolocationViaPlaywright({ cdpUrl, targetId: tab.targetId, latitude, longitude, accuracy, origin, clear }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/media", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const schemeRaw = toStringOrEmpty(body.colorScheme); const colorScheme = schemeRaw === "dark" || schemeRaw === "light" || schemeRaw === "no-preference" ? schemeRaw : schemeRaw === "none" ? null : void 0; if (colorScheme === void 0) return jsonError(res, 400, "colorScheme must be dark|light|no-preference|none"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "media emulation", run: async ({ cdpUrl, tab, pw }) => { await pw.emulateMediaViaPlaywright({ cdpUrl, targetId: tab.targetId, colorScheme }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/timezone", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const timezoneId = toStringOrEmpty(body.timezoneId); if (!timezoneId) return jsonError(res, 400, "timezoneId is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "timezone", run: async ({ cdpUrl, tab, pw }) => { await pw.setTimezoneViaPlaywright({ cdpUrl, targetId: tab.targetId, timezoneId }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/locale", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const locale = toStringOrEmpty(body.locale); if (!locale) return jsonError(res, 400, "locale is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "locale", run: async ({ cdpUrl, tab, pw }) => { await pw.setLocaleViaPlaywright({ cdpUrl, targetId: tab.targetId, locale }); res.json({ ok: true, targetId: tab.targetId }); } }); }); app.post("/set/device", async (req, res) => { const body = readBody(req); const targetId = resolveTargetIdFromBody(body); const name = toStringOrEmpty(body.name); if (!name) return jsonError(res, 400, "name is required"); await withPlaywrightRouteContext({ req, res, ctx, targetId, feature: "device emulation", run: async ({ cdpUrl, tab, pw }) => { await pw.setDeviceViaPlaywright({ cdpUrl, targetId: tab.targetId, name }); res.json({ ok: true, targetId: tab.targetId }); } }); }); } //#endregion //#region src/browser/routes/agent.ts function registerBrowserAgentRoutes(app, ctx) { registerBrowserAgentSnapshotRoutes(app, ctx); registerBrowserAgentActRoutes(app, ctx); registerBrowserAgentDebugRoutes(app, ctx); registerBrowserAgentStorageRoutes(app, ctx); } //#endregion //#region src/browser/profiles-service.ts const HEX_COLOR_RE = /^#[0-9A-Fa-f]{6}$/; function createBrowserProfilesService(ctx) { const listProfiles = async () => { return await ctx.listProfiles(); }; const createProfile = async (params) => { const name = params.name.trim(); const rawCdpUrl = params.cdpUrl?.trim() || void 0; const driver = params.driver === "extension" ? "extension" : void 0; if (!isValidProfileName(name)) throw new Error("invalid profile name: use lowercase letters, numbers, and hyphens only"); const state = ctx.state(); const resolvedProfiles = state.resolved.profiles; if (name in resolvedProfiles) throw new Error(`profile "${name}" already exists`); const cfg = loadConfig(); const rawProfiles = cfg.browser?.profiles ?? {}; if (name in rawProfiles) throw new Error(`profile "${name}" already exists`); const usedColors = getUsedColors(resolvedProfiles); const profileColor = params.color && HEX_COLOR_RE.test(params.color) ? params.color : allocateColor(usedColors); let profileConfig; if (rawCdpUrl) profileConfig = { cdpUrl: parseHttpUrl(rawCdpUrl, "browser.profiles.cdpUrl").normalized, ...driver ? { driver } : {}, color: profileColor }; else { const cdpPort = allocateCdpPort(getUsedPorts(resolvedProfiles), deriveDefaultBrowserCdpPortRange(state.resolved.controlPort)); if (cdpPort === null) throw new Error("no available CDP ports in range"); profileConfig = { cdpPort, ...driver ? { driver } : {}, color: profileColor }; } await writeConfigFile({ ...cfg, browser: { ...cfg.browser, profiles: { ...rawProfiles, [name]: profileConfig } } }); state.resolved.profiles[name] = profileConfig; const resolved = resolveProfile(state.resolved, name); if (!resolved) throw new Error(`profile "${name}" not found after creation`); return { ok: true, profile: name, cdpPort: resolved.cdpPort, cdpUrl: resolved.cdpUrl, color: resolved.color, isRemote: !resolved.cdpIsLoopback }; }; const deleteProfile = async (nameRaw) => { const name = nameRaw.trim(); if (!name) throw new Error("profile name is required"); if (!isValidProfileName(name)) throw new Error("invalid profile name"); const cfg = loadConfig(); const profiles = cfg.browser?.profiles ?? {}; if (!(name in profiles)) throw new Error(`profile "${name}" not found`); if (name === (cfg.browser?.defaultProfile ?? DEFAULT_BROWSER_DEFAULT_PROFILE_NAME)) throw new Error(`cannot delete the default profile "${name}"; change browser.defaultProfile first`); let deleted = false; const state = ctx.state(); if (resolveProfile(state.resolved, name)?.cdpIsLoopback) { try { await ctx.forProfile(name).stopRunningBrowser(); } catch {} const userDataDir = resolveOpenClawUserDataDir(name); const profileDir = path.dirname(userDataDir); if (fs.existsSync(profileDir)) { await movePathToTrash(profileDir); deleted = true; } } const { [name]: _removed, ...remainingProfiles } = profiles; await writeConfigFile({ ...cfg, browser: { ...cfg.browser, profiles: remainingProfiles } }); delete state.resolved.profiles[name]; state.profiles.delete(name); return { ok: true, profile: name, deleted }; }; return { listProfiles, createProfile, deleteProfile }; } //#endregion //#region src/browser/routes/basic.ts async function withBasicProfileRoute(params) { const profileCtx = resolveProfileContext(params.req, params.res, params.ctx); if (!profileCtx) return; try { await params.run(profileCtx); } catch (err) { jsonError(params.res, 500, String(err)); } } function registerBrowserBasicRoutes(app, ctx) { app.get("/profiles", async (_req, res) => { try { const profiles = await createBrowserProfilesService(ctx).listProfiles(); res.json({ profiles }); } catch (err) { jsonError(res, 500, String(err)); } }); app.get("/", async (req, res) => { let current; try { current = ctx.state(); } catch { return jsonError(res, 503, "browser server not started"); } const profileCtx = getProfileContext(req, ctx); if ("error" in profileCtx) return jsonError(res, profileCtx.status, profileCtx.error); const [cdpHttp, cdpReady] = await Promise.all([profileCtx.isHttpReachable(300), profileCtx.isReachable(600)]); const profileState = current.profiles.get(profileCtx.profile.name); let detectedBrowser = null; let detectedExecutablePath = null; let detectError = null; try { const detected = resolveBrowserExecutableForPlatform(current.resolved, process.platform); if (detected) { detectedBrowser = detected.kind; detectedExecutablePath = detected.path; } } catch (err) { detectError = String(err); } res.json({ enabled: current.resolved.enabled, profile: profileCtx.profile.name, running: cdpReady, cdpReady, cdpHttp, pid: profileState?.running?.pid ?? null, cdpPort: profileCtx.profile.cdpPort, cdpUrl: profileCtx.profile.cdpUrl, chosenBrowser: profileState?.running?.exe.kind ?? null, detectedBrowser, detectedExecutablePath, detectError, userDataDir: profileState?.running?.userDataDir ?? null, color: profileCtx.profile.color, headless: current.resolved.headless, noSandbox: current.resolved.noSandbox, executablePath: current.resolved.executablePath ?? null, attachOnly: current.resolved.attachOnly }); }); app.post("/start", async (req, res) => { await withBasicProfileRoute({ req, res, ctx, run: async (profileCtx) => { await profileCtx.ensureBrowserAvailable(); res.json({ ok: true, profile: profileCtx.profil