framework/dist/browser-stream.js

The (AI) Framework: turnkey, zero-config AI orchestration that wraps a coding-agent CLI (Claude Code) as a black box and takes you from an idea to a running app. Vite for AI.

import { createServer } from 'node:http';
/**
 * The page a human should be looking at: the agent's current one.
 *
 * Chrome lists targets most-recently-used first, so the first `page` is the one the agent is
 * working in. Picking by position is what keeps the pane from going blind when the agent opens
 * a tab — the failure the spike hit. Ignores targets with no socket (a crashed or detached
 * tab) rather than returning something unusable.
 */
export function pickActivePage(targets) {
    return targets.find(t => t.type === 'page' && !!t.webSocketDebuggerUrl);
}
/**
 * The CDP calls one input maps to, or `[]` for anything unrecognized — a malformed POST must
 * never reach Chrome. A click is press + release (Chrome ignores a lone `mousePressed`), and
 * text goes through `insertText` so it types the character rather than a key code, which is
 * what makes non-ASCII and password managers behave.
 */
export function inputToCdp(input) {
    switch (input?.type) {
        case 'click': {
            if (!Number.isFinite(input.x) || !Number.isFinite(input.y))
                return [];
            const base = { x: input.x, y: input.y, button: 'left', clickCount: 1 };
            return [
                { method: 'Input.dispatchMouseEvent', params: { ...base, type: 'mousePressed' } },
                { method: 'Input.dispatchMouseEvent', params: { ...base, type: 'mouseReleased' } },
            ];
        }
        case 'key': {
            if (typeof input.text !== 'string' || input.text === '')
                return [];
            return [{ method: 'Input.insertText', params: { text: input.text } }];
        }
        case 'scroll': {
            if (!Number.isFinite(input.deltaY))
                return [];
            return [
                {
                    method: 'Input.dispatchMouseEvent',
                    params: { type: 'mouseWheel', x: input.x ?? 0, y: input.y ?? 0, deltaX: 0, deltaY: input.deltaY },
                },
            ];
        }
        case 'navigate': {
            if (typeof input.url !== 'string' || !/^https?:\/\//i.test(input.url))
                return [];
            return [{ method: 'Page.navigate', params: { url: input.url } }];
        }
        default:
            return [];
    }
}
/** The MJPEG part header for one frame. */
export function framePart(boundary, jpeg) {
    return Buffer.concat([
        Buffer.from(`--${boundary}\r\nContent-Type: image/jpeg\r\nContent-Length: ${jpeg.length}\r\n\r\n`),
        jpeg,
        Buffer.from('\r\n'),
    ]);
}
const BOUNDARY = 'frame';
/**
 * Start the bridge. Returns undefined when Chrome has no page to stream — the caller carries
 * on without a pane rather than failing the agent.
 *
 * The stream is bound to loopback explicitly: the frames can contain whatever the human is
 * typing, including a password, so this must not be reachable from the network. For the same
 * reason no frame is ever written to disk or into the agent's event log.
 */
export async function startBrowserStream(opts) {
    const listTargets = opts.listTargets ?? defaultListTargets;
    const targets = await listTargets(opts.browserUrl).catch(() => []);
    const page = pickActivePage(targets);
    if (!page?.webSocketDebuggerUrl)
        return undefined;
    let latest;
    const viewers = new Set();
    // What onPage last said, so a poll that sees the same page again stays silent.
    let announcedUrl;
    const announce = (url) => {
        if (!/^https?:\/\//i.test(url) || url === announcedUrl)
            return;
        announcedUrl = url;
        opts.onPage?.(url);
    };
    /** Attach the screencast to one page. Frames land in `latest` and go straight to viewers. */
    const attach = async (target) => {
        const session = await opts.connect(target.webSocketDebuggerUrl);
        session.on('Page.screencastFrame', ({ data, sessionId }) => {
            latest = Buffer.from(data, 'base64');
            for (const res of viewers)
                res.write(framePart(BOUNDARY, latest));
            void session.send('Page.screencastFrameAck', { sessionId }).catch(() => { });
        });
        await session.send('Page.startScreencast', { format: 'jpeg', quality: 60, maxWidth: 1280, maxHeight: 720 });
        return session;
    };
    let current = page;
    let session = await attach(page);
    announce(page.url);
    /**
     * Follow the agent when it opens or switches tabs. Without this the pane shows whichever
     * page happened to be first while the agent works somewhere else — the exact failure the
     * #609 spike reproduced. Cheap: one `/json/list` on an interval, re-attach only on change.
     */
    /**
     * Re-send the newest frame while anyone is watching (#818).
     *
     * Chrome does not finalize a `multipart/x-mixed-replace` part until the next boundary arrives,
     * so the most recent frame is always held back unpainted. A still page never produces that next
     * frame, which left the pane blank while holding a perfectly good JPEG — and a still page is
     * exactly the case this exists for: an agent parked on a login wall is not repainting itself.
     *
     * Repeating the frame supplies the boundary. Loopback only, and only while a viewer is attached.
     */
    const repeat = setInterval(() => {
        if (!latest || viewers.size === 0)
            return;
        for (const res of viewers)
            res.write(framePart(BOUNDARY, latest));
    }, opts.repeatIntervalMs ?? 1000);
    repeat.unref?.();
    const followMs = opts.followIntervalMs ?? 2000;
    const follow = followMs
        ? setInterval(() => {
            void (async () => {
                const next = pickActivePage(await listTargets(opts.browserUrl).catch(() => []));
                if (!next?.webSocketDebuggerUrl)
                    return;
                if (next.id === current.id) {
                    // Same tab, possibly a new page: the screencast keeps itself current, but the
                    // navigation is still worth announcing (#1455 item 6b).
                    current = next;
                    announce(next.url);
                    return;
                }
                const previous = session;
                try {
                    session = await attach(next);
                    current = next;
                    announce(next.url);
                    await previous.send('Page.stopScreencast').catch(() => { });
                    previous.close();
                }
                catch {
                    // Keep streaming the page we already have rather than dropping the pane.
                }
            })();
        }, followMs)
        : undefined;
    follow?.unref?.();
    const server = createServer((req, res) => {
        if (req.method === 'GET' && req.url?.startsWith('/stream')) {
            res.writeHead(200, {
                'content-type': `multipart/x-mixed-replace; boundary=${BOUNDARY}`,
                'cache-control': 'no-store',
                connection: 'close',
            });
            // Node holds the headers back until the first write, so a pane opened before any frame
            // exists would hang waiting for a response rather than showing an empty stream.
            res.flushHeaders();
            // Chrome only emits a frame when the page changes, so a pane opened on a still page
            // would sit blank. Send the last one we have immediately.
            if (latest)
                res.write(framePart(BOUNDARY, latest));
            viewers.add(res);
            req.on('close', () => viewers.delete(res));
            return;
        }
        if (req.method === 'POST' && req.url?.startsWith('/input')) {
            let body = '';
            req.on('data', chunk => {
                body += chunk;
                if (body.length > 8192)
                    req.destroy(); // an input payload is tiny; anything else is not input
            });
            req.on('end', () => {
                let calls = [];
                try {
                    calls = inputToCdp(JSON.parse(body));
                }
                catch {
                    calls = [];
                }
                for (const call of calls)
                    void session.send(call.method, call.params).catch(() => { });
                res.writeHead(calls.length ? 204 : 400).end();
            });
            return;
        }
        res.writeHead(404).end();
    });
    await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
    const port = server.address().port;
    let closed = false;
    return {
        url: `http://127.0.0.1:${port}`,
        port,
        close: async () => {
            if (closed)
                return;
            closed = true;
            if (follow)
                clearInterval(follow);
            clearInterval(repeat);
            for (const res of viewers)
                res.end();
            viewers.clear();
            await session.send('Page.stopScreencast').catch(() => { });
            session.close();
            await new Promise(resolve => server.close(() => resolve()));
        },
    };
}
/**
 * Talk CDP to Chrome over its debugger socket. Node's global WebSocket is enough, which is
 * what keeps this dependency-free.
 */
export const connectCdp = async (webSocketDebuggerUrl) => {
    const ws = new WebSocket(webSocketDebuggerUrl);
    await new Promise((resolve, reject) => {
        ws.addEventListener('open', () => resolve(), { once: true });
        ws.addEventListener('error', () => reject(new Error(`could not open ${webSocketDebuggerUrl}`)), { once: true });
    });
    let nextId = 1;
    const pending = new Map();
    const frameHandlers = [];
    ws.addEventListener('message', ev => {
        let msg;
        try {
            msg = JSON.parse(String(ev.data));
        }
        catch {
            return;
        }
        if (typeof msg.id === 'number') {
            const p = pending.get(msg.id);
            if (!p)
                return;
            pending.delete(msg.id);
            msg.error ? p.reject(new Error(msg.error.message ?? 'CDP error')) : p.resolve(msg.result);
            return;
        }
        if (msg.method === 'Page.screencastFrame') {
            for (const handler of frameHandlers)
                handler(msg.params);
        }
    });
    return {
        send: (method, params = {}) => new Promise((resolve, reject) => {
            const id = nextId++;
            pending.set(id, { resolve, reject });
            try {
                ws.send(JSON.stringify({ id, method, params }));
            }
            catch (err) {
                pending.delete(id);
                reject(err instanceof Error ? err : new Error(String(err)));
            }
        }),
        on: (_event, handler) => void frameHandlers.push(handler),
        close: () => ws.close(),
    };
};
/** The real target list: Chrome's own `/json/list`. */
async function defaultListTargets(browserUrl) {
    const res = await fetch(`${browserUrl}/json/list`);
    if (!res.ok)
        return [];
    const body = (await res.json());
    return Array.isArray(body) ? body : [];
}
//# sourceMappingURL=browser-stream.js.map