donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
378 lines • 15.1 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.flushTbdSessions = void 0;
exports.tbd = tbd;
const crypto_1 = require("crypto");
const playwright_1 = require("playwright");
const Logger_1 = require("../../utils/Logger");
const MiscUtils_1 = require("../../utils/MiscUtils");
const actionRecorder_1 = require("./tbd/actionRecorder");
const callSiteCapture_1 = require("./tbd/callSiteCapture");
var fileRewriter_1 = require("./tbd/fileRewriter");
Object.defineProperty(exports, "flushTbdSessions", { enumerable: true, get: function () { return fileRewriter_1.flushTbdSessions; } });
const TBD_HTML_PATH = MiscUtils_1.MiscUtils.getResourceFilePath('control-panel-tbd.html');
// ---------------------------------------------------------------------------
// TbdControlPanel — bridges the DonobuFlow control-panel protocol to the
// tbd panel window.
// ---------------------------------------------------------------------------
/**
* A {@link ControlPanel} implementation backed by the tbd panel window.
*
* When `page.ai(instruction)` creates a DonobuFlow, the flow calls
* `update()` and `popLatestUserAction()` on this panel. Updates are
* forwarded to the panel UI via Playwright evaluate; user actions
* (pause/resume) are queued by the IPC handler and returned here.
*/
class TbdControlPanel {
constructor(panelPage) {
this.latestAction = null;
this.panelPage = panelPage;
}
/** Called by the IPC handler when the panel sends a user action. */
setLatestAction(action) {
this.latestAction = action;
}
// -- ControlPanel interface -------------------------------------------
update(data) {
this.panelPage
.evaluate((d) => {
const listener = window.__controlPanelListeners?.stateUpdate;
if (listener) {
listener(null, d);
}
}, { state: data.state, headline: data.headline })
.catch(() => {
// Panel may have been closed — ignore.
});
}
popLatestUserAction() {
const action = this.latestAction;
this.latestAction = null;
return action;
}
close() {
// No-op — the tbd() orchestrator manages the panel lifecycle.
}
}
function createEventQueue() {
const queue = [];
let resolver = null;
return {
push(event) {
if (resolver) {
resolver(event);
resolver = null;
}
else {
queue.push(event);
}
},
next() {
if (queue.length > 0) {
return Promise.resolve(queue.shift());
}
return new Promise((resolve) => {
resolver = resolve;
});
},
};
}
// ---------------------------------------------------------------------------
// Main entry point
// ---------------------------------------------------------------------------
/**
* Opens an interactive mini-session that pauses test execution and lets
* the user explore the page and/or give AI instructions.
*
* - In headed mode the user interacts directly with the visible browser.
* - In headless mode a live CDP screencast is mirrored into the panel.
* - The user can type an instruction and hit Enter → AI takes over.
* - The user can pause the AI, revise instructions, or take over manually.
* - Closing the panel window ends the session.
*
* All interactions (manual and AI) are recorded. When the test completes,
* `await page.tbd()` is replaced in the source file with the equivalent
* Playwright code.
*/
async function tbd(page) {
// Capture the call site *first*, before any async work.
const callSite = (0, callSiteCapture_1.captureCallSite)();
const extPage = page;
const flowId = (0, crypto_1.randomUUID)();
const isHeadless = await detectHeadless(page);
// Install the action recorder (Donobu's standard recording pipeline).
const recorder = await (0, actionRecorder_1.installRecorder)(page, extPage._dnb.persistence, extPage._dnb.donobuFlowMetadata);
// Ordered list of actions for code generation.
const recordedActions = [];
// Launch the control panel browser.
const panelBrowser = await playwright_1.chromium.launch({
headless: false,
args: [
'--disable-extensions',
'--disable-component-extensions-with-background-pages',
'--disable-background-networking',
'--no-first-run',
],
});
const panelContext = await panelBrowser.newContext({
viewport: isHeadless
? { width: 1024, height: 768 }
: { width: 400, height: 300 },
colorScheme: 'dark',
});
const panelPage = await panelContext.newPage();
// Create the control panel that bridges to the panel window.
const tbdControlPanel = new TbdControlPanel(panelPage);
// Event queue for the main loop.
const events = createEventQueue();
// Track whether an AI flow is currently running so IPC messages are
// routed correctly: either to the flow's control panel (when running)
// or to our event queue (when idle).
let aiRunning = false;
// CDP session for screencast (headless) and input forwarding.
let cdpSession = null;
// --- IPC bridge --------------------------------------------------------
await panelPage.exposeFunction('__controlPanelSend', (rawMessage) => {
try {
const { channel, data } = JSON.parse(rawMessage);
if (channel === 'userAction') {
const action = data.action;
if (aiRunning) {
// AI is running — route pause/resume/end to the flow's
// control panel so DonobuFlow picks it up on the next poll.
tbdControlPanel.setLatestAction(action);
}
else {
// Idle — interpret RESUME-with-instruction as a new AI
// instruction to start.
if (action.type === 'RESUME' && action.userInstruction) {
events.push({
type: 'instruction',
instruction: action.userInstruction,
});
}
}
}
else if (channel === 'cdpInput' && cdpSession) {
void dispatchCdpInput(cdpSession, data);
}
}
catch (error) {
Logger_1.appLogger.warn('tbd: failed to parse control panel IPC message', error);
}
});
// Detect panel window close → end the session.
panelPage.on('close', () => {
if (aiRunning) {
// Tell the running flow to stop.
tbdControlPanel.setLatestAction({ type: 'END' });
}
events.push({ type: 'close' });
});
await panelPage.goto(`file://${TBD_HTML_PATH}?flow=${encodeURIComponent(flowId)}`);
// --- CDP screencast (headless mode only) --------------------------------
if (isHeadless) {
try {
cdpSession = await page.context().newCDPSession(page);
cdpSession.on('Page.screencastFrame', (params) => {
void (async () => {
await cdpSession.send('Page.screencastFrameAck', {
sessionId: params.sessionId,
});
await panelPage.evaluate(({ data, metadata }) => {
const listener = window.__controlPanelListeners
?.screencastFrame;
if (listener) {
listener(null, { data, metadata });
}
}, { data: params.data, metadata: params.metadata });
})().catch(() => {
// Panel may have been closed — ignore.
});
});
await cdpSession.send('Page.startScreencast', {
format: 'jpeg',
quality: 80,
maxWidth: 1280,
maxHeight: 960,
everyNthFrame: 1,
});
}
catch (error) {
Logger_1.appLogger.warn('tbd: failed to start CDP screencast — falling back to non-mirror mode', error);
cdpSession = null;
}
}
// --- Temporarily install our control panel factory ---------------------
// This makes `page.ai()` use our TbdControlPanel so the user can
// pause/resume the AI from the panel window.
const originalFactory = extPage._dnb.controlPanelFactory;
extPage._dnb.controlPanelFactory = (async () => tbdControlPanel);
// --- Main event loop ---------------------------------------------------
try {
while (true) {
const event = await events.next();
if (event.type === 'close') {
break;
}
if (event.type === 'instruction') {
// Drain manual interactions that happened before this instruction
// so they appear in the correct chronological position.
for (const tc of recorder.drainRecordedActions()) {
recordedActions.push({ type: 'toolCall', toolCall: tc });
}
recordedActions.push({
type: 'aiInstruction',
instruction: event.instruction,
});
aiRunning = true;
try {
await extPage.ai(event.instruction);
}
catch (error) {
Logger_1.appLogger.warn('tbd: AI instruction failed', error);
// Tell the panel the AI finished (with an error).
// Use FAILED so the panel transitions back to idle state.
tbdControlPanel.update({
state: 'FAILED',
headline: `AI error: ${error.message?.slice(0, 80)}`,
});
}
aiRunning = false;
// Tell the panel we're back to idle.
// Use SUCCESS so the panel transitions back to idle state.
tbdControlPanel.update({
state: 'SUCCESS',
headline: 'Explore the page, or give the AI an instruction',
});
}
}
}
finally {
// Restore the original control panel factory.
extPage._dnb.controlPanelFactory = originalFactory;
// Drain any remaining manual interactions.
for (const tc of recorder.drainRecordedActions()) {
recordedActions.push({ type: 'toolCall', toolCall: tc });
}
// Store the session on the page for post-test file rewriting.
if (callSite) {
extPage._dnb.tbdSessions.push({
callSite,
recordedActions,
});
Logger_1.appLogger.info(`tbd: recorded ${recordedActions.length} action(s) at ${callSite.file}:${callSite.line}`);
}
else {
Logger_1.appLogger.warn('tbd: could not capture call site — recorded actions will not be spliced into source');
}
// --- Cleanup ----------------------------------------------------------
if (cdpSession) {
try {
await cdpSession.send('Page.stopScreencast');
await cdpSession.detach();
}
catch {
// Ignore — session may already be gone.
}
}
await panelBrowser.close().catch(() => { });
}
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/**
* Detect whether the browser context is running headless by checking
* the CDP Browser.getVersion product string and window dimensions.
*/
async function detectHeadless(page) {
try {
const cdp = await page.context().newCDPSession(page);
const { result } = await cdp.send('Runtime.evaluate', {
expression: 'navigator.webdriver',
returnByValue: true,
});
const version = await cdp.send('Browser.getVersion');
await cdp.detach();
return (version.product.toLowerCase().includes('headless') ||
(result.value === true && !(await hasVisibleWindow(page))));
}
catch {
// If CDP isn't available (e.g. Firefox), assume headed.
return false;
}
}
/**
* Heuristic: in headed Chromium the outer dimensions are larger than the
* viewport (window chrome). In headless they are identical.
*/
async function hasVisibleWindow(page) {
try {
const dims = await page.evaluate(() => ({
outerWidth: window.outerWidth,
innerWidth: window.innerWidth,
outerHeight: window.outerHeight,
innerHeight: window.innerHeight,
}));
return (dims.outerWidth > dims.innerWidth || dims.outerHeight > dims.innerHeight);
}
catch {
return true;
}
}
/**
* Forward mouse/keyboard input from the control panel canvas to the
* headless page via CDP Input domain.
*/
async function dispatchCdpInput(cdp, input) {
try {
switch (input.type) {
case 'mousePressed':
case 'mouseReleased':
case 'mouseMoved':
await cdp.send('Input.dispatchMouseEvent', {
type: input.type,
x: input.x,
y: input.y,
button: input.button ?? 'none',
clickCount: input.clickCount ?? 0,
});
break;
case 'mouseWheel':
await cdp.send('Input.dispatchMouseEvent', {
type: 'mouseWheel',
x: input.x,
y: input.y,
deltaX: input.deltaX ?? 0,
deltaY: input.deltaY ?? 0,
});
break;
case 'keyDown': {
const params = {
type: 'keyDown',
key: input.key,
code: input.code,
modifiers: input.modifiers ?? 0,
};
if (input.text) {
params.text = input.text;
}
await cdp.send('Input.dispatchKeyEvent', params);
break;
}
case 'keyUp':
await cdp.send('Input.dispatchKeyEvent', {
type: 'keyUp',
key: input.key,
code: input.code,
modifiers: input.modifiers ?? 0,
});
break;
}
}
catch (error) {
Logger_1.appLogger.debug('tbd: CDP input dispatch failed', error);
}
}
//# sourceMappingURL=tbd.js.map