donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
531 lines (493 loc) • 15.4 kB
HTML
<html>
<head>
<meta charset="utf-8" />
<title>Donobu — Interactive Session</title>
<style>
:root {
font-family:
-apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial,
sans-serif;
font-size: 13px;
--accent: #fbbc05;
--panel-bg: #1a1a1a;
}
* {
box-sizing: border-box;
margin: 0;
padding: 0;
}
html,
body {
margin: 0;
height: 100%;
background: var(--panel-bg);
color: #fff;
overflow: hidden;
}
body {
display: flex;
flex-direction: column;
}
/* Toolbar --------------------------------------------------------- */
#toolbar {
display: flex;
align-items: center;
padding: 8px 12px;
background: rgba(255, 255, 255, 0.03);
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
flex-shrink: 0;
gap: 6px;
}
#headline {
font-weight: 500;
font-size: 13px;
letter-spacing: -0.2px;
flex: 1;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.icon-btn {
display: none;
cursor: pointer;
opacity: 0.4;
transition:
opacity 0.2s ease,
transform 0.1s ease,
background-color 0.2s ease;
border-radius: 6px;
padding: 4px;
background: transparent;
flex-shrink: 0;
}
.icon-btn:hover {
opacity: 0.7;
background: rgba(255, 255, 255, 0.1);
}
.icon-btn.visible {
display: flex;
opacity: 1;
background: rgba(251, 188, 5, 0.15);
}
.icon-btn.visible:hover {
background: rgba(251, 188, 5, 0.25);
}
.icon-btn:active {
transform: scale(0.9);
transition: transform 0.05s;
}
.icon-btn svg {
width: 20px;
height: 20px;
fill: none;
stroke: currentColor;
stroke-width: 2;
stroke-linecap: round;
stroke-linejoin: round;
}
.pending {
color: var(--accent);
animation: pulse 1.5s ease-in-out infinite;
}
@keyframes pulse {
0%,
100% {
opacity: 1;
}
50% {
opacity: 0.7;
}
}
/* Screencast viewport ---------------------------------------------- */
#viewport {
flex: 1;
position: relative;
overflow: hidden;
background: #000;
cursor: default;
}
#viewport.no-mirror {
display: flex;
align-items: center;
justify-content: center;
background: var(--panel-bg);
}
#viewport.no-mirror::after {
content: 'Browser is visible — interact directly';
color: rgba(255, 255, 255, 0.3);
font-size: 14px;
}
#screencast {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
object-fit: contain;
image-rendering: -webkit-optimize-contrast;
}
/* Instructions bar ------------------------------------------------- */
#instructions-bar {
display: flex;
align-items: center;
padding: 8px 12px;
gap: 8px;
border-top: 1px solid rgba(255, 255, 255, 0.08);
background: rgba(255, 255, 255, 0.03);
flex-shrink: 0;
}
#msgBox {
flex: 1;
min-height: 32px;
max-height: 80px;
resize: none;
background: rgba(255, 255, 255, 0.95);
color: #333;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 6px;
padding: 6px 10px;
font-family: inherit;
font-size: 12px;
}
#msgBox:focus {
border-color: rgba(251, 188, 5, 0.8);
box-shadow: 0 0 0 2px rgba(251, 188, 5, 0.2);
outline: none;
}
#msgBox::placeholder {
color: #999;
}
#msgBox:disabled {
background: rgba(255, 255, 255, 0.3);
color: #666;
}
#sendBtn {
display: flex;
cursor: pointer;
padding: 4px;
border-radius: 6px;
background: rgba(251, 188, 5, 0.15);
opacity: 0.8;
transition:
opacity 0.2s,
background-color 0.2s;
flex-shrink: 0;
}
#sendBtn:hover {
opacity: 1;
background: rgba(251, 188, 5, 0.3);
}
#sendBtn:active {
transform: scale(0.9);
}
#sendBtn.hidden {
display: none;
}
#sendBtn svg {
width: 20px;
height: 20px;
fill: none;
stroke: currentColor;
stroke-width: 2;
stroke-linecap: round;
stroke-linejoin: round;
}
</style>
</head>
<body>
<!-- Toolbar -->
<div id="toolbar">
<!-- Pause: shown when AI is running -->
<span id="btnPause" class="icon-btn" title="Pause AI">
<svg viewBox="0 0 24 24">
<circle cx="12" cy="12" r="10" />
<line x1="10" y1="8" x2="10" y2="16" />
<line x1="14" y1="8" x2="14" y2="16" />
</svg>
</span>
<!-- Play: shown when AI is paused (resume) -->
<span id="btnPlay" class="icon-btn" title="Resume AI">
<svg viewBox="0 0 24 24">
<circle cx="12" cy="12" r="10" />
<polygon points="10 8 16 12 10 16 10 8" />
</svg>
</span>
<span id="headline">Explore the page, or give the AI an instruction</span>
</div>
<!-- Screencast viewport (only used in headless mirror mode) -->
<div id="viewport" class="no-mirror">
<canvas id="screencast"></canvas>
</div>
<!-- Instructions bar -->
<div id="instructions-bar">
<input id="msgBox" type="text" placeholder="Tell the AI what to do…" />
<span id="sendBtn" title="Send instruction">
<svg viewBox="0 0 24 24">
<line x1="22" y1="2" x2="11" y2="13" />
<polygon points="22 2 15 22 11 13 2 9 22 2" />
</svg>
</span>
</div>
<script>
const ipc = (() => {
if (window.electron) {
return { send: window.electron.send, on: window.electron.on };
}
if (window.__controlPanelSend) {
window.__controlPanelListeners = {};
return {
send: (channel, data) => {
window.__controlPanelSend(JSON.stringify({ channel, data }));
},
on: (channel, listener) => {
window.__controlPanelListeners[channel] = listener;
},
};
}
throw new Error('IPC bridge missing');
})();
const { send, on } = ipc;
const flowId = new URLSearchParams(location.search).get('flow');
const viewport = document.getElementById('viewport');
const canvas = document.getElementById('screencast');
const ctx = canvas.getContext('2d');
const btnPause = document.getElementById('btnPause');
const btnPlay = document.getElementById('btnPlay');
const headline = document.getElementById('headline');
const msgBox = document.getElementById('msgBox');
const sendBtn = document.getElementById('sendBtn');
// --- Panel state management ----------------------------------------
let panelState = 'idle'; // 'idle' | 'running' | 'paused'
function setPanelState(newState, headlineText) {
panelState = newState;
headline.classList.remove('pending');
if (newState === 'idle') {
headline.textContent =
headlineText || 'Explore the page, or give the AI an instruction';
msgBox.disabled = false;
btnPause.classList.remove('visible');
btnPlay.classList.remove('visible');
sendBtn.classList.remove('hidden');
} else if (newState === 'running') {
headline.textContent = headlineText || 'AI is working…';
headline.classList.add('pending');
msgBox.disabled = true;
btnPause.classList.add('visible');
btnPlay.classList.remove('visible');
sendBtn.classList.add('hidden');
} else if (newState === 'paused') {
headline.textContent =
headlineText ||
'AI paused — interact manually or revise instructions';
msgBox.disabled = false;
btnPause.classList.remove('visible');
btnPlay.classList.add('visible');
sendBtn.classList.remove('hidden');
}
}
// --- State updates from backend ------------------------------------
on('stateUpdate', (_event, data) => {
const state = data.state;
const hl = data.headline;
if (state === 'PAUSED') {
setPanelState('paused', hl);
} else if (
state === 'RUNNING_ACTION' ||
state === 'QUERYING_LLM_FOR_NEXT_ACTION' ||
state === 'INITIALIZING' ||
state === 'RESUMING'
) {
setPanelState('running', hl);
} else if (
state === 'IDLE' ||
state === 'SUCCESS' ||
state === 'FAILED'
) {
setPanelState('idle', hl);
}
});
// --- Screencast rendering -----------------------------------------
let screencastActive = false;
let imgScale = { x: 1, y: 1 };
let screencastOffsetX = 0;
let screencastOffsetY = 0;
let screencastWidth = 0;
let screencastHeight = 0;
on('screencastFrame', (_event, { data, metadata }) => {
if (!screencastActive) {
screencastActive = true;
viewport.classList.remove('no-mirror');
}
const img = new Image();
img.onload = () => {
const vw = viewport.clientWidth;
const vh = viewport.clientHeight;
const iw = metadata.deviceWidth;
const ih = metadata.deviceHeight;
const scale = Math.min(vw / iw, vh / ih);
screencastWidth = iw * scale;
screencastHeight = ih * scale;
screencastOffsetX = (vw - screencastWidth) / 2;
screencastOffsetY = (vh - screencastHeight) / 2;
canvas.width = vw;
canvas.height = vh;
canvas.style.width = vw + 'px';
canvas.style.height = vh + 'px';
ctx.clearRect(0, 0, vw, vh);
ctx.drawImage(
img,
screencastOffsetX,
screencastOffsetY,
screencastWidth,
screencastHeight,
);
imgScale.x = iw / screencastWidth;
imgScale.y = ih / screencastHeight;
};
img.src = 'data:image/jpeg;base64,' + data;
});
// --- Input forwarding (screencast mode only) ----------------------
function translateCoords(clientX, clientY) {
const rect = canvas.getBoundingClientRect();
const cx = clientX - rect.left - screencastOffsetX;
const cy = clientY - rect.top - screencastOffsetY;
return {
x: Math.round(cx * imgScale.x),
y: Math.round(cy * imgScale.y),
};
}
function isInBounds(clientX, clientY) {
const rect = canvas.getBoundingClientRect();
const cx = clientX - rect.left;
const cy = clientY - rect.top;
return (
cx >= screencastOffsetX &&
cx <= screencastOffsetX + screencastWidth &&
cy >= screencastOffsetY &&
cy <= screencastOffsetY + screencastHeight
);
}
viewport.addEventListener('mousedown', (e) => {
if (!screencastActive || !isInBounds(e.clientX, e.clientY)) return;
const { x, y } = translateCoords(e.clientX, e.clientY);
send('cdpInput', {
type: 'mousePressed',
x,
y,
button: 'left',
clickCount: 1,
});
});
viewport.addEventListener('mouseup', (e) => {
if (!screencastActive || !isInBounds(e.clientX, e.clientY)) return;
const { x, y } = translateCoords(e.clientX, e.clientY);
send('cdpInput', {
type: 'mouseReleased',
x,
y,
button: 'left',
clickCount: 1,
});
});
viewport.addEventListener('mousemove', (e) => {
if (!screencastActive || !isInBounds(e.clientX, e.clientY)) return;
const { x, y } = translateCoords(e.clientX, e.clientY);
send('cdpInput', { type: 'mouseMoved', x, y });
});
viewport.addEventListener(
'wheel',
(e) => {
if (!screencastActive) return;
e.preventDefault();
const { x, y } = translateCoords(e.clientX, e.clientY);
send('cdpInput', {
type: 'mouseWheel',
x,
y,
deltaX: e.deltaX,
deltaY: e.deltaY,
});
},
{ passive: false },
);
// Keyboard input — only forward when canvas area is focused
viewport.tabIndex = 0;
viewport.addEventListener('keydown', (e) => {
if (!screencastActive) return;
e.preventDefault();
send('cdpInput', {
type: 'keyDown',
key: e.key,
code: e.code,
modifiers: getModifiers(e),
text: e.key.length === 1 ? e.key : '',
});
});
viewport.addEventListener('keyup', (e) => {
if (!screencastActive) return;
e.preventDefault();
send('cdpInput', {
type: 'keyUp',
key: e.key,
code: e.code,
modifiers: getModifiers(e),
});
});
function getModifiers(e) {
let m = 0;
if (e.altKey) m |= 1;
if (e.ctrlKey) m |= 2;
if (e.metaKey) m |= 4;
if (e.shiftKey) m |= 8;
return m;
}
// --- User actions --------------------------------------------------
function submitInstruction() {
const text = msgBox.value.trim();
if (!text) return;
msgBox.value = '';
if (panelState === 'idle') {
// Start a new AI instruction
send('userAction', {
flowId,
action: { type: 'RESUME', userInstruction: text },
});
setPanelState('running');
} else if (panelState === 'paused') {
// Resume AI with revised instructions
send('userAction', {
flowId,
action: { type: 'RESUME', userInstruction: text },
});
setPanelState('running');
}
}
// Send button
sendBtn.onclick = submitInstruction;
// Enter to submit (only when idle or paused)
msgBox.addEventListener('keydown', (e) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
submitInstruction();
}
});
// Pause button — pauses the AI mid-execution
btnPause.onclick = () => {
send('userAction', {
flowId,
action: { type: 'PAUSE' },
});
setPanelState('paused');
};
// Play button — resume without new instructions
btnPlay.onclick = () => {
send('userAction', {
flowId,
action: { type: 'RESUME' },
});
setPanelState('running');
};
</script>
</body>
</html>