naisys
Version:
NAISYS - Autonomous AI agent runner with built-in context management and cost tracking
375 lines • 15.5 kB
JavaScript
/**
* Google-specific computer use helpers.
* Handles coordinate normalization (0-999 ↔ native pixels), action format
* conversion between Google named functions and internal (Anthropic-compatible)
* format, desktop action extraction from responses, image resizing for
* screenshots, and context formatting with function_call/function_response.
*/
import { getTargetScaleFactor, mapCoordinateBetweenSpaces, } from "../computerService.js";
import { canonicalizeKeyCombo } from "../keyCombo.js";
// --- Coordinate normalization ---
// Google uses a 0-999 normalized grid regardless of screen resolution.
const NORMALIZED_MAX = 1000;
// --- Known Google Computer Use action names ---
const GOOGLE_CU_ACTIONS = new Set([
"click_at",
"hover_at",
"type_text_at",
"key_combination",
"scroll_document",
"scroll_at",
"drag_and_drop",
"open_web_browser",
"wait_5_seconds",
"go_back",
"go_forward",
"search",
"navigate",
]);
function isGoogleComputerUseAction(name) {
return GOOGLE_CU_ACTIONS.has(name);
}
// --- Google action → internal format conversion ---
function convertGoogleActionToInternal(name, args, scaledWidth, scaledHeight) {
const denormalize = (x, y) => mapCoordinateBetweenSpaces([x, y], NORMALIZED_MAX, NORMALIZED_MAX, scaledWidth, scaledHeight);
switch (name) {
case "click_at":
return [
{
action: "left_click",
coordinate: denormalize(args.x, args.y),
},
];
case "hover_at":
return [
{
action: "mouse_move",
coordinate: denormalize(args.x, args.y),
},
];
case "type_text_at": {
const actions = [];
actions.push({
action: "left_click",
coordinate: denormalize(args.x, args.y),
});
if (args.clear_before_typing !== false) {
actions.push({ action: "key", text: "ctrl+a" });
actions.push({ action: "key", text: "backspace" });
}
actions.push({ action: "type", text: args.text });
if (args.press_enter !== false) {
actions.push({ action: "key", text: "enter" });
}
return actions;
}
case "key_combination":
return [{ action: "key", text: args.keys }];
case "scroll_document":
return [
{
action: "scroll",
coordinate: [
Math.round(scaledWidth / 2),
Math.round(scaledHeight / 2),
],
scroll_direction: args.direction,
scroll_amount: 3,
},
];
case "scroll_at": {
const magnitude = args.magnitude || 800;
const amount = Math.max(1, Math.ceil(magnitude / 200));
return [
{
action: "scroll",
coordinate: denormalize(args.x, args.y),
scroll_direction: args.direction,
scroll_amount: amount,
},
];
}
case "drag_and_drop":
return [
{
action: "left_click_drag",
start_coordinate: denormalize(args.x, args.y),
coordinate: denormalize(args.destination_x, args.destination_y),
},
];
case "wait_5_seconds":
return [{ action: "wait" }];
case "go_back":
return [{ action: "key", text: "alt+left" }];
case "go_forward":
return [{ action: "key", text: "alt+right" }];
case "navigate": {
const actions = [];
actions.push({ action: "key", text: "ctrl+l" });
actions.push({ action: "type", text: args.url });
actions.push({ action: "key", text: "enter" });
return actions;
}
case "open_web_browser":
case "search":
return []; // No-op in desktop context
default:
return [];
}
}
// --- Internal format → Google args reconstruction ---
function reconstructGoogleArgs(googleFuncName, internalActions, scaledWidth, scaledHeight) {
const normalize = (x, y) => mapCoordinateBetweenSpaces([x, y], scaledWidth, scaledHeight, NORMALIZED_MAX, NORMALIZED_MAX);
switch (googleFuncName) {
case "click_at":
case "hover_at": {
const a = internalActions[0];
if (a?.action !== "left_click" && a?.action !== "mouse_move")
return {};
const [x, y] = normalize(a.coordinate[0], a.coordinate[1]);
return { x, y };
}
case "type_text_at": {
const clickAction = internalActions.find((a) => a.action === "left_click");
const typeAction = internalActions.find((a) => a.action === "type");
if (clickAction?.action !== "left_click")
return {};
const hasClear = internalActions.some((a) => a.action === "key" && canonicalizeKeyCombo(a.text) === "ctrl+a");
const hasEnter = internalActions.some((a) => a.action === "key" && canonicalizeKeyCombo(a.text) === "enter");
const [x, y] = normalize(clickAction.coordinate[0], clickAction.coordinate[1]);
return {
x,
y,
text: typeAction?.action === "type" ? typeAction.text : "",
press_enter: hasEnter,
clear_before_typing: hasClear,
};
}
case "key_combination": {
const a = internalActions[0];
return { keys: a?.action === "key" ? a.text : "" };
}
case "scroll_document": {
const a = internalActions[0];
return {
direction: a?.action === "scroll" ? a.scroll_direction : "down",
};
}
case "scroll_at": {
const a = internalActions[0];
if (a?.action !== "scroll")
return {};
const [x, y] = normalize(a.coordinate[0], a.coordinate[1]);
return {
x,
y,
direction: a.scroll_direction,
magnitude: a.scroll_amount * 200,
};
}
case "drag_and_drop": {
const a = internalActions[0];
if (a?.action !== "left_click_drag")
return {};
const [x, y] = normalize(a.start_coordinate[0], a.start_coordinate[1]);
const [destination_x, destination_y] = normalize(a.coordinate[0], a.coordinate[1]);
return { x, y, destination_x, destination_y };
}
case "navigate": {
const typeAction = internalActions.find((a) => a.action === "type");
return {
url: typeAction?.action === "type" ? typeAction.text : "",
};
}
case "go_back":
case "go_forward":
case "open_web_browser":
case "search":
case "wait_5_seconds":
return {};
default:
return {};
}
}
/**
* Derive the scaled dimensions to use when replaying a stored action back
* to the Google API as 0-999 normalized coords. Reads the viewport stamp
* `attachViewportToActions` puts on every emitted action, so a later focus
* change cannot misalign replayed coords. Falls back to the native display
* only for malformed/legacy input that lacks a stamp.
*/
function getReplayScaledDimensions(input, desktopConfig) {
const stamped = input.viewport;
const hasStamp = !!stamped &&
typeof stamped.width === "number" &&
typeof stamped.height === "number" &&
stamped.width > 0 &&
stamped.height > 0;
const width = hasStamp
? stamped.width
: desktopConfig.nativeDisplayWidth;
const height = hasStamp
? stamped.height
: desktopConfig.nativeDisplayHeight;
const scaleFactor = getTargetScaleFactor(width, height);
return {
scaledWidth: Math.floor(width * scaleFactor),
scaledHeight: Math.floor(height * scaleFactor),
};
}
// --- Public API ---
/**
* Extract desktop actions from Google response **parts** (not bare function calls).
* Accepts full Part objects so we can capture `thoughtSignature` which lives
* at the Part level, not inside the FunctionCall object.
* Converts normalized coordinates to native screen space and
* maps named functions to internal action format.
*/
export function extractDesktopActions(parts, scaledWidth, scaledHeight) {
const actions = [];
for (const part of parts) {
const fc = part?.functionCall;
if (!fc || typeof fc !== "object")
continue;
const name = fc.name;
const id = fc.id ||
`google-cu-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
// Capture thoughtSignature from the Part level (not inside functionCall).
// Gemini 3 Flash requires it when replaying function_call parts in context.
// It flows through ToolUseBlock.input (Record<string, unknown>) untouched.
const thoughtSignature = part.thoughtSignature ?? part.thought_signature;
// Unknown Google function names used to be silently skipped, which let
// the model retry the same unsupported call indefinitely. Surface as a
// validationError-bearing DesktopAction so the model gets feedback in
// the next turn's tool_result.
if (!isGoogleComputerUseAction(name)) {
actions.push({
id,
name,
input: {
actions: [],
...(thoughtSignature ? { thoughtSignature } : {}),
},
validationError: `Unsupported Google computer-use function: ${name}`,
});
continue;
}
const args = (fc.args || {});
const internalActions = convertGoogleActionToInternal(name, args, scaledWidth, scaledHeight);
actions.push({
id,
name, // Preserve Google function name for context reconstruction
input: {
actions: internalActions,
...(thoughtSignature ? { thoughtSignature } : {}),
},
});
}
return actions;
}
/**
* Format the full context for the Google API with computer use support.
* Converts internal ToolUseBlock/ToolResultBlock content blocks to
* Google's function_call/function_response parts, resizing screenshot
* images for token efficiency.
*/
export function formatContextWithComputerUse(context, desktopConfig, formatPartsForGoogle) {
// Map tool_use IDs to their Google function names for function_response reconstruction
const toolUseIdToName = new Map();
const formattedMessages = [];
for (const msg of context) {
if (typeof msg.content === "string") {
formattedMessages.push({
role: msg.role === "assistant" ? "model" : "user",
parts: formatPartsForGoogle(msg.content),
});
continue;
}
const content = msg.content;
const hasToolUse = content.some((b) => b.type === "tool_use");
const hasToolResult = content.some((b) => b.type === "tool_result");
// Assistant message with tool_use → model message with function_call parts
if (msg.role === "assistant" && hasToolUse) {
const parts = [];
for (const block of content) {
if (block.type === "text") {
parts.push({ text: block.text });
}
else if (block.type === "tool_use") {
toolUseIdToName.set(block.id, block.name);
const input = block.input;
const { scaledWidth, scaledHeight } = getReplayScaledDimensions(input, desktopConfig);
const googleArgs = reconstructGoogleArgs(block.name, input.actions, scaledWidth, scaledHeight);
parts.push({
functionCall: {
name: block.name,
id: block.id,
args: googleArgs,
},
// thoughtSignature lives at the Part level, not inside functionCall.
// Gemini 3 Flash requires it when replaying function_call parts.
...(input.thoughtSignature
? { thoughtSignature: input.thoughtSignature }
: {}),
});
}
}
formattedMessages.push({ role: "model", parts });
continue;
}
// User message with tool_result → merge into the previous user message
// if it also contains function_responses. Google requires ALL function_responses
// for a batch of function_calls in a SINGLE user message.
if (msg.role === "user" && hasToolResult) {
const parts = [];
for (const block of content) {
if (block.type === "tool_result") {
const funcName = toolUseIdToName.get(block.toolUseId) || "computer_action";
// Google's computer use model requires a 'url' field in every function response
const response = { url: "" };
if (block.isError) {
const textContent = block.resultContent?.find((c) => c.type === "text");
response.error =
textContent?.type === "text" ? textContent.text : "Action failed";
}
const frParts = [];
for (const rc of block.resultContent) {
if (rc.type === "image") {
frParts.push({
inlineData: { mimeType: rc.mimeType, data: rc.base64 },
});
}
}
parts.push({
functionResponse: {
name: funcName,
id: block.toolUseId,
response,
parts: frParts.length > 0 ? frParts : undefined,
},
});
}
else if (block.type === "text") {
parts.push({ text: block.text });
}
}
// Merge with previous user message if it has function_responses
const prev = formattedMessages[formattedMessages.length - 1];
if (prev?.role === "user" &&
prev.parts?.some((p) => p.functionResponse)) {
prev.parts.push(...parts);
}
else {
formattedMessages.push({ role: "user", parts });
}
continue;
}
// Regular ContentBlock[] message (no tool blocks)
formattedMessages.push({
role: msg.role === "assistant" ? "model" : "user",
parts: formatPartsForGoogle(content),
});
}
return formattedMessages;
}
//# sourceMappingURL=google-computer-use.js.map