UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

580 lines (579 loc) 21.3 kB
import { randomUUID } from "node:crypto"; import { Type } from "typebox"; import { Compile } from "typebox/compile"; //#region src/plugins/computer-use-contract.ts const COMPUTER_USE_V2_ACTION_NAMES = [ "screenshot", "left_click", "right_click", "middle_click", "double_click", "triple_click", "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", "type", "key", "hold_key", "wait", "list_apps", "list_windows", "get_accessibility_tree", "get_cursor_position", "get_window_state", "launch_app", "kill_app", "bring_to_front", "set_value", "zoom", "get_browser_state", "browser_prepare", "browser_navigate", "browser_click", "browser_type", "browser_dialog", "browser_set_input_files", "browser_download", "browser_pointer", "escalate_scope", "get_recording_state", "start_recording", "stop_recording", "replay_trajectory", "invoke_menu" ]; const COMPUTER_USE_V1_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(0, 15); const COMPUTER_ACT_V1_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14); const COMPUTER_CONTRACT_MISMATCH = "COMPUTER_CONTRACT_MISMATCH"; const COMPUTER_STALE_OBSERVATION = "COMPUTER_STALE_OBSERVATION"; const SCROLL_DIRECTIONS = [ "up", "down", "left", "right" ]; const DELIVERY_MODES = ["background", "foreground"]; const ESCALATION_REASONS = [ "ax_tree_pixel_mismatch", "background_delivery_failed", "foreground_ineffective", "no_window_target", "other" ]; const COMPUTER_RESOURCE_HANDLE_PATTERN = "^openclaw:computer-resource:v1:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"; const COMPUTER_EXECUTION_ID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"; const optionalScreenFields = { screenIndex: Type.Optional(Type.Integer({ minimum: 0 })), refWidth: Type.Optional(Type.Integer({ minimum: 1 })) }; const optionalReferenceFields = { windowRef: Type.Optional(Type.String({ minLength: 1 })), elementRef: Type.Optional(Type.String({ minLength: 1 })), observationId: Type.Optional(Type.String({ minLength: 1 })), deliveryMode: Type.Optional(Type.Enum(DELIVERY_MODES, { type: "string" })) }; function actionObject(actions, properties) { return Type.Object({ action: Type.Enum(actions, { type: "string" }), executionId: Type.Optional(Type.String({ pattern: COMPUTER_EXECUTION_ID_PATTERN })), ...properties }, { additionalProperties: false }); } const ComputerActV1ParamsSchema = Type.Union([ actionObject([ "left_click", "right_click", "middle_click", "double_click", "triple_click", "mouse_move" ], { displayFrameId: Type.Optional(Type.String()), x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })), modifiers: Type.Optional(Type.String()), ...optionalScreenFields, ...optionalReferenceFields }), actionObject(["left_click_drag"], { displayFrameId: Type.Optional(Type.String()), x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })), fromX: Type.Optional(Type.Number({ minimum: 0 })), fromY: Type.Optional(Type.Number({ minimum: 0 })), durationMs: Type.Optional(Type.Integer({ minimum: 0 })), ...optionalScreenFields, ...optionalReferenceFields }), actionObject(["left_mouse_down", "left_mouse_up"], { displayFrameId: Type.Optional(Type.String()), x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })), modifiers: Type.Optional(Type.String()), ...optionalScreenFields, ...optionalReferenceFields }), actionObject(["scroll"], { displayFrameId: Type.Optional(Type.String()), x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })), modifiers: Type.Optional(Type.String()), scrollDirection: Type.Optional(Type.Enum(SCROLL_DIRECTIONS, { type: "string" })), scrollAmount: Type.Optional(Type.Integer({ minimum: 1 })), ...optionalScreenFields, ...optionalReferenceFields }), actionObject(["type"], { text: Type.Optional(Type.String()), ...optionalScreenFields, ...optionalReferenceFields }), actionObject(["key"], { keys: Type.Optional(Type.String()), ...optionalScreenFields, ...optionalReferenceFields }), actionObject(["hold_key"], { keys: Type.Optional(Type.String()), durationMs: Type.Optional(Type.Integer({ minimum: 0 })), ...optionalScreenFields, ...optionalReferenceFields }) ]); /** Canonical inner payload accepted by the `computer.act` node command. */ const ComputerActParamsSchema = Type.Union([ ...ComputerActV1ParamsSchema.anyOf, actionObject([ "list_apps", "list_windows", "get_cursor_position" ], {}), actionObject(["get_accessibility_tree"], { windowRef: Type.Optional(Type.String({ minLength: 1 })), query: Type.Optional(Type.String()), depth: Type.Optional(Type.Integer({ minimum: 0, maximum: 64 })), maxElements: Type.Optional(Type.Integer({ minimum: 1, maximum: 2e3 })) }), actionObject(["get_window_state"], { windowRef: Type.String({ minLength: 1 }), query: Type.Optional(Type.String()), depth: Type.Optional(Type.Integer({ minimum: 0, maximum: 64 })), maxElements: Type.Optional(Type.Integer({ minimum: 1, maximum: 2e3 })) }), actionObject(["launch_app", "kill_app"], { app: Type.String({ minLength: 1 }) }), actionObject(["bring_to_front"], { windowRef: Type.String({ minLength: 1 }) }), actionObject(["set_value"], { windowRef: Type.String({ minLength: 1 }), elementRef: Type.String({ minLength: 1 }), observationId: Type.String({ minLength: 1 }), value: Type.String(), deliveryMode: Type.Optional(Type.Enum(DELIVERY_MODES, { type: "string" })) }), actionObject(["invoke_menu"], { windowRef: Type.String({ minLength: 1 }), path: Type.Array(Type.String({ minLength: 1, maxLength: 200 }), { minItems: 1, maxItems: 16 }), deliveryMode: Type.Optional(Type.Enum(DELIVERY_MODES, { type: "string" })) }), actionObject(["zoom"], { windowRef: Type.String({ minLength: 1 }), observationId: Type.String({ minLength: 1 }), x1: Type.Number({ minimum: 0 }), y1: Type.Number({ minimum: 0 }), x2: Type.Number({ minimum: 0 }), y2: Type.Number({ minimum: 0 }) }), actionObject(["get_browser_state"], { windowRef: Type.String({ minLength: 1 }) }), actionObject(["get_browser_state"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), snapshotFormat: Type.Optional(Type.Enum(["dom_refs_v1", "semantic_v2"], { type: "string" })), elementRef: Type.Optional(Type.String({ minLength: 1 })), observationId: Type.Optional(Type.String({ minLength: 1 })), query: Type.Optional(Type.String()), continuation: Type.Optional(Type.String({ minLength: 1 })), includeScreenshot: Type.Optional(Type.Boolean()) }), actionObject(["browser_prepare"], { windowRef: Type.String({ minLength: 1 }), profile: Type.Optional(Type.Enum(["isolated_new", "isolated_named"], { type: "string" })), profileName: Type.Optional(Type.String({ minLength: 1, maxLength: 64, pattern: "^[A-Za-z0-9._-]+$" })) }), actionObject(["browser_navigate"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), url: Type.String({ minLength: 1 }) }), actionObject(["browser_click"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), observationId: Type.String({ minLength: 1 }), elementRef: Type.Optional(Type.String({ minLength: 1 })), x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })), inputRoute: Type.Optional(Type.Enum(["trusted", "dom_event"], { type: "string" })) }), actionObject(["browser_type"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), observationId: Type.String({ minLength: 1 }), elementRef: Type.String({ minLength: 1 }), text: Type.String(), mode: Type.Optional(Type.Enum(["insert_text", "keystrokes"], { type: "string" })), replace: Type.Optional(Type.Boolean()) }), actionObject(["browser_dialog"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), dialogAction: Type.Literal("inspect") }), actionObject(["browser_dialog"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), dialogAction: Type.Literal("accept"), dialogRef: Type.String({ minLength: 1 }), promptText: Type.Optional(Type.String()), deliveryMode: Type.Optional(Type.Enum(DELIVERY_MODES, { type: "string" })) }), actionObject(["browser_dialog"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), dialogAction: Type.Literal("dismiss"), dialogRef: Type.String({ minLength: 1 }), deliveryMode: Type.Optional(Type.Enum(DELIVERY_MODES, { type: "string" })) }), actionObject(["browser_set_input_files"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), observationId: Type.String({ minLength: 1 }), elementRef: Type.String({ minLength: 1 }), resourceHandles: Type.Array(Type.String({ pattern: COMPUTER_RESOURCE_HANDLE_PATTERN }), { minItems: 1, maxItems: 32 }) }), actionObject(["browser_download"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), observationId: Type.String({ minLength: 1 }), elementRef: Type.String({ minLength: 1 }) }), actionObject(["browser_pointer"], { browserRef: Type.String({ minLength: 1 }), pageRef: Type.String({ minLength: 1 }), observationId: Type.String({ minLength: 1 }), pointerAction: Type.Enum([ "hover", "right_click", "double_click", "scroll", "drag" ], { type: "string" }), inputRoute: Type.Optional(Type.Enum(["trusted", "dom_event"], { type: "string" })), elementRef: Type.Optional(Type.String({ minLength: 1 })), x: Type.Optional(Type.Number({ minimum: 0 })), y: Type.Optional(Type.Number({ minimum: 0 })), destinationElementRef: Type.Optional(Type.String({ minLength: 1 })), toX: Type.Optional(Type.Number({ minimum: 0 })), toY: Type.Optional(Type.Number({ minimum: 0 })), deltaX: Type.Optional(Type.Number()), deltaY: Type.Optional(Type.Number()) }), actionObject(["escalate_scope"], { reason: Type.Enum(ESCALATION_REASONS, { type: "string" }) }), actionObject(["get_recording_state", "stop_recording"], {}), actionObject(["start_recording"], { recordVideo: Type.Optional(Type.Boolean()) }), actionObject(["replay_trajectory"], { resourceHandle: Type.String({ pattern: COMPUTER_RESOURCE_HANDLE_PATTERN }), delayMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 1e4 })), stopOnError: Type.Optional(Type.Boolean()) }) ]); const COMPUTER_ACT_RESULT_MAX_ELEMENTS = 2e3; const COMPUTER_ACT_RESULT_MAX_DETAIL_KEYS = 64; const ComputerBoundsSchema = Type.Object({ x: Type.Number(), y: Type.Number(), width: Type.Number({ minimum: 0 }), height: Type.Number({ minimum: 0 }) }, { additionalProperties: false }); const ComputerObservationSchema = Type.Object({ kind: Type.Enum([ "window", "screen", "browser" ], { type: "string" }), base64: Type.Optional(Type.String()), format: Type.Optional(Type.Enum(["jpeg", "png"], { type: "string" })), width: Type.Optional(Type.Integer({ minimum: 1 })), height: Type.Optional(Type.Integer({ minimum: 1 })), observationId: Type.Optional(Type.String({ minLength: 1 })), elements: Type.Optional(Type.Array(Type.Object({ elementRef: Type.String({ minLength: 1 }), role: Type.String({ minLength: 1 }), label: Type.Optional(Type.String()), value: Type.Optional(Type.String()), bounds: ComputerBoundsSchema }, { additionalProperties: false }), { maxItems: COMPUTER_ACT_RESULT_MAX_ELEMENTS })) }, { additionalProperties: false }); const ComputerActResultSchema = Type.Object({ ok: Type.Boolean(), effect: Type.Optional(Type.Enum([ "confirmed", "unverifiable", "suspected_noop" ], { type: "string" })), observation: Type.Optional(ComputerObservationSchema), escalation: Type.Optional(Type.Object({ recommended: Type.Enum([ "window-pixel", "foreground", "desktop" ], { type: "string" }), reasonCode: Type.String({ minLength: 1 }) }, { additionalProperties: false })), details: Type.Optional(Type.Record(Type.String({ minLength: 1, maxLength: 128 }), Type.Unknown(), { maxProperties: COMPUTER_ACT_RESULT_MAX_DETAIL_KEYS })) }, { additionalProperties: false }); const ComputerUseCapabilityDescriptorSchema = Type.Object({ contractVersion: Type.Literal(2), provider: Type.Object({ id: Type.String({ minLength: 1, maxLength: 128 }), label: Type.String({ minLength: 1, maxLength: 256 }), generation: Type.String({ minLength: 1, maxLength: 256 }) }, { additionalProperties: false }), actions: Type.Array(Type.Enum(COMPUTER_USE_V2_ACTION_NAMES, { type: "string" }), { maxItems: COMPUTER_USE_V2_ACTION_NAMES.length, uniqueItems: true }), targets: Type.Array(Type.Enum([ "screen", "window", "element", "browser" ]), { maxItems: 4, uniqueItems: true }), deliveryModes: Type.Array(Type.Enum(DELIVERY_MODES, { type: "string" }), { maxItems: DELIVERY_MODES.length, uniqueItems: true }), observations: Type.Array(Type.Enum([ "image", "accessibility", "browser" ], { type: "string" }), { maxItems: 3, uniqueItems: true }), features: Type.Object({ recording: Type.Boolean(), agentCursor: Type.Boolean(), multiDisplay: Type.Boolean() }, { additionalProperties: false }) }, { additionalProperties: false }); /** Canonical inner payload accepted by the `screen.snapshot` node command. */ const ScreenSnapshotParamsSchema = Type.Object({ executionId: Type.Optional(Type.String({ pattern: COMPUTER_EXECUTION_ID_PATTERN })), screenIndex: Type.Optional(Type.Integer({ minimum: 0 })), maxWidth: Type.Optional(Type.Integer({ minimum: 1 })), quality: Type.Optional(Type.Number()), format: Type.Optional(Type.Enum(["jpeg", "png"], { type: "string" })) }, { additionalProperties: false }); /** Canonical inner payload returned by the `screen.snapshot` node command. */ const ScreenSnapshotResultSchema = Type.Object({ format: Type.Enum(["jpeg", "png"], { type: "string" }), base64: Type.String({ minLength: 1 }), displayFrameId: Type.Optional(Type.String()), screenIndex: Type.Optional(Type.Number()), width: Type.Optional(Type.Number()), height: Type.Optional(Type.Number()), capturedAtMs: Type.Optional(Type.Integer({ minimum: 0 })) }); /** Compile one Computer Use wire schema into a reusable type-guard validator. */ function compileComputerUseValidator(schema) { const validator = Compile(schema); return (value) => validator.Check(value); } const validateComputerActParams = compileComputerUseValidator(ComputerActParamsSchema); const validateComputerActResult = compileComputerUseValidator(ComputerActResultSchema); const validateComputerUseCapabilityDescriptor = compileComputerUseValidator(ComputerUseCapabilityDescriptorSchema); const validateScreenSnapshotParams = compileComputerUseValidator(ScreenSnapshotParamsSchema); const validateScreenSnapshotResult = compileComputerUseValidator(ScreenSnapshotResultSchema); function parseParamsJSON(paramsJSON, validate) { let value; try { value = JSON.parse(paramsJSON ?? "{}"); } catch { throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON"); } if (!validate(value)) throw new Error("COMPUTER_INVALID_REQUEST: invalid params"); return value; } function parseComputerActParamsJSON(paramsJSON) { return parseParamsJSON(paramsJSON, validateComputerActParams); } function parseScreenSnapshotParamsJSON(paramsJSON) { return parseParamsJSON(paramsJSON, validateScreenSnapshotParams); } /** Validate one provider result envelope. */ function parseComputerActResult(value) { if (!validateComputerActResult(value)) throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: invalid computer.act result`); return value; } /** Validate one bounded Computer Use declaration carried by a node connect. */ function parseComputerUseCapabilityDescriptor(value) { if (!validateComputerUseCapabilityDescriptor(value)) throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: invalid capability descriptor`); return value; } /** Validate and project a `screen.snapshot` result without retaining unknown fields. */ function parseScreenSnapshotResult(value) { if (!validateScreenSnapshotResult(value)) throw new Error("invalid screen.snapshot payload"); return { format: value.format, base64: value.base64, ...value.displayFrameId ? { displayFrameId: value.displayFrameId } : {}, ...value.screenIndex !== void 0 ? { screenIndex: value.screenIndex } : {}, ...value.width !== void 0 ? { width: value.width } : {}, ...value.height !== void 0 ? { height: value.height } : {}, ...value.capturedAtMs !== void 0 ? { capturedAtMs: value.capturedAtMs } : {} }; } /** Register the canonical node-host command pair for one node-local provider. */ function registerComputerUseProvider(api, provider) { let execution; let closingPromise = Promise.resolve(); const executionEnvelopeFromParams = (paramsJSON) => { let value; try { value = JSON.parse(paramsJSON ?? "{}"); } catch { throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON"); } const executionId = value && typeof value === "object" && !Array.isArray(value) ? value.executionId : void 0; if (executionId === void 0) return { executionId: void 0, value }; if (typeof executionId !== "string" || !new RegExp(COMPUTER_EXECUTION_ID_PATTERN, "u").test(executionId)) throw new Error("COMPUTER_INVALID_REQUEST: executionId is required"); return { executionId, value }; }; const getExecution = async (paramsJSON, context) => { const { executionId } = executionEnvelopeFromParams(paramsJSON); if (!executionId) throw new Error("COMPUTER_INVALID_REQUEST: executionId is required"); await closingPromise; if (execution && execution.id !== executionId) throw new Error("COMPUTER_HOST_BUSY: another provider execution owns this computer"); if (!execution) { const opened = provider.openExecution(context?.sessionKey ? { executionId, sessionKey: context.sessionKey } : { executionId }); opened.catch(() => { if (execution?.promise === opened) execution = void 0; }); execution = { id: executionId, promise: opened }; } return execution.promise; }; const closeExecution = async (executionId, reason) => { await closingPromise; const current = execution; if (!current || executionId !== void 0 && current.id !== executionId) return; execution = void 0; if (current) { const close = current.promise.then(async (opened) => await opened.close(reason)); closingPromise = close.catch(() => {}); await close; } }; api.registerNodeHostCommand({ command: "screen.snapshot", cap: "screen", dangerous: false, prepare: (context) => provider.prepare?.(context), isAvailable: () => provider.isAvailable(), watchAvailability: (context, onChange) => { const stopWatching = provider.watchAvailability?.(context, onChange); return () => { stopWatching?.(); closeExecution(void 0, "node-host-stop"); }; }, onDisconnect: async () => await closeExecution(void 0, "gateway-disconnect"), handle: async (paramsJSON, _io, context) => { if (executionEnvelopeFromParams(paramsJSON).executionId) return await (await getExecution(paramsJSON, context)).snapshot(paramsJSON, context?.signal); const executionId = randomUUID(); const opened = await provider.openExecution(context?.sessionKey ? { executionId, sessionKey: context.sessionKey } : { executionId }); try { return await opened.snapshot(paramsJSON, context?.signal); } finally { await opened.close("snapshot-complete"); } } }); api.registerNodeHostCommand({ command: "computer.act", cap: "computer", dangerous: true, computerUse: () => provider.capabilities(), isAvailable: () => provider.isAvailable(), handle: async (paramsJSON, _io, context) => { const envelope = executionEnvelopeFromParams(paramsJSON); if (!envelope.executionId) throw new Error("COMPUTER_INVALID_REQUEST: executionId is required"); if (envelope.value && typeof envelope.value === "object" && !Array.isArray(envelope.value) && envelope.value.action === "__close_execution") { const reason = envelope.value.reason; await closeExecution(envelope.executionId, typeof reason === "string" && reason.trim() ? reason.slice(0, 64) : "completion"); return JSON.stringify({ ok: true }); } return await (await getExecution(paramsJSON, context)).act(paramsJSON, context?.signal); } }); } //#endregion export { registerComputerUseProvider as _, COMPUTER_USE_V2_ACTION_NAMES as a, ComputerUseCapabilityDescriptorSchema as c, compileComputerUseValidator as d, parseComputerActParamsJSON as f, parseScreenSnapshotResult as g, parseScreenSnapshotParamsJSON as h, COMPUTER_USE_V1_ACTION_NAMES as i, ScreenSnapshotParamsSchema as l, parseComputerUseCapabilityDescriptor as m, COMPUTER_CONTRACT_MISMATCH as n, ComputerActParamsSchema as o, parseComputerActResult as p, COMPUTER_STALE_OBSERVATION as r, ComputerActResultSchema as s, COMPUTER_ACT_V1_ACTION_NAMES as t, ScreenSnapshotResultSchema as u };