@gacua/backend/dist/services/computer-use/tool-computer/groundable-tool.d.ts

GACUA Backend

/**
 * @license
 * Copyright 2025 MuleRun
 * SPDX-License-Identifier: Apache-2.0
 */
import type { FunctionDeclaration, Part } from '@google/genai';
import { type Box, type Image } from '../screen.js';
export interface GroundedToolCall {
    getDescription(saveImage: (imageBuffer: Buffer, nameSuffix: string) => Promise<string>): Promise<({
        text: string;
    } | {
        imageFileName: string;
    })[]>;
    value(): {
        name: string;
        args: Record<string, unknown>;
    };
}
export interface GroundableTool<TArgs = unknown> {
    readonly functionDeclaration: FunctionDeclaration;
    validate(args: TArgs): string | null;
    ground(args: TArgs, screenshot: Image, croppedScreenshotParts: {
        imagePart: Part;
    }[], detectElement: (imagePart: Part, elementDescription: string) => Promise<Box>): Promise<GroundedToolCall | string>;
}
export declare abstract class BaseGroundableTool<TArgs = unknown> implements GroundableTool<TArgs> {
    abstract readonly functionDeclaration: FunctionDeclaration;
    abstract ground(args: TArgs, screenshot: Image, croppedScreenshotParts: {
        imagePart: Part;
    }[], detectElement: (imagePart: Part, elementDescription: string) => Promise<Box>): Promise<GroundedToolCall | string>;
    validate(args: unknown): string | null;
    protected validateImageElementPair(args: unknown): string | null;
    protected detectAndTransform(imageId: number, croppedScreenshotParts: {
        imagePart: Part;
    }[], detectElement: (imagePart: Part) => Promise<Box>): Promise<string | {
        indexAndBox: {
            index: number;
            box: Box;
        };
        screenCoordinate: import("../screen.js").Coordinate;
    }>;
}