smooth-operator-agent-tools
Version:
Node.js client library for Smooth Operator Agent Tools - a toolkit for programmers developing Computer Use Agents on Windows systems
561 lines • 21.8 kB
JavaScript
"use strict";
/**
* Type definitions for Smooth Operator Agent Tools
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.OverviewResponse = exports.ChromeElementInfo = exports.FocusInformation = exports.TabData = exports.ChromeOverview = exports.WindowDetailInfosDTO = exports.WindowDetailResponse = exports.WindowInfoDTO = exports.ControlDTO = exports.InstalledProgramDTO = exports.TaskbarIconDTO = exports.DesktopIconDTO = exports.SimpleResponse = exports.CSharpCodeResponse = exports.ChromeScriptResponse = exports.ChromeTabDetails = exports.ChromeTab = exports.ScreenGrasp2Response = exports.ActionResponse = exports.Point = exports.ScreenshotResponse = exports.MechanismType = void 0;
/**
* Specifies the mechanism to use for AI-based UI element interaction.
*/
var MechanismType;
(function (MechanismType) {
MechanismType["ScreenGrasp2"] = "screengrasp2";
MechanismType["ScreenGrasp2Low"] = "screengrasp2-low";
MechanismType["ScreenGraspMedium"] = "screengrasp-medium";
MechanismType["ScreenGraspHigh"] = "screengrasp-high";
MechanismType["LLabs"] = "llabs";
MechanismType["AnthropicComputerUse"] = "anthropic-computer-use";
MechanismType["OpenAIComputerUse"] = "openai-computer-use";
MechanismType["Qwen25Vl72b"] = "qwen25-vl-72b";
})(MechanismType || (exports.MechanismType = MechanismType = {}));
/**
* Response from the screenshot endpoint
*/
class ScreenshotResponse {
constructor(data) {
var _a;
this.success = data.success;
this.imageBase64 = data.imageBase64;
this.timestamp = data.timestamp;
this.message = (_a = data.message) !== null && _a !== void 0 ? _a : null;
}
/** Raw image bytes */
get imageBytes() {
return Buffer.from(this.imageBase64, 'base64');
}
/** Mime type of the image */
get imageMimeType() {
// Assuming JPEG as per C# comment, though this could be dynamic in future
return "image/jpeg";
}
toJSON() {
return {
success: this.success,
imageBase64: this.imageBase64,
timestamp: this.timestamp,
message: this.message,
};
}
}
exports.ScreenshotResponse = ScreenshotResponse;
/**
* Represents a point on the screen with X and Y coordinates
*/
class Point {
constructor(data) {
this.x = data.x;
this.y = data.y;
}
toJSON() {
return {
x: this.x,
y: this.y,
};
}
}
exports.Point = Point;
/**
* Generic response for action endpoints
*/
class ActionResponse {
constructor(data) {
var _a, _b;
this.success = data.success;
this.message = (_a = data.message) !== null && _a !== void 0 ? _a : null;
this.data = (_b = data.data) !== null && _b !== void 0 ? _b : null;
}
toJSON() {
return {
success: this.success,
message: this.message,
data: this.data,
};
}
}
exports.ActionResponse = ActionResponse;
/**
* Response from the ScreenGrasp2 endpoint (find-ui-element-by-description)
*/
class ScreenGrasp2Response extends ActionResponse {
constructor(data) {
var _a, _b, _c;
super({ success: data.success, message: data.message, data: data.data });
this.x = (_a = data.x) !== null && _a !== void 0 ? _a : null;
this.y = (_b = data.y) !== null && _b !== void 0 ? _b : null;
this.status = (_c = data.status) !== null && _c !== void 0 ? _c : null;
}
// Override toJSON to include specific properties
toJSON() {
return {
...super.toJSON(),
x: this.x,
y: this.y,
status: this.status,
};
}
}
exports.ScreenGrasp2Response = ScreenGrasp2Response;
/**
* Information about a Chrome browser tab
*/
class ChromeTab {
constructor(data) {
this.id = data.id;
this.title = data.title;
this.url = data.url;
this.isActive = data.isActive;
}
toJSON() {
return {
id: this.id,
title: this.title,
url: this.url,
isActive: this.isActive,
};
}
}
exports.ChromeTab = ChromeTab;
/**
* Detailed information about a Chrome tab
*/
class ChromeTabDetails {
constructor(data) {
this.title = data.title;
this.url = data.url;
this.content = data.content;
this.elements = data.elements;
this.summary = data.summary;
}
toJSON() {
return {
title: this.title,
url: this.url,
content: this.content,
elements: this.elements,
summary: this.summary,
};
}
}
exports.ChromeTabDetails = ChromeTabDetails;
/**
* Response from Chrome script execution
*/
class ChromeScriptResponse extends ActionResponse {
constructor(data) {
var _a;
super({ success: data.success, message: data.message, data: data.data });
this.result = (_a = data.result) !== null && _a !== void 0 ? _a : null;
}
toJSON() {
return {
...super.toJSON(),
result: this.result,
};
}
}
exports.ChromeScriptResponse = ChromeScriptResponse;
/**
* Response from C# code execution
*/
class CSharpCodeResponse extends ActionResponse {
constructor(data) {
var _a;
super({ success: data.success, message: data.message, data: data.data });
this.result = (_a = data.result) !== null && _a !== void 0 ? _a : null;
}
toJSON() {
return {
...super.toJSON(),
result: this.result,
};
}
}
exports.CSharpCodeResponse = CSharpCodeResponse;
/**
* Simple response with a message
*/
class SimpleResponse {
constructor(data) {
var _a, _b;
this.message = (_a = data.message) !== null && _a !== void 0 ? _a : null;
this.internalMessage = (_b = data.internalMessage) !== null && _b !== void 0 ? _b : null;
}
toJSON() {
return {
message: this.message,
internalMessage: this.internalMessage,
};
}
}
exports.SimpleResponse = SimpleResponse;
/**
* Information about a desktop icon
*/
class DesktopIconDTO {
constructor(data) {
this.name = data.name;
this.path = data.path;
}
toJSON() {
return {
name: this.name,
path: this.path,
};
}
}
exports.DesktopIconDTO = DesktopIconDTO;
/**
* Information about a taskbar icon
*/
class TaskbarIconDTO {
constructor(data) {
this.name = data.name;
this.path = data.path;
}
toJSON() {
return {
name: this.name,
path: this.path,
};
}
}
exports.TaskbarIconDTO = TaskbarIconDTO;
/**
* Information about an installed program
*/
class InstalledProgramDTO {
constructor(data) {
this.name = data.name;
this.executablePath = data.executablePath;
}
toJSON() {
return {
name: this.name,
executablePath: this.executablePath,
};
}
}
exports.InstalledProgramDTO = InstalledProgramDTO;
/**
* Information about a UI control
*/
// Note: We deliberately exclude recursive properties and parent from toJSON
class ControlDTO {
constructor(data) {
var _a, _b, _c, _d, _e, _f;
this.id = data.id;
this.name = (_a = data.name) !== null && _a !== void 0 ? _a : null;
this.creationDate = data.creationDate;
this.controlType = (_b = data.controlType) !== null && _b !== void 0 ? _b : null;
this.supportsSetValue = (_c = data.supportsSetValue) !== null && _c !== void 0 ? _c : null;
this.supportsInvoke = (_d = data.supportsInvoke) !== null && _d !== void 0 ? _d : null;
this.currentValue = (_e = data.currentValue) !== null && _e !== void 0 ? _e : null;
// Reconstruct children as ControlDTO instances if they exist
this.children = data.children ? data.children.map(cData => new ControlDTO(cData)) : null;
// IMPORTANT: Avoid creating a new parent instance here to prevent infinite loops
// The parent should ideally be assigned after creation if needed for navigation,
// but the raw data shouldn't contain nested parent ControlDTOs.
this.parent = (_f = data.parent) !== null && _f !== void 0 ? _f : null;
this.isSmoothOperator = data.isSmoothOperator;
// Set parent for children after construction
if (this.children) {
this.children.forEach(child => child.parent = this);
}
}
/** Recursively get all descendant controls. */
get childrenRecursive() {
const descendants = [];
if (!this.children) {
return descendants;
}
for (const child of this.children) {
if (child) { // Check if child is not null
descendants.push(child);
descendants.push(...child.childrenRecursive); // Recursive call
}
}
return descendants;
}
/** Get all ancestor controls. */
get parentsRecursive() {
const ancestors = [];
let current = this.parent;
while (current) {
ancestors.push(current);
current = current.parent; // Move up the hierarchy
}
return ancestors;
}
/** Find the closest ancestor control that is a Window. */
get parentWindow() {
let current = this.parent;
while (current) {
if (current.controlType === "Window") {
return current;
}
current = current.parent;
}
return null;
}
// We don't need allChildrenRecursive or allParentsRecursive as separate properties;
// the getters above compute them on demand.
/** Custom serialization to avoid circular references and large payloads */
toJSON() {
var _a, _b;
// Explicitly list properties to include, excluding 'parent' and computed getters
return {
id: this.id,
name: this.name,
creationDate: this.creationDate,
controlType: this.controlType,
supportsSetValue: this.supportsSetValue,
supportsInvoke: this.supportsInvoke,
currentValue: this.currentValue,
// Serialize children, but ensure they also use toJSON to avoid parent recursion
children: (_b = (_a = this.children) === null || _a === void 0 ? void 0 : _a.map((child) => (child === null || child === void 0 ? void 0 : child.toJSON) ? child.toJSON() : child)) !== null && _b !== void 0 ? _b : null,
isSmoothOperator: this.isSmoothOperator,
};
}
}
exports.ControlDTO = ControlDTO;
/**
* Information about a window
*/
class WindowInfoDTO {
constructor(data) {
var _a, _b, _c, _d, _e;
this.id = data.id;
this.title = (_a = data.title) !== null && _a !== void 0 ? _a : null;
this.executablePath = (_b = data.executablePath) !== null && _b !== void 0 ? _b : null;
this.isForeground = (_c = data.isForeground) !== null && _c !== void 0 ? _c : null;
this.processName = (_d = data.processName) !== null && _d !== void 0 ? _d : null;
this.isMinimized = (_e = data.isMinimized) !== null && _e !== void 0 ? _e : null;
// Reconstruct nested object if data is provided
this.detailInfos = data.detailInfos ? new WindowDetailResponse(data.detailInfos) : null;
}
toJSON() {
var _a;
return {
id: this.id,
title: this.title,
executablePath: this.executablePath,
isForeground: this.isForeground,
processName: this.processName,
isMinimized: this.isMinimized,
// Ensure nested objects also use toJSON if available
detailInfos: ((_a = this.detailInfos) === null || _a === void 0 ? void 0 : _a.toJSON) ? this.detailInfos.toJSON() : this.detailInfos,
};
}
}
exports.WindowInfoDTO = WindowInfoDTO;
/**
* Response containing window details
*/
class WindowDetailResponse {
constructor(data) {
var _a;
// Reconstruct nested object if data is provided
this.details = data.details ? new WindowDetailInfosDTO(data.details) : null;
this.message = (_a = data.message) !== null && _a !== void 0 ? _a : null;
}
toJSON() {
var _a;
return {
// Ensure nested objects also use toJSON if available
details: ((_a = this.details) === null || _a === void 0 ? void 0 : _a.toJSON) ? this.details.toJSON() : this.details,
message: this.message,
};
}
}
exports.WindowDetailResponse = WindowDetailResponse;
/**
* Detailed information about a window
*/
class WindowDetailInfosDTO {
constructor(data) {
var _a;
this.note = (_a = data.note) !== null && _a !== void 0 ? _a : null;
// Reconstruct nested objects
this.window = new WindowInfoDTO(data.window);
this.userInterfaceElements = new ControlDTO(data.userInterfaceElements);
}
toJSON() {
var _a, _b;
return {
note: this.note,
// Ensure nested objects also use toJSON if available
window: ((_a = this.window) === null || _a === void 0 ? void 0 : _a.toJSON) ? this.window.toJSON() : this.window,
userInterfaceElements: ((_b = this.userInterfaceElements) === null || _b === void 0 ? void 0 : _b.toJSON) ? this.userInterfaceElements.toJSON() : this.userInterfaceElements,
};
}
}
exports.WindowDetailInfosDTO = WindowDetailInfosDTO;
/**
* Information about a Chrome instance
*/
class ChromeOverview {
constructor(data) {
this.instanceId = data.instanceId;
// Reconstruct nested objects, default to empty array if null/undefined
this.tabs = data.tabs ? data.tabs.map(tData => new TabData(tData)) : [];
this.lastUpdate = data.lastUpdate;
}
toJSON() {
var _a, _b;
return {
instanceId: this.instanceId,
// Ensure nested objects also use toJSON if available
tabs: (_b = (_a = this.tabs) === null || _a === void 0 ? void 0 : _a.map(tab => (tab === null || tab === void 0 ? void 0 : tab.toJSON) ? tab.toJSON() : tab)) !== null && _b !== void 0 ? _b : null,
lastUpdate: this.lastUpdate,
};
}
}
exports.ChromeOverview = ChromeOverview;
/**
* Information about a Chrome tab
*/
class TabData {
constructor(data) {
var _a, _b, _c, _d, _e;
this.id = data.id;
this.url = (_a = data.url) !== null && _a !== void 0 ? _a : null;
this.isActive = (_b = data.isActive) !== null && _b !== void 0 ? _b : null;
this.html = (_c = data.html) !== null && _c !== void 0 ? _c : null;
this.text = (_d = data.text) !== null && _d !== void 0 ? _d : null;
this.idString = (_e = data.idString) !== null && _e !== void 0 ? _e : null;
this.tabNr = data.tabNr;
}
toJSON() {
return {
id: this.id,
url: this.url,
isActive: this.isActive,
html: this.html,
text: this.text,
idString: this.idString,
tabNr: this.tabNr,
};
}
}
exports.TabData = TabData;
/**
* Information about the focused element
*/
class FocusInformation {
constructor(data) {
var _a;
// Reconstruct nested objects
this.focusedElement = data.focusedElement ? new ControlDTO(data.focusedElement) : null;
this.focusedElementParentWindow = data.focusedElementParentWindow ? new WindowInfoDTO(data.focusedElementParentWindow) : null;
this.someOtherElementsInSameWindowThatMightBeRelevant = data.someOtherElementsInSameWindowThatMightBeRelevant
? data.someOtherElementsInSameWindowThatMightBeRelevant.map(elData => new ControlDTO(elData))
: null;
this.currentChromeTabMostRelevantElements = data.currentChromeTabMostRelevantElements
? data.currentChromeTabMostRelevantElements.map(elData => new ChromeElementInfo(elData))
: null;
this.isChrome = data.isChrome;
this.note = (_a = data.note) !== null && _a !== void 0 ? _a : null;
}
toJSON() {
var _a, _b, _c, _d, _e, _f;
return {
// Ensure nested objects also use toJSON if available
focusedElement: ((_a = this.focusedElement) === null || _a === void 0 ? void 0 : _a.toJSON) ? this.focusedElement.toJSON() : this.focusedElement,
focusedElementParentWindow: ((_b = this.focusedElementParentWindow) === null || _b === void 0 ? void 0 : _b.toJSON) ? this.focusedElementParentWindow.toJSON() : this.focusedElementParentWindow,
someOtherElementsInSameWindowThatMightBeRelevant: (_d = (_c = this.someOtherElementsInSameWindowThatMightBeRelevant) === null || _c === void 0 ? void 0 : _c.map(el => (el === null || el === void 0 ? void 0 : el.toJSON) ? el.toJSON() : el)) !== null && _d !== void 0 ? _d : null,
currentChromeTabMostRelevantElements: (_f = (_e = this.currentChromeTabMostRelevantElements) === null || _e === void 0 ? void 0 : _e.map(el => (el === null || el === void 0 ? void 0 : el.toJSON) ? el.toJSON() : el)) !== null && _f !== void 0 ? _f : null,
isChrome: this.isChrome,
note: this.note,
};
}
}
exports.FocusInformation = FocusInformation;
/**
* Information about a Chrome element
*/
class ChromeElementInfo {
constructor(data) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
this.smoothOpId = data.smoothOpId;
this.tagName = data.tagName;
this.cssSelector = data.cssSelector;
this.innerText = (_a = data.innerText) !== null && _a !== void 0 ? _a : null;
this.isVisible = (_b = data.isVisible) !== null && _b !== void 0 ? _b : null;
this.score = data.score;
this.role = (_c = data.role) !== null && _c !== void 0 ? _c : null;
this.value = (_d = data.value) !== null && _d !== void 0 ? _d : null;
this.type = (_e = data.type) !== null && _e !== void 0 ? _e : null;
this.name = (_f = data.name) !== null && _f !== void 0 ? _f : null;
this.className = (_g = data.className) !== null && _g !== void 0 ? _g : null;
this.semantic = (_h = data.semantic) !== null && _h !== void 0 ? _h : null;
this.dataAttributes = (_j = data.dataAttributes) !== null && _j !== void 0 ? _j : null;
this.truncatedHtml = (_k = data.truncatedHtml) !== null && _k !== void 0 ? _k : null;
this.boundingRect = (_l = data.boundingRect) !== null && _l !== void 0 ? _l : null;
// Reconstruct nested object
this.centerPoint = data.centerPoint ? new Point(data.centerPoint) : null;
}
toJSON() {
var _a;
return {
smoothOpId: this.smoothOpId,
tagName: this.tagName,
cssSelector: this.cssSelector,
innerText: this.innerText,
isVisible: this.isVisible,
score: this.score,
role: this.role,
value: this.value,
type: this.type,
name: this.name,
className: this.className,
semantic: this.semantic,
dataAttributes: this.dataAttributes,
truncatedHtml: this.truncatedHtml,
boundingRect: this.boundingRect,
// Ensure nested objects also use toJSON if available
centerPoint: ((_a = this.centerPoint) === null || _a === void 0 ? void 0 : _a.toJSON) ? this.centerPoint.toJSON() : this.centerPoint,
};
}
}
exports.ChromeElementInfo = ChromeElementInfo;
/**
* System overview response
*/
class OverviewResponse {
constructor(data) {
var _a;
// Reconstruct nested objects, default to empty arrays if null/undefined
this.windows = data.windows ? data.windows.map(wData => new WindowInfoDTO(wData)) : [];
this.chromeInstances = data.chromeInstances ? data.chromeInstances.map(cData => new ChromeOverview(cData)) : [];
this.focusInfo = data.focusInfo ? new FocusInformation(data.focusInfo) : null;
this.topPinnedTaskbarIcons = data.topPinnedTaskbarIcons ? data.topPinnedTaskbarIcons.map(tData => new TaskbarIconDTO(tData)) : [];
this.topDesktopIcons = data.topDesktopIcons ? data.topDesktopIcons.map(dData => new DesktopIconDTO(dData)) : [];
this.topInstalledPrograms = data.topInstalledPrograms ? data.topInstalledPrograms.map(pData => new InstalledProgramDTO(pData)) : [];
this.importantNote = (_a = data.importantNote) !== null && _a !== void 0 ? _a : null;
}
toJSON() {
var _a;
return {
// Ensure nested objects also use toJSON if available
windows: this.windows.map(w => (w === null || w === void 0 ? void 0 : w.toJSON) ? w.toJSON() : w),
chromeInstances: this.chromeInstances.map(c => (c === null || c === void 0 ? void 0 : c.toJSON) ? c.toJSON() : c),
focusInfo: ((_a = this.focusInfo) === null || _a === void 0 ? void 0 : _a.toJSON) ? this.focusInfo.toJSON() : this.focusInfo,
topPinnedTaskbarIcons: this.topPinnedTaskbarIcons.map(t => (t === null || t === void 0 ? void 0 : t.toJSON) ? t.toJSON() : t),
topDesktopIcons: this.topDesktopIcons.map(d => (d === null || d === void 0 ? void 0 : d.toJSON) ? d.toJSON() : d),
topInstalledPrograms: this.topInstalledPrograms.map(p => (p === null || p === void 0 ? void 0 : p.toJSON) ? p.toJSON() : p),
importantNote: this.importantNote,
};
}
}
exports.OverviewResponse = OverviewResponse;
//# sourceMappingURL=models.js.map