magnitude-core/dist/connectors/desktopConnector.js

Magnitude e2e testing agent

import { desktopActions } from '@/actions/desktopActions';
import { Observation } from "@/memory/observation";
import { Image } from "@/memory/image";
import logger from "@/logger";
import sharp from 'sharp';
export class DesktopConnector {
    id = "desktop";
    desktopInterface;
    options;
    logger;
    constructor(options) {
        this.options = options;
        this.desktopInterface = options.desktopInterface;
        this.logger = logger.child({
            name: `connectors.${this.id}`
        });
    }
    async onStart() {
        // Desktop interface should already be initialized by the service
        this.logger.info("Desktop connector started");
    }
    async onStop() {
        // Cleanup handled by the interface provider
        this.logger.info("Desktop connector stopped");
    }
    getActionSpace() {
        return desktopActions;
    }
    async collectObservations() {
        const observations = [];
        // Always collect screenshot
        const screenshot = await this.desktopInterface.screenshot();
        const sharpImage = sharp(screenshot);
        const image = new Image(sharpImage);
        // Apply virtual screen dimensions if configured
        const transformedImage = this.options.virtualScreenDimensions
            ? await image.resize(this.options.virtualScreenDimensions.width, this.options.virtualScreenDimensions.height)
            : image;
        observations.push(Observation.fromConnector(this.id, transformedImage, {
            type: 'screenshot',
            limit: this.options.minScreenshots ?? 2,
            dedupe: true
        }));
        // Optional: Window information
        if (this.desktopInterface.getOpenWindows) {
            try {
                const windows = await this.desktopInterface.getOpenWindows();
                const windowInfo = this.formatWindowInfo(windows);
                observations.push(Observation.fromConnector(this.id, windowInfo, { type: 'window-info', limit: 1 }));
            }
            catch (error) {
                this.logger.warn('Failed to get window information', error);
            }
        }
        return observations;
    }
    formatWindowInfo(windows) {
        let info = "Open Windows:\n";
        windows.forEach(window => {
            info += `${window.isActive ? '[ACTIVE] ' : ''}${window.title} (${window.app})\n`;
        });
        return info;
    }
    // Expose interface for actions to use
    getInterface() {
        return this.desktopInterface;
    }
}