embody/src/provider/robotjs.ts

A digital consciousness training environment

import robot from 'robotjs'
import Jimp from 'jimp'

import * as path from 'path'

class RobotJSProviderOptions { }

class RobotJSProvider {
  options: RobotJSProviderOptions

  recordScreenSubscribers: {
    [subscriberID: string]: {
      callback: (data: any) => void
      options: {}
    }
  } = {}
  recordScreenPublisher?: NodeJS.Timeout

  x: number = 0
  y: number = 0

  boundaryWidth: number = 600
  boundaryHeight: number = 500

  constructor (options: RobotJSProviderOptions) {
    this.options = options

    const { width, height } = robot.getScreenSize()

    this.x = (width / 2) - this.boundaryWidth
    this.y = (height / 2) - (this.boundaryHeight / 2)
  }

  initScreenPublisher () {

    // Ensure the screen data publisher isn't already initialized.
    // Ensure there are subscribers available to send data to.
    if (this.recordScreenPublisher || !this.recordScreenSubscribers) {
      return
    }

    const frameRate = 60

    // The screen publisher is a repeated call to capture the screen data and
    // send it to the subscribers.
    const screenPublisher = setInterval(async () => {

      const screenImage = await this.getScreenImage()
      // const screenImage = 0

      for (const subscriberID of Object.keys(this.recordScreenSubscribers)) {
        if (!this.recordScreenSubscribers[subscriberID]) {
          continue
        }

        this.recordScreenSubscribers[subscriberID].callback(screenImage)
      }

    }, 1000 / frameRate)

    this.recordScreenPublisher = screenPublisher
  }

  startRecordingScreen (subscriberID: string, callback: (data: any) => void, options?: {}): void {

    this.recordScreenSubscribers[subscriberID] = {
      callback,
      options: options || {}
    }

    this.initScreenPublisher()
  }

  stopRecordingScreen (subscriberID: string): void {

    // Remove the subscriber.
    delete this.recordScreenSubscribers[subscriberID]

    // If there are no subscribers, stop the publisher.
    if (Object.keys(this.recordScreenSubscribers).length < 1) {
      clearInterval(this.recordScreenPublisher!)
      this.recordScreenPublisher = undefined
    }
  }

  getScreenImage (): Promise<string|Buffer> {

    const picture = robot.screen.capture(this.x, this.y, this.boundaryWidth, this.boundaryHeight)

    return new Promise((resolve, reject) => {
      try {
        const image = new Jimp(picture.width, picture.height)
        let pos = 0
        image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
          image.bitmap.data[idx + 2] = picture.image.readUInt8(pos++)
          image.bitmap.data[idx + 1] = picture.image.readUInt8(pos++)
          image.bitmap.data[idx + 0] = picture.image.readUInt8(pos++)
          image.bitmap.data[idx + 3] = picture.image.readUInt8(pos++)
        })

        image.getBuffer(Jimp.MIME_PNG, (err2, png) => {
          resolve(png)
        })
      } catch (err) {
        reject(err)
      }
    })
  }

  moveMousePosition (dx: number, dy: number): void {
    const { x, y } = robot.getMousePos()
    robot.moveMouse(x - dx, y - dy)
  }
}


export {
  RobotJSProvider,
  RobotJSProviderOptions
}