siesta-lite/src/nodejs/simulator/robotjs/Server.ts

Stress-free JavaScript unit testing and functional testing tool, works in NodeJS and browsers

import * as point from '../../../generic/util/Point'
import * as types from '../../../generic/simulator/Types'
import {ServerEndPoint} from "../../channel/websocket/ServerEndPoint.js";
import {Channel, Envelop, Message} from "../../../generic/channel/Types.js";
import {SimulatorCommand} from "../../../generic/simulator/Commands.js";
import * as commands from "../../../generic/simulator/Commands.js";
import {filterPathAccordingToPrecision} from "../../../generic/util/Point.js";
import {delay} from "../../../generic/util/helper/Delay.js";

let robotjs

let noRobot = false

try {
    robotjs = require('robotjs')

    // this can probably be reduced on Linux (or even set to 0)
    // once https://github.com/octalmage/robotjs/issues/347 gets fixed
    robotjs.setMouseDelay(process.platform == 'win32' ? 10 : 10)
    robotjs.setKeyboardDelay(process.platform == 'win32' ? 20 : 20)
} catch (e) {
    noRobot = true
}

export class SimulatorServerRobotJs extends ServerEndPoint {
    displayNumber       : string

    mouseMovePrecision  : number = 3

    // see https://github.com/octalmage/robotjs/issues/347
    afterActionDelay    : number = 100


    setXDisplayName (displayNumber : string) {
        this.displayNumber = displayNumber

        robotjs.setXDisplayName(':' + displayNumber + '.0')
    }


    connect(channel : Channel) : Promise<any> {
        if ((process.platform == 'linux' || process.platform == 'freebsd')) {
            // running on Linux w/o X11 server?
            // no point to even setup the WebSocket server - trying to call any RobotJS method, like `moveMouse` below
            // will segfault
            if (!process.env.DISPLAY || noRobot) return Promise.resolve()

            if (this.displayNumber) this.setXDisplayName(this.displayNumber)
        }

        // move mouse left-top corner
        robotjs.moveMouse(0, 0)

        return super.connect(channel)
    }


    doDispatchEnvelop (envelop : Envelop) {
        const cmd           = envelop.payload as SimulatorCommand

        let action

        switch (cmd.type) {
            case 'calibrate'            : action = this.doCalibrate(<commands.Calibrate>cmd); break
            case 'move_pointer'         : action = this.doMovePointer(<commands.MovePointer>cmd); break
            case 'set_pointer_state'    : action = this.doSetPointerState(<commands.SetPointerState>cmd); break
            case 'pointer_click'        : action = this.doPointerClick(<commands.PointerClick>cmd); break
            case 'pointer_double_click' : action = this.doPointerDoubleClick(<commands.PointerDoubleClick>cmd); break
            case 'mouse_wheel'          : action = this.doMouseWheel(<commands.MouseWheel>cmd); break
            case 'pointer_drag'         : action = this.doPointerDrag(<commands.PointerDrag>cmd); break
            case 'type'                 : action = this.doType(<commands.Type>cmd); break
            case 'reset'                : action = this.doSimulationReset(<commands.Reset>cmd); break
            case 'set_key_state'        : action = this.doSetKeyState(<commands.SetKeyState>cmd); break
        }

        action.then(result => this.replyWith(envelop, result), reason => {
            this.info("Simulator command failed: " + reason)

            this.replyWith(envelop, reason, true)
        })
    }


    doCalibrate (command : commands.Calibrate) : Promise<any> {
        const testPointX        = Math.round(command.left + command.width / 2)
        const testPointY        = Math.round(command.top + command.height / 2)

        robotjs.moveMouse(0, 0)

        return delay(100).then(() => {
            robotjs.moveMouse(testPointX, testPointY)

            return delay(100)

        }).then(() => {
            return Promise.resolve({ x : testPointX, y : testPointY })
        })
    }


    doMovePointer (command : commands.MovePointer) : Promise<any> {
        this.doWithModifierKeys(() => {
            if (command.moveKind == 'instant')
                robotjs.moveMouse(command.x, command.y)
            else if (command.moveKind == 'smooth') {
                this.moveMouseTo(command.x, command.y, command.mouseMovePrecision)
            }
        }, command.modifierKey)

        return Promise.resolve()
    }


    doSetKeyState (command : commands.SetKeyState) : Promise<any> {
        robotjs.keyToggle(command.key, command.state)

        return delay(this.afterActionDelay)
    }


    doSetPointerState (command : commands.SetPointerState) : Promise<any> {
        this.doWithModifierKeys(() => {
            robotjs.mouseToggle(command.state, command.modifier)
        }, command.modifierKey)

        return delay(this.afterActionDelay)
    }


    doPointerClick (command : commands.PointerClick) : Promise<any> {
        this.doWithModifierKeys(() => {
            robotjs.mouseClick(command.modifier, false)
        }, command.modifierKey)

        return delay(this.afterActionDelay)
    }


    doPointerDoubleClick (command : commands.PointerDoubleClick) : Promise<any> {
        this.doWithModifierKeys(() => {
            robotjs.mouseClick(command.modifier, true)
        }, command.modifierKey)

        return delay(this.afterActionDelay)
    }


    // not used currently but might be useful
    doInBatches (processor : Function, input: any[], batchSize : number, delayAfterBatch : number) : Promise<any> {
        if (input.length === 0) return Promise.resolve()

        for (let i = 0; i < batchSize; i++) {
            if (input.length === 0) break

            processor(input.shift())
        }

        return delay(delayAfterBatch).then(() => this.doInBatches(processor, input, batchSize, delayAfterBatch))
    }


    doSimulationReset (command : commands.Reset) : Promise<any> {

        '`1234567890-=qwertyuiop[]\\asdfghjkl;\'zxcvbnm,./ '.split('').concat(
            Object.keys(SpecialKey).filter(
                key => isNaN(Number(key))
                    && !key.match(/^lights_/) // fails on linux
                    && !key.match(/^numpad_/) // fails on linux
                    && !key.match(/^audio_/) // fails on linux

                    && !(process.platform == 'darwin' && (key == 'printscreen' || key == 'insert'))
            )
        ).forEach(key => {
            // console.log("Key up: " + key)

            robotjs.keyToggle(key, types.KeyState.Up)
        })

        if (process.platform != 'win32')
            for (let button in types.PointerModifier) {
                if (isNaN(Number(button))) {
                    robotjs.mouseToggle(types.PointerState.Up, types.PointerModifier[ button ])
                }
            }

        robotjs.moveMouse(0, 0)

        return delay(this.afterActionDelay)
    }


    doMouseWheel (command : commands.MouseWheel) : Promise<any> {
        this.doWithModifierKeys(() => {
            // https://github.com/octalmage/robotjs/issues/318
            // https://github.com/octalmage/robotjs/issues/303 ??
            robotjs.scrollMouse(command.deltaX, 0)
        }, command.modifierKey)

        return delay(this.afterActionDelay)
    }


    doPointerDrag (command : commands.PointerDrag) : Promise<any> {

        this.doWithModifierKeys(() => {
            robotjs.moveMouse(command.fromX, command.fromY)

            robotjs.mouseToggle(types.PointerState.Down, command.modifier)

            this.moveMouseTo(command.toX, command.toY, 1)

            if (!command.dragOnly) robotjs.mouseToggle(types.PointerState.Up, command.modifier)

        }, command.modifierKey)

        return Promise.resolve()
    }


    doWithModifierKeys (action : Function, modifierKeys : types.ModifierKey[] = []) {
        this.toggleModifierKeys(modifierKeys, types.KeyState.Down)

        action()

        this.toggleModifierKeys(modifierKeys, types.KeyState.Up)
    }


    toggleModifierKeys (modifierKeys : types.ModifierKey[] = [], state : types.KeyState) {
        modifierKeys.forEach(modKey => robotjs.keyToggle(this.siestaKeyToSimulatorKey(modKey), state))
    }


    siestaKeyToSimulatorKey (key : string) : string {
        if (key.length == 1) return key

        return siestaToRobotJsKeys[ key ]
    }


    doType (command : commands.Type) : Promise<any> {
        const text              = command.text

        for (let i = 0; i < text.length; i++) {
            this.typeSingleChar(this.siestaKeyToSimulatorKey(text[ i ]), command.modifierKey)
        }

        // this setTimeout is required, because for some reason when typing just "delete" in the input field
        // the callback is called, but the delete has not been processed yet
        // (the it=`keyPress` method should support special chars like [SMTH]` in keyevents/043_special_keys.t.js
        // created a ticket for robotjs: https://github.com/octalmage/robotjs/issues/347 but project is very poorly maintained..
        return delay(2 * this.afterActionDelay)
    }


    // `key` is a robotjs-recognizable key
    typeSingleChar (key : string, modifierKey : types.ModifierKey[] = []) {
        if (key.length > 1)
            // special key - we just tap it
            // see also comment below
            this.doWithModifierKeys(() => {
                robotjs.keyTap(key)
            }, modifierKey)
        else {
            // a single char
            let needShift   = false

            if (key == '!') { key = "1"; needShift = true }
            if (key == '@') { key = "2"; needShift = true }
            if (key == '#') { key = "3"; needShift = true }
            if (key == '$') { key = "4"; needShift = true }
            if (key == '%') { key = "5"; needShift = true }
            if (key == '^') { key = "6"; needShift = true }
            if (key == '&') { key = "7"; needShift = true }
            if (key == '*') { key = "8"; needShift = true }
            if (key == '(') { key = "9"; needShift = true }
            if (key == ')') { key = "0"; needShift = true }
            if (key == '_') { key = "-"; needShift = true }
            if (key == '+') { key = "="; needShift = true }
            if (key == '{') { key = "["; needShift = true }
            if (key == '}') { key = "]"; needShift = true }
            if (key == '|') { key = "\\"; needShift = true }
            if (key == ':') { key = ";"; needShift = true }
            if (key == '"') { key = "'"; needShift = true }
            if (key == '<') { key = ","; needShift = true }
            if (key == '>') { key = "."; needShift = true }
            if (key == '?') { key = "/"; needShift = true }

            // the `keyTap` method of robotjs support modifiers array, but we can't use it, because
            // lets say for CTRL + A (keyTap('a', [ 'control' ]), the order of events will be:
            // keydown CTRL / keydown a / keyup CTRL / keyup a
            // which is obviously wrong, at least for us, we want:
            // keydown CTRL / keydown a / keyup a / keyup CTRL
            // so we use our "doWithModifierKeys" wrapper
            this.doWithModifierKeys(() => {

                robotjs.keyTap(key)

            }, needShift || key != key.toLowerCase() ? [ <types.ModifierKey>'SHIFT' ].concat(modifierKey) : modifierKey)
        }
    }


    // Of course, the "moveMouseSmooth" method in RobotJS is a total mess:
    // https://github.com/octalmage/robotjs/issues/238
    // we have to introduce our own
    moveMouseTo (x : number, y : number, mouseMovePrecision : number) {
        var currentPos  = robotjs.getMousePos()

        var path        = point.getPathBetweenPoints([ currentPos.x, currentPos.y ], [ x, y ])

        path            = filterPathAccordingToPrecision(path, mouseMovePrecision || this.mouseMovePrecision)

        for (var i = 0; i < path.length; i++) {
            var step    = path[ i ]

            robotjs.moveMouse(step[ 0 ], step[ 1 ])
        }
    }
}


export enum SpecialKey {
    'backspace',
    'delete',
    'enter',
    'tab',
    'escape',
    'up',
    'down',
    'right',
    'left',
    'home',
    'end',
    'pageup',
    'pagedown',
    'f1',
    'f2',
    'f3',
    'f4',
    'f5',
    'f6',
    'f7',
    'f8',
    'f9',
    'f10',
    'f11',
    'f12',
    'command',
    'alt',
    'control',
    'shift',
    'right_shift',
    'space',
    'printscreen',
    'insert',
    'audio_mute',
    'audio_vol_down',
    'audio_vol_up',
    'audio_play',
    'audio_stop',
    'audio_pause',
    'audio_prev',
    'audio_next',
    'audio_rewind',
    'audio_forward',
    'audio_repeat',
    'audio_random',
    'numpad_0',
    'numpad_1',
    'numpad_2',
    'numpad_3',
    'numpad_4',
    'numpad_5',
    'numpad_6',
    'numpad_7',
    'numpad_8',
    'numpad_9',
    'lights_mon_up',
    'lights_mon_down',
    'lights_kbd_toggle',
    'lights_kbd_up',
    'lights_kbd_down'
}


const siestaToRobotJsKeys = {
    'BACKSPACE'     : 'backspace',

    'TAB'           : 'tab',

    'RETURN'        : 'enter',
    'ENTER'         : 'enter',

    //special
    'SHIFT'         : 'shift',
    'CTRL'          : 'control',
    'ALT'           : 'alt',
    'CMD'           : 'command', // Mac

    //weird
    'PAUSE-BREAK'   : null,
    'CAPS'          : null,

    'ESCAPE'        : 'escape',
    'ESC'           : 'escape',
    'NUM-LOCK'      : null,
    'SCROLL-LOCK'   : null,
    'PRINT'         : 'printscreen', // No Mac support

    //navigation
    'PAGE-UP'       : 'pageup',
    'PAGE-DOWN'     : 'pagedown',
    'PAGEUP'        : 'pageup',
    'PAGEDOWN'      : 'pagedown',
    'END'           : 'end',
    'HOME'          : 'home',
    'LEFT'          : 'left',
    'ARROWLEFT'     : 'left',
    'UP'            : 'up',
    'ARROWUP'       : 'up',
    'RIGHT'         : 'right',
    'ARROWRIGHT'    : 'right',
    'DOWN'          : 'down',
    'ARROWDOWN'     : 'down',
    'INSERT'        : 'insert', // No Mac support
    'DELETE'        : 'delete',

    //NORMAL-CHARACTERS, NUMPAD
    'NUM0'          : 'numpad_0',
    'NUM1'          : 'numpad_1',
    'NUM2'          : 'numpad_2',
    'NUM3'          : 'numpad_3',
    'NUM4'          : 'numpad_4',
    'NUM5'          : 'numpad_5',
    'NUM6'          : 'numpad_6',
    'NUM7'          : 'numpad_7',
    'NUM8'          : 'numpad_8',
    'NUM9'          : 'numpad_9',

    'F1'            : 'f1',
    'F2'            : 'f2',
    'F3'            : 'f3',
    'F4'            : 'f4',
    'F5'            : 'f5',
    'F6'            : 'f6',
    'F7'            : 'f7',
    'F8'            : 'f8',
    'F9'            : 'f9',
    'F10'           : 'f10',
    'F11'           : 'f11',
    'F12'           : 'f12'
}
// eof robotJSKeys

export const SimulatorServer = SimulatorServerRobotJs