dcp-client/lib/standaloneWorker.js

Core libraries for accessing DCP network

/**
 *  @file       standaloneWorker.js     A Node module which implements the class standaloneWorker,
 *                                      which knows how to execute jobs over the network in a
 *                                      standalone worker.
 *
 *  @author     Wes Garland, wes@sparc.network
 *  @date       March 2018
 */
'use strict';

const { setBackoffInterval, clearBackoffInterval, Reference } = require('dcp/dcp-timers');
const { leafMerge, deepClone } = require('dcp/utils');

var debugging = require('dcp/internal/debugging').scope('saw');

/**
 * StandaloneWorker constructor
 * The running configuration is derived by merging:
 *  - baked-in defaults
 *  - dcpConfig.evaluator
 *  - option.hostname/port/location
 *
 *  @param      options         Options for the Worker constructor.  These can be options per the specification
 *                              for Web Workers (note: not current propagated) or any of these options:
 *
 *                              hostname:     The hostname of the Evaluator server; default to dcpConfig.eveluator.hostname or localhost.
 *                              port:         The port number of the Evaluator server; default to dcpConfig.evaluator.port or 9000;
 *                              location:     URL which contains hostname and port
 *                              readStream:   An instance of Stream.readable connected to an Evaluator
 *                              writeStream:  An instance of Stream.writeable connected to the same Evaluator, default=readStream
 *
 *  @returns an object that inherits from dcp-timers::References, having the following
 *  - methods:
 *    . addEventListener
 *    . removeEventListener
 *    . postMessage
 *    . terminate
 *    . create
 *  - properties:
 *    . onmessage
 *    . onerror
 *    . serial                  sequence number
 *    . serialize               current serialization function
 *    . deserialize             current deserialization function
 *    . config                  derived configuration
 *  - events:
 *    . error
 *    . message
 */
function StandaloneWorker(options = {})
{
  const that = this;
  var readStream, writeStream;
  var ee = new (require('events').EventEmitter)('StandaloneWorker');
  var readBuf = ''
  var connected = false
  var dieTimer
  var shutdown;

  Reference.call(this, debugging, () => `sa-worker@${this.serial}/${this.config.location}`);
  const defaultConfig = {
    tuning: {
      noDelay: true,
      connectBackoff:
      {
        maxInterval:   5 * 60 * 1000, // max: 5 minutes
        baseInterval:      10 * 1000, // start: 10s
        backoffFactor: 1.1            // each fail, back off by 10%
      },
    },
  };
  this.config = leafMerge(
    defaultConfig,
    deepClone(dcpConfig.evaluator),
  );
  delete this.config.listen;
  if (options.location)
    this.config.location = options.location;
  if (options.hostname)
    this.config.location.hostname = options.hostname;
  if (options.port)
    this.config.location.port = options.port;

  if (typeof options === 'object' && options.readStream) {
    debugging('lifecycle') && console.debug('Connecting via supplied streams');
    readStream = options.readStream;
    writeStream = options.writeStream || options.readStream;
    delete options.readStream;
    delete options.writeStream;
  }

  if (!readStream) {
    /* No supplied streams - reach out and connect over TCP/IP */
    let socket = readStream = writeStream = new (require('net')).Socket().unref();
    debugging() && console.debug('Connecting to evaluator on', this.config.location.hostname + ':' + this.config.location.port);

    socket.setNoDelay(this.config.tuning.noDelay ?? true);
    socket.connect(this.config.location.port, this.config.location.hostname);
    socket.on('connect', beginSession.bind(this));

    let alreadySignaled = false;
    const sandboxDeath = () => {
      connected = false;
      if (!alreadySignaled) {
        ee.emit('end');
        alreadySignaled = true;
      }
    };
    // common culprits include ECONNRESET, ECONNREFUSED, ECONNABORTED, ETIMEDOUT
    socket.on('error', sandboxDeath);
    // when the evaluator half closes the connection, the sandbox can't be useful anymore
    socket.on('end', sandboxDeath);

    socket.on('error', function saWorker$$socket$errorHandler(e) {
      debugging() && console.debug(`Evaluator ${this.serial} - socket error:`,
                                   e.code === 'ECONNREFUSED' ? e.message : e);
      if (options.onsocketerror)
        options.onsocketerror(e, this);
    }.bind(this));
  } else {
    setImmediate(beginSession.bind(this));
  }

  this.addEventListener = ee.addListener.bind(ee)
  this.removeEventListener = ee.removeListener.bind(ee)
  this.removeAllListeners = ee.removeAllListeners.bind(ee);
  this.serial = StandaloneWorker.lastSerial = (StandaloneWorker.lastSerial || 0) + 1
  this.serialize = JSON.stringify
  this.deserialize = JSON.parse

  /** @param   data   Buffer or string */
  function writeOrQueue(data) {
    /* We queue writes in pendingWrites between the call to connect() and
     * the actual establishment of the TCP socket. Once connected, we drain that
     * queue into the write queue in Node's net module.
     */
    let realWrite = writeOrQueue.realWrite;
    let pendingWrites = writeOrQueue.pendingWrites;

    if (!connected) {
      if (data !== null)
        pendingWrites.push(data);
      return;
    }

    if (typeof data === 'object' && data instanceof Buffer)
      data = Buffer.from(data);

    while (pendingWrites.length && !writeStream.destroyed) {
      realWrite(pendingWrites.shift());
    }

    if (data && writeStream.destroyed) {
      console.log(`Warning; tried to write data ${data} after writeStream was destroyed`, new Error());
    }

    if (data !== null) /* pass null to force flush pending if connected */
      realWrite(data);
  }
  writeOrQueue.pendingWrites = [];
  writeOrQueue.realWrite = writeStream.write.bind(writeStream);
  delete writeOrQueue.write;

  function beginSession () {
    connected = true

    readStream.on ('error', shutdown);
    readStream.on ('end',   shutdown);
    readStream.on ('close', shutdown);
    if (readStream !== writeStream) {
      writeStream.on('error', shutdown);
      writeStream.on('end',   shutdown);
      writeStream.on('close', shutdown);
    }

    readStream.setEncoding('utf-8');
    if (writeStream.setEncoding)
      writeStream.setEncoding('utf-8');
    debugging() && console.debug('Connected; ' + writeOrQueue.pendingWrites.length + ' pending messages.');

    writeOrQueue(null); /* drain pending */
    /* @todo Make this auto-detected /wg jul 2018
     * changeSerializer.bind(this)("/var/dcp/lib/serialize.js")
     */
  }

  /** Change the protocol's serialization implementation. Must be in
   *  a format which returns an 'exports' object on evaluation that
   *  has serialize and deserialize methods that are call-compatible
   *  with JSON.stringify and JSON.parse.
   *
   *  @param   filename     The path to the serialization module, or an exports object
   *  @param   charset      [optional]   The character set the code is stored in
   */
  this.changeSerializer = (filename, charset) => {
    if (this.newSerializer) {
      throw new Error('Outstanding serialization change on worker #' + this.serial)
    }

    try {
      let code

      if (typeof filename === 'object') {
        let expo = filename
        code = '({ serialize:' + expo.serialize + ',deserialize:' + expo.deserialize + '})'
      } else {
        code = require('fs').readFileSync(filename, charset || 'utf-8')
      }
      this.newSerializer = eval(code)
      if (typeof this.newSerializer !== 'object') {
        throw new TypeError('newSerializer code evaluated as ' + typeof this.newSerializer)
      }
      writeOrQueue(this.serialize({ type: 'newSerializer', payload: code }) + '\n');
      this.serialize = this.newSerializer.serialize /* do not change deserializer until worker acknowledges change */
    } catch (e) {
      console.log('Cannot change serializer', e)
    }
  }

  /* Receive data from the network, turning it into debug output,
   * remote exceptions, worker messages, etc.
   */
  readStream.on('data', function standaloneWorker$$Worker$recvData (data) {
    var line, lineObj /* line of data coming over the network */
    var nl
    readBuf += data
    while ((nl = readBuf.indexOf('\n')) !== -1) {
      try {
        line = readBuf.slice(0, nl)
        readBuf = readBuf.slice(nl + 1)
        if (!line.length) { continue }

        if (line.match(/^DIE: */)) {
          /* Remote telling us they are dying */
          debugging('lifecycle') && console.debug('Worker is dying (', line + ')');
          shutdown();
          clearTimeout(dieTimer)
          break
        }

        if (line.match(/^LOG: */)) {
          debugging('log') && console.log('Worker', this.serial, 'Log:', line.slice(4));
          continue
        }

        if (!line.match(/^MSG: */)) {
          debugging('messages') && console.debug('worker:', line);
          continue
        }

        lineObj = this.deserialize(line.slice(4))
        switch (lineObj.type) {
          case 'workerMessage': /* Remote posted message */
            ee.emit('message', {data: lineObj.message})
            break
          case 'nop':
            break
          case 'result':
            if (lineObj.hasOwnProperty('exception')) { /* Remote threw exception */
              let e2 = new Error(lineObj.exception.message)
              e2.stack = 'Worker #' + this.serial + ' ' + lineObj.exception.stack + '\n   via' + e2.stack.split('\n').slice(1).join('\n').slice(6)
              e2.name = 'Worker' + lineObj.exception.name
              if (lineObj.exception.fileName) { e2.fileName = lineObj.exception.fileName }
              if (lineObj.exception.lineNumber) { e2.lineNumber = lineObj.exception.lineNumber }
              ee.emit('error', e2)
              continue
            } else {
              if (lineObj.success && lineObj.origin === 'newSerializer') { /* Remote acknowledges change of serialization */
                this.deserialize = this.newSerializer.deserialize
                delete this.newSerializer
              } else {
                debugging() && console.log('Worker', this.serial, 'returned result object: ', lineObj.result);
              }
            }
            break
          default:
            ee.emit('error', new Error('Unrecognized message type from worker #' + this.serial + ', \'' + lineObj.type + '\''))
        }
      } catch (e) {
        ee.emit('error', 'Error processing remote response: \'' + line + '\' (' + e.name + ': ' + e.message + e.stack.split('\n')[1].replace(/^  */, ' ') + ')')
      }
    }
  }.bind(this))

  ee.on('error', function standaloneWorker$$Worker$recvData$error (e) {
    debugging() && console.error("Evaluator threw an error:", e);
  })

  ee.on('message', function standaloneWorker$$Worker$recvData$message (ev) {
    debugging() && console.log("Worker relayed a message:", ev);
  });

  /**
   * Exclude certain socket errors from reporting.
   * @Wes Please verify that the 5 error codes below should not be reported to a worker or a dcp-client.
   * @param {Error} error
   * @returns (boolean) -- true when error is reportable
   */
  const isErrorReportable = function standaloneWorker$$Worker$isErrorReportable (error)
  {
    switch(error.code)
    {
      case 'ECONNREFUSED':  /* remote not listening - evaluator not running? */
      case 'ECONNRESET':    /* RST packet sent: remote closed - throttled? */
      case 'ECONNABORTED':  /* unexpectedly aborted by the other side */
      case 'EPIPE':
      case 'ETIMEDOUT':
        debugging() && console.debug('Non-reportable error:', error)
        return false;
      default:
        return true;
    }
  }

  /* Shutdown the stream(s) which are connected to the evaluator */
  shutdown = (e) => {

    if ((e instanceof Error) && isErrorReportable(e))
      console.error(e);
    debugging('lifecycle') && console.debug('Shutting down evaluator connection ' + this.serial + '');
    try {
      writeOrQueue(null);
      readStream.destroy();
      if (readStream !== writeStream)
        writeStream.destroy();
    } catch(error) {
      console.error(`Warning: could not shutdown evaluator connection ${e.code}`, error);
    };
    connected = false;
  }

  debugging('lifecycle') && console.debug('Connecting to', this.config.location.hostname + ':' + this.config.location.port);

  /** Send a message over the network to a standalone worker */
  this.postMessage = function standaloneWorker$$Worker$postMessage (message) {
    var wrappedMessage = this.serialize({ type: 'workerMessage', message: message }) + '\n'
    writeOrQueue(wrappedMessage);
  }

  /* Tell the sandbox die.  The sandbox should respond with a message back of type DIE:, provided the
   * work function hops off the event loop before the timeout expires. Afterwards, we will close the
   * socket which should cause something like an NMI at the evaluator end that will terminate any
   * running work.
   */
  this.terminate = function standaloneWorker$$Worker$terminate () {
    debugging() && console.log(`saw.terminate on Evaluator ${this.serial}`);
    var wrappedMessage = this.serialize({ type: 'die' }) + '\n'

    try {
      writeOrQueue(wrappedMessage);
    } catch (e) {
      // Socket may have already been destroyed
    }
    finally
    {
      this.unref();
    }

    /* If DIE: response doesn't arrive in a reasonable time -- clean up */
    dieTimer = setTimeout(shutdown, 7000);
    dieTimer.unref();
  }
}
StandaloneWorker.prototype = Object.create(Reference.prototype);

/**
 * Function to create constructors which behave very much like the WindowOrWorkerGlobalScope Worker
 * constructor, except they instanciate StandaloneWorker in place of a Web Worker. This factory
 * captures the options parameter, so that invocation of the returned constructor does not need to
 *
 * @param       options         options with which to invoke the StandaloneWorker constructor
 *                              when constructing workers.
 */
exports.workerFactory = function standaloneWorker$$WorkerFactory(options)
{
  const KVIN = new (require('kvin').KVIN)();

  function Worker()
  {
    if (!new.target)
      throw new TypeError('Worker constructor: \'new\' is required');
    return new StandaloneWorker(options);
  }

  /**
   * There is a data encapsulation design bug in our stack which has accidentally made KVIN part of the
   * evaluator -> sa-worker communication stack.  The net effect of this is that node programs using
   * the StandaloneWorker class must decode messages from the evaluator using the same serialization as
   * that evaluator used to encode them.  This is correctly specified in the sandbox-definitions.js file
   * as kvin/kvin.js (May 2025).
   *
   * This work-around function decodes messages exactly how they were encoded, and returns the plain
   * object containing the message.  Eventually, this function will go away, leaving behind something
   * like "return arguments[0]" as a backwards compat API entry point.
   */
  Worker.decodeMessageEvent = function saWorker$$decodeMessageEvent(event)
  {
    const eventData = KVIN.unmarshal(event.data);
    return { data: eventData };
  }

  /**
   * See comments for decodeWorkerMessageEvent. This same design flaw also requires an encoding function
   * that will also eventually become a NOP.
   */
  Worker.encodeMessage = function saWorker$$encodeMessage(message)
  {
    return KVIN.marshal(message);
  }

  return Worker;
}

/* Attach setters for onmessage, onerror, etc on the Worker.prototype
 * which are implemented with this.addEventListener.
 */
const onHandlerTypes = ['message', 'error']
StandaloneWorker.prototype.onHandlers = {}

for (let i = 0; i < onHandlerTypes.length; i++) {
  let onHandlerType = onHandlerTypes[i]
  Object.defineProperty(StandaloneWorker.prototype, 'on' + onHandlerType, {
    enumerable: true,
    configurable: false,
    set: function (cb) {
      /* maintain on{eventName} singleton pattern */
      if (this.onHandlers.hasOwnProperty(onHandlerType)) {
        this.removeEventListener(onHandlerType, this.onHandlers[onHandlerType])
      }
      this.addEventListener(onHandlerType, cb)
      this.onHandlers[onHandlerType] = cb
    },
    get: function () {
      return this.onHandlers[onHandlerType]
    }
  })
}