async-await-parallel/index.js

Concurrency control for awaiting an array of async results

module.exports = parallelMap
module.exports.pool = parallelPool

/**
 * Invokes an array of async functions in parallel with a limit to the maximum
 * number of concurrent invocations. Async functions are executed in-order and
 * the results are mapped to the return array.
 *
 * Acts as a replacement for `await Promise.all([ ... ])` by limiting the max
 * concurrency of the array of function invocations.
 *
 * If any single task fails (eg, returns a rejected Promise), the pool will drain
 * any remaining active tasks and reject the resulting Promsie.
 *
 * @param {Array<async Function(Void) => Any>} thunks
 * @param {Number?} concurrency Max concurrency (defaults to 5)
 *
 * @return {Promise<Array<Any>>}
 */
async function parallelMap (thunks, concurrency = 5) {
  if (concurrency > 0) {
    concurrency = Math.min(concurrency, thunks.length)
  } else {
    concurrency = thunks.length
  }

  let index = 0
  let results = [ ]

  await parallelPool(concurrency, async () => {
    if (index < thunks.length) {
      const currentIndex = index++
      const thunk = thunks[currentIndex]
      results[currentIndex] = await thunk.call(this)
    }

    return (index < thunks.length)
  })

  return results
}

/**
 * Executes a given async `task` multiple times in parallel with a guaranteed
 * max concurrency given by `size`.
 *
 * The task should be an async function which resolves to a boolean for whether
 * or not there are more tasks to process.
 *
 * If any single task fails (eg, returns a rejected Promise), the pool will drain
 * any remaining active tasks and reject the resulting Promsie.
 *
 * @param {Number} size
 * @param {async Function(Void) => Boolean} task
 *
 * @return {Promise<Void>}
 */
async function parallelPool (size, task) {
  let done = false
  let active = 0
  let errors = [ ]

  return new Promise((resolve, reject) => {
    function poolIterator () {
      while (active < size && !done) {
        active++

        task()
          .then((more) => {
            if (--active <= 0 && (done || !more)) {
              if (errors.length > 0) {
                // at least one task failed
                return reject(new Error(errors))
              } else {
                // all tasks completed successfully
                return resolve()
              }
            } else if (more) {
              poolIterator()
            } else {
              done = true
            }
          })
          .catch((err) => {
            errors.push(err)
            done = true

            if (--active <= 0) {
              return reject(new Error(errors))
            } else {
              // wait until all active tasks are drained before rejecting the
              // final result (no new tasks will be started now that we're in
              // this error state)
            }
          })
      }
    }

    poolIterator()
  })
}