@sap/cds/libx/queue/processing.js

SAP Cloud Application Programming Model - CDS for Node.js

/*
 * new scheduling-based event queue processing
 */

const cds = require('../../lib')
const LOG = cds.log('queue|outbox')

const { inspect } = require('util')

const { INTERNAL_USER, OUTBOX_MESSAGES, PROCESSING, QUEUE, SCHEDULER_SERVICE } = require('./consts')
const { create_task, is_provider_tenant } = require('./utils')

const JobExecutor = require('./JobExecutor')
const jobExecutor = new JobExecutor()
cds.on('shutdown', () => (jobExecutor._shutdown = true))

const { expBkfFix: waitingTime } = require('../_runtime/common/utils/waitingTime')

const $queued = Symbol('queued')
const $unqueued = Symbol('unqueued')
const $stored_reqs = Symbol('stored_reqs')
const $error = Symbol('error')
const $service = Symbol('service')

const MAX_WAITING_TIME = 1480000
const CAPATTEMPTS = Math.ceil(Math.log(MAX_WAITING_TIME / 1000 + 1) / Math.log(1.5))

let _processableTasksQuery = {}

const _addParamFalse = w => {
  if (w.xpr) return w.xpr.map(xpr => _addParamFalse(xpr))
  if (w.val === undefined) return w
  w.param = false
  return w
}

const _getProcessableTasksQuery = ({ maxAttempts, chunkSize }, queueName) => {
  if (_processableTasksQuery[queueName]) return _processableTasksQuery[queueName]

  const entity = cds.model.definitions[OUTBOX_MESSAGES]
  // Build attempt filter conditions bounded by maxAttempts
  const attemptConditions = [`attempts = 0 AND timestamp < ?`]
  if (maxAttempts > 1) {
    const to = maxAttempts > CAPATTEMPTS ? CAPATTEMPTS - 1 : maxAttempts - 2
    for (let i = 1; i <= to; i++) {
      attemptConditions.push(`attempts = ${i} AND lastAttemptTimestamp <= ?`)
    }
    attemptConditions.push(`attempts > ${to} AND lastAttemptTimestamp <= ?`)
  }

  const appid = cds.env.appid

  const sql = `
    SELECT ID FROM ${entity.name}
    WHERE ${appid ? `appid = '${appid}' AND ` : ''}target = '${queueName}'
      AND attempts < ${maxAttempts}
      AND (status is NULL OR lastAttemptTimestamp < ?)
      AND (${attemptConditions.join(' OR ')})
    ORDER BY timestamp ASC, ID ASC
    LIMIT ${chunkSize}`

  _processableTasksQuery[queueName] = cds.ql(sql)
  _processableTasksQuery[queueName].SELECT.limit.rows.param = false
  _processableTasksQuery[queueName].SELECT.where.map(w => _addParamFalse(w))
  _processableTasksQuery[queueName].SELECT.columns = Object.keys(entity.elements).map(k => ({ ref: [k] }))
  _processableTasksQuery[queueName].forUpdate()
  _processableTasksQuery[queueName].SELECT.forUpdate.ignoreLocked = true

  return _processableTasksQuery[queueName]
}

const _getProcessableTasksQueryParams = (now, shifted, maxAttempts) => {
  const maxWaitingTime = maxAttempts > CAPATTEMPTS ? MAX_WAITING_TIME : (Math.pow(1.5, maxAttempts - 1) - 1) * 1000
  const params = [shifted, new Date(now).toISOString()]
  if (maxAttempts > 1) {
    const to = maxAttempts > CAPATTEMPTS ? CAPATTEMPTS - 1 : maxAttempts - 2
    params.push(
      ...[...Array(to)].map((_, i) => new Date(now - (Math.pow(1.5, i + 1) - 1) * 1000).toISOString()),
      new Date(now - maxWaitingTime).toISOString()
    )
  }
  return params
}

let _flushTasksQuery = {}
const _getFlushTasksQuery = ({ maxAttempts, chunkSize }, queueName) => {
  if (_flushTasksQuery[queueName]) return _flushTasksQuery[queueName]
  else {
    const entity = cds.model.definitions[OUTBOX_MESSAGES]
    const appid = cds.env.appid
    const targetFilter = appid ? `appid = '${appid}' AND target = '${queueName}'` : `target = '${queueName}'`

    _flushTasksQuery[queueName] = cds.ql(`
      SELECT ID FROM ${entity.name}
      WHERE ${targetFilter}
        AND attempts < ${maxAttempts}
        AND (status is NULL OR lastAttemptTimestamp < ?)
        AND timestamp < ?
        AND msg LIKE '%"event":"flush"%'
        AND msg LIKE '%"target":"${queueName}"%'
      ORDER BY timestamp ASC, ID ASC
      LIMIT ${chunkSize}`)
    _flushTasksQuery[queueName].SELECT.limit.rows.param = false
    _flushTasksQuery[queueName].SELECT.where.map(w => _addParamFalse(w))
    _flushTasksQuery[queueName].SELECT.columns = Object.keys(entity.elements).map(k => ({ ref: [k] }))
    _flushTasksQuery[queueName].forUpdate()
    _flushTasksQuery[queueName].SELECT.forUpdate.ignoreLocked = true
  }
  return _flushTasksQuery[queueName]
}

const _safeJSONParse = string => {
  try {
    return string && JSON.parse(string)
  } catch {
    // Don't throw
  }
}

// REVISIT: move _processTasks to task runnter/ a new task processor
// REVISIT: update the description below
// Note: This function can also run for each tenant on startup
//
// tx1: Fetch messages which are not in process and are not locked (SELECT FOR UPDATE)
// tx1: Set those which are processable (not 'processing' or timed out) to 'processing'
// Process messages (in parallel or sequentially)
// tx2 : Update/Delete messages based on outcome, set status to null
//
const _processTasks = (queueName, tenant, _opts = {}) => {
  const opts = Object.assign({ attempt: 0 }, _opts)
  if (!opts.parallel) opts.chunkSize = 1

  // Add the current done callback if provided
  if (opts.done) {
    jobExecutor.addCallback({ target: queueName, tenant }, opts.done)
    delete opts.done // Remove to avoid re-adding in recursive calls
  }

  const OUTBOX = cds.model.definitions[OUTBOX_MESSAGES]

  let letAppCrash = false

  const _done = () => {
    if (letAppCrash) cds.exit(1)
    // REVISIT: In case of an error this keeps scheduling the processing, even though it will likely fail again.
    //          We should have a better strategy for that (e.g., max attempts for scheduling as well, or a delay which increases with each attempt).
    // REVISIT: Why do we need to provide a callback here at all?
    jobExecutor.end({ target: queueName, tenant }, () => _processTasks(queueName, tenant, opts))
  }

  return jobExecutor.run({ target: queueName, tenant }, () => {
    const config = tenant ? { tenant, user: cds.User.privileged } : { user: cds.User.privileged }
    config.after = 1 // make sure spawn puts its cb on the `timer` queue (via setTimeout), which is also used by `taskRunner`

    // REVISIT: do we really need cds.spawn (e.g., for extended model)?
    const _spawn = (fn, o) => setTimeout(() => cds._with(o, fn), 1).unref()
    const _begin = cds.db.kind === 'sqlite' ? _spawn : cds.spawn.bind(cds)
    const _tx = cds.tx.bind(cds)

    const processing_fn = async () => {
      const scheduler = await cds.connect.to(SCHEDULER_SERVICE)

      let selectedTasks
      let more_chunks
      let currMinWaitingTime

      const is_t0 = is_provider_tenant(tenant)

      const _now = Date.now()
      const _now_iso_string = new Date(_now).toISOString()
      const _timeout = cds.utils.ms4(opts.timeout)

      const _setWaitingTime = time => {
        if (time <= 0) return
        if (currMinWaitingTime === undefined) currMinWaitingTime = time
        else currMinWaitingTime = Math.min(currMinWaitingTime, time)
      }

      /*
       * SELECT tasks for processing
       */
      const _shifted = new Date(Math.max(0, _now - _timeout)).toISOString()

      LOG._debug && LOG.debug(`${tenant ? tenant + ' ' : ''}${queueName}: Fetch messages`)
      try {
        // Use dedicated transaction to fetch relevant messages
        // and _immediately_ set their status to 'processing' and commit
        // thus keeping the database lock as short as possible
        selectedTasks = await _tx(async () => {
          let selectedTasks
          try {
            if (is_t0) {
              const query = _getFlushTasksQuery(opts, queueName)
              const params = [_shifted, _now_iso_string]
              selectedTasks = await cds.db.run(query, params)
            } else {
              const query = _getProcessableTasksQuery(opts, queueName)
              const params = _getProcessableTasksQueryParams(_now, _shifted, opts.maxAttempts)
              selectedTasks = await cds.db.run(query, params)
            }
          } catch (e) {
            LOG.warn(`${queueName}: Failed to fetch messages for processing`, e)
            throw e
          }
          if (!selectedTasks.length) return []

          // for triggering the next chunk later
          more_chunks = selectedTasks.length === opts.chunkSize

          // dedupe tenants to flush
          if (is_t0) {
            const tenants = new Set()
            selectedTasks = selectedTasks.filter(task => {
              const tenant = JSON.parse(task.msg).data.tenant
              if (tenants.has(tenant)) return false
              tenants.add(tenant)
              return true
            })
          }

          // prettier-ignore
          LOG._debug && LOG.debug(`${tenant ? tenant + ' ' : ''}${queueName}: Process ${selectedTasks.length} ${selectedTasks.length > 1 ? 'messages' : 'message'}`)

          // set to processing and commit as fast as possible to release locks
          const where = { ID: { in: selectedTasks.map(t => t.ID) } }
          const data = { status: PROCESSING }
          await UPDATE(OUTBOX).set(data).where(where)

          return selectedTasks
        })
      } catch (e) {
        // server is shutting down, no need to reschedule, just exit
        if (jobExecutor._shutdown) return _done()

        // could potentially be a timeout
        const attempt = opts.attempt + 1
        const _waitingTime = waitingTime(attempt)
        // prettier-ignore
        LOG.error(`${queueName}: Message retrieval failed`, e, `Retry attempt ${attempt} in ${Math.round(_waitingTime / 1000)} s`)
        // Note: This must not be awaited!
        scheduler.scheduleFlush({
          target: queueName,
          tenant,
          flushOpts: { after: _waitingTime === 0 ? undefined : _waitingTime, attempt }
        })
        return _done()
      }

      const tasksGen = function* () {
        for (const _task of selectedTasks) {
          const _msg = _safeJSONParse(_task.msg)

          const context = _msg.context || {}
          // REVISIT: controlled context propagation
          // delete _msg.context

          const userId = _msg[INTERNAL_USER]
          delete _msg[INTERNAL_USER]
          const user = new cds.User.Privileged(userId)
          context.user = user

          // The latter is for backwards compatibility where `service` was not stored and `target` was the service name
          const srvName = _msg.service || _task.target
          delete _msg.service

          const fromSend = _msg._fromSend
          delete _msg._fromSend

          const msg = fromSend ? new cds.Request(_msg) : new cds.Event(_msg)
          if (!msg) continue

          const task = {
            ID: _task.ID,
            msg,
            _msg: _task.msg,
            context,
            srvName,
            attempts: _task.attempts || 0,
            _task
          }

          yield task
        }
      }

      const succeeded = []
      const toBeUpdated = []
      const afters = new Set()

      // Helper to create a new message from an existing one for callbacks
      const _newMsgForCallbackFrom = msg => {
        const _fromSend = msg instanceof cds.Request
        const newMsg = { ...msg }
        if (msg.entity) newMsg.entity = msg.entity // needed for proper `h.for(msg)` handling
        newMsg._fromSend = _fromSend
        if (!newMsg.queue) return newMsg
        newMsg.queue = { ...newMsg.queue }
        delete newMsg.queue.task // follow-ups must not inherit the singleton task ID
        delete newMsg.queue.every
        delete newMsg.queue.after
        return newMsg
      }

      try {
        const _handleWithErr = async task => {
          const service = cds.unqueued(await cds.connect.to(task.srvName))
          try {
            if (task.msg.query) {
              const q = (task.msg.query = cds.ql(task.msg.query))
              q.bind(service)
              task.msg.target = cds.infer.target(q)
            }

            // REVISIT: Shouldn't that work like a standard inbound adapter? I.e. either of:
            // - cds._with({...}, ()=> srv.dispatch(task.msg)) // instead of srv.handle(task.msg)
            // - cds.tx({...}, ()=> srv.dispatch(task.msg)) // instead of srv.handle(task.msg)
            // Problem: If task involves db, dedicated transactions will block on SQLite if an outer transaction is open
            const _run = service.tx.bind(service)
            const result = await _run({ ...task.context, tenant }, async () => {
              // REVISIT: why the queued service?
              //          -> looks like for "custom processor" -> remove
              const queued = cds.queued(service)
              const result = queued.outboxed.handle
                ? await queued.outboxed.handle.call(service, task.msg)
                : await service.handle(task.msg)

              // delete after successful processing in same tx for exactly once guarantee
              if (is_t0) {
                await DELETE.from(OUTBOX)
                  .where({ timestamp: { '<': _now_iso_string } })
                  .where({ msg: { like: '%"event":"flush"%' } })
                  .where({ msg: { like: `%"tenant":"${task.msg.data.tenant}"%` } })
                  .where({ msg: { like: `%"target":"${task.msg.data.target}"%` } })
              } else {
                await DELETE.from(OUTBOX).where({ ID: task.ID })

                // For atomicity: create follow-up tasks in same transaction as delete
                const follow_ups = []

                // #succeeded callback (only if not already a callback event)
                if (!task.msg.event.endsWith('/#succeeded') && !task.msg.event.endsWith('/#failed')) {
                  const succeededMsg = _newMsgForCallbackFrom(task.msg)
                  succeededMsg.event = succeededMsg.event + '/#succeeded'
                  succeededMsg.results = result
                  if (
                    service.handlers.on.some(h => h.for(succeededMsg)) ||
                    service.handlers.after.some(h => h.for(succeededMsg))
                  ) {
                    follow_ups.push({
                      task: create_task(queueName, succeededMsg, task.context, task.srvName, opts),
                      after: 0
                    })
                    afters.add(0)
                  }
                }

                // Next recurrence for .every() tasks
                if (task.msg.queue?.every) {
                  const _m = { ...task.msg }
                  _m._fromSend = task.msg instanceof cds.Request
                  const next_iteration_task = create_task(queueName, _m, task.context, task.srvName, opts, true)
                  // Date.parse() only accepts up to 3 ms digits -> first 23 chars
                  const after = Math.max(Date.parse(next_iteration_task.timestamp.slice(0, 23) + 'Z') - Date.now(), 0)
                  follow_ups.push({ task: next_iteration_task, after })
                  afters.add(after)
                }

                if (follow_ups.length) {
                  await scheduler.scheduleTask(
                    follow_ups.map(f => f.task),
                    queueName,
                    { after: follow_ups.map(f => f.after) }
                  )
                  // prettier-ignore
                  LOG._debug && LOG.debug(`${tenant ? tenant + ' ' : ''}${queueName}: Messages modified (-1, ~0, +${follow_ups.length})`)
                }
              }

              return result
            })
            task.results = result
            succeeded.push(task)
          } catch (e) {
            task[$error] = e
            if (cds.error.isSystemError(e)) {
              LOG.error(`${service.name}: Programming error detected:`, e)
              task.updateData = { attempts: opts.maxAttempts }
              toBeUpdated.push(task)
              throw new Error(`${service.name}: Programming error detected.`, { cause: e })
            }
            if (e.unrecoverable) {
              LOG.error(`${service.name}: Unrecoverable error:`, e)
              task.updateData = { attempts: opts.maxAttempts }
              toBeUpdated.push(task)
            } else {
              cds.repl || LOG.error(`${service.name}: Emit failed:`, e.message)
              task.updateData = { attempts: task.attempts + 1 }
              toBeUpdated.push(task)
              return false
            }
          }
        }

        const tasks = tasksGen()

        const res = await Promise.allSettled([...tasks].map(_handleWithErr))
        const errors = res.filter(r => r.status === 'rejected').map(r => r.reason)
        if (errors.length) throw new Error(`${queueName}: Programming errors detected.`)
      } catch (e) {
        LOG.error(e)
        letAppCrash = true
      }

      // Handle failed tasks: update attempts and create #failed callbacks
      if (toBeUpdated.length) {
        await _tx(async () => {
          const follow_up_tasks = []

          for (const each of toBeUpdated) {
            // REVISIT: when is this the case?
            if (succeeded.some(d => d.ID === each.ID)) continue

            each.updateData.status = null
            if (opts.storeLastError !== false) each.updateData.lastError = inspect(each[$error])
            if (each.updateData.lastError && typeof each.updateData.lastError !== 'string') {
              each.updateData.lastError = inspect(each.updateData.lastError)
            }
            each.updateData.lastAttemptTimestamp = cds.context.timestamp
            follow_up_tasks.push(Object.assign({}, each._task, each.updateData))
            afters.add(waitingTime(each.updateData.attempts))
          }

          const _errorToObj = error => {
            if (typeof error === 'string') return { message: error }
            return {
              name: error.name,
              message: error.message,
              stack: error.stack,
              code: error.code,
              ...error
            }
          }

          // Create #failed callbacks (only if max attempts is reached)
          for (const task of toBeUpdated) {
            if (
              !is_t0 &&
              !task.msg.event.endsWith('/#succeeded') &&
              !task.msg.event.endsWith('/#failed') &&
              task.updateData.attempts >= opts.maxAttempts
            ) {
              const msg = _newMsgForCallbackFrom(task.msg)
              msg.event = msg.event + '/#failed'
              msg.results = _errorToObj(task[$error])
              // No local cache needed as cds.connect.to caches services in cds.services
              const service = await cds.connect.to(task.srvName)
              if (service.handlers.on.some(h => h.for(msg)) || service.handlers.after.some(h => h.for(msg))) {
                const failed_task = create_task(queueName, msg, task.context, task.srvName, opts)
                follow_up_tasks.push(failed_task)
                afters.add(0)
              }
              _setWaitingTime(waitingTime(1))
            }
          }

          if (follow_up_tasks.length) {
            await scheduler.scheduleTask(follow_up_tasks, queueName, { after: Array.from(afters).sort() })
          }
        })
      }

      // prettier-ignore
      LOG._debug && LOG.debug(`${tenant ? tenant + ' ' : ''}${queueName}: Messages modified (-${succeeded.length}, ~${toBeUpdated.length})`)

      if (letAppCrash) return _done()

      if (toBeUpdated.length) LOG.warn(`${queueName}: Some messages could not be processed`)

      // continue with next chunk?
      if (more_chunks) _setWaitingTime(1) //> 1 to have a tiny waitingTime

      // do not retry when triggered by task runner t0
      if (currMinWaitingTime !== undefined) {
        // Note: This must not be awaited!
        scheduler.scheduleFlush({ target: queueName, tenant, flushOpts: { after: currMinWaitingTime } })
        return _done()
      }

      LOG._debug && LOG.debug(`${tenant ? tenant + ' ' : ''}${queueName}: Done`)

      return _done()
    }

    _begin(processing_fn, config)
  })
}

const _hasPersistentQueue = tenant => {
  if (!cds.db) return false // no persistence configured
  if (cds.requires.multitenancy && tenant && is_provider_tenant(tenant) && !cds.env.requires.scheduling) return false // no persistence for provider account
  return true
}

exports.queued = function queued(srv, opts) {
  if (srv[$queued]) return srv[$queued]

  if (opts && typeof opts !== 'object') cds.error(`Queue 'opts' must be an object, but got ${typeof opts}.`)

  // Have the original & queued service reference each other
  const originalSrv = srv[$unqueued] ?? srv
  const queuedSrv = Object.create(originalSrv)
  Object.defineProperty(queuedSrv, $unqueued, { value: originalSrv })
  Object.defineProperty(srv, $queued, { value: queuedSrv })

  // Get service-specific queued configurations
  // REVISIT: remove with cds^11
  if (queuedSrv.options?.outbox)
    cds.utils.deprecated({ kind: 'Config', old: 'cds.requires.<srv>.outbox', use: 'cds.requires.<srv>.outboxed' })
  if (queuedSrv.options?.outboxed && queuedSrv.options?.queued)
    LOG.warn('Options "outboxed" and "queued" are both set. Using "outboxed" and ignoring "queued".')
  let serviceQOpts = queuedSrv.options?.outbox ?? queuedSrv.options?.outboxed ?? queuedSrv.options?.queued
  // > At this point serviceQOpts can conceivably be either object, string, or nullish

  let queueName, queueOpts
  // NOTE: only `outboxed: true` is officially supported
  if (typeof serviceQOpts === 'string') {
    queueName = serviceQOpts
    queueOpts = Object.assign({}, cds.env.requires.queue, cds.env.requires[queueName])
  } else if (!serviceQOpts || typeof serviceQOpts !== 'object') {
    queueName = QUEUE
    queueOpts = Object.assign({}, cds.env.requires.queue, opts)
  } else {
    queueName = srv.name
    queueOpts = Object.assign({}, cds.env.requires.queue, serviceQOpts, opts)
  }

  // REVISIT: Deprecate 'persistent-outbox' and remove this normalization
  if (queueOpts.kind === 'persistent-outbox') queueOpts.kind = 'persistent-queue'

  if (queueOpts.kind !== 'persistent-queue')
    throw new Error(`Unsupported queue kind: ${queueOpts.kind}. Only 'persistent-queue' is supported.`)

  if (queueOpts.targetPrefix) throw new Error('Unsupported queue config "targetPrefix". Use "cds.appid" instead.')

  // Store effective queue configuration at queued and outboxed (e.g. used in telemetry)
  queuedSrv.queued = queuedSrv.outboxed = queueOpts

  // REVISIT: remove in-memory queue support
  queuedSrv.handle = async function (req) {
    // REVISIT: why is req.context preferred and when would they differ?
    const context = req.context || cds.context

    // persistent queue
    if (queueOpts.kind === 'persistent-queue' && _hasPersistentQueue(context.tenant)) {
      const scheduler = await cds.connect.to(SCHEDULER_SERVICE)
      const task = create_task(queueName, req, context, srv.name, queueOpts)
      await scheduler.scheduleTask(task, queueName, { after: req.queue?.after }, context)
      return
    }

    // in-memory queue
    if (!context[$stored_reqs]) {
      context[$stored_reqs] = []
      context.on('succeeded', async () => {
        // REVISIT: Also allow maxAttempts for in-memory queue?
        for (const _req of context[$stored_reqs]) {
          try {
            if (_req.reply) await originalSrv.send(_req)
            else await originalSrv.emit(_req)
          } catch (e) {
            LOG.error('Emit failed', { event: _req.event, cause: e })
            if (cds.error.isSystemError(e)) {
              await cds.shutdown(e)
              return //> ?
            }
          }
        }
        delete context[$stored_reqs]
      })
    }
    context[$stored_reqs].push(req)
  }

  // REVISIT: do we even need srv.flush() or is cds.flush() sufficient?
  // REVISIT:
  // Not great to have `flush` on the service, because it will flush the corresponding queue.
  // A better API would be: (await cds.connect.to('queueName')).flush()
  // But the current logic of connect.to doesn't allow having a model but no service.
  queuedSrv.flush = function flush(opts) {
    queueOpts[$service] = srv.name
    const tenant = cds.context?.tenant
    return new Promise(resolve => _processTasks(queueName, tenant, Object.assign({ done: resolve }, queueOpts, opts)))
  }

  queuedSrv._unschedule = async function (task) {
    const query = DELETE.from(OUTBOX_MESSAGES).where({ task })
    if (cds.env.appid) query.where({ appid: cds.env.appid })
    // NOTE: target should and will be the service name in the future instead of the "queue name".
    //       until then, the service name is encoded in msg. selecting with OR ensures it works for both variants.
    query.where`target = ${srv.name} OR msg LIKE ${`%"service":"${srv.name}"%`}`
    return await query
  }

  return queuedSrv
}

exports.unqueued = function unqueued(srv) {
  return srv[$unqueued] || srv
}

exports.cdsFlush = async function cdsFlush(target, opts) {
  // NOTE: in cds-serve tests, credentials === {} and the SELECT below never comes back
  //       should never be the case in a real project, but flushing without a db doesn't make sense so check is fine
  if (!cds.db?.options.credentials || !Object.keys(cds.db.options.credentials).length) return

  if (!opts && typeof target !== 'string') {
    opts = target
    target = undefined
  }

  const tenant = cds.context?.tenant

  let queues
  if (target) {
    queues = [target]
  } else {
    // REVISIT: why not simply await SELECT?
    queues = await cds.tx(async () => {
      try {
        return await SELECT.from(OUTBOX_MESSAGES).groupBy('target').columns('target')
      } catch (e) {
        LOG.warn('Failed to fetch targets for flush', e)
        return []
      }
    })
    queues = queues.map(t => t.target)
  }

  const promises = []
  for (const queueName of queues) {
    const queueOpts = Object.assign({}, cds.requires.queue, cds.requires[queueName] || {}, opts || {})
    promises.push(new Promise(resolve => _processTasks(queueName, tenant, Object.assign({ done: resolve }, queueOpts))))
  }
  await Promise.all(promises)
}