@sap/cds/libx/queue/index.js

SAP Cloud Application Programming Model - CDS for Node.js

/*
 * old non-scheduling-based event queue processing
 */

const cds = require('../_runtime/cds')
const LOG = cds.log('persistent-queue|queue|persistent-outbox|outbox')

const { inspect } = require('util')

const TaskRunner = require('./TaskRunner')
const taskRunner = new TaskRunner()
cds.on('shutdown', () => (taskRunner._shutdown = true))

const { INTERNAL_USER, PROCESSING, QUEUE } = require('./consts')
const { targetName, create_task } = require('./utils')

const { expBkfFix: waitingTime } = require('../_runtime/common/utils/waitingTime')

const $taskProcessorRegistered = Symbol('task processor registered')
const $queued = Symbol('queued')
const $unqueued = Symbol('unqueued')
const $stored_reqs = Symbol('stored_reqs')
const $error = Symbol('error')
const $service = Symbol('service')

const _timestampPlusOneMS = timestamp => new Date(timestamp.getTime() + 1).toISOString()

const _getTasksEntity = () => {
  const tasksDbName = 'cds.outbox.Messages'
  const tasksEntity = cds.model.definitions[tasksDbName]
  if (!tasksEntity) throw new Error(`The entity '${tasksDbName}' is missing but needed for persistent tasks.`)
  return tasksEntity
}

// REVISIT: Is this always a reliable way to identify the provider tenant?
//          Are there scenarios where the credentials have a different format?
const _isProviderTenant = tenant =>
  (cds.requires.auth && cds.requires.auth.credentials && cds.requires.auth.credentials.identityzoneid === tenant) ||
  cds.requires.multitenancy.t0 === tenant

const _hasPersistentQueue = tenant => {
  if (!cds.db) return false // no persistence configured
  if (cds.requires.multitenancy && tenant && _isProviderTenant(tenant)) return false // no persistence for provider account
  return true
}

const _safeJSONParse = string => {
  try {
    return string && JSON.parse(string)
  } catch {
    // Don't throw
  }
}

// Note: This function can also run for each tenant on startup
//
// tx1: Fetch messages which are not in process and are not locked (SELECT FOR UPDATE)
// tx1: Set those which are processable (not 'processing' or timed out) to 'processing'
// Process messages (in parallel or sequentially)
// tx2 (legacyLocking: tx1): Update/Delete messages based on outcome, set status to null
//
const _processTasks = (target, tenant, _opts = {}) => {
  const opts = Object.assign({ attempt: 0 }, _opts)
  if (!opts.parallel) opts.chunkSize = 1

  // Add the current done callback if provided
  if (opts.done) {
    taskRunner.addCallback({ name: target, tenant }, opts.done)
    delete opts.done // Remove to avoid re-adding in recursive calls
  }

  const tasksEntity = _getTasksEntity()

  let letAppCrash = false

  const __done = () => {
    if (letAppCrash) cds.exit(1)
    // REVISIT: In case of an error this keeps scheduling the processing, even though it will likely fail again.
    //          We should have a better strategy for that (e.g., max attempts for scheduling as well, or a delay which increases with each attempt).
    // REVISIT: Why do we need to provide a callback here at all?
    taskRunner.end({ name: target, tenant }, () => _processTasks(target, tenant, opts))
  }
  const _done = () => {
    if (!opts.legacyLocking) __done()
    // else will be handled in spawn
  }

  return taskRunner.run({ name: target, tenant }, () => {
    const config = tenant ? { tenant, user: cds.User.privileged } : { user: cds.User.privileged }
    config.after = 1 // make sure spawn puts its cb on the `timer` queue (via setTimeout), which is also used by `taskRunner`

    const _begin = opts.legacyLocking ? cds.spawn.bind(cds) : cb => cb()
    const _end = opts.legacyLocking ? s => s.on('done', __done) : () => {}
    const _tx = opts.legacyLocking ? cb => cb() : cds.tx.bind(cds)

    const spawn = _begin(async () => {
      let selectedTasks
      const currTime = Date.now()
      const _timeout = cds.utils.ms4(opts.timeout)

      let currMinWaitingTime

      const _setWaitingTime = time => {
        if (currMinWaitingTime === undefined) currMinWaitingTime = time
        else currMinWaitingTime = Math.min(currMinWaitingTime, time)
      }

      // Get tasks to process from the database

      const whereClause = []
      if (cds.env.appid && !opts.targetPrefix) {
        whereClause.push(...cds.ql.where({ appid: cds.env.appid, target: target }).where)
      } else if (cds.env.appid && opts.targetPrefix) {
        whereClause.push(...cds.ql.where({ appid: cds.env.appid, target: target }).where)
        whereClause.push('or')
        whereClause.push(...cds.ql.where(`appid is null and startswith(target, '${opts.targetPrefix}') = true`).where)
      } else {
        whereClause.push(...cds.ql.where({ target: targetName(target, opts) }).where)
      }

      // Legacy Fallback: To flush messages which have been stored with target = service.name
      if (target === QUEUE && opts[$service]) {
        whereClause.push('or')
        whereClause.push(...cds.ql.where({ appid: null, target: targetName(opts[$service], opts) }).where)
      }

      const tasksQuery = SELECT.from(tasksEntity)
        .where(whereClause)
        .orderBy(opts.parallel ? ['status', 'timestamp', 'ID'] : ['timestamp', 'status', 'ID'])
        .limit(opts.chunkSize)
        .forUpdate()

      if (opts.maxAttempts) tasksQuery.where({ attempts: { '<': opts.maxAttempts } })
      if (opts.parallel) {
        tasksQuery.SELECT.forUpdate.ignoreLocked = true
        const shifted = new Date(Math.max(0, currTime - _timeout)).toISOString()
        tasksQuery.where({ status: null, or: { lastAttemptTimestamp: { '<': shifted } } })
        // Note: If there are messages which are not yet timed out, but will hang eventually,
        // there will be no scheduled processing. One could remove that filter, but
        // it would (in the worst case) select <chunkSize> messages which are all in process
        // and not yet timed out, hence only <chunkSize> messages could be processed at any given time.
      }

      LOG._debug && LOG.debug(`${target}: Fetch messages`)

      try {
        // Use dedicated transaction to fetch relevant messages
        // and _immediately_ set their status to 'processing' and commit
        // thus keeping the database lock as short as possible
        selectedTasks = await _tx(async () => {
          const selectedTasks = await tasksQuery
          const shifted = currTime - _timeout
          const processableTasks = []

          // filter those which must not be processed
          for (const t of selectedTasks) {
            // break at the first one which is in the future (they're ordered by timestamp)
            const taskTimestamp = new Date(t.timestamp).getTime()
            const _waitingTimePlanned = taskTimestamp - currTime
            if (_waitingTimePlanned > 0) {
              _setWaitingTime(_waitingTimePlanned)
              break // everything afterwards is even further in the future
            }

            // ignore those which should have a longer waiting time
            // remember the minimum waiting time to retrigger the processing (will be compared to failed messages later)
            const lastAttemptTimestamp = t.lastAttemptTimestamp && new Date(t.lastAttemptTimestamp).getTime()
            if (lastAttemptTimestamp && t.attempts) {
              const _alreadyWaited = currTime - lastAttemptTimestamp
              const _shouldHaveWaited = waitingTime(t.attempts)
              const _remainingTime = _shouldHaveWaited - _alreadyWaited
              if (_remainingTime > 0) {
                _setWaitingTime(_remainingTime)
                continue
              }
            }
            if (t.status === null || new Date(t.lastAttemptTimestamp).getTime() < shifted) processableTasks.push(t)
          }
          // Note: There's also no scheduling for tasks which are not yet timed out.
          if (!processableTasks.length) return [] // all in process
          // prettier-ignore
          LOG._debug && LOG.debug(`${target}: Process ${processableTasks.length} ${processableTasks.length > 1 ? 'messages' : 'message'}`)
          if (!opts.legacyLocking) {
            await UPDATE(tasksEntity)
              .set({ status: PROCESSING })
              .where({ ID: { in: processableTasks.filter(t => t.status === null).map(t => t.ID) } })
          }
          return processableTasks
        })
      } catch (e) {
        // REVISIT: server is shutting down, no need to reschedule, just exit
        if (taskRunner._shutdown) return _done()

        // could potentially be a timeout
        const attempt = opts.attempt + 1
        const _waitingTime = waitingTime(attempt)
        // prettier-ignore
        LOG.error(`${target}: Message retrieval failed`, e, `Retry attempt ${attempt} in ${Math.round(_waitingTime / 1000)} s`)
        taskRunner.plan({ name: target, tenant, waitingTime: _waitingTime }, () =>
          _processTasks(target, tenant, { ...opts, attempt })
        )
        return _done()
      }

      const tasksGen = function* () {
        for (const _task of selectedTasks) {
          const _msg = _safeJSONParse(_task.msg)

          const context = _msg.context || {}
          // REVISIT: controlled context propagation
          // delete _msg.context

          const userId = _msg[INTERNAL_USER]
          delete _msg[INTERNAL_USER]
          const user = new cds.User.Privileged(userId)
          context.user = user

          // The latter is for backwards compatibility where `service` was not stored and `target` was the service name
          const srvName = _msg.service || _task.target
          delete _msg.service

          const fromSend = _msg._fromSend
          delete _msg._fromSend

          const msg = fromSend ? new cds.Request(_msg) : new cds.Event(_msg)
          if (!msg) continue

          const task = {
            ID: _task.ID,
            msg,
            context,
            srvName,
            attempts: _task.attempts || 0
          }

          yield task
        }
      }

      const succeeded = []
      const toBeDeleted = []
      const toBeUpdated = []
      const toBeCreated = []

      // Remember the failed message with the minimum current attempts (-> determines when the next processing shall be planned)
      let minAttemptFailed
      try {
        const _handleWithErr = async task => {
          const service = cds.unqueued(await cds.connect.to(task.srvName))
          try {
            if (task.msg.query) {
              const q = (task.msg.query = cds.ql(task.msg.query))
              q.bind(service)
              task.msg.target = cds.infer.target(q)
            }

            // REVISIT: Shouldn't that work like a standard inbound adapter? I.e. either of:
            // - cds._with({...}, ()=> srv.dispatch(task.msg)) // instead of srv.handle(task.msg)
            // - cds.tx({...}, ()=> srv.dispatch(task.msg)) // instead of srv.handle(task.msg)
            // Problem: If task involves db, dedicated transactions will block on SQLite if an outer transaction is open
            const _run = opts.legacyLocking && cds.db?.kind === 'sqlite' ? cds._with : service.tx.bind(service)
            const result = await _run({ ...task.context, tenant }, async () => {
              const result = opts.handle ? await opts.handle.call(service, task.msg) : await service.handle(task.msg)
              // we can only delete in the current tx if legacyLocking (i.e., long-lasting db locks) is false
              if (!opts.legacyLocking) {
                const _delete = DELETE.from(tasksEntity).where({ ID: task.ID })
                // only delete singleton task if not modified in the meantime
                if (task.msg.queue?.task) _delete.where({ timestamp: { '<': _timestampPlusOneMS(task.msg.timestamp) } })
                await _delete
              }
              return result
            })
            task.results = result
            succeeded.push(task)
          } catch (e) {
            if (!minAttemptFailed) minAttemptFailed = task.attempts
            else minAttemptFailed = Math.min(minAttemptFailed, task.attempts)
            task[$error] = e
            if (cds.error.isSystemError(e)) {
              LOG.error(`${service.name}: Programming error detected:`, e)
              task.updateData = { attempts: opts.maxAttempts }
              toBeUpdated.push(task)
              throw new Error(`${service.name}: Programming error detected.`, { cause: e })
            }
            if (e.unrecoverable) {
              LOG.error(`${service.name}: Unrecoverable error:`, e)
              if (opts.maxAttempts) {
                task.updateData = { attempts: opts.maxAttempts }
                toBeUpdated.push(task)
              } else toBeDeleted.push(task)
            } else {
              cds.repl || LOG.error(`${service.name}: Emit failed:`, e)
              task.updateData = { attempts: task.attempts + 1 }
              toBeUpdated.push(task)
              return false
            }
          }
        }
        const tasks = tasksGen()
        // REVISIT: Maybe we can also support handleMany and provide the iterator (for batch processing)
        if (opts.parallel) {
          const res = await Promise.allSettled([...tasks].map(_handleWithErr))
          const errors = res.filter(r => r.status === 'rejected').map(r => r.reason)
          if (errors.length) {
            throw new Error(`${target}: Programming errors detected.`)
          }
        } else {
          // In principle, this branch is not needed as for `parallel == false`, there's only one chunk at a time,
          // hence the `Promise.allSettled` above would be sufficient.
          // Let's keep it if we want to change it in the future.
          for (const task of tasks) {
            if ((await _handleWithErr(task)) === false) break
          }
        }
      } catch (e) {
        LOG.error(e)
        letAppCrash = true
      }

      const queries = []
      const _toBeDeleted = opts.legacyLocking ? toBeDeleted.concat(succeeded) : toBeDeleted
      if (_toBeDeleted.length) {
        const where = []
        for (const task of _toBeDeleted) {
          if (where.length) where.push('or')
          where.push({ ref: ['ID'] }, '=', { val: task.ID })
          // only delete singleton task if not modified in the meantime
          if (task.msg.queue?.task)
            where.push('and', { ref: ['timestamp'] }, '<', { val: _timestampPlusOneMS(task.msg.timestamp) })
        }
        queries.push(DELETE.from(tasksEntity).where(where))
      }

      // There can be tasks which are not updated / deleted, their status must be set back to `null`
      const updateTasks = selectedTasks.filter(
        task =>
          !toBeDeleted.some(t => t.ID === task.ID) &&
          !succeeded.some(t => t.ID === task.ID) &&
          !toBeUpdated.some(t => t.ID === task.ID)
      )
      if (updateTasks.length) {
        queries.push(
          UPDATE(tasksEntity)
            .where({ ID: { in: updateTasks.map(t => t.ID) } })
            .set({ status: null })
        )
      }

      for (const each of toBeUpdated) {
        if (toBeDeleted.some(d => d.ID === each.ID) || succeeded.some(d => d.ID === each.ID)) continue
        each.updateData.status = null
        if (opts.storeLastError !== false) each.updateData.lastError = inspect(each[$error])
        if (each.updateData.lastError && typeof each.updateData.lastError !== 'string') {
          each.updateData.lastError = inspect(each.updateData.lastError)
        }
        each.updateData.lastAttemptTimestamp = cds.context.timestamp
        queries.push(UPDATE(tasksEntity).where({ ID: each.ID }).set(each.updateData))
      }

      // Helper to create a new message from an existing one for callbacks
      const _newMsgForCallbackFrom = msg => {
        const _fromSend = msg instanceof cds.Request
        const newMsg = { ...msg }
        if (msg.entity) newMsg.entity = msg.entity // needed for proper `h.for(msg)` handling
        newMsg._fromSend = _fromSend
        if (!newMsg.queue) return newMsg
        newMsg.queue = { ...newMsg.queue }
        delete newMsg.queue.task // follow-ups must not inherit the singleton task ID
        delete newMsg.queue.every
        delete newMsg.queue.after
        return newMsg
      }

      const _failed = async task => {
        const msg = _newMsgForCallbackFrom(task.msg)
        msg.event = msg.event + '/#failed'
        const _errorToObj = error => {
          if (typeof error === 'string') return { message: error }
          return {
            name: error.name,
            message: error.message,
            stack: error.stack,
            code: error.code,
            ...error
          }
        }
        msg.results = _errorToObj(task[$error])
        const service = await cds.connect.to(task.srvName)
        if (service.handlers.on.some(h => h.for(msg)) || service.handlers.after.some(h => h.for(msg))) {
          toBeCreated.push(create_task(target, msg, task.context, task.srvName, opts))
        }
      }

      const _succeeded = async task => {
        const msg = _newMsgForCallbackFrom(task.msg)
        msg.event = msg.event + '/#succeeded'
        const service = await cds.connect.to(task.srvName)
        if (service.handlers.on.some(h => h.for(msg)) || service.handlers.after.some(h => h.for(msg))) {
          toBeCreated.push(create_task(target, msg, task.context, task.srvName, opts))
        }
      }

      for (const task of succeeded) {
        // invoke succeeded handlers
        if (!task.msg.event.endsWith('/#succeeded') && !task.msg.event.endsWith('/#failed')) {
          if (!task.error) {
            // skip programming errors & unrecoverable without maxAttempts
            await _succeeded(task)
          }
        }

        // handle `every`
        if (task.msg.queue?.every) {
          const _m = { ...task.msg }
          if (task.msg.queue.task) {
            // Read from database as the singleton task might have been modified/deleted in the meantime
            const dbTask = await SELECT.one(tasksEntity).where({ ID: task.ID })
            if (!dbTask) continue //> was unscheduled -> no next occurrence
            Object.assign(_m, JSON.parse(dbTask.msg))
          }
          _m._fromSend = task.msg instanceof cds.Request
          const _task = create_task(target, _m, task.context, task.srvName, opts, true)
          toBeCreated.push(_task)
        }
      }
      // invoke failed handlers (only if max attempts is reached)
      for (const task of toBeUpdated) {
        if (
          !task.msg.event.endsWith('/#succeeded') &&
          !task.msg.event.endsWith('/#failed') &&
          opts.maxAttempts &&
          task.updateData.attempts >= opts.maxAttempts
        ) {
          await _failed(task)
        }
      }

      if (toBeCreated.length) queries.push(INSERT.into(tasksEntity).entries(toBeCreated))

      if (queries.length) {
        await _tx(() => Promise.all(queries))
        // prettier-ignore
        LOG._debug && LOG.debug(`${target}: Messages modified (-${toBeDeleted.length + succeeded.length}, ~${toBeUpdated.length + updateTasks.length}, +${toBeCreated.length})`)
      }

      if (letAppCrash) return _done()

      if (toBeUpdated.length) {
        LOG.error(`${target}: Some messages could not be processed`)
        _setWaitingTime(waitingTime(minAttemptFailed + 1))
      }
      if (toBeDeleted.length + succeeded.length === opts.chunkSize || toBeCreated.length) {
        _setWaitingTime(0)
      }
      if (currMinWaitingTime !== undefined) {
        // prettier-ignore
        LOG._debug && LOG.debug(`${target}: Process${currMinWaitingTime > 0 ? ` in ${Math.round(currMinWaitingTime / 1000)} s` : ''}`)
        taskRunner.plan(
          {
            name: target,
            tenant,
            waitingTime: currMinWaitingTime
          },
          () => _processTasks(target, tenant, opts)
        )
        return _done()
      }

      LOG._debug && LOG.debug(`${target}: Done`)
      return _done()
    }, config)
    _end(spawn)
  })
}

const _registerTaskProcessor = (name, context) => {
  const registry = context[$taskProcessorRegistered] || (context[$taskProcessorRegistered] = new Set())
  if (!registry.has(name)) {
    registry.add(name)
    return true
  }
  return false
}

const _writeInQueue = async (name, msg, context, serviceName, taskOpts) => {
  const taskMsg = create_task(name, msg, context, serviceName, taskOpts)
  const tasksEntity = _getTasksEntity()
  LOG._debug && LOG.debug(`${name}: Write message to queue`)
  if (msg.queue?.task) return UPSERT.into(tasksEntity).entries(taskMsg)
  return INSERT.into(tasksEntity).entries(taskMsg)
}

exports.unqueued = function unqueued(srv) {
  return srv[$unqueued] || srv
}

exports.queued = function queued(srv, opts) {
  if (srv[$queued]) return srv[$queued]

  if (opts && typeof opts !== 'object') cds.error(`Queue 'opts' must be an object, but got ${typeof opts}.`)

  // Have the original & queued service reference each other
  const originalSrv = srv[$unqueued] ?? srv
  const queuedSrv = Object.create(originalSrv)
  Object.defineProperty(queuedSrv, $unqueued, { value: originalSrv })
  Object.defineProperty(srv, $queued, { value: queuedSrv })

  // Get service-specific queued configurations
  if (queuedSrv.options?.outbox)
    cds.utils.deprecated({ kind: 'Config', old: 'cds.requires.<srv>.outbox', use: 'cds.requires.<srv>.outboxed' })
  if (queuedSrv.options?.outboxed && queuedSrv.options?.queued)
    LOG.warn('Options "outboxed" and "queued" are both set. Using "outboxed" and ignoring "queued".')
  let serviceQOpts = queuedSrv.options?.outbox ?? queuedSrv.options?.outboxed ?? queuedSrv.options?.queued
  // > At this point serviceQOpts can conceivably be either object, string, or nullish

  let queueName, queueOpts
  // NOTE: only `outboxed: true` is officially supported
  if (typeof serviceQOpts === 'string') {
    queueName = serviceQOpts
    queueOpts = Object.assign({}, cds.env.requires.queue, cds.env.requires[queueName])
  } else if (!serviceQOpts || typeof serviceQOpts !== 'object') {
    queueName = QUEUE
    queueOpts = Object.assign({}, cds.env.requires.queue, opts)
  } else {
    queueName = srv.name
    queueOpts = Object.assign({}, cds.env.requires.queue, serviceQOpts, opts)
  }

  // REVISIT: Deprecate 'persistent-outbox' and remove this normalization
  if (queueOpts.kind === 'persistent-outbox') queueOpts.kind = 'persistent-queue'

  if (queueOpts.targetPrefix)
    cds.utils.deprecated({ kind: 'Configuration', old: 'cds.requires.queue.targetPrefix', use: 'cds.appid' })

  // Store effective queue configuration at queued and outboxed (e.g. used in telemetry)
  queuedSrv.queued = queuedSrv.outboxed = queueOpts

  queuedSrv.handle = async function (req) {
    const context = req.context || cds.context
    if (queueOpts.kind === 'persistent-queue' && _hasPersistentQueue(context.tenant)) {
      // returns true if not yet registered
      if (_registerTaskProcessor(srv.name, context)) {
        // NOTE: What if there are different queue options for the same service?!
        //       There could be tasks for srv1 with { maxAttempts: 1 }
        //       and tasks for srv1 with { maxAttempts: 9 }.
        //       How would they be processed? I'd rather not have dedicated
        //       service names or store serialized options for each task.
        context.on('succeeded', () => _processTasks(queueName, context.tenant, queueOpts))
      }
      await _writeInQueue(queueName, req, context, srv.name, queueOpts)
      return
    }

    if (!context[$stored_reqs]) {
      context[$stored_reqs] = []
      context.on('succeeded', async () => {
        // REVISIT: Also allow maxAttempts for in-memory queue?
        for (const _req of context[$stored_reqs]) {
          try {
            if (_req.reply) await originalSrv.send(_req)
            else await originalSrv.emit(_req)
          } catch (e) {
            LOG.error('Emit failed', { event: _req.event, cause: e })
            if (cds.error.isSystemError(e)) {
              await cds.shutdown(e)
              return
            }
          }
        }
        delete context[$stored_reqs]
      })
    }
    context[$stored_reqs].push(req)
  }

  // REVISIT:
  // Not great to have `flush` on the service, because it will flush the corresponding queue.
  // A better API would be: (await cds.connect.to('queueName')).flush()
  // But the current logic of connect.to doesn't allow having a model but no service.
  queuedSrv.flush = function flush(opts) {
    queueOpts[$service] = srv.name
    const tenant = cds.context?.tenant
    return new Promise(resolve => _processTasks(queueName, tenant, Object.assign({ done: resolve }, queueOpts, opts)))
  }

  queuedSrv._unschedule = async function (task) {
    const query = DELETE.from(_getTasksEntity()).where({ task })
    const target = targetName(srv.name, queueOpts)
    if (cds.env.appid) {
      if (queueOpts.targetPrefix)
        query.where`appid = ${cds.env.appid} OR appid IS NULL AND startswith(target, '${queueOpts.targetPrefix}') = true`
      else query.where({ appid: cds.env.appid })
    }
    // NOTE: target should and will be the service name in the future instead of the "queue name".
    //       until then, the service name is encoded in msg. selecting with OR ensures it works for both variants.
    query.where`target = ${target} OR msg LIKE ${`%"service":"${srv.name}"%`}`
    return await query
  }

  return queuedSrv
}

exports.cdsFlush = function cdsFlush(target, opts) {
  if (typeof target === 'object') [opts, target] = [target, QUEUE]
  const tenant = cds.context?.tenant
  const queueOpts = Object.assign({}, cds.requires.queue, cds.requires[target] || {}, opts || {})
  return new Promise(resolve => _processTasks(target, tenant, Object.assign({ done: resolve }, queueOpts)))
}