@sap/cds-mtxs/srv/jobs-service.js

SAP Cloud Application Programming Model - Multitenancy library

const { inspect } = require('util')
const cds = require('@sap/cds'), { uuid } = cds.utils
const main = require('./config')
const LOG = cds.log('mtx'), DEBUG = cds.debug('mtx')

const Jobs = 'cds.xt.Jobs', Tasks = 'cds.xt.Tasks'

const { t0_ } = require('../lib/utils')

const {
  queueSize = 100, clusterSize = 3, workerSize = 4, poolSize = 1
} = {
  ...(cds.env.requires.multitenancy?.jobs ?? {}),
  ...(cds.env.requires['cds.xt.SaasProvisioningService']?.jobs ?? {}),
  ...(cds.env.requires['cds.xt.SmsProvisioningService']?.jobs ?? {})
}

const RUNNING = 'RUNNING', FINISHED = 'FINISHED', FAILED = 'FAILED', QUEUED = 'QUEUED', TIMEOUT = 'TIMEOUT'

// A queue, implemented as a circular buffer for O(1) insert + delete
class Queue {
  constructor(capacity) {
    this.buffer = new Array(capacity)
    this.pointer = 0
    this.size = 0
    this.capacity = capacity
    DEBUG?.('initialized tenant operation job queue with capacity', capacity)
  }
  enqueue(value) {
    if (this.size === this.capacity) cds.error('Tenant operation job queue is full. Please try again later.', { status: 429 })
    this.buffer[(this.pointer + this.size) % this.capacity] = value
    this.size++
  }
  peek() {
    return this.buffer[this.pointer]
  }
  dequeue() {
    const value = this.buffer[this.pointer]
    this.pointer = (this.pointer + 1) % this.capacity
    this.size--
    return value
  }
}
const jobQueue = new Queue(queueSize)
let runningJobs = new Map()

module.exports = class JobsService extends cds.ApplicationService {

  async init() {
    if (cds.requires.multitenancy?.diagnostics) _diagnostics4(this)
    cds.on('listening', () => _scheduled(this))

    this.on('READ', 'Jobs', async () => {
      const jobs = await t0_(
        SELECT.from(Jobs, j => { j.ID, j.service, j.op, j.status, j.error, j.tasks(t => { t.ID, t.status, t.tenant, t.database, t.error } )}).orderBy('createdAt desc')
      )
      return jobs.map(job => ({ ...job, tasks: job.tasks.sort((a, b) => a.tenant.localeCompare(b.tenant))}))
    })
    this.on('config', () => {
      const conf = main.requires.extensibility, es = main.requires['cds.xt.ExtensibilityService']
      const allowlist = 'extension-allowlist', blocklist = 'namespace-blocklist'
      return {
        pool: main.requires.db.pool,
        poolMtx: cds.requires.db.pool,
        jobs: cds.requires.multitenancy.jobs,
        extensibility: {
          tenantCheckInterval: conf?.tenantCheckInterval,
          [allowlist]: conf?.[allowlist] ?? es?.[allowlist],
          [blocklist]: conf?.[blocklist] ?? es?.[blocklist]
        }
      }
    })

    return super.init()
  }

  async enqueue(service, op, clusters, args, onJobDone) {
    const _inspect = obj => obj && Object.values(obj).filter(Boolean).length > 0 ? inspect(obj, { depth: 5, colors: cds.utils.colors.enabled }) : []
    const inspected = Object.entries(args).reduce((acc, [k, v]) => {
      const inspectedValue = _inspect(v)
      if (Buffer.isBuffer(v)) acc.push(`${k}: <Buffer>`)
      else if (v?.length || v && typeof v === 'object' && Object.keys(v).length > 0) acc.push(`${k}: ${inspectedValue}`)
      return acc
    }, [])
    const _args = inspected.length ? ['with', ...inspected] : []
    const _format = clusters => {
      if (Array.isArray(clusters)) return clusters.flatMap(c => [...c])
      return Object.fromEntries(Object.entries(clusters).map(([k, v]) => [k, [...v]]))
    }

    LOG.info('enqueuing', { service, op }, 'for', _format(clusters), ..._args)

    const job_ID = uuid()
    const job = { ID: job_ID, service, op, args, status: QUEUED }
    const clustered = Object.entries(clusters).map(([database, cluster]) => {
      if (database !== '0') return Array.from(cluster).map(tenant => ({ job_ID, ID: uuid(), tenant, service, op, args, database, status: QUEUED }))
      else return Array.from(cluster).map(tenant => ({ job_ID, ID: uuid(), tenant, service, op, args, status: QUEUED }))
    })
    const tasks = clustered.flat()
    await t0_(async () => {
      await INSERT.into(Jobs, job)
      if (tasks.length) {
        await INSERT.into(Tasks, tasks)
        job.tasks = tasks
        await this.emit('jobUpdate', { job })
      } else {
        await UPDATE(Jobs, { ID: job_ID }).with({ status: FINISHED })
        await this.emit('jobUpdate', { job: { ...job, status: FINISHED } })
      }
    })

    if (tasks.length) {
      if (cds.services[service]) {
        const fn = task => cds.services[service].tx({ tenant: task.tenant }, tx => tx[op](task.tenant, ...Object.values(args)))
        jobQueue.enqueue({ job_ID, clusters: clustered, fn, onJobDone })
      } else {
        jobQueue.enqueue({ job_ID, clusters: clustered, fn: () => {
          LOG.warn('service', service, 'is not available, skipping job')
        }, onJobDone })
      }
      this.pickJob()
    }

    const url = process.env.VCAP_APPLICATION ? 'https://' + JSON.parse(process.env.VCAP_APPLICATION).uris?.[0] : cds.server.url
    cds.context.http?.res.set('Location', `${url}/-/cds/jobs/pollJob(ID='${job_ID}')`)
    cds.context.http?.res.set('x-job-id', job_ID)
    const { headers } = cds.context.http?.req ?? {}
    if (headers?.prefer?.includes('respond-async') || headers?.status_callback || headers?.mtx_status_callback) {
      cds.context.http.res.status(202)
    }
    return {
      ...job,
      tenants: Object.fromEntries(tasks.map(task =>
        [task.tenant, { ...task, job_ID: undefined, tenant: undefined, op: undefined }]
      )),
      tasks: Object.fromEntries(tasks.sort((a, b) => a.tenant.localeCompare(b.tenant)).map(task =>
        [task.tenant, { ...task, job_ID: job.ID, tenant: task.tenant, service: job.service, op: job.op }]
      ))
    }
  }

  async pollJob(ID) {
    const job = await t0_(
      SELECT.one.from(Jobs, j => { j.ID, j.service, j.op, j.error, j.status, j.tasks(t => { t.ID, t.status, t.tenant, t.error } )}).where({ ID })
    )
    if (!job) cds.error(`No job found for ID ${ID}`, { status: 404 })
    job.tasks.sort((a, b) => a.tenant.localeCompare(b.tenant)) // REVISIT: Ideally j.tasks supports orderBy
    job.tenants = Object.fromEntries(job.tasks.map(task => [task.tenant ?? task.TENANT, {
      status: task.status ?? task.STATUS,
      error: task.error ?? task.ERROR ?? undefined
    }]))

    return job
  }

  async pollTask(ID) {
    const task = await t0_(SELECT.one.from(Tasks).where({ ID }))
    if (!task) cds.error(`No task found for ID ${ID}`, { status: 404 })
    return {
      status: task.status ?? task.STATUS,
      op: task.op ?? task.OP,
      service: task.service ?? task.SERVICE,
      error: task.error ?? task.ERROR ?? undefined
    }
  }

  async limiter(limit, payloads, fn, asTask = false) {
    const pending = [], all = []
    for (const payload of payloads) {
      const { ID, tenant, job_ID, service } = payload
      if (asTask) {
        await t0_(async () => {
          await UPDATE(Tasks, { ID, tenant }).with({ status: RUNNING })
          await this.emit('taskUpdate', { task: { ID, job_ID, tenant, service, status: RUNNING }})
        })
      }
      const execute = asTask ? this._nextTask(payload, fn(payload)) : fn(payload)
      all.push(execute)
      const executeAndRemove = execute.finally(() => pending.splice(pending.indexOf(executeAndRemove), 1))
      pending.push(executeAndRemove)
      if (pending.length >= limit) {
        await Promise.race(pending)
      }
    }
    return Promise.allSettled(all)
  }

  async pickJob() {
    if (jobQueue.size === 0) return

    const next = new Set(jobQueue.peek().clusters.flat().map(t => t.tenant).flat())
    const running = [...runningJobs.values()].flatMap(j => j.clusters.flat().map(t => t.tenant))
    if (running.some(t => next.has(t))) return

    const job = jobQueue.dequeue()
    const { job_ID, clusters, fn, onJobDone } = job
    try {
      runningJobs.set(job_ID, job)
      await t0_(async () => {
        await UPDATE(Jobs, { ID: job_ID }).with({ status: RUNNING })
        await this.emit('jobUpdate', { job: { ID: job_ID, status: RUNNING }})
      })
      await this._nextJob(clusters, fn, onJobDone)
    } catch (e) {
      await t0_(async () => {
        await UPDATE(Jobs, { ID: job_ID }).with({ status: FAILED, error: _errorMessage(e) })
        await this.emit('jobUpdate', { job: { ID: job_ID, status: FAILED, error: _errorMessage(e) }})
      })
    } finally {
      runningJobs.delete(job_ID)
    }
    setImmediate(() => this.pickJob())
  }

  async _nextJob(clusters, fn, onJobDone) {
    if (clusters.length > 1) {
      await this.limiter(clusterSize, clusters, cluster => this.limiter(workerSize ?? poolSize, Array.from(cluster), fn, true))
    } else {
      await this.limiter(workerSize ?? poolSize, Array.from(clusters[0]), fn, true)
    }

    const { job_ID } = clusters[0][0] // all tasks have the same job ID -> just take the first
    const failed = await t0_(SELECT.one.from(Tasks).where ({ job_ID, and: { status: FAILED }}))
    const running = await t0_(SELECT.one.from(Tasks).where ({ job_ID, and: { status: RUNNING }}))

    const patch = failed ? { status: FAILED } : { status: FINISHED }
    await t0_(async () => {
      await UPDATE(Jobs, { ID: job_ID }).with(patch)
    })
    if (failed) {
      await onJobDone?.(failed.error ?? failed.ERROR ?? 'Unknown error')
    } else if (!running) {
      await onJobDone?.()
    }
    await this.emit('jobUpdate', { job: { ID: job_ID, ...patch } })
  }

  async _nextTask(task, _fn) {
    const { ID, tenant, job_ID, op, service, args } = task
    try {
      await _fn
      await t0_(async () => {
        await UPDATE(Tasks, { ID, tenant }).with({ status: FINISHED })
        await this.emit('taskUpdate', { task: { ID, job_ID, tenant, op, service, args, status: FINISHED}})
      })
    } catch (e) {
      LOG.error(e)
      const error = _errorMessage(e) ?? 'Unknown error'
      await t0_(async () => {
        await UPDATE(Tasks, { ID, tenant }).with({ status: FAILED, error })
        await this.emit('taskUpdate', { task: { ID, job_ID, tenant, op, service, args, status: FAILED, error }})
      })
    }
  }
}

function _errorMessage(e) {
  let message = e.message ?? 'Unknown error'
  if (e.error) message += ' ' + e.error
  if (e.description) message += ': ' + e.description
  return message.length > 5000 ? message.substring(0, 4997) + '...' : message // TODO use different datatype for errors -> next release
}

/*********************************************
 * Clean-up
 *********************************************/

function _scheduled(srv) {
  const minutes = 1000 * 60, hours = 60*minutes

  const { jobs:conf, /* compat */ jobCleanup, jobCleanupInterval, jobCleanupAge } = cds.requires.multitenancy ?? {}
  const {
    heartbeatInterval, heartbeatAge,
    cleanupInterval = jobCleanupInterval, cleanupAge = jobCleanupAge,
    noCleanup = jobCleanup === false
  } = conf ?? {}

  const heartbeat = setInterval(async () => {
    try {
      const cutoff = new Date(new Date - (heartbeatAge ?? 90*minutes))
      await t0_(async () => {
        const timedOut = { status: { in: [RUNNING, QUEUED] }, and: { modifiedAt: { '<': cutoff.toISOString() }}}
        const affectedJobs = await SELECT.from(Jobs, j => { j.ID, j.status }).where(timedOut)
        const affectedTasks = await SELECT.from(Tasks, t => { t.ID, t.status, t.job_ID }).where(timedOut)
        await UPDATE(Jobs, timedOut).with({ status: TIMEOUT })
        await UPDATE(Tasks, timedOut).with({ status: TIMEOUT })
        await Promise.all(affectedJobs.map(job => srv.emit('jobUpdate', { job: { ...job, status: TIMEOUT }})))
        await Promise.all(affectedTasks.map(task => srv.emit('taskUpdate', { task: { ...task, status: TIMEOUT }})))
      })
    } catch (error) {
      LOG.error('Error in heartbeat interval:', error)
    }
  }, heartbeatInterval ?? 5*minutes).unref()

  let cleanup
  if (!noCleanup) {
    cleanup = setInterval(async () => {
      try {
        const cutoff = new Date(new Date - (cleanupAge ?? 24*hours))
        await t0_(DELETE.from(Jobs, { status: FAILED, or: { status: FINISHED, and: { modifiedAt: { '<': cutoff.toISOString() }}}}))
      } catch (error) {
        LOG.error('Error in cleanup interval:', error)
      }
    }, cleanupInterval ?? 24*hours).unref()
  }

  cds.on('shutdown', () => {
    clearInterval(heartbeat)
    if (cleanup) clearInterval(cleanup)
  })
}

/*********************************************
 * Opt-in Websockets for diagnostics
 *********************************************/
const _diagnostics4 = srv => {
  const WebSocket = require('ws')
  let wss
  cds.on('listening', async app => {
    ;['jobUpdate', 'taskUpdate'].forEach(event => srv.on(event, forwardToWebSocketClients))
    ;['hana:create', 'hana:destroy', 'hana:validate',
      'pool:acquire', 'pool:release', 'pool:release:after',
      'pool:destroy', 'pool:destroy:after', 'pool:drain', 'pool:drain:after',
      'pool:createResource', 'pool:createResource:after', 'pool:dispense', 'pool:dispense:after',
      'pool:scheduleEviction', 'pool:scheduleEviction:after'
    ].forEach(event => cds.on(event, forwardToWebSocketClients))

    wss = new WebSocket.Server({ server: app.server })
    wss.on('connection', ws => {
      DEBUG?.('Websocket client for jobs service connected')
      ws.on('close', () => {
        DEBUG?.('Websocket client for jobs service disconnected')
      })
    })
  })

  function forwardToWebSocketClients(data = {}) {
    const message = JSON.stringify(data)
    DEBUG?.('> to socket:', message)
    wss.clients.forEach(client => {
      if (client.readyState === WebSocket.OPEN) {
        client.send(message)
      }
    })
  }
}