@sap/cds-mtxs/srv/extensibility-service.js

SAP Cloud Application Programming Model - Multitenancy library

const cds = require('@sap/cds/lib'), { fs, path, tar, rimraf, read, readdir } = cds.utils
const { token, authMeta } = require('./extensibility/token')
const linter = require('./extensibility/linter')
const { getMigratedProjects } = require('../lib/migration/migration')
const { SELECT, DELETE } = require('@sap/cds/lib/ql/cds-ql')
const { mkdirp, write } = require('@sap/cds/lib/utils')

const TOMBSTONE_ID = '__tombstone'
const TEMP_DIR = fs.realpathSync(require('os').tmpdir())
const LOG = cds.log('mtx')
const production = process.env.NODE_ENV === 'production'

const ACTIVE = 2, DRAFT = 1, INACTIVE = 0

const _async = () => {
  return cds.context.http?.req?.headers?.prefer === 'respond-async'
}

const _includeExt = () => {
  return (cds.env.requires['cds.xt.ExtensibilityService']?.['check-existing-extensions'] === true)
}
class DbAdapter {

  static activated(status) {
    return ['inactive', 'draft', 'database'][status]
  }

  static status(activated) {
    const level = {
      inactive: INACTIVE,
      draft: DRAFT,
      database: ACTIVE
    }
    return level[activated]
  }

  // compatible put
  static async put(ext) {

    // status 'active' is only set by the activation later
    if (ext.status === ACTIVE) {
      throw Error('Internal error: direct activation')
    }

    // TODO must be urgently changed in favor of additional database field for csvs
    if (ext.csvs && !ext.archive) {
      const temp = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
      try {
        const root = await mkdirp(path.join(temp, 'data'))
        for (const csv in ext.csvs) {
          await write(path.join(root, csv), ext.csvs[csv])
        }
        ext.archive = await tar.cz(temp)
      } finally {
        await rimraf(temp)
      }
    }

    // insert extension
    const tenant = ext.tenant
    const ID = cds.utils.uuid()
    const csn = JSON.stringify(ext.csn)
    const i18n = ext.i18n ? JSON.stringify(ext.i18n) : null
    const tag = ext.ID
    await cds.tx({ tenant, user: new cds.User.Privileged() }, async tx => {
      // remove current extension with tag
      if (tag) await tx.run(DELETE.from('cds.xt.Extensions').where({ tag, activated: { '!=': this.activated(ACTIVE) } }))
      await tx.run(INSERT.into('cds.xt.Extensions').entries({ ID, csn, i18n, sources: ext.archive, activated: this.activated(ext.status), tag }))
    })
    return ID
  }

  static async delete(tag) {
    const ID = cds.utils.uuid()
    const result = !tag ? await DELETE.from('cds.xt.Extensions') : await DELETE.from('cds.xt.Extensions').where({ tag })
    // leave tombstone for deployment - ID cannot be used in case of all extensions (no ID passed)
    await DELETE.from('cds.xt.Extensions').where({ tag: TOMBSTONE_ID }) // cleanup
    await INSERT.into('cds.xt.Extensions').entries({ ID, csn: '{}', i18n: null, sources: null, activated: null, tag: TOMBSTONE_ID })
    return result
  }
}

module.exports = class ExtensibilityService extends cds.ApplicationService {

  async init() {

    cds.on('serving:cds.xt.JobsService', js => {
      js.on('taskUpdate', async (msg) => {
        if (msg.data?.task?.service === 'cds.xt.ExtensibilityService') { // Only filter for service. There is only one operation that triggers jobs
          await this.emit('activated', { tenant: msg.tenant, job_ID: msg.data.task.job_ID, status: msg.data.task.status, error: msg.data.task.error })
        }
      })
    })

    // import compatibility handlers
    require('./extensibility/compat/update')(this)

    this.before('push', async req => {
      let { extension, tag } = req.data
      if (!req.user.is('internal-user') && req.data.tenant && req.data.tenant !== req.tenant)
        req.reject(403, `No permission to push extensions to tenants other than ${production ? 'the user\'s tenant' : req.tenant}`)
      const tenant = (req.user.is('internal-user') && req.data.tenant) || req.tenant // TODO: revisit magic
      if (tenant) cds.context = { tenant }

      if (!extension) req.reject(400, 'Missing extension')
      if (!tag) tag = null // REVISIT: can never be empty

      const { root, archive } = await _unpackExtension(extension, tenant) // REVISIT part of handler contract?
      req.data.sources = root
      req.data.archive = archive
    })

    // REVISIT - change to after handler or rely on handler cooperation with the right use of next()
    this.on('push', async req => {
      const { sources } = req.data
      req.data.csn = await _readExtension(req, sources)
      req.data.csvs = await _readInitialData(sources)
      req.data.i18n = await _readI18n(sources)
      req.data.ID = req.data.tag ?? null
      req.data.status = ACTIVE // immediately activate

      try {
        const { 'cds.xt.ExtensibilityService': es } = cds.services
        await es.put(`/Extensions`, req.data)
      } finally {
        rimraf(req.data.sources)
      }
    })

    async function _unpackExtension(extension) {
      // REVISIT: string ?
      const sources = typeof extension === 'string' ? Buffer.from(extension, 'base64') : extension
      const root = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
      await tar.xvz(sources).to(root)
      return { root, archive: sources }
    }

    // retrieves extension from archive
    async function _readExtension(req, root) {
      try {
        const extCsn = JSON.parse(await read(path.join(root, 'extension.csn')))
        return [JSON.stringify(extCsn)]
      } catch (e) {
        if (e.code !== 'ENOENT') throw e
        req.reject(400, 'Missing or bad extension')
      }
    }

    // retrieves i18n from archive
    async function _readI18n(root) {
      try { return await read(path.join(root, 'i18n', 'i18n.json')) }
      catch (e) { if (e.code !== 'ENOENT') throw e }
    }

    // retrieves initial data from archive
    async function _readInitialData(root) {
      const csvs = []
      try {
        const dirents = await readdir(path.join(root, 'data'))
        for (const dirent of dirents) {
          const basename = path.basename(dirent)
          csvs.push({ name: basename, content: await read(path.join(root, 'data', dirent)) })
        }
      }
      catch (e) { if (e.code !== 'ENOENT') throw e }

      return csvs
    }

    this.on(['UPDATE', 'UPSERT'], 'Extensions', async req => {
      req.data.status = req.data.status ?? ACTIVE // set the default manually as it had to be removed from the model
      req.data.internalExtensionId = await DbAdapter.put({ ...req.data, status: req.data.status === ACTIVE ? DRAFT : req.data.status })
    })

    this.on('DELETE', 'Extensions', async req => {
      const tenant = _tenant(req)
      if (tenant) cds.context = { tenant }

      // put empty extension instead of deletion
      req.data.internalExtensionId = await DbAdapter.put({ ID: req.data.ID, tenant: cds.context.tenant, csn: {}, status: DRAFT })
    })

    // final activation
    this.after(['UPDATE', 'UPSERT'], 'Extensions', async (results, req) => { // TODO populate sources
      const async = _async()
      const { ID, status } = req.data

      // direct activation
      if (status === ACTIVE) await this.activate(ID, ACTIVE, { _internalExtensionId: req.data.internalExtensionId }) // TODO: what status? How to enforce activation?

      if (async) { // 202 is set by the job service
        return
      }
      const res = await SELECT.one.from('cds.xt.Extensions').where({ tag: ID })
      req.results = { ID: res.tag, csn: res.csn, i18n: res.i18n !== '{}' ? res.i18n : undefined, timestamp: res.timestamp } // REVISIT: return csvs?
    })

    this.after('DELETE', 'Extensions', async (results, req) => {
      const async = _async(req)
      const { ID } = req.data

      // delegate to activate action
      if (!req.http.req.query.status || req.http.req.query?.status === ACTIVE)
        await this.activate(ID, ACTIVE, { _internalExtensionId: req.data.internalExtensionId })
      if (async) {
        return
      }
      return results
    })

    // difference to direct activation: no insert of extensions into db
    this.on('activate', async req => {
      const { ID: tag, level: status = ACTIVE, options } = req.data
      const tenant = _tenant(req)
      if (tenant) cds.context = { tenant }

      // TODO: check if level promotion is allowed
      // TODO get full csn from model provider for other handlers? Or leave it to them to run the model provider?

      const async = _async()

      const internalEntryId = req.data.options?._internalExtensionId
      try {
        const js = await cds.connect.to('cds.xt.JobsService')

        async function cleanup(tag, internalEntryId) {
          if (internalEntryId) { // cleanup only for direct activation
            await cds.tx({ tenant, user: new cds.User.Privileged() }, async () => DELETE.from('cds.xt.Extensions').where({ tag, ID: internalEntryId }))
          }
        }

        const postProcess = async (error) => {
          if (error) {
            await cleanup(tag, internalEntryId)
          }
        }

        // eslint-disable-next-line no-async-promise-executor
        return await new Promise(async (resolve, reject) => {
          const cb = !async ? async error => {
            if (error) {
              try {
                const errorObject = _decodeError(error)
                await cleanup(tag, internalEntryId)
                return reject(errorObject)
              } catch {
                return reject(error)
              }
            }
            return resolve()
          } : postProcess

          // separate user needed as job service is protected
          // protection is necessary because parts of the service are exposed to the outside
          const tx = js.tx({ tenant: cds.context.tenant, user: new cds.User.Privileged() })
          const job = await tx.enqueue('cds.xt.ExtensibilityService', 'activateExtension', [new Set([cds.context.tenant])], { tag, status, options }, cb)
          if (async) {
            resolve(job)
          }
        })
      } catch (err) {
        if (err.code === 'ERR_CDS_COMPILATION_FAILURE') req.reject({ status: 422, message: _getCompilerError(err.messages), messages: err.messages })
        else req.reject(err)
      }
    })

    this.on('validate', async req => {
      let tenant = _tenant(req)
      if (tenant) cds.context = { tenant }
      else tenant = cds.context.tenant

      let messages
      try {
        LOG.info(`validating extension '${req.data.ID}' ...`)

        const { 'cds.xt.ModelProviderService': mps } = cds.services
        const extFullCsn = await mps.getCsn(tenant, ['*'], undefined, undefined, false, false, { inactive: [req.data.ID]})
        const extCsn = _includeExt()
          ? await mps.getExtensions(tenant, { inactive: [req.data.ID]})
          : await mps.getExtensions(tenant, { inactive: [req.data.ID], compat: true }) // only check current extension

        messages = linter.lint(extCsn, extFullCsn, cds.env, /* clone */ true)
      } catch (error) {
        let validationError = error
        if (error.code === 'ERR_CDS_COMPILATION_FAILURE') messages = error.messages
        else throw validationError
      }
      return {
        errors: messages.filter(m => m.severity === 'Error').map(m => ({ message: m.message, severity: m.severity ?? 'Error' })),
        messages: messages.filter(m => m.severity !== 'Error')
      }
    })

    this.on('discard', async req => {
      await DELETE.from('cds.xt.Extensions').where({ tag: req.data.ID, activated: { '!=': 'database' } })
    })

    this.on('set', req => {
      let { extension, tag, resources, activate = 'database' } = req.data
      // if (!req.user.is('internal-user') && req.data.tenant && req.data.tenant !== req.tenant)
      //   req.reject(403, `No permission to add extensions to tenants other than ${req.tenant}`)
      const tenant = (req.user.is('internal-user') && req.data.tenant) || req.tenant || '' // REVISIT: magic

      const { 'cds.xt.ExtensibilityService': es } = cds.services

      return es.put(`/Extensions`,
        {
          ID: tag,
          csn: extension,
          i18n: resources,
          csvs: resources,
          tenant: tenant,
          status: (activate === 'database') ? ACTIVE : DRAFT
        })

    })

    this.on('pull', async req => {
      LOG.info(`pulling latest model for tenant '${req.tenant}'`)
      const { tag } = req.data
      const { 'cds.xt.ModelProviderService': mps } = cds.services
      const csn = await mps.getCsn({
        tenant: req.tenant,
        toggles: cds.context.features, // with all enabled feature extensions
        base: !_includeExt(), // without any custom extensions
        flavor: 'xtended',
        options: {
          skipExt: [tag]
        }
      })
      // filter @impl from csn - danger, must be cloned as it comes from the cache
      const extProjectBaseCsn = cds.clone(csn)
      for (const def of Object.values(extProjectBaseCsn.definitions)) {
        delete def['@impl']
      }
      req.http.res?.set('content-type', 'application/octet-stream; charset=binary')
      const temp = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
      try {
        await fs.promises.writeFile(path.join(temp, 'index.csn'), cds.compile.to.json(extProjectBaseCsn))
        const config = linter.configCopyFrom(cds.env)
        await fs.promises.writeFile(path.join(temp, '.cdsrc.json'), JSON.stringify(config, null, 2))
        return await tar.cz(temp)
      } finally {
        rimraf(temp)
      }
    })

    // TODO read csvs -> will fail because csvs are not yet stored
    this.on('READ', 'Extensions', async req => {
      const tenant = _tenant(req) // TODO add to compat later
      if (tenant) cds.context = { tenant }
      const ext = !req.data?.ID ? await SELECT.from('cds.xt.Extensions') : await SELECT.one.from('cds.xt.Extensions').where({ tag: req.data.ID })
      if (Array.isArray(ext)) return ext.filter(e => e.tag !== TOMBSTONE_ID).map(e => ({ ID: e.tag, csn: e.csn, i18n: e.i18n !== '{}' ? e.i18n : undefined, timestamp: e.timestamp }))
      if (ext) return { ID: ext.tag, csn: ext.csn, i18n: ext.i18n !== '{}' ? ext.i18n : undefined, timestamp: ext.timestamp } // REVISIT: csvs?
      return ext
    })

    this.on('getMigratedProjects', (req) => {
      let { tagRule, defaultTag } = req.data
      const tenant = req.tenant // REVISIT: check if access for arbitrary tenants needed
      if (!tenant) req.reject(401, 'User not assigned to any tenant')
      return getMigratedProjects(req, tagRule || undefined, defaultTag || undefined, tenant)
    })

    cds.on('served', () => {
      cds.app?.post('/-/cds/login/token', token)
      cds.app?.get('/-/cds/login/authorization-metadata', authMeta)
    })

    // REVISIT duplicate
    const _tenant = function (req) {
      const tenant = req.headers['x-tenant-id']
      if (!req.user.is('internal-user') && tenant && tenant !== req.tenant)
        req.reject(403, `No permission to activate extensions by tenants other than ${req.tenant}`)
      return tenant
    }

    return super.init()
  }

  // REMEMBER: changing extension table schema will potentially require migration
  // REMEMBER: async execution -> ensure correct transaction handling
  // REMEMBER: async execution -> avoid using cds.context.tenant
  /**
   * Final activation called by the job
   */
  async activateExtension(tenant, tag, status, options) {
    let extEntry
    try {
      // TODO: handle '*'?
      extEntry = await SELECT.one.from('cds.xt.Extensions').where({ tag, activated: { '!=': 'database' } }) // select only entries with activated != 'database'
      if (!extEntry) {
        LOG.info(`Extension '${tag}' not found or already activated`) // REVISIT: in case of parallel activation, any entry for 'tag' is used
        return
      }

      // validate extension before activation
      // TODO: uses cds.context.tenant -> potentially dangerous in async jobs
      // -> REVISIT: Why not always check inactive extensions before activation?
      const validationResult = await this.validate(tag)
      if (validationResult.errors.length > 0) {
        const findings = [...validationResult.errors, ...validationResult.messages]
        let message = `Validation for tag '${tag}' failed with ${findings.length} finding(s)\n\n`
        message += findings.map(f => '  - ' + f.message).join('\n') + '\n'
        throw new cds.error({ message, status: 422 })
      }

      const deleted = extEntry.csn === '{}' // REVISIT mark differently?
      await DELETE.from('cds.xt.Extensions').where({ tag, activated: 'database' }) // remove current extension with tag
      if (deleted) await DbAdapter.delete(tag) // delete entries and add tombstone
      else await UPDATE('cds.xt.Extensions', { tag, ID: extEntry.ID }).with({ activated: DbAdapter.activated(status) }) // update status

      // really activate extension
      if (status === ACTIVE && !(cds.env.requires['cds.xt.ExtensibilityService']?.activate?.['skip-db'] === true)) {
        const { 'cds.xt.DeploymentService': ds } = cds.services
        if (!ds) {
          LOG.info(`database activation skipped for extension '${tag}' - no deployment service`)
          return
        }
        LOG.info(`activating extension '${tag}' ...`)
        await ds.extend(tenant, options)
      }
    } catch (error) {
      let activationError = error
      // needs to be serialized because it is stored in the db by the job service - TODO check for HDI error somehow?
      if (!error.code && !error.status) activationError = { status: 422, message: error.message } // for HDI errors
      throw new Error(_encodeError(activationError))
    }
  }
}

const _getCompilerError = messages => {
  const defaultMsg = 'Error while compiling extension'
  if (!messages) return defaultMsg
  for (const msg of messages) {
    if (msg.severity === 'Error') return msg.message
  }
  return defaultMsg
}

const _encodeError = (activationError) => {
  if (activationError.messages) {
    // cut out first 10 elements of the messages array to avoid db problems
    activationError.messages = activationError.messages.slice(0, 10)
  }
  return JSON.stringify(activationError, (element, value) => {
    if (element === 'message') return Buffer.from(value).toString('base64')
    return value
  })
}

const _decodeError = (activationError) => {
  return JSON.parse(activationError, (element, value) => {
    if (element === 'message') return Buffer.from(value, 'base64').toString('utf8')
    return value
  })
}