@sap/cds-mtxs/srv/extensibility-service.js

SAP Cloud Application Programming Model - Multitenancy library

const express = require('express')
const cds = require('@sap/cds/lib')
const { SELECT, DELETE } = cds.ql
const { fs, path, tar, rimraf, read, readdir, mkdirp, write } = cds.utils
const { token: authToken } = require('../lib/utils')
const linter = require('./extensibility/linter')
const { getMigratedProjects } = require('../lib/migration/migration')


const BUILT_IN_NAMESPACES = ['cds.core', 'cds.outbox', 'cds.xt'] // REVISIT: move to a more central place to be consumed also by linter
const TOMBSTONE_ID = '__tombstone'
const TEMP_DIR = fs.realpathSync(require('os').tmpdir())
const LOG = cds.log('mtx')
const DEBUG = cds.debug('mtx')
const production = process.env.NODE_ENV === 'production'

const ACTIVE = 2, DRAFT = 1, INACTIVE = 0

// determine appid for separation of extensions
//const packageJson = require(main.root + '/package.json');
const APPID = require('../lib/utils').getAppId()

const _async = () => {
  return cds.context.http?.req?.headers?.prefer === 'respond-async'
}

const _includeExt = () => {
  return (cds.env.requires['cds.xt.ExtensibilityService']?.['check-existing-extensions'] !== false)
}
class DbAdapter {

  static activated(status) {
    return ['inactive', 'draft', 'database'][status]
  }

  static status(activated) {
    const level = {
      inactive: INACTIVE,
      draft: DRAFT,
      database: ACTIVE
    }
    return level[activated]
  }

  // compatible put
  static async put(ext) {

    // status 'active' is only set by the activation later
    if (ext.status === ACTIVE) {
      throw Error('Internal error: direct activation')
    }

    // TODO must be urgently changed in favor of additional database field for csvs
    if (ext.csvs && !ext.archive) {
      const temp = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
      try {
        const root = await mkdirp(path.join(temp, 'data'))
        for (const csv in ext.csvs) {
          await write(path.join(root, csv), ext.csvs[csv])
        }
        ext.archive = await tar.cz(temp)
      } finally {
        await rimraf(temp)
      }
    }

    if (ext.csvs && Object.keys(ext.csvs).length > 0 && cds.env.requires['cds.xt.ExtensibilityService']?.activate?.['skip-csv']) throw Error('CSV deployment is disabled')

    // insert extension
    const tenant = ext.tenant
    const ID = cds.utils.uuid()
    const csn = JSON.stringify(ext.csn)
    const i18n = ext.i18n ? JSON.stringify(ext.i18n) : null
    const tag = ext.ID
    await cds.tx({ tenant, user: new cds.User.Privileged() }, async tx => {
      // remove current inactive extension with tag
      // use native table names to avoid issues with missing appid
      if (tag) await tx.run(DELETE.from('cds_xt_Extensions').where({ tag, activated: { '!=': this.activated(ACTIVE) } }))
      try {
        await tx.run(INSERT.into('cds_xt_Extensions').entries({ ID, csn, i18n, sources: ext.archive ?? null, activated: this.activated(ext.status), tag, appid: ext.appid ?? APPID, timestamp: new Date().toISOString() }))
      } catch (error) {
        DEBUG?.('Trying legacy request without appid', error)
        await tx.run(INSERT.into('cds_xt_Extensions').entries({ ID, csn, i18n, sources: ext.archive ?? null, activated: this.activated(ext.status), tag, timestamp: new Date().toISOString() }))
      }
    })
    return ID
  }

  static async delete(tag) {
    const ID = cds.utils.uuid()
    const result = !tag ? await DELETE.from('cds.xt.Extensions') : await DELETE.from('cds.xt.Extensions').where({ tag })
    // leave tombstone for deployment - ID cannot be used in case of all extensions (no ID passed)
    await DELETE.from('cds.xt.Extensions').where({ tag: TOMBSTONE_ID }) // cleanup
    await INSERT.into('cds.xt.Extensions').entries({ ID, csn: '{}', i18n: null, sources: null, activated: null, tag: TOMBSTONE_ID })
    return result
  }
}

module.exports = class ExtensibilityService extends cds.ApplicationService {

  async init() {

    cds.on('serving:cds.xt.JobsService', js => {
      js.on('taskUpdate', async (msg) => {
        if (msg.data?.task?.service === 'cds.xt.ExtensibilityService' && msg.data?.task?.status === 'FINISHED') { // Only filter for service. There is only one operation that triggers jobs
          await this.emit(
            'activated',
            { tenant: msg.tenant, job_ID: msg.data.task.job_ID, args: msg.data.task.args, status: msg.data.task.status, error: msg.data.task.error },
            { 'x-tenant-id': msg.tenant }
          )
        }
      })
    })

    // import compatibility handlers
    require('./extensibility/compat/update')(this)

    this.before('push', async req => {
      let { extension } = req.data
      if (!req.user.is('internal-user') && req.data.tenant && req.data.tenant !== req.tenant)
        req.reject(403, `No permission to push extensions to tenants other than ${production ? 'the user\'s tenant' : req.tenant}`)
      const tenant = (req.user.is('internal-user') && req.data.tenant) || req.tenant // TODO: revisit magic
      if (tenant) cds.context = { tenant }

      if (!extension) req.reject(400, 'Missing extension')

      const { root, archive } = await _unpackExtension(extension, tenant) // REVISIT part of handler contract?
      req.data.sources = root
      req.data.archive = archive
    })

    // REVISIT - change to after handler or rely on handler cooperation with the right use of next()
    this.on('push', async req => {
      const { sources } = req.data
      req.data.csn = await _readExtension(req, sources)
      req.data.csvs = await _readInitialData(sources)
      req.data.i18n = await _readI18n(sources)
      req.data.ID = req.data.tag ?? null
      req.data.status = ACTIVE // immediately activate

      try {
        const { 'cds.xt.ExtensibilityService': es } = cds.services
        await es.put(`/Extensions`, req.data)
      } finally {
        rimraf(req.data.sources)
      }
    })

    async function _unpackExtension(extension) {
      // REVISIT: string ?
      const sources = typeof extension === 'string' ? Buffer.from(extension, 'base64') : extension
      const root = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
      await tar.xvz(sources).to(root)
      return { root, archive: sources }
    }

    // retrieves extension from archive
    async function _readExtension(req, root) {
      try {
        const extCsn = JSON.parse(await read(path.join(root, 'extension.csn')))
        return [JSON.stringify(extCsn)]
      } catch (e) {
        if (e.code !== 'ENOENT') throw e
        req.reject(400, 'Missing or bad extension')
      }
    }

    // retrieves i18n from archive
    async function _readI18n(root) {
      try { return await read(path.join(root, 'i18n', 'i18n.json')) }
      catch (e) { if (e.code !== 'ENOENT') throw e }
    }

    // retrieves initial data from archive
    async function _readInitialData(root) {
      const csvs = []
      try {
        const dirents = await readdir(path.join(root, 'data'))
        for (const dirent of dirents) {
          const basename = path.basename(dirent)
          csvs.push({ name: basename, content: await read(path.join(root, 'data', dirent)) })
        }
      }
      catch (e) { if (e.code !== 'ENOENT') throw e }

      return csvs
    }

    this.on(['UPDATE', 'UPSERT'], 'Extensions', async req => {
      const tenant = _tenant(req)
      if (tenant) cds.context = { tenant }

      req.data.status = req.data.status ?? ACTIVE // set the default manually as it had to be removed from the model
      req.data.internalExtensionId = await DbAdapter.put({ ...req.data, status: req.data.status === ACTIVE ? DRAFT : req.data.status })
    })

    this.on('DELETE', 'Extensions', async req => {
      const tenant = _tenant(req)
      if (tenant) cds.context = { tenant }

      // put empty extension instead of deletion
      req.data.internalExtensionId = await DbAdapter.put({ ID: req.data.ID, tenant: cds.context.tenant, csn: {}, status: DRAFT })
    })

    // final activation
    this.after(['UPDATE', 'UPSERT'], 'Extensions', async (results, req) => { // TODO populate sources
      const async = _async()
      const { ID, status } = req.data

      // direct activation
      if (status === ACTIVE) await this.activate(ID, ACTIVE, { _internalExtensionId: req.data.internalExtensionId }) // TODO: what status? How to enforce activation?

      if (async) { // 202 is set by the job service
        return
      }
      const res = await _fetchExtensions(req.data.ID)
      req.results = { ID: res?.tag, csn: res?.csn, i18n: res?.i18n !== '{}' ? res?.i18n : undefined, timestamp: res?.timestamp, appid: res?.appid, status: DbAdapter.status(res?.activated) } // REVISIT: return csvs?
    })

    this.after('DELETE', 'Extensions', async (results, req) => {
      const async = _async(req)
      const { ID } = req.data

      // delegate to activate action
      if (!req.http.req.query.status || req.http.req.query?.status === ACTIVE)
        await this.activate(ID, ACTIVE, { _internalExtensionId: req.data.internalExtensionId })
      if (async) {
        return
      }
      return results
    })

    // difference to direct activation: no insert of extensions into db
    this.on('activate', async req => {
      const { ID: tag, status = ACTIVE, options } = req.data
      const tenant = _tenant(req)
      if (tenant) cds.context = { tenant }

      // TODO: check if level promotion is allowed
      // TODO get full csn from model provider for other handlers? Or leave it to them to run the model provider?

      const async = _async()

      const internalEntryId = req.data.options?._internalExtensionId
      try {
        const js = await cds.connect.to('cds.xt.JobsService')

        async function cleanup(tag, internalEntryId) {
          if (internalEntryId) { // cleanup only for direct activation
            await cds.tx({ tenant, user: new cds.User.Privileged() }, async () => DELETE.from('cds.xt.Extensions').where({ tag, ID: internalEntryId }))
          }
        }

        const postProcess = async (error) => {
          if (error) {
            await cleanup(tag, internalEntryId)
          }
        }

        // eslint-disable-next-line no-async-promise-executor
        return await new Promise(async (resolve, reject) => {
          const cb = !async ? async error => {
            if (error) {
              try {
                const errorObject = _decodeError(error)
                await cleanup(tag, internalEntryId)
                return reject(errorObject)
              } catch {
                return reject(error)
              }
            }
            return resolve()
          } : postProcess

          // separate user needed as job service is protected
          // protection is necessary because parts of the service are exposed to the outside
          const tx = js.tx({ tenant: cds.context.tenant, user: new cds.User.Privileged() })
          const job = await tx.enqueue('cds.xt.ExtensibilityService', 'activateExtension', [new Set([cds.context.tenant])], { tag, status, options }, cb)
          if (async) {
            resolve(job)
          }
        })
      } catch (err) {
        if (err.code === 'ERR_CDS_COMPILATION_FAILURE') req.reject({ status: 422, message: _getCompilerError(err.messages), messages: err.messages })
        else req.reject(err)
      }
    })

    this.on('validate', async req => {
      let tenant = _tenant(req)
      if (tenant) cds.context = { tenant }
      else tenant = cds.context.tenant

      let messages
      try {
        LOG.info('validating extension', req.data.ID, '...')

        const { 'cds.xt.ModelProviderService': mps } = cds.services
        const extFullCsn = await mps.getCsn(tenant, ['*'], undefined, undefined, false, false, { inactive: [req.data.ID]})
        const extCsn = _includeExt()
          ? await mps.getExtensions(tenant, { inactive: [req.data.ID]})
          : await mps.getExtensions(tenant, { inactive: [req.data.ID], compat: true }) // only check current extension

        // verify edmx
        cds.compile.to.edmx(extFullCsn, { service: 'all', version: cds.env.odata?.version ?? 'v4' })

        messages = linter.lint(extCsn, extFullCsn, cds.env, /* clone */ true)
      } catch (error) {
        let validationError = error
        if (error.code === 'ERR_CDS_COMPILATION_FAILURE') messages = error.messages
        else throw validationError
      }
      return {
        errors: messages.filter(m => m.severity === 'Error').map(m => ({ message: m.message, severity: m.severity ?? 'Error' })),
        messages: messages.filter(m => m.severity !== 'Error')
      }
    })

    this.on('discard', async req => {
      const tenant = _tenant(req)
      if (tenant) cds.context = { tenant }
      await DELETE.from('cds.xt.Extensions').where({ tag: req.data.ID, activated: { '!=': 'database' } })
    })

    this.on('set', req => {
      let { extension, tag, resources, activate = 'database' } = req.data
      const tenant = (req.user.is('internal-user') && req.data.tenant) || req.tenant || '' // REVISIT: magic

      const { 'cds.xt.ExtensibilityService': es } = cds.services

      return es.put(`/Extensions`,
        {
          ID: tag,
          csn: extension,
          i18n: resources,
          csvs: resources,
          tenant: tenant,
          status: (activate === 'database') ? ACTIVE : DRAFT
        })

    })

    this.on('pull', async req => {
      LOG.info('pulling latest model for tenant', req.tenant)
      const { tag } = req.data
      const { 'cds.xt.ModelProviderService': mps } = cds.services
      const csn = await mps.getCsn({
        tenant: req.tenant,
        toggles: cds.context.features, // with all enabled feature extensions
        base: !_includeExt(), // without any custom extensions
        flavor: 'xtended',
        for: 'pull',
        options: {
          skipExt: [tag]
        }
      })
      // filter system entries
      // filter @impl from csn - danger, must be cloned as it comes from the cache
      const extProjectBaseCsn = cds.clone(csn)
      for (const key of Object.keys(extProjectBaseCsn.definitions)) {
        for (const namespace of BUILT_IN_NAMESPACES) {
          if (key.startsWith(`${namespace}.`)) {
            delete extProjectBaseCsn.definitions[key]
            break
          }
          const def = extProjectBaseCsn.definitions[key]
          if (def) delete def['@impl']
        }
      }
      req.http.res?.set('content-type', 'application/octet-stream; charset=binary')
      const temp = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
      try {
        await fs.promises.writeFile(path.join(temp, 'index.csn'), cds.compile.to.json(extProjectBaseCsn))
        const config = linter.configCopyFrom(cds.env)
        await fs.promises.writeFile(path.join(temp, '.cdsrc.json'), JSON.stringify(config, null, 2))
        return await tar.cz(temp)
      } finally {
        rimraf(temp)
      }
    })

    async function _fetchExtensions(ID) {
      try {
        return !ID ? await SELECT.from('cds.xt.Extensions') : await SELECT.one.from('cds.xt.Extensions').where({ tag: ID })
      } catch (error) {
        DEBUG?.('Trying legacy request without appid', error)
        const fields = ['ID', 'csn', 'tag', 'activated', 'timestamp', 'i18n']
        return !ID ? await SELECT(fields).from('cds.xt.Extensions') : await SELECT(fields).one.from('cds.xt.Extensions').where({ tag: ID })
      }
    }

    // TODO read csvs -> will fail because csvs are not yet stored
    this.on('READ', 'Extensions', async req => {
      const tenant = _tenant(req)
      if (tenant) cds.context = { tenant }
      const ext = await _fetchExtensions(req.data?.ID)
      if (Array.isArray(ext)) return ext.filter(e => e.tag !== TOMBSTONE_ID).map(e => ({ ID: e.tag, csn: e.csn, i18n: e.i18n !== '{}' ? e.i18n : undefined, timestamp: e.timestamp, status: DbAdapter.status(e.activated), appid: e.appid })) // REVISIT: csvs?
      if (ext) return { ID: ext.tag, csn: ext.csn, i18n: ext.i18n !== '{}' ? ext.i18n : undefined, timestamp: ext.timestamp, status: DbAdapter.status(ext.activated),  appid: ext.appid } // REVISIT: csvs?
      return ext


    })

    this.on('getMigratedProjects', req => {
      const { tagRule, defaultTag } = req.data
      const tenant = req.tenant // REVISIT: check if access for arbitrary tenants needed
      if (!tenant)  req.reject(401, 'User not assigned to any tenant')
      return getMigratedProjects(req, tagRule || undefined, defaultTag || undefined, tenant)
    })

    cds.on('served', () => {
      cds.app?.post('/-/cds/login/token', express.urlencoded({ extended: false }), async (req, res) => {
        try {
          if (req.method === 'HEAD') return res.status(204).send()

          const cred = cds.env.requires.auth.credentials || {}
          if (!cred.clientid) throw cds.error('missing authentication service binding', { status: 500 })

          const query = req.method === 'POST' ? (req.body || {}) : (req.query || {})

          const form = query.refresh_token ? { grant_type: 'refresh_token', refresh_token: query.refresh_token } :
                       query.passcode      ? { grant_type: 'password', passcode: query.passcode } : {}
          form.scope = `${cred.xsappname}.cds.ExtensionDeveloper`
          const raw = await authToken(cred, { form, subdomain: query.subdomain })
          const data = JSON.parse(raw)
          if (data.error) return res.status(401).json(data)
          res.json(data)
        } catch (err) {
          const status = err.status || err.statusCode || 500
          res.status(status).json({
            error: err.message,
            ...(err.error_description && { error_description: err.error_description })
          })
        }
      })

      cds.app?.get('/-/cds/login/authorization-metadata', authMeta)
    })

    const _tenant = function (req) {
      const tenant = req.headers['x-tenant-id']?? req.data.tenant // the latter can be provided by 'set' when delegating to 'UPDATE'
      if (!req.user.is('internal-user') && tenant && tenant !== req.tenant)
        req.reject(403, `No permission to activate extensions by tenants other than ${req.tenant}`)
      return tenant
    }

    return super.init()
  }

  // REMEMBER: changing extension table schema will potentially require migration
  // REMEMBER: async execution -> ensure correct transaction handling
  // REMEMBER: async execution -> avoid using cds.context.tenant
  /**
   * Final activation called by the job
   */
  async activateExtension(tenant, tag, status, options = {}) {
    let extEntry
    try {
      // TODO: handle '*'?
      await cds.tx(async () => {
        // use native table names to avoid issues with missing appid
        extEntry = await SELECT(['ID','csn', 'tag','activated']).one.from('cds.xt.Extensions').where({ tag, activated: { '!=': 'database' } }).forUpdate() // select only entries with activated != 'database'
        if (!extEntry) {
          LOG.info('Extension', tag, 'not found or already activated') // REVISIT: in case of parallel activation, any entry for 'tag' is used
          return
        }

        // validate extension before activation
        // TODO: uses cds.context.tenant -> potentially dangerous in async jobs
        // -> REVISIT: Why not always check inactive extensions before activation?
        const validationResult = await this.validate(tag)
        if (validationResult.errors.length > 0) {
          const findings = [...validationResult.errors, ...validationResult.messages]
          const text = findings.length === 1 ? 'finding' : 'findings'
          let message = `Validation for tag '${tag}' failed with ${findings.length} ${text}:\n\n`
          message += findings.map(f => '  - ' + f.message).join('\n') + '\n'
          throw new cds.error({ message, status: 422 })
        }

        const deleted = extEntry.csn === '{}' // REVISIT mark differently?
        await DELETE.from('cds.xt.Extensions').where({ tag, activated: 'database' }) // remove current extension with tag
        if (deleted) await DbAdapter.delete(tag) // delete entries and add tombstone
        else await UPDATE('cds.xt.Extensions', { tag, ID: extEntry.ID }).with({ activated: DbAdapter.activated(status) }) // update status

        // really activate extension
        if (status === ACTIVE && !(cds.env.requires['cds.xt.ExtensibilityService']?.activate?.['skip-db'] === true)) {
          const { 'cds.xt.DeploymentService': ds } = cds.services
          if (!ds) {
            LOG.info('database activation skipped for extension', tag, '- no deployment service')
            return
          }
          LOG.info('activating extension', tag, '...')
          await ds.extend(tenant, options)
        }
      })
    } catch (error) {
      let activationError = error
      // needs to be serialized because it is stored in the db by the job service - TODO check for HDI error somehow?
      if (!error.code && !error.status) activationError = { status: 422, message: error.message } // for HDI errors
      throw new Error(_encodeError(activationError), { cause: error })
    }
  }
}

function _passcodeUrl(c, sub) {
  const u = new URL(c.url)
  if (sub) u.hostname = u.hostname.replace(/^[^.]+/, sub)
  u.pathname = '/passcode'
  return u.toString()
}

function authMeta(req, res) {
  const cred = cds.env.requires?.auth?.credentials || {}
  res.send({ passcode_url: _passcodeUrl(cred, req.query?.subdomain) })
}

const _getCompilerError = messages => {
  const defaultMsg = 'Error while compiling extension'
  if (!messages) return defaultMsg
  for (const msg of messages) {
    if (msg.severity === 'Error') return msg.message
  }
  return defaultMsg
}

const _encodeError = (activationError) => {
  if (activationError.messages) {
    // cut out first 10 elements of the messages array to avoid db problems
    activationError.messages = activationError.messages.slice(0, 10)
  }
  return JSON.stringify(activationError, (element, value) => {
    if (element === 'message') return JSON.stringify(value)
    return value
  })
}

const _decodeError = (activationError) => {
  return JSON.parse(activationError, (element, value) => {
    if (element === 'message') {
      try {
        return JSON.parse(value)
      } catch (e) {
        // in case of problems with parsing because of HANA bug
        LOG.error('Failed to parse error. Please use at least HANA database version (2024-26)', e)
        return value
      }
    }
    return value
  })
}