@sap/cds-dk/lib/build/index.js

Command line client and development toolkit for the SAP Cloud Application Programming Model

const path = require('node:path')
const { inspect } = require('node:util')
// @ts-expect-error - _log not declared in cds-types
const cds = require('../cds'), { _log: log } = cds
const { exists, isdir, rimraf, mkdirp, write, fs, colors: { YELLOW, RESET } } = cds.utils
const Plugin = require('./plugins/plugin')
const { relativePaths, BuildError, BuildMessage, resolveRequiredSapModels, normalizePath, getDefaultModelPaths, hasOptionValue } = require('./util')
const { sortMessagesSeverityAware, deduplicateMessages, CompilationError } = cds.compiler
const { SEVERITIES, SEVERITY_ERROR, LOG_LEVELS, DEFAULT_SRC_FOLDER, OPTION_WS, IGNORE_DEFAULT_MODELS, NODEJS_MODEL_EXCLUDE_LIST, BUILD_TASK_NODEJS, BUILD_TASK_NODE_CF, BUILD_TASK_JAVA_CF, BUILD_TASK_JAVA } = require("./constants")
const COMPILATION_ERROR = 'CompilationError'
const COMPILE_MESSAGE = 'CompileMessage'
const DEBUG = cds.debug('cli|build')
const LOG_LIMIT = 30

/**
 * @typedef {import('./plugins/plugin')} Plugin_
 * @typedef {import('./plugins/plugin').Task} Task
 * @typedef {import('../env/schemas/cds-rc')['$defs']['buildTaskType']['enum'][number]} BuildFor
 * using (string & {}) prevents widening of `'x' | string` to just `string`
 * @typedef {{
 *  clean?: boolean,
 *  for?: BuildFor | (string & {}),
 *  src?: string,
 *  dest?: string,
 *  model?: string[],
 *  root?: string
 * } & {[key:string]:unknown}} BuildOptions
 */

/**
 * Executes cds build with specified options.
 *
 * @param {BuildOptions} options - command options
 */
async function build({ root = cds.root, tasks, ...options } = {}) {
    if (!exists(root) || !isdir(root)) {
        throw `Project folder '${root}' does not exist`
    }

    return await new BuildTaskEngine(options).processTasks(tasks)
}

function register(id, plugin) {
    return require('./plugins').register(id, plugin)
}

class BuildTaskEngine {

    /**
     * @param {BuildOptions} options
     */
    constructor(options = {}) {
        if (options.clean === undefined) {
            options.clean = true
        }
        for (const key of ['for', 'src', 'dest']) if (options[key]) {
            if (typeof options[key] !== 'string') {
                throw new BuildError(`Invalid build options - property '${key}' must be a string`)
            }
            if (key === 'src' || key === 'dest') {
                // normalize path separator and remove trailing slash
                options[key] = normalizePath(options[key])
            }
        }
        /** @type {{ options: BuildOptions, tasks: Task[]}} */
        this._context = { options, tasks: [] } // resolved tasks
        const plugins = require('./plugins')
        this._plugins = plugins.all
        plugins.registerDefaults()
    }

    get options() {
        return this._context.options
    }
    get context() {
        return this._context
    }
    get plugins() {
        return this._plugins
    }

    async processTasks(tasks) {
        const startTime = Date.now()

        tasks ??= await this.getTasks()
        console.log(`building project with`, inspect({
            versions: { cds: cds.version, compiler: cds.compiler.version(), dk: require('../../package.json').version },
            target: cds.env.build.target,
            tasks,
            ...(!this.options.clean && { clean: this.options.clean }),
        }, { depth: 11, colors: cds.utils.colors.enabled, compact: 3, breakLength: 250 }))

        // validate required @sap namespace models - log only
        const unresolved = BuildTaskEngine._resolveRequiredSapServices(tasks)
        if (unresolved.length > 0) {
            throw new BuildError(`Required CDS models [${unresolved.join(', ')}] cannot be resolved. Run the 'npm install' command to install up-to-date versions of the missing packages.`)
        }

        // create build plugins
        const plugins = []
        tasks.forEach((task) => {
            if (task) {
                const plugin = this.createPlugin(task)
                plugin.init()
                plugins.push(plugin)
            }
        })

        try {
            await this._executeCleanBuildTasks(plugins)

            // throwing Exception in case of compilation errors
            const buildResult = await this._executeBuildTasks(plugins)

            await this._writeGenerationLog(plugins)
            this._logOutput(plugins)
            this._logMessages(plugins.reduce((acc, plugin) => acc.concat(plugin.messages), []))
            console.log(`build completed in ${Date.now() - startTime} ms`)

            return buildResult
        } catch (error) {
            this._logOutput(plugins)
            throw error
        }
    }

    async getTasks() {
        const tasks = await this._createTasks()
        const resolvedTasks = this.resolveTasks(tasks)
        if (this.options.resolve) return resolvedTasks
        return tasks
    }

    /**
     * Create a Plugin instance for the given build task.
     * @param {Task} task
     */
    createPlugin(task) {
        const PluginClass = this.getPlugin(task)
        const resolvedTask = this._resolveTask(task)
        DEBUG?.(`loaded build plugin [${resolvedTask.for}]`)

        const plugin = new PluginClass()
        if (!(plugin instanceof PluginClass)) {
            throw new Error(`Invalid Build plugin type ${task.for}`)
        }
        plugin._task = resolvedTask
        plugin._context = this.context
        this.context.tasks.push(resolvedTask)
        DEBUG?.(`created build plugin [${resolvedTask.for}]`)
        return plugin
    }

    async _createTasks() {
        DEBUG?.(`determining build tasks for project [${cds.root}].`)

        let tasks = Array.isArray(cds.env.build?.tasks) ? cds.clone(cds.env.build.tasks) : []
        if (tasks.length === 0) {
            tasks = await this.lookupTasks()
            this._applyCliOptions(tasks)
        } else {
            this._applyCliOptions(tasks)
            // 1. apply default values including task.for and ensure that for all tasks a provider exists - throwing error otherwise
            await this.applyTaskDefaults(tasks)
            // ensure that dependencies get wired up before filtering
            await this.lookupTasks(tasks, true)
        }

        // 2. filters the list of build tasks
        let existingTasks = tasks
        tasks = await this._filterTasksForCli(tasks)
        if (tasks.length === 0) return tasks

        // 3. add dependencies
        existingTasks = [...tasks]
        await this.lookupTasks(tasks, true)
        if (tasks.length > existingTasks.length) {
            const newTasks = tasks.filter(task => !existingTasks.includes(task))
            this._applyCliOptions(newTasks)
        }

        await BuildTaskEngine._applyCommonTaskDefaults(tasks)

        // Java projects use "." as the default build target folder
        if (cds.env['project-nature'] === 'java' && cds.env.build.target !== ".") {
            const userEnv = cds.env.for("cds", cds.root, false)
            if (!userEnv.build?.target) {
                cds.env.build.target = "."
                DEBUG?.("using inplace build for java project instead of default staging build")
            }
        }
        return tasks
    }

    static async _applyCommonTaskDefaults(tasks) {
        tasks.forEach(task => {
            if (task.options?.model && !Array.isArray(task.options.model)) {
                task.options.model = [task.options.model]
            }
        })
        const modelPaths = await getDefaultModelPaths(false)
        let wsModelPaths
        if (cds.cli.options?.[OPTION_WS] || tasks.some(task => hasOptionValue(task.options?.[OPTION_WS], true))) {
            wsModelPaths = await getDefaultModelPaths(true)
        }
        tasks.forEach(task => {
            if (!task.src) {
                throw new BuildError(`Mandatory property 'src' not defined for build task '${task.for}'.`)
            }
            const optionWs = cds.cli.options?.[OPTION_WS] || hasOptionValue(task.options?.[OPTION_WS], true)
            this._setTaskModelOptions(task, optionWs ? [...wsModelPaths] : [...modelPaths])
        })
    }

    async _filterTasksForCli(tasks) {
        const options = this.options
        let resultTasks = tasks.filter(task => {
            return (!options.for || options.for === task.for)
                && (!options.src || options.src === task.src)
        })
        if (resultTasks.length === 0) {
            if (options.for) {
                const task = this.getTask({ for: options.for })
                if (options.src) task.src = options.src
                resultTasks.push(task)
                this._applyCliOptions(resultTasks)
                await this.applyTaskDefaults(resultTasks)
            }
        } else if (resultTasks.length <= tasks.length) {
            tasks.length = 0
            resultTasks.forEach(task => tasks.push(task))
            resultTasks = tasks
        }
        return resultTasks
    }

    _applyCliOptions(tasks) {
        const options = this.options
        tasks.forEach(task => {
            if (options.dest) task.dest = options.dest
            if (options.taskOptions) {
                const taskOptions = cds.clone(options.taskOptions)
                task.options = task.options ? Object.assign(task.options, taskOptions) : taskOptions
            }
        })
    }

    static escapeRegExpInput(str) {
        return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
    }

    static _setTaskModelOptions(task, defaultModelPaths) {
        if (task.for !== BUILD_TASK_NODEJS && task.for !== BUILD_TASK_NODE_CF) {
            defaultModelPaths = defaultModelPaths.filter(p => !NODEJS_MODEL_EXCLUDE_LIST.includes(p))
        }

        let taskModelPaths = task.options?.model
        if (taskModelPaths && !Array.isArray(taskModelPaths)) {
            taskModelPaths = [taskModelPaths]
        }
        const allowList = new RegExp(`^@sap/cds${cds.env.requires.toggles
            ? '|^' + BuildTaskEngine.escapeRegExpInput(cds.env.features.folders.replace('*', ''))
            : ''}`)
        task.options = task.options || {}
        let modelPaths = []
        if (taskModelPaths?.length) {
            if (!hasOptionValue(task.options?.[IGNORE_DEFAULT_MODELS], true)) {
                modelPaths = defaultModelPaths.filter(p => p.match(allowList))
            }
        } else {
            modelPaths = [...defaultModelPaths]
            if (!modelPaths.includes(task.src)) modelPaths.push(task.src)
            if (hasOptionValue(task.options?.[IGNORE_DEFAULT_MODELS], true)) {
                modelPaths = modelPaths.filter(p => !p.match(allowList))
            }
        }
        task.options.model = [...new Set(taskModelPaths?.length ? taskModelPaths.concat(modelPaths) : modelPaths)]
    }

    getTask(key) {
        this._normalizeKey(key)
        this._assertProvided(key)
        const task = cds.clone(this.plugins.get(key.for)?.taskDefaults ?? {})
        task.for = key.for
        task.src ??= DEFAULT_SRC_FOLDER
        task.src = task.src.replace(/\/$/, '')
        return task
    }

    async applyTaskDefaults(tasks) {
        return Promise.all(tasks.map(async (task) => {
            if (!task.for) throw new BuildError(`Mandatory property 'for' not defined for build task.`)
            return this._applyTaskDefaults(task)
        }))
    }

    async lookupTasks(tasks = [], dependencies) {
        const existingTasks = [...tasks]

        if (!dependencies) {
            for (const [id, PluginClass] of this.plugins) {
                if (PluginClass.hasTask()) tasks.push(this.getTask({ for: id }))
            }
        } else {
            const toAdd = []
            for (const task of tasks) {
                const PluginClass = this.getPlugin(task)
                const reqs = new PluginClass().requires || []
                for (const req of reqs) {
                    const reqId = this._normalizeKey({ for: req }).for
                    const ReqPluginClass = this.plugins.get(reqId)
                    if (ReqPluginClass && !tasks.some(t => t.for === reqId) && !toAdd.some(t => t.for === reqId)) {
                        const defaultTaskSrc = this.getTask({ for: reqId }).src
                        const configuredTasks = cds.env.build?.tasks?.filter(t => t.for === reqId)
                        let configuredTask
                        if (configuredTasks?.length) {
                            configuredTask = configuredTasks.find(t => !t.src || t.src === defaultTaskSrc) || configuredTasks[0]
                        }

                        if (configuredTask) {
                            configuredTask = cds.clone(configuredTask)
                            await this._applyTaskDefaults(configuredTask)
                            toAdd.push(configuredTask)
                        } else if (ReqPluginClass.hasTask?.()) {
                            toAdd.push(this.getTask({ for: reqId }))
                        }
                    }
                }
            }
            tasks.push(...toAdd)
        }

        if (existingTasks.length < tasks.length) {
            const newTasks = tasks.filter(task => !existingTasks.includes(task))
            await Promise.all(newTasks.map(t => this._applyTaskDefaults(t)))
            DEBUG?.(`Build task provider ${this.constructor.name} returned build tasks ${JSON.stringify(newTasks)}`)
        }
        return tasks
    }

    /**
     * Loads the build plugin implementation for the given build task.
     * @param {Task} task
     * @returns {typeof import('./plugins/plugin')}
     */
    getPlugin(task) {
        this._normalizeKey(task)
        this._assertProvided(task)
        try {
            return this.plugins.get(task.for)
        } catch (e) {
            console.error(`Provider failed to load build plugin class ${task.for} - provider: ${this.constructor.name}`)
            throw e
        }
    }

    /**
     * @param {Task[]} tasks
     */
    resolveTasks(tasks) {
        return tasks.map(task => this._resolveTask(task))
    }

    /**
     * @param {Task} task
     */
    _resolveTask(task) {
        const resolvedTask = cds.clone(task)
        resolvedTask.src = path.resolve(cds.root, task.src)
        if (!this.options.ws) {
            try {
                fs.realpathSync(resolvedTask.src)
            } catch {
                throw new BuildError(`The 'src' folder '${path.resolve(cds.root, task.src)}' for build task '${resolvedTask.for}' does not exist`)
            }
        }
        resolvedTask.dest = path.resolve(cds.root, cds.env.build.target, task.dest || task.src)
        resolvedTask.options = task.options || {}
        return resolvedTask
    }

    _normalizeKey(key) {
        const compat = { [BUILD_TASK_NODE_CF]: BUILD_TASK_NODEJS, [BUILD_TASK_JAVA_CF]: BUILD_TASK_JAVA }
        if (compat[key.for]) {
            console.warn(`${YELLOW}Build task '${key.for}' is deprecated, use '${compat[key.for]}' instead${RESET}`)
            key.for = compat[key.for]
        }
        return key
    }

    _assertProvided(key) {
        if (!this.plugins.has(key.for)) {
            throw new BuildError(`No provider found for build task '${key.for}'. Ensure that all required dependencies have been added and 'npm install' has been executed.`)
        }
    }

    async _applyTaskDefaults(task) {
        const defaultTask = this.getTask(task)
        const names = Object.getOwnPropertyNames(defaultTask)
        names.forEach(name => {
            task[name] ??= defaultTask[name]
        })
    }

    async _executeCleanBuildTasks(plugins) {
        if (!this.options.clean) return

        if (cds.env.build.target !== '.') {
            const target = path.resolve(cds.root, cds.env.build.target)
            DEBUG?.(`cleaning staging folder ${target}`)
            await rimraf(target)
        }

        const results = await Promise.allSettled(plugins.map((plugin) => {
            DEBUG?.(`cleaning, plugin [${plugin.constructor.name}], src [${relativePaths(cds.root, plugin.task.src)}]`)
            return plugin.clean()
        }))
        this._resolveHandlerResponse(results)
    }

    async _executeBuildTasks(plugins) {
        const levels = new Map()
        const getLevel = (PluginClass) => {
            if (levels.has(PluginClass)) return levels.get(PluginClass)
            const plugin = new PluginClass()
            if (plugin.priority && plugin.priority < 0) { // cds10: compat for cds-plugin-ui5 using undocumented priority
                const level = Number.MAX_SAFE_INTEGER
                levels.set(PluginClass, level)
                return level
            }
            let maxDepLevel = -1
            const reqs = plugin.requires || []
            for (const req of reqs) {
                const reqId = this._normalizeKey({ for: req }).for
                const ReqPluginClass = this.plugins.get(reqId)
                if (ReqPluginClass) maxDepLevel = Math.max(maxDepLevel, getLevel(ReqPluginClass))
            }
            const level = maxDepLevel + 1
            levels.set(PluginClass, level)
            return level
        }

        plugins.forEach(p => getLevel(p.constructor))

        const groupsMap = new Map()
        plugins.forEach(p => {
            const lvl = levels.get(p.constructor)
            if (!groupsMap.has(lvl)) groupsMap.set(lvl, [])
            groupsMap.get(lvl).push(p)
        })

        const sortedLevels = [...groupsMap.keys()].sort((a, b) => a - b)
        const pipeline = sortedLevels.map(lvl => groupsMap.get(lvl))

        const results = await this._executePipeline(pipeline)
        return this._resolveHandlerResponse(results, plugins.flatMap(plugin => plugin.messages))
    }

    async _executePipeline(pipeline) {
        let allResults = []
        for (const group of pipeline) {
            const results = await Promise.allSettled(group.map((plugin) => {
                DEBUG?.(`building, plugin [${plugin.constructor.name}], src [${relativePaths(cds.root, plugin.task.src)}]`)
                return plugin.build().then(pluginResult => Promise.resolve({
                    task: plugin.task,
                    result: pluginResult,
                    messages: this._sortMessagesUnique(plugin.messages)
                }))
            }))
            allResults = allResults.concat(results)
        }
        return allResults
    }

    _resolveHandlerResponse(results, pluginMessages = []) {
        /** @type {(BuildError | Error)[]}*/
        const errors = []
        const resolvedResults = results.reduce((acc, r) => {
            if (r.status === 'fulfilled') acc.push(r.value)
            if (r.status === 'rejected' && r.reason) errors.push(r.reason)
            return acc
        }, [])

        if (errors.length > 0) {
            const error = errors.find(e => e.constructor.name !== COMPILATION_ERROR)
            if (error) {
                if (!(error instanceof BuildError)) throw error
                error.messages = error.messages?.map(m => typeof m === 'string' ? new BuildMessage(m, SEVERITY_ERROR) : m)
                error.messages = this._sortMessagesUnique(pluginMessages.filter(m => m instanceof BuildMessage), error.messages)
                throw error
            }

            const compileErrors = errors.filter(e => e.constructor.name === COMPILATION_ERROR)
            const compileMessages = pluginMessages.filter(message => message.constructor.name === COMPILE_MESSAGE)
            if (compileErrors.length) {
                throw new CompilationError(this._sortMessagesUnique(BuildTaskEngine._getErrorMessages(compileErrors), compileMessages))
            }
        }
        return resolvedResults
    }

    _logOutput(plugins) {
        const files = BuildTaskEngine._getOutput(plugins)
        if (files.length > 0) {
            const term = require('../util/term')
            if (!process.env.DEBUG && files.length > LOG_LIMIT) {
                const length = files.length
                files.length = LOG_LIMIT
                files.push(`... ${length - files.length} more. Run with DEBUG=build to show all files.`)
            }
            console.log(`done > wrote output to:\n   ${term.dim(files.join("\n   "))}\n`)
        }
    }

    async _writeGenerationLog(plugins) {
        const outputFile = cds.env.build.outputfile || process.env.GENERATION_LOG
        if (outputFile) {
            const files = BuildTaskEngine._getOutput(plugins)
            console.log(`writing generation log to [${outputFile}]\n`)
            try {
                await mkdirp(path.dirname(outputFile))
                await write(outputFile, files.join('\n'))
            } catch (error) {
                console.error(`failed to write generation log`)
                console.error(error.stack || error)
            }
        }
    }

    static _getOutput(plugins) {
        const files = plugins.flatMap(plugin => plugin.files).sort()
        return files.map(file => {
            if (path.isAbsolute(cds.env.build.target)) return file
            return path.relative(cds.root, file)
        })
    }

    _logMessages(messages) {
        if (messages.length > 0) {
            const options = {
                log: console.log,
                "log-level": this._getLogLevel()
            }
            deduplicateMessages(messages)
            sortMessagesSeverityAware(messages)
            log(messages, options)
        }
    }

    static _resolveRequiredSapServices(tasks) {
        const taskModelPaths = tasks.reduce((acc, task) => {
            const model = task.options?.model
            if (model) {
                if (Array.isArray(model)) model.forEach(m => acc.add(m))
                else acc.add(model)
            }
            return acc
        }, new Set())

        return resolveRequiredSapModels([...taskModelPaths])
    }

    /**
     * @param {(string | BuildError)[]} errors
     */
    static _getErrorMessages(errors) {
        let messages = []
        errors.forEach(error => {
            if (typeof error !== 'string' && Array.isArray(error.errors) && error.errors.length > 0) {
                messages = messages.concat(this._getErrorMessages(error.errors))
            } else {
                messages.push(error)
            }
        })
        return messages
    }

    _sortMessagesUnique(...messages) {
        const logLevelIdx = LOG_LEVELS.indexOf(this._getLogLevel())
        messages = messages.reduce((acc, m) => acc.concat(m), [])
        const filteredMessages = messages.filter(message => message && (!message.severity || logLevelIdx >= SEVERITIES.indexOf(message.severity)))
        // @ts-expect-error - formally expects CompileMessage, but actually, an .severity is the only expected property
        deduplicateMessages(filteredMessages)
        // @ts-expect-error - see above
        return sortMessagesSeverityAware(filteredMessages)
    }

    _getLogLevel() {
        return this.options["log-level"] || cds.env["log-level"]
    }
}

module.exports = { build, register, Plugin, BuildError, BuildTaskEngine }