@sap/cds-dk/lib/build/util.js

Command line client and development toolkit for the SAP Cloud Application Programming Model

// @ts-check
const cp = require('node:child_process')
const fs = require('node:fs')
const path = require('node:path')

const cds = require('../cds')
const { exists } = cds.utils
const execAsync = require('node:util').promisify(cp.exec)
const { isRoot } = require('../util/fs')

const IS_WIN = require('node:os').platform() === 'win32'
const { SEVERITY_ERROR, FILE_EXT_CDS, NODEJS_MODEL_EXCLUDE_LIST } = require('./constants');
const DEBUG = cds.debug('cli|build')

// failsafe to avoid too deep recursions
// increased from 2 to 5, adding a config options in the future
const WS_MAX_DEPTH = 5
const FLATTEN_MAX_DEPTH = 20 // should suffice, was infinite in earlier versions

/** @typedef {{col?: number, endCol?: number, line?: number, endLine?: number, file: string}} SourceLocation */
/** @typedef {object[] | object} Credentials */
/** @typedef {(string | NestedStringArray)[]} NestedStringArray */

/**
 * Determines whether the both values are identical.
 * @template {number | boolean} T
 * @param {T} actual
 * @param {T} expected
 */
function hasOptionValue(actual, expected) {
    if (typeof expected === 'undefined') {
        return actual !== undefined
    }
    if (typeof actual === 'undefined') {
        return false
    }
    if (typeof expected === 'boolean') {
        if (typeof actual === 'string') {
            return String(expected) === actual
        }
    }
    if (typeof expected === 'number' && typeof actual === 'number') {
        return (actual & expected) !== 0
    }
    return actual === expected
}

/**
 * Returning the project relative path representation of the given path(s),
 * @param {string} root - project root
 * @param {string | string[]} qualifiedPaths - fully qualified path(s)
 */
function relativePaths(root, qualifiedPaths) {
    qualifiedPaths = typeof qualifiedPaths === "string" ? [qualifiedPaths] : qualifiedPaths
    return Array.isArray(qualifiedPaths)
        ? qualifiedPaths.map(qualifiedPath => path.relative(root, qualifiedPath) ?? ".")
        : qualifiedPaths
}

/**
 * Returns a list of fully qualified model names belonging to the '@sap' namespace that cannot be resolved.
 * E.g. the module might NOT have been installed.
 * @param {string[]} modelPaths
 * @returns {boolean | string[]}
 */
function resolveRequiredSapModels(modelPaths) {
    return Array.isArray(modelPaths) && modelPaths.filter(p => {
        if (p.startsWith('@sap/')) {
            const files = cds.resolve(p)
            return !files || files.length === 0
        }
    })
}

/**
 * Returns the default model paths for the given workspace.
 * @param {boolean} ws - whether to include workspace models
 * @returns {Promise<string[]>} array of model paths
 */
async function getDefaultModelPaths(ws = false) {
    const { toggles } = cds.env.requires, { folders } = cds.env.features
    const wildcards = toggles && folders ? ['*', folders] : '*'
    if (!cds.resolve.locations) return _compatGetDefaultModelPaths(ws, wildcards)

    let paths = _filterModelPaths(cds.resolve.locations(wildcards))
    if (ws) {
        const workspaces = (await getWorkspaces())
          .map(ws => ws.workspace)
        workspaces.forEach(workspace => {
            const root = path.join(cds.root, workspace)
            const env = cds.env.for('cds', root)
            const resolved = cds.resolve.locations(env.requires.toggles ? wildcards : '*', { env, root })
            paths.push(..._filterModelPaths(resolved, { root, env })
                .map(p => normalizePath(path.join(workspace, p))))
        })
        // exclude redundant built-in models starting with workspace path, e.g. 'books/@sap/cds-mtxs/srv/bootstrap'
        paths = paths.filter(p => NODEJS_MODEL_EXCLUDE_LIST.includes(p) || !NODEJS_MODEL_EXCLUDE_LIST.some(ex => p.endsWith(ex)))
    }
    return paths
}

// cds10: remove
async function _compatGetDefaultModelPaths(ws, wildcards) {
    let paths = _filterModelPaths(cds.resolve(wildcards, false))

    if (ws) {
        const workspaces = (await getWorkspaces())
          .map(ws => ws.workspace)
        const env = cds.env
        const root = cds.root
        try {
            workspaces.forEach(workspace => {
                cds.root = path.join(root, workspace)
                cds.env = cds.env.for('cds', cds.root)
                const fromWorkspace = _filterModelPaths(cds.resolve(cds.env.requires.toggles ? wildcards : '*', false))
                paths.push(...fromWorkspace.map(p => normalizePath(path.join(workspace, p))))
            })
        } finally {
            cds.root = root
            cds.env = env
        }
        // exclude redundant built-in models starting with workspace path, e.g. 'books/@sap/cds-mtxs/srv/bootstrap'
        paths = paths.filter(p => NODEJS_MODEL_EXCLUDE_LIST.includes(p) || !NODEJS_MODEL_EXCLUDE_LIST.some(ex => p.endsWith(ex)))
    }
    return paths
}

/**
 * @overload
 * @param {string} dir
 * @returns {string}
 */
/**
 * @overload
 * @param {unknown} dir
 * @returns {unknown}
 */
/**
 * For valid paths replace '\\' with '/' and remove trailing '/'. Otherwise return as is
 * @param {string | unknown} dir
 * @returns {string | unknown}
 */
function normalizePath(dir) {
    return typeof dir === "string" ? dir.replace(/\\/g, '/').replace(/\/$/, '') : dir
}

function getI18nDefaultFolder() {
    // keep '_i18n' as preferred folder for compatibility reasons
    // if (cds.env.i18n.folders.includes('_i18n')) return '_i18n'
    // if (cds.env.i18n.folders.includes('i18n')) return 'i18n'
    return cds.env.i18n.folders?.[0] ?? '_i18n'
}

async function getWorkspacePaths() {
    const wsRoot = findWorkspaceRoot(cds.root)
    if (!wsRoot) {
        return []
    }
    return (await getWorkspaces(wsRoot))
        .map(ws => ws.workspace)
        .filter(ws => cds.root !== path.join(wsRoot, ws))
        .map(ws => path.join(wsRoot, ws))
}

/**
 * @typedef {{
 *   path: string,
 *   workspace: string,
 *   packageName: string
 * }} Workspace
 *
 * Returns a map of workspaces from a root project, with their paths and workspace names,
 * I.e. for a monorepo setup this command finds all workspaces defined therein.
 * @param {string} root - the root directory to search for workspaces
 * @returns {Promise<Record<string, Workspace>>} - a map of workspaces with their paths and workspace names
 */
async function getWorkspacesDeclarations (root) {
    const rootPackageJson = path.join(root, 'package.json')
    if (!fs.existsSync(rootPackageJson)) return {}
    const pkg = require(rootPackageJson)
    if (!pkg?.workspaces) return {}

    const workspaces = []
    for (const workspace of pkg.workspaces) {
        if (/[*?[\]]/.test(workspace)) {
            // contains globs
            const { glob } = require('../util/glob')
            for await (const file of glob(workspace, { cwd: root })) {
                try {
                    const projectPath = path.join(root, file)
                    const projectName = require(path.join(projectPath, 'package.json')).name
                    workspaces.push([projectName, {
                        packageName: projectName,
                        path: projectPath,
                        workspace: file
                    }])
                } catch {
                    // not a directory or doesn't contain a package.json -> not a workspace -> skip
                }
            }
        } else {
            // no glob, just a path literal
            const projectPath = path.join(root, workspace)
            try {
                const projectName = require(path.join(projectPath, 'package.json')).name
                workspaces.push([projectName, { path: projectPath, workspace }])
            } catch {
                // not a directory or doesn't contain a package.json -> not a workspace -> skip
            }
        }
    }
    return Object.fromEntries(workspaces)
}

/**
 * Returns the list of workspaces defined for the current project. A workspace dependency
 * needs to be defined resulting in a corresponding symlink.
 * @returns {Promise<Workspace[]>} the paths are relative to 'wsRoot'.
 */
async function getWorkspaces(wsRoot, resolve = false, devDependencies = true, visited = new Set(), root = cds.root) {
    wsRoot ??= findWorkspaceRoot(root)
    if (!wsRoot) {
        return []
    }
    const pkgJsonWsRoot = require(path.join(wsRoot, 'package.json'))
    let pkgJson = pkgJsonWsRoot
    if (wsRoot !== root) {
        pkgJson = require(path.join(root, 'package.json'))
    }
    const workspaceDependencies = []
    if (pkgJsonWsRoot.workspaces) {
        if (resolve) {
            _validateWsDependencies(wsRoot, pkgJson.dependencies, devDependencies ? pkgJson.devDependencies : {})
        }
        // read npm workspaces
        const workspaces = await getWorkspacesDeclarations(wsRoot)
        const pkgs = await _execNpmWs(wsRoot)
        for (const pkg of pkgs) {
            // only create tarball archives for dependencies with version '*'
            if (!resolve || devDependencies && pkgJson.devDependencies?.[pkg] === '*' || pkgJson.dependencies?.[pkg] === '*') {
                const subproject = workspaces[pkg]
                if (subproject && !visited.has(pkg)) {
                    visited.add(pkg)
                    workspaceDependencies.push(subproject)
                    // recursively resolve transitive deps with the subproject as root
                    workspaceDependencies.push(...(await getWorkspaces(wsRoot, resolve, devDependencies, visited, subproject.path)))
                }
            }
        }
    }
    DEBUG?.(`Found ${workspaceDependencies.length} workspace dependencies ${workspaceDependencies.join(', ')}`)
    return workspaceDependencies
}

function findWorkspaceRoot(currentPath, depth = 0) {
    if (isRoot(currentPath) || depth > WS_MAX_DEPTH) {
        DEBUG?.(`No workspaces found, filesystem root or max depth of ${WS_MAX_DEPTH} reached`)
        return
    }

    const packageJsonPath = path.join(currentPath, 'package.json')
    try {
        const packageJsonContent = fs.readFileSync(packageJsonPath, 'utf8')
        const packageJson = JSON.parse(packageJsonContent)
        if (packageJson.workspaces?.length > 0) {
            return currentPath
        }
    } catch (e) {
        if (e.code !== 'ENOENT') {
            if (e.name === 'SyntaxError') {
                throw `Corrupt package.json found at ${packageJsonPath}: ${e.message}`
            }
            throw e
        }
    }

    if (fs.existsSync(path.join(currentPath, '.gitmodules'))) {
        return // project root reached
    }

    return findWorkspaceRoot(path.dirname(currentPath), depth + 1)
}

/**
 * This JavaScript function filterFeaturePaths is used to filter and categorize file paths
 * from a given model into two categories: base and features.
 * @param {*} model - represents a data model. It's expected to have a $sources property which is an array of file paths.
 * @param {*} modelPaths - an array of file paths.
 * @returns
 */
function filterFeaturePaths(model, modelPaths) {
    const ftsName = path.dirname(cds.env.features.folders || 'fts/*')
    const regex = new RegExp(`[/|\\\\]+${ftsName}[/|\\\\]+(?<ftName>[^/|\\\\]*)`)
    const sources = { base: [] }

    // add ROOT source file paths for the base model
    sources.base = cds.resolve(modelPaths).reduce((acc, file) => {
        const match = file.match(regex)
        if (!match) {
            acc.push(file)
        }
        return acc
    }, [])

    // add source file paths for the features
    sources.features = model['$sources'].reduce((acc, file) => {
        const match = file.match(regex)
        if (match) {
            const { ftName } = match.groups
            //feature
            if (!acc[ftName]) {
                acc[ftName] = []
            }
            acc[ftName].push(file)
        }
        return acc
    }, {})

    return sources
}

// spawning npm ls is expensive. In projects with many workspaces,
// this can significantly increase build time, despite the result being the same.
const _cachedWsRoots = new Map()
/**
 * Read installed npm workspaces using 'npm ls -ws --json'.
 * There is currently no easier way for this, as workspaces
 * could contain globs. Glob resolution will be supported.
 * The result is cached by workspace root!
 * natively in the future (fs.glob is in experimental state as of Node23).
 * @param {import('fs').PathLike} wsRoot - the root directory of the workspace
 * @returns {Promise<string[]>} list of package paths, or an empty array if either
 * no workspaces are defined or have not been installed.
 */
async function _execNpmWs(wsRoot) {
    const cached = _cachedWsRoots.get(wsRoot)
    if (cached) {
        return cached
    }
    /** @type {string[]} */
    let result = []
    try {
        const cmdLine = 'npm ls -ws --json'
        DEBUG?.(`execute ${cmdLine}`)
        // @ts-expect-error - TODO: shell should be a string. But can probably be removed altogether?
        const { stdout } = await execAsync(cmdLine, { shell: IS_WIN, stdio: ['inherit', 'pipe', 'inherit'], cwd: wsRoot })
        const jsonResult = JSON.parse(stdout)
        if (jsonResult.dependencies) {
            result = Object.keys(jsonResult.dependencies)
        } else if (jsonResult.name) {
            result = [jsonResult.name]
        }
    } catch (/** @type {{stderr: string}} */e) {
        e.stderr = e.stderr?.replace(/npm ERR! /gi, '')
        throw e
    }
    _cachedWsRoots.set(wsRoot, result)
    return result
}

function _validateWsDependencies(wsRoot, dependencies, devDependencies) {
    const allDependencies = { ...dependencies, ...devDependencies }
    for (const name in allDependencies) {
        if (allDependencies[name] === '*') {
            try {
                require.resolve(path.join(name, 'package.json'), { paths: [wsRoot] })
            } catch (e) {
                if (e.code === 'MODULE_NOT_FOUND') {
                    throw new BuildError(`npm packaging failed, module '${name}' not found. Make sure configured npm workspaces are installed.`)
                }
                throw e
            }
        }
    }
}

/**
 * Returns a flat array of existing files and folders, trailing '/' or '\\' characters have been removed.
 * @param {NestedStringArray | object | undefined } modelPaths array of files and folders - optionally nested
 * @returns {string[] } flat array of existing files and folders
 */
function _filterModelPaths(modelPaths, { env = cds.env, root = cds.root } = {}) {
    const model = /** @type {Set<string>}*/(new Set())
    // may contain nested arrays
    const flatModelPaths = Array.isArray(modelPaths) ? modelPaths.flat(FLATTEN_MAX_DEPTH) : modelPaths ? [modelPaths] : []
    const { roots } = env
    flatModelPaths.forEach(m => {
        if (m && !model.has(m) && !model.has(m + "/")) {
            // filter roots and absolute model paths that do not exist
            if (roots.includes(m)) {
                const dirent = path.resolve(root, m)
                // might be cds file name, compatibility to old build configs
                if (exists(dirent) || exists(dirent + FILE_EXT_CDS) || exists(dirent.replaceAll('*', ''))) {
                    model.add(m)
                }
            } else {
                model.add(m)
            }
        }
    })
    return [...model]
        .map(m => normalizePath(m))
}

/**
 * Return gnu-style error string for location `loc`:
 *  - 'File:Line:Col' without `loc.end`
 *  - 'File:Line:StartCol-EndCol' if Line = start.line = end.line
 *  - 'File:StartLine.StartCol-EndLine.EndCol' otherwise
 *
 * @param {SourceLocation | undefined} loc
 * @private
 */
function _locationString(loc) {
    if (!loc)
        return '<???>';
    if (!(loc instanceof Object))
        return loc;
    if (!loc.line) {
        return loc.file;
    }
    else if (!loc.endLine) {
        return (loc.col)
            ? `${loc.file}:${loc.line}:${loc.col}`
            : `${loc.file}:${loc.line}`;
    }

    return (loc.line === loc.endLine)
        ? `${loc.file}:${loc.line}:${loc.col}-${loc.endCol}`
        : `${loc.file}:${loc.line}.${loc.col}-${loc.endLine}.${loc.endCol}`;
}

/**
 * Class for individual build message.
 */
class BuildMessage {
    /**
     * Creates an instance of BuildMessage.
     * @param {string} message The message text
     * @param {import('./constants').Severity} [severity='Error'] severity - severity of the message
     * @param {SourceLocation | undefined} location Location of the message
     *
     * @memberOf BuildMessage
     */
    constructor(message, severity = SEVERITY_ERROR, location = undefined) {
        /** @type {string} */
        this.message = message
        /** @type {string} */
        this.name = "BuildMessage"
        /** @type {import('./constants').Severity} */
        this.severity = severity
        /** @type {SourceLocation} */
        this.$location = location
    }
    toString() {
        return `${this.$location?.file ? _locationString(this.$location) + ':' : ''} ${this.severity}: ${this.message}`
    }
}

/**
 * Class for combined build and compiler errors.
 * Additional members:
 *   messages: vector of detailed build messages
 */
class BuildError extends Error {
    /** @type {string[] | {message: string}[]} */
    messages

    /**
     * @param {string} message
     * @param {string | string[]} messages
     */
    constructor(message, messages = []) {
        super(message)
        this.name = "BuildError"
        this.messages = Array.isArray(messages) ? messages : [messages]
    }

    // for compatibility reasons
    get errors() {
        return this.messages
    }

    toString() {
        return this.message + (this.messages.length > 0 ? '\n' + this.messages.map(m => m.toString()).join('\n') : '')
    }
}

module.exports = {
    hasOptionValue,
    relativePaths,
    resolveRequiredSapModels,
    getDefaultModelPaths,
    getI18nDefaultFolder,
    normalizePath,
    getWorkspacePaths,
    getWorkspaces,
    findWorkspaceRoot,
    filterFeaturePaths,
    BuildMessage,
    BuildError
}