@sap/cds-dk/lib/build/util.js

Command line client and development toolkit for the SAP Cloud Application Programming Model

// @ts-check
const fs = require('fs')
const path = require('path')
const cds = require('../cds')
const cp = require('child_process');
const execAsync = require('util').promisify(cp.exec)
const IS_WIN = require('os').platform() === 'win32'
const { SEVERITY_ERROR, FILE_EXT_CDS, NODEJS_MODEL_EXCLUDE_LIST } = require('./constants');
const DEBUG = cds.debug('cli|build')
const WS_MAX_DEPTH = 2

/** @typedef {{col?: number, endCol?: number, line?: number, endLine?: number, file: string}} SourceLocation */
/** @typedef {object[] | object} Credentials */
/** @typedef {(string | NestedStringArray)[]} NestedStringArray */

/**
 * @param {object} src - object to get property from
 * @param {string | string[]} segments - property path
 */
function getProperty(src, segments) {
    segments = Array.isArray(segments) ? segments : segments.split('.')
    return segments.reduce((p, n) => p && p[n], src)
}

/**
 * @param {object} src - object to set property on
 * @param {string | string[]} segments - property path
 * @param {unknown} value - value to set
 */
function setProperty(src, segments, value) {
    segments = Array.isArray(segments) ? segments : segments.split('.')
    segments.reduce((p, n, idx) => {
        if (segments.length === idx + 1) {
            p[n] = value
        } else if (p[n] === undefined) {
            p[n] = {}
        }
        return p[n]
    }, src)
}

/**
 * Returns whether this project is a java project or not.
*/
function hasJavaNature() {
    return [cds.env.folders.srv, '.'].some(f => f && fs.existsSync(path.join(cds.root, f, 'pom.xml')))
}

/**
 * Determines whether the both values are identical.
 * @template {number | boolean} T
 * @param {T} actual
 * @param {T} expected
 */
function hasOptionValue(actual, expected) {
    if (typeof expected === 'undefined') {
        return actual !== undefined
    }
    if (typeof actual === 'undefined') {
        return false
    }
    if (typeof expected === 'boolean') {
        if (typeof actual === 'string') {
            return String(expected) === actual
        }
    }
    if (typeof expected === 'number' && typeof actual === 'number') {
        return (actual & expected) !== 0
    }
    return actual === expected
}

/**
 * Returning the project relative path representation of the given path(s),
 * @param {string} root - project root
 * @param {string | string[]} qualifiedPaths - fully qualified path(s)
 */
function relativePaths(root, qualifiedPaths) {
    qualifiedPaths = typeof qualifiedPaths === "string" ? [qualifiedPaths] : qualifiedPaths
    return Array.isArray(qualifiedPaths)
        ? qualifiedPaths.map(qualifiedPath => path.relative(root, qualifiedPath) ?? ".")
        : qualifiedPaths
}

/**
 * Returns a list of fully qualified model names belonging to the '@sap' namespace that cannot be resolved.
 * E.g. the module might NOT have been installed.
 * @param {string[]} modelPaths
 * @returns {string[]}
 */
function resolveRequiredSapModels(modelPaths) {
    return Array.isArray(modelPaths) && modelPaths.filter(p => {
        if (p.startsWith('@sap/')) {
            const files = cds.resolve(p)
            return !files || files.length === 0
        }
    })
}

/**
 * Returns the default model options for the given workspace.
 * @param {boolean} ws - whether to include workspace models
 * @param {Function | undefined} filter - filter function for model paths
 */
async function getDefaultModelOptions(ws = false, filter = undefined) {
    // Note: requires.toggles holds the actual kind representation and not the value 'true'
    const wildcards = cds.env.requires.toggles && cds.env.features.folders ? ['*', cds.env.features.folders] : '*'
    let paths = _filterModelPaths(cds.resolve(wildcards, false), filter)

    if (ws) {
        const workspaces = (await getWorkspaces())
          .map(ws => ws.workspace)
        const env = cds.env
        const root = cds.root
        try {
            workspaces.forEach(workspace => {
                cds.root = path.join(root, workspace)
                cds.env = cds.env.for('cds', cds.root)
                paths = paths.concat(_filterModelPaths(cds.resolve(cds.env.requires.toggles ? wildcards : '*', false), filter).map(p => normalizePath(path.join(workspace, p))))
            })
        } finally {
            cds.root = root
            cds.env = env
        }
        // exclude redundant built-in models starting with workspace path, e.g. 'books/@sap/cds-mtxs/srv/bootstrap'
        paths = paths
            .filter(p => typeof(p) === 'string' && (NODEJS_MODEL_EXCLUDE_LIST.includes(p) || !NODEJS_MODEL_EXCLUDE_LIST.some(ex => p.endsWith(ex))))
    }
    return paths
}

/**
 * @overload
 * @param {string} dir
 * @returns {string}
 */
/**
 * @overload
 * @param {unknown} dir
 * @returns {unknown} 
 */
/**
 * For valid paths replace '\\' with '/' and remove trailing '/'. Otherwise return as is
 * @param {string | unknown} dir
 * @returns {string | unknown}
 */
function normalizePath(dir) {
    return typeof dir === "string" ? dir.replace(/\\/g, '/').replace(/\/$/, '') : dir
}

/**
 * @param {NestedStringArray} modelPaths
 */
function flatten(modelPaths) {
    // TODO: I guess this could be modelPaths.flat(Infinity) in the future
    return /** @type{string[]}*/(modelPaths.reduce((/** @type {string[]}*/acc, m) => {
        if (Array.isArray(m)) {
            acc = acc.concat(flatten(m))
        } else if (m) {
            acc.push(m)
        }
        return acc
    }, []))
}

function getI18nDefaultFolder() {
    // keep '_i18n' as preferred folder for compatibility reasons
    if (cds.env.i18n.folders.includes('_i18n')) return '_i18n'
    if (cds.env.i18n.folders.includes('i18n')) return 'i18n'
    return cds.env.i18n.folders[0] ?? 'i18n'
}

async function pathExists(path) {
    return fs.promises.access(path).then(() => true).catch(() => false);
}

async function getWorkspacePaths() {
    const wsRoot = findWorkspaceRoot(cds.root)
    if (!wsRoot) {
        return []
    }
    return (await getWorkspaces(wsRoot))
        .map(ws => ws.workspace)
        .filter(ws => cds.root !== path.join(wsRoot, ws))
        .map(ws => path.join(wsRoot, ws))
}

/**
 * @typedef {{
 *   path: string,
 *   workspace: string,
 *   packageName: string
 * }} Workspace
 *
 * Returns a map of workspaces from a root project, with their paths and workspace names,
 * I.e. for a monorepo setup this command finds all workspaces defined therein.
 * @param {string} root - the root directory to search for workspaces
 * @returns {Promise<Record<string, Workspace>>} - a map of workspaces with their paths and workspace names
 */
async function getWorkspacesDeclarations (root) {
    const rootPackageJson = path.join(root, 'package.json')
    if (!fs.existsSync(rootPackageJson)) return {}
    const pkg = require(rootPackageJson)
    if (!pkg?.workspaces) return {}

    const workspaces = []
    for (const workspace of pkg.workspaces) {
        if (/[*?[\]]/.test(workspace)) {
            // contains globs
            const { glob } = require('../util/glob')
            for await (const file of glob(workspace, { cwd: root })) {
                try {
                    const projectPath = path.join(root, file)
                    const projectName = require(path.join(projectPath, 'package.json')).name
                    workspaces.push([projectName, {
                        packageName: projectName,
                        path: projectPath,
                        workspace: file
                    }])
                } catch {
                    // not a directory or doesn't contain a package.json -> not a workspace -> skip
                }
            }
        } else {
            // no glob, just a path literal
            const projectPath = path.join(root, workspace)
            try {
                const projectName = require(path.join(projectPath, 'package.json')).name
                workspaces.push([projectName, { path: projectPath, workspace }])
            } catch {
                // not a directory or doesn't contain a package.json -> not a workspace -> skip
            }
        }
    }
    return Object.fromEntries(workspaces)
}

/**
 * Returns the list of workspaces defined for the current project. A workspace dependency
 * needs to be defined resulting in a corresponding symlink.
 * @returns {Promise<Workspace[]>} the paths are relative to 'wsRoot'.
 */
async function getWorkspaces(wsRoot, resolve = false, devDependencies = true, visited = new Set()) {
    wsRoot ??= findWorkspaceRoot(cds.root)
    if (!wsRoot) {
        return []
    }
    const pkgJsonWsRoot = require(path.join(wsRoot, 'package.json'))
    let pkgJson = pkgJsonWsRoot
    if (wsRoot !== cds.root) {
        pkgJson = require(path.join(cds.root, 'package.json'))
    }
    const workspaceDependencies = []
    if (pkgJsonWsRoot.workspaces) {
        if (resolve) {
            _validateWsDependencies(wsRoot, pkgJson.dependencies, devDependencies ? pkgJson.devDependencies : {})
        }
        // read npm workspaces
        const workspaces = await getWorkspacesDeclarations(wsRoot)
        const pkgs = await _execNpmWs(wsRoot)
        for (const pkg of pkgs) {
            // only create tarball archives for dependencies with version '*'
            if (!resolve || devDependencies && pkgJson.devDependencies?.[pkg] === '*' || pkgJson.dependencies?.[pkg] === '*') {
                const subproject = workspaces[pkg]
                if (subproject && !visited.has(pkg)) {
                    visited.add(pkg)
                    workspaceDependencies.push(subproject)
                    // temporarily set cds.root to the dependency to resolve transitive deps thereof
                    const cdsr = cds.root
                    cds.root = subproject.path
                    workspaceDependencies.push(...(await getWorkspaces(wsRoot, resolve, devDependencies, visited)))
                    cds.root = cdsr
                }
            }
        }
    }
    DEBUG?.(`Found ${workspaceDependencies.length} workspace dependencies ${workspaceDependencies.join(', ')}`)
    return workspaceDependencies
}

function findWorkspaceRoot(currentPath, depth = 0) {
    if (depth > WS_MAX_DEPTH) {
        DEBUG?.(`No workspaces exist, max depth of ${WS_MAX_DEPTH} reached`)
        return
    }
    const packageJson = path.join(currentPath, 'package.json')
    if (fs.existsSync(packageJson)) {
        if (require(path.join(currentPath, 'package.json')).workspaces) {
            return currentPath
        }
    }
    if (fs.existsSync(path.join(currentPath, '.gitmodules'))) {
        return // project root reached
    }
    if (path.dirname(currentPath) !== currentPath) {
        return findWorkspaceRoot(path.dirname(currentPath), depth + 1)
    }
}

/**
 * This JavaScript function filterFeaturePaths is used to filter and categorize file paths
 * from a given model into two categories: base and features.
 * @param {*} model - represents a data model. It's expected to have a $sources property which is an array of file paths.
 * @param {*} modelPaths - an array of file paths.
 * @returns
 */
function filterFeaturePaths(model, modelPaths) {
    const ftsName = path.dirname(cds.env.features.folders || 'fts/*')
    const regex = new RegExp(`[/|\\\\]+${ftsName}[/|\\\\]+(?<ftName>[^/|\\\\]*)`)
    const sources = { base: [] }

    // add ROOT source file paths for the base model
    sources.base = cds.resolve(modelPaths).reduce((acc, file) => {
        const match = file.match(regex)
        if (!match) {
            acc.push(file)
        }
        return acc
    }, [])

    // add source file paths for the features
    sources.features = model['$sources'].reduce((acc, file) => {
        const match = file.match(regex)
        if (match) {
            const { ftName } = match.groups
            //feature
            if (!acc[ftName]) {
                acc[ftName] = []
            }
            acc[ftName].push(file)
        }
        return acc
    }, {})

    return sources
}

/**
 * Read installed npm workspaces using 'npm ls -ws --json'.
 * There is currently no easier way for this, as workspaces
 * could contain globs. Glob resolution will be supported
 * natively in the future (fs.glob is in experimental state as of Node23).
 * @param {import('fs').PathLike} wsRoot - the root directory of the workspace
 * @returns {Promise<string[]>} list of package paths, or an empty array if either
 * no workspaces are defined or have not been installed.
 */
async function _execNpmWs(wsRoot) {
    try {
        const cmdLine = 'npm ls -ws --json'
        DEBUG?.(`execute ${cmdLine}`)
        // @ts-expect-error - TODO: shell should be a string. But can probably be removed altogether?
        const result = await execAsync(cmdLine, { shell: IS_WIN, stdio: ['inherit', 'pipe', 'inherit'], cwd: wsRoot })
        // DEBUG?.(result.stdout)
        const jsonResult = JSON.parse(result.stdout)
        if (jsonResult.dependencies) {
            return Object.keys(jsonResult.dependencies)
        } else if (jsonResult.name) {
            return [jsonResult.name]
        }
    } catch (/** @type {{stderr: string}} */e) {
        e.stderr = e.stderr?.replace(/npm ERR! /g, '')
        throw e
    }
    return []
}

function _validateWsDependencies(wsRoot, dependencies, devDependencies) {
    const allDependencies = { ...dependencies, ...devDependencies }
    for (const name in allDependencies) {
        if (allDependencies[name] === '*') {
            try {
                require.resolve(path.join(name, 'package.json'), { paths: [wsRoot] })
            } catch (e) {
                if (e.code === 'MODULE_NOT_FOUND') {
                    throw new BuildError(`npm packaging failed, module '${name}' not found. Make sure configured npm workspaces are installed.`)
                }
                throw e
            }
        }
    }
}

/**
 * Returns a flat array of existing files and folders, trailing '/' or '\\' characters have been removed.
 * @param  {NestedStringArray} modelPaths array of files and folders - optionally nested
 * @param  {((m: string) => boolean) | undefined} filter filter function to apply to the found files
 * @returns 
 */
function _filterModelPaths(modelPaths, filter = undefined) {
    const model = /** @type {Set<string>}*/(new Set())
    // may contain nested arrays
    const flatModelPaths = flatten(modelPaths)
    const { roots } = cds.env
    flatModelPaths.forEach(m => {
        if (m && !model.has(m) && !model.has(m + "/")) {
            // filter roots and absolute model paths that do not exist
            if (roots.includes(m)) {
                const dir = path.resolve(cds.root, m)
                // dir + FILE_EXT_CDS: might be cds file name, compatibility to old build configs
                if (fs.existsSync(dir) || fs.existsSync(dir + FILE_EXT_CDS)) {
                    model.add(m)
                }
            } else {
                model.add(m)
            }
        }
    })
    return [...model]
        .filter(m => filter ? filter(m) : true)
        .map(m => normalizePath(m))
}

/**
 * Returns a set of files found in the given directory and its subdirectories.
 * @param {string} srcDir - the directory to search for files
 * @param {(file: string) => boolean} [filter] - the filter function to apply to the found files
 * @param {string[]} files - the set of files to add the found files to
 * @returns {string[]} files found in the given directory and its subdirectories
 */
function getFiles(srcDir, filter, files = []) {
    function handleResource(/** @type {string} entry */ entry) {
        if (!filter || filter.call(this, entry)) {
            const stats = fs.lstatSync(entry)
            if (cds.utils.isdir(entry)) {
                getFiles(entry, filter, files)
            } else if (stats.isFile() || stats.isSymbolicLink()) {
                files.push(entry)
            }
        }
    }
    let entries = []
    try {
        entries = fs.readdirSync(srcDir)
    } catch {
        // ignore if not existing
    }
    entries.map(subDirEntry => path.join(srcDir, subDirEntry)).forEach((entry) => {
        handleResource(entry)
    })
    return files
}

/**
 * Return gnu-style error string for location `loc`:
 *  - 'File:Line:Col' without `loc.end`
 *  - 'File:Line:StartCol-EndCol' if Line = start.line = end.line
 *  - 'File:StartLine.StartCol-EndLine.EndCol' otherwise
 *
 * @param {SourceLocation | undefined} loc
 * @private
 */
function _locationString(loc) {
    if (!loc)
        return '<???>';
    if (!(loc instanceof Object))
        return loc;
    if (!loc.line) {
        return loc.file;
    }
    else if (!loc.endLine) {
        return (loc.col)
            ? `${loc.file}:${loc.line}:${loc.col}`
            : `${loc.file}:${loc.line}`;
    }

    return (loc.line === loc.endLine)
        ? `${loc.file}:${loc.line}:${loc.col}-${loc.endCol}`
        : `${loc.file}:${loc.line}.${loc.col}-${loc.endLine}.${loc.endCol}`;
}

/**
 * Class for individual build message.
 */
class BuildMessage {
    /**
     * Creates an instance of BuildMessage.
     * @param {string} message The message text
     * @param {import('./constants').Severity} [severity='Error'] severity - severity of the message
     * @param {SourceLocation | undefined} location Location of the message
     * 
     * @memberOf BuildMessage
     */
    constructor(message, severity = SEVERITY_ERROR, location = undefined) {
        /** @type {string} */
        this.message = message
        /** @type {string} */
        this.name = "BuildMessage"
        /** @type {import('./constants').Severity} */
        this.severity = severity
        /** @type {SourceLocation} */
        this.$location = location
    }
    toString() {
        return `${this.$location?.file ? _locationString(this.$location) + ':' : ''} ${this.severity}: ${this.message}`
    }
}

/**
 * Class for combined build and compiler errors.
 * Additional members:
 *   messages: vector of detailed build messages
 */
class BuildError extends Error {
    /** @type {string[] | {message: string}[]} */
    messages

    /** 
     * @param {string} message
     * @param {string | string[]} messages
     */
    constructor(message, messages = []) {
        super(message)
        this.name = "BuildError"
        this.messages = Array.isArray(messages) ? messages : [messages]
    }

    // for compatibility reasons
    get errors() {
        return this.messages
    }

    toString() {
        return this.message + (this.messages.length > 0 ? '\n' + this.messages.map(m => m.toString()).join('\n') : '')
    }
}

module.exports = {
    getProperty,
    setProperty,
    hasJavaNature,
    hasOptionValue,
    relativePaths,
    resolveRequiredSapModels,
    getDefaultModelOptions,
    flatten,
    getI18nDefaultFolder,
    normalizePath,
    pathExists,
    getWorkspacePaths,
    getWorkspaces,
    findWorkspaceRoot,
    filterFeaturePaths,
    getFiles,
    BuildMessage,
    BuildError
}