@sap/cds-dk/lib/build/util.js

Command line client and development toolkit for the SAP Cloud Application Programming Model

const fs = require('fs')
const path = require('path')
const cds = require('../cds')
const cp = require('child_process');
const execAsync = require('util').promisify(cp.exec)
const IS_WIN = require('os').platform() === 'win32'
const { SEVERITY_ERROR, FILE_EXT_CDS, NODEJS_MODEL_EXCLUDE_LIST } = require('./constants');
const DEBUG = cds.debug('cli|build')
const SECRETS = /(passw)|(cert)|(ca)|(secret)|(key)/i
const WS_MAX_DEPTH = 2

function getProperty(src, segments) {
    segments = Array.isArray(segments) ? segments : segments.split('.')
    return segments.reduce((p, n) => p && p[n], src)
}

function setProperty(src, segments, value) {
    segments = Array.isArray(segments) ? segments : segments.split('.')
    segments.reduce((p, n, idx) => {
        if (segments.length === idx + 1) {
            p[n] = value
        } else {
            if (p[n] === undefined) {
                p[n] = {}
            }
        }
        return p[n]
    }, src)
}

/**
 * Returns whether this project is a java project or not.
*/
function hasJavaNature() {
    return [cds.env.folders.srv, '.'].some(f => f && fs.existsSync(path.join(cds.root, f, 'pom.xml')))
}

/**
 * Determines whether the both values are identical.
 * @param {*} actual
 * @param {*} expected
 */
function hasOptionValue(actual, expected) {
    if (typeof expected === 'undefined') {
        return actual !== undefined
    }
    if (typeof actual === 'undefined') {
        return false
    }
    if (typeof expected === 'boolean') {
        if (typeof actual === 'string') {
            return String(expected) === actual
        }
    }
    if (typeof expected === 'number') {
        return (actual & expected) !== 0
    }
    return actual === expected
}

// Returning the project relative path representation of the given path(s),
function relativePaths(root, qualifiedPaths) {
    qualifiedPaths = typeof qualifiedPaths === "string" ? [qualifiedPaths] : qualifiedPaths
    if (Array.isArray(qualifiedPaths)) {
        return qualifiedPaths.map(qualifiedPath => {
            const relPath = path.relative(root, qualifiedPath)
            return relPath || "."
        })
    }
    return qualifiedPaths
}

function redactCredentials(config) {
    if (typeof config !== 'object' || config === null) {
        return config;
    }

    if (Array.isArray(config)) {
        return config.map(item => redactCredentials(item));
    } else {
        const newConfig = Object.assign({}, config);
        for (let key in newConfig) {
            if (key === 'credentials') {
                newConfig[key] = _redacted(newConfig[key]);
            } else {
                newConfig[key] = redactCredentials(newConfig[key]);
            }
        }
        return newConfig;
    }
}

/**
 * Masks password-like strings, also reducing clutter in output
 * @param {any} cred - object or array with credentials
 * @returns {any}
 */
function _redacted(cred) {
    if (!cred) return cred
    if (Array.isArray(cred)) return cred.map(c => typeof c === 'string' ? '...' : _redacted(c))
    if (typeof cred === 'object') {
        const newCred = Object.assign({}, cred)
        Object.keys(newCred).forEach(k => (typeof newCred[k] === 'string' && SECRETS.test(k)) ? (newCred[k] = '...') : (newCred[k] = _redacted(newCred[k])))
        return newCred
    }
    return cred
}

/**
 * Returns a list of fully qualified model names belonging to the '@sap' namespace that cannot be resolved.
 * E.g. the module might NOT have been installed.
 * @param {Array} modelPaths
 * @returns {Array}
 */
function resolveRequiredSapModels(modelPaths) {
    return Array.isArray(modelPaths) && modelPaths.filter(p => {
        if (p.startsWith('@sap/')) {
            const files = cds.resolve(p)
            return !files || files.length === 0
        }
    })
}

async function getDefaultModelOptions(ws = false, filter) {
    // Note: requires.toggles holds the actual kind representation and not the value 'true'
    const wildcards = cds.env.requires.toggles && cds.env.features.folders ? ['*', cds.env.features.folders] : '*'
    let paths = _filterModelPaths(cds.resolve(wildcards, false), filter)

    if (ws) {
        const workspaces = await getWorkspaces()
        const env = cds.env
        const root = cds.root
        try {
            workspaces.forEach(workspace => {
                cds.root = path.join(root, workspace)
                cds.env = cds.env.for('cds', cds.root)
                paths = paths.concat(_filterModelPaths(cds.resolve(cds.env.requires.toggles ? wildcards : '*', false), filter).map(p => normalizePath(path.join(workspace, p))))
            })
        } finally {
            cds.root = root
            cds.env = env
        }
        // exclude redundant built-in models starting with workspace path, e.g. 'books/@sap/cds-mtxs/srv/bootstrap'
        paths = paths.filter(p => NODEJS_MODEL_EXCLUDE_LIST.includes(p) || !NODEJS_MODEL_EXCLUDE_LIST.some(ex => p.endsWith(ex)))
    }
    return paths
}

/**
 * For valid paths replace '\\' with '/' and remove trailing '/'. Otherwise return as is
 * @param {*} dir
 */
function normalizePath(dir) {
    return typeof dir === "string" ? dir.replace(/\\/g, '/').replace(/\/$/, '') : dir
}

function flatten(modelPaths) {
    return modelPaths.reduce((acc, m) => {
        if (Array.isArray(m)) {
            acc = acc.concat(flatten(m))
        } else if (m) {
            acc.push(m)
        }
        return acc
    }, [])
}

function getI18nDefaultFolder() {
    // keep '_i18n' as preferred folder for compatibility reasons
    if (cds.env.i18n.folders.includes('_i18n')) return '_i18n'
    if (cds.env.i18n.folders.includes('i18n')) return 'i18n'
    return cds.env.i18n.folders[0] ?? 'i18n'
}

async function pathExists(path) {
    return fs.promises.access(path).then(() => true).catch(() => false);
}

async function getWorkspacePaths() {
    const wsRoot = findWorkspaceRoot(cds.root)
    if (!wsRoot) {
        return []
    }
    return (await getWorkspaces(wsRoot)).filter(ws => cds.root !== path.join(wsRoot, ws)).map(ws => path.join(wsRoot, ws))
}

/**
 * Returns the list of workspaces defined for the current project. A workspace dependency
 * needs to be defined resulting in a corresponding sym-link.
 * @returns the paths are relative to 'wsRoot'.
 */
async function getWorkspaces(wsRoot, resolve = false, devDependencies = true) {
    wsRoot ??= findWorkspaceRoot(cds.root)
    if (!wsRoot) {
        return []
    }
    const pgkJsonWsRoot = require(path.join(wsRoot, 'package.json'))
    let pgkJson = pgkJsonWsRoot
    if (wsRoot !== cds.root) {
        pgkJson = require(path.join(cds.root, 'package.json'))
    }
    const workspaces = []
    if (pgkJsonWsRoot.workspaces) {
        if (resolve) {
            _validateWsDependencies(wsRoot, pgkJson.dependencies, devDependencies ? pgkJson.devDependencies : {})
        }
        // read npm workspaces
        const pkgs = await _execNpmWs(wsRoot)
        pkgs.forEach(pkg => {
            // only create tarball archives for dependencies with version '*'
            if (!resolve || devDependencies && pgkJson.devDependencies?.[pkg] === '*' || pgkJson.dependencies?.[pkg] === '*') {
                const workspace = _getWorkspace(wsRoot, pkg)
                if (workspace) {
                    workspaces.push(workspace)
                }
            }
        })
    }
    DEBUG?.(`Found ${workspaces.length} workspaces ${workspaces.join(', ')}`)
    return workspaces
}

function findWorkspaceRoot(currentPath, depth = 0) {
    if (depth > WS_MAX_DEPTH) {
        DEBUG?.(`No workspaces exist, max depth of ${WS_MAX_DEPTH} reached`)
        return
    }
    const packageJson = path.join(currentPath, 'package.json')
    if (fs.existsSync(packageJson)) {
        if (require(path.join(currentPath, 'package.json')).workspaces) {
            return currentPath
        }
    }
    if (fs.existsSync(path.join(currentPath, '.gitmodules'))) {
        return // project root reached
    }
    if (path.dirname(currentPath) !== currentPath) {
        return findWorkspaceRoot(path.dirname(currentPath), depth + 1)
    }
}

/**
 * This JavaScript function filterFeaturePaths is used to filter and categorize file paths
 * from a given model into two categories: base and features.
 * @param {*} model - represents a data model. It's expected to have a $sources property which is an array of file paths.
 * @param {*} modelPaths - an array of file paths.
 * @returns
 */
function filterFeaturePaths(model, modelPaths) {
    const ftsName = path.dirname(cds.env.features.folders || 'fts/*')
    const regex = new RegExp(`[/|\\\\]+${ftsName}[/|\\\\]+(?<ftName>[^/|\\\\]*)`)
    const sources = { base: [] }

    // add ROOT source file paths for the base model
    sources.base = cds.resolve(modelPaths).reduce((acc, file) => {
        const match = file.match(regex)
        if (!match) {
            acc.push(file)
        }
        return acc
    }, [])

    // add source file paths for the features
    sources.features = model['$sources'].reduce((acc, file) => {
        const match = file.match(regex)
        if (match) {
            const { ftName } = match.groups
            //feature
            if (!acc[ftName]) {
                acc[ftName] = []
            }
            acc[ftName].push(file)
        }
        return acc
    }, {})

    return sources
}

/**
 * Read installed npm workspaces using 'npm ls -ws --json'.
 * @returns {Array} list of package paths, or an empty array if either
 * no workspaces are defined or have not been installed.
 */
async function _execNpmWs(wsRoot) {
    try {
        const cmdLine = 'npm ls -ws --json'
        DEBUG?.(`execute ${cmdLine}`)
        let result = await execAsync(cmdLine, { shell: IS_WIN, stdio: ['inherit', 'pipe', 'inherit'], cwd: wsRoot })
        // DEBUG?.(result.stdout)
        result = JSON.parse(result.stdout)
        if (result.dependencies) {
            return Object.keys(result.dependencies)
        } else if (result.name) {
            return [result.name]
        }
    } catch (e) {
        e.stderr = e.stderr?.replace(/npm ERR! /g, '')
        throw e
    }
    return []
}

function _getWorkspace(wsRoot, pkg) {
    try {
        const pkgPath = require.resolve(path.join(pkg, 'package.json'), { paths: [wsRoot] })
        if (!pkgPath.match(/(\/|\\)node_modules(\/|\\)?/) && pkgPath.startsWith(wsRoot)) {
            return path.relative(wsRoot, path.dirname(pkgPath))
        }
    } catch (e) {
        // ignore - workspace dependencies have already been validated
    }
}

function _validateWsDependencies(wsRoot, dependencies, devDependencies) {
    const allDependencies = { ...dependencies, ...devDependencies }
    for (const name in allDependencies) {
        if (allDependencies[name] === '*') {
            try {
                require.resolve(path.join(name, 'package.json'), { paths: [wsRoot] })
            } catch (e) {
                if (e.code === 'MODULE_NOT_FOUND') {
                    throw new BuildError(`npm packaging failed, module '${name}' not found. Make sure configured npm workspaces are installed.`)
                }
                throw e
            }
        }
    }
}

/**
 * Returns a flat array of existing files and folders, trailing '/' or '\\' characters have been removed.
 * @param  {Array} modelPaths array of files and folders - optionally nested
 * @returns
 */
function _filterModelPaths(modelPaths, filter) {
    const model = new Set()
    // may contain nested arrays
    modelPaths = flatten(modelPaths)
    const { roots } = cds.env
    modelPaths.forEach(m => {
        if (m && !model.has(m) && !model.has(m + "/")) {
            // filter roots and absolute model paths that do not exist
            if (roots.includes(m)) {
                const dir = path.resolve(cds.root, m)
                if (fs.existsSync(dir)) {
                    model.add(m)
                } else if (fs.existsSync(dir + FILE_EXT_CDS)) { //might be cds file name, compatibility to old build configs
                    model.add(m)
                }
            } else {
                model.add(m)
            }
        }
    })
    return [...model].filter(m => filter ? filter(m) : true).map(m => normalizePath(m))
}

/**
 * Returns a set of files found in the given directory and its subdirectories.
 * @param {string} srcDir - the directory to search for files
 * @param {Function} filter - the filter function to apply to the found files
 * @param {Array} files - the set of files to add the found files to
 * @returns {Array} files found in the given directory and its subdirectories
 */
function getFiles(srcDir, filter, files = []) {
    function handleResource(entry, filter, files) {
        if (!filter || filter.call(this, entry)) {
            const stats = fs.lstatSync(entry)
            if (stats.isDirectory()) {
                getFiles(entry, filter, files)
            } else if (stats.isFile() || stats.isSymbolicLink()) {
                files.push(entry)
            }
        }
    }
    let entries = []
    try {
        entries = fs.readdirSync(srcDir)
    } catch (e) {
        // ignore if not existing
    }
    entries.map(subDirEntry => path.join(srcDir, subDirEntry)).forEach((entry) => {
        handleResource(entry, filter, files)
    })
    return files
}

/**
 * Return gnu-style error string for location `loc`:
 *  - 'File:Line:Col' without `loc.end`
 *  - 'File:Line:StartCol-EndCol' if Line = start.line = end.line
 *  - 'File:StartLine.StartCol-EndLine.EndCol' otherwise
 *
 * @param {CSN.Location|CSN.Location} location
 */
function _locationString(loc) {
    if (!loc)
        return '<???>';
    if (!(loc instanceof Object))
        return loc;
    if (!loc.line) {
        return loc.file;
    }
    else if (!loc.endLine) {
        return (loc.col)
            ? `${loc.file}:${loc.line}:${loc.col}`
            : `${loc.file}:${loc.line}`;
    }

    return (loc.line === loc.endLine)
        ? `${loc.file}:${loc.line}:${loc.col}-${loc.endCol}`
        : `${loc.file}:${loc.line}.${loc.col}-${loc.endLine}.${loc.endCol}`;
}

/**
 * Class for individual build message.
 *
 * @class BuildMessage
 */
class BuildMessage {
    /**
     * Creates an instance of BuildMessage.
     * @param {string} message The message text
     * @param {string} [severity='Error'] Severity: Debug, Info, Warning, Error
     * @param {any} location Location of the message
     *
     * @memberOf BuildMessage
     */
    constructor(message, severity = SEVERITY_ERROR, location) {
        this.message = message
        this.name = "BuildMessage"
        this.severity = severity
        this.$location = location
    }
    toString() {
        return `${this.$location?.file ? _locationString(this.$location) + ':' : ''} ${this.severity}: ${this.message}`
    }
}

/**
 * Class for combined build and compiler errors.
 * Additional members:
 *   messages: array of detailed build messages
 * @class BuildError
 * @extends {Error}
 */
class BuildError extends Error {
    constructor(message, messages = []) {
        super(message)
        this.name = "BuildError"
        this.messages = Array.isArray(messages) ? messages : [messages]
    }

    // for compatibility reasons
    get errors() {
        return this.messages
    }

    toString() {
        return this.message + (this.messages.length > 0 ? '\n' + this.messages.map(m => m.toString()).join('\n') : '')
    }
}

module.exports = {
    getProperty,
    setProperty,
    hasJavaNature,
    redactCredentials,
    hasOptionValue,
    relativePaths,
    resolveRequiredSapModels,
    getDefaultModelOptions,
    flatten,
    getI18nDefaultFolder,
    normalizePath,
    pathExists,
    getWorkspacePaths,
    getWorkspaces,
    findWorkspaceRoot,
    filterFeaturePaths,
    getFiles,
    BuildMessage,
    BuildError
}