@sap/cds-dk/lib/init/merge.js

Command line client and development toolkit for the SAP Cloud Application Programming Model

const { EOL } = require('node:os')
const { inspect } = require('node:util')
const cds = require('../cds')
const { copy, read, write, exists, path: {resolve, join}, yaml: YAML } = cds.utils
const { readYAML, writeYAML, readJSON } = require('../util/fs')
const { detectIndent } = require('./indent')
const { colors } = require('../util/term')

function _mergeObject(target, source) {
  const _isObject = item => item && typeof item === 'object' && !Array.isArray(item)
  const unique = array => [...new Set(array.map(JSON.stringify))].map(JSON.parse)
  if (_isObject(target) && _isObject(source)) {
      for (const key in source) {
          if (_isObject(source[key])) {
              if (!target[key]) Object.assign(target, { [key]: source[key] })
              else _mergeObject(target[key], source[key])
          } else if (Array.isArray(source[key]) && Array.isArray(target[key])) {
              target[key] = unique([...source[key], ...target[key]])
          } else {
              Object.assign(target, { [key]: target[key] || source[key] })
          }
      }
  } else if (Array.isArray(target) && Array.isArray(source)) {
      target = unique([...source, ...target])
  }
  return target ?? source
}

module.exports.sort = async function(path, key) {
  const json = await read(path)
  json[key] = Object.fromEntries(
      Object.entries(json[key] ?? {}).sort(([a],[b]) => a>b ? 1 : -1)
  )
  await write(path, json, { spaces: 2 });
}

/**
 * @param {string} into - file path to merge into
 * @param {string | string[]} from - either path to file (string) or lines from source gitignore (string[])
 * @returns {Promise<string[]>} the lines of the updated .gitignore
 */
module.exports.mergeGitignore = async function(into, from) {
    const source = typeof from === 'string' ? (await read(from)).split(EOL) : from

    if (typeof into === 'string' && !exists(into)) {
      return write(into, source.join(EOL))
    }

    const target = (await read(into)).split(EOL)
    const enhanced = target.concat(source.filter(line => !target.includes(line)))
    await write(into, enhanced.join(EOL))
    return enhanced
  }

/**
 * @typedef {Object} MatchedNode
 * @property {string | MatchedNode} in - key path or matched node to start search from
 * @property {Object} where - list of constraints
 */

/**
 * @typedef {Object} Overwrites
 * @property {(string | MatchedNode)[]} item - path to item to be overwritten
 * @property {string} withValue - replacement string
 */

/**
 * @typedef {Object} Deletion
 * @property {MatchedNode} item - matched node to delete
 * @property {MatchedNode} [key] - key path to property to delete in node (optional)
 * @property {Object[]} [relationships] - list of relationships to delete
 * @property {string} relationships[].removeProperty - key path to to-be-removed property
 * @property {string[]} relationships[].allWithin - key path to relationship
 */

/**
 * @typedef {Object} Relationship
 * @property {(string | MatchedNode)[]} insert - source path to matched node
 * @property {(string | MatchedNode)[]} into - target path for inserted relationship
 */

/**
 * @typedef {Object} MergingSemantics
 * @property {boolean} [forceOverwrite] - overwrite values for existing keys
 * @property {MatchedNode[]} [additions] - matched nodes to be added from source template
 * @property {Overwrites[]} [overwrites] - matched keys to be overwritten with given string
 * @property {Deletion[]} [deletions] - matched nodes to be deleted
 * @property {Relationship[]} [relationships] - relationships to be inserted
 */

/**
 * @param {string | JSON} into - file path or JSON to merge into
 * @param {string | JSON} from - file path or JSON to merge from
 * @param {import('./projectReader').readProject} [project] - project descriptor used for Mustache replacements
 * @param {MergingSemantics} [semantics] - merging algorithm semantics
 * @returns {Promise<JSON>} - merged JSON
 */
async function _mergeJSON(into, from, project, semantics) {
    const source = typeof from === 'string' ? await readJSON(from, project) : from

    let result
    if (typeof into === 'string' && !exists(into)) {
        result = source
    } else if (semantics) {
        const target = await read(into)
        const targetYAML = YAML.parseDocument(YAML.stringify(target))
        const sourceYAML = YAML.parseDocument(YAML.stringify(source))
        const resultYAML = await _mergeYAML(targetYAML, sourceYAML, project, semantics)
        result = YAML.parse(YAML.stringify(resultYAML))
    } else {
        const target = typeof into === 'string' ? await read(into, 'json') : into
        result = _mergeObject(target, source)
    }
    // we need to stringify result on site to be able to apply the appropriate indentation.
    // cds-utils.write is currently hardcoded to use spaces for indentation.
    if (typeof into === 'string') {
        const indent = exists(into) && detectIndent((await read(into, 'utf-8')).split(EOL)) || '  '
        await write(into, JSON.stringify(result, null, indent) + EOL)
    }
    return result
}

/**
 * @param {string | YAML.Document} into - file path or YAML to merge into
 * @param {string | YAML.Document} from - file path or YAML to merge from
 * @param {Object} [project] - project descriptor used for Mustache replacements
 * @param {MergingSemantics} [semantics] - merging algorithm semantics
 * @returns {Promise<YAML.Document>} - merged YAML
 * @throws {Error} Throws an error if empty target YAML is passed in
 */
async function _mergeYAML(into, from, project, semantics = {}) {
    const target = typeof into === 'string'      ? await readYAML(into) :
                   into instanceof YAML.Document ? into :
                   new YAML.Document(into)
    const source = typeof from === 'string'      ? await readYAML(from, project) :
                   from instanceof YAML.Document ? from :
                   new YAML.Document(from)

    if (!target?.contents) {
        if (typeof into === 'string' && typeof from === 'string') {
            if (project) await writeYAML(into, source)
            else await copy(from, into)
            return source
        }
        throw new Error(`Target YAML doesn't exist`)
    }

    const entryMap = new Map, templateEntryMap = new Map

    const { additions, overwrites, deletions, relationships } = semantics
    for (const entry of additions ?? []) {
        entryMap.set(entry, undefined)
        templateEntryMap.set(entry, undefined)
    }
    for (const deletion of deletions ?? []) {
        entryMap.set(deletion.item, undefined)
    }
    for (const relationship of relationships ?? []) {
        const [entry] = relationship.into
        const [insertEntry] = relationship.insert
        const hash = JSON.stringify(insertEntry) + ' -> ' + JSON.stringify(entry)
        entryMap.set(hash, undefined)
    }

    let collectionStack = [target.contents]

    const _getProperty = (object, keyPath) => keyPath.split('.').filter(k => k).reduce((p, k) => p && p[k], object)
    const _getYAMLProperty = (object, keyPath) => keyPath.split('.').filter(k => k).reduce((p, k) => p && p.get(k), object)

    const _validateWhere = (node, dict, index) => {
        const _addToMap = entries => entries?.filter(entry => {
                if (typeof entry.in === 'string') return true // just a key path on the document root
                const neededParent = dict.get(entry.in[0])?.node
                return collectionStack.includes(neededParent) // lookbehind in parent collection stack
                // REVISIT: Only look behind until sequence is reached?
        }).forEach(item => {
            const json = JSON.parse(String(node))
            const whereFulfilled = Object.entries(item.where).every(([key, value]) =>
                _getProperty(json, key) === value
            )
            if (whereFulfilled) {
                const [collection] = collectionStack
                dict.set(item, { json, node, index, collection })
            }
        })
        if (additions) _addToMap(additions)
        if (deletions) _addToMap(deletions.map(deletion => deletion.item))
        if (relationships) _addToMap(relationships?.map(({into: [entry]}) => entry).filter(e => e))
        if (overwrites) _addToMap(overwrites.map(overwrite => overwrite.item[0]))
    }

    function _traverseYAMLNode(node, index, actions, templateNode) {
        let shifted = false
        if (YAML.isMap(node)) {
            actions.visitMap?.(node, index)
            const [collection] = collectionStack
            if (YAML.isSeq(collection)) {
                collectionStack.unshift(node)
                shifted = true
            }
        } else if (YAML.isPair(node) && node.value?.items) {
            collectionStack.unshift(node.value)
        } else if (YAML.isScalar(node) && semantics.forceOverwrite && templateNode && node.comment != 'cds.noOverwrite') {
            node.value = templateNode.value ?? templateNode.items?.[0]?.value?.value
        }

        if (node.items) {
            if (YAML.isSeq(node) && templateNode?.items) {
                actions.mergeCollection?.(node, templateNode)
                _traverseYAMLCollection(node, actions, templateNode)
            } else {
                _traverseYAMLCollection(node, actions, templateNode)
            }
        } else if(node.value?.source === '' && YAML.isSeq(templateNode?.value)) {
            node.value = new YAML.YAMLSeq()
            collectionStack.unshift(node.value)
            _traverseYAMLNode(node.value, index, actions, templateNode?.value)
        } else if (node.value?.items || node.value && semantics.forceOverwrite) {
            _traverseYAMLNode(node.value, index, actions, templateNode?.value)
        }

        if (node.value?.items || shifted) {
            collectionStack.shift()
        }
    }
    _traverseYAMLNode.bind(this)

    function _traverseYAMLCollection (collection, actions, templateCollection) {
        if (!collection) return

        const keyToIndex = new Map, templateIndexToIndex = new Map

        // Map collection items to their semantic counterpart(s)
        collection.items.forEach((node, i) => {
            if (node.key) keyToIndex.set(node.key.value, i)
            const entry = [...entryMap.entries()].find(([,value]) => value?.node === node)
            if (entry) {
                const [entryKey] = entry
                const templateEntry = entryKey && templateEntryMap.get(entryKey)
                const templateIndex = templateEntry?.index
                if (templateIndex !== undefined) templateIndexToIndex.set(templateIndex, i)
            }
            if (!templateCollection) _traverseYAMLNode(node, i, actions)
        })

        if (!templateCollection?.items) return

        templateCollection.items?.forEach((templateNode, templateIndex) => {
            if (YAML.isScalar(templateNode)) {
                if (!collection.items.map(item => item.value).includes(templateNode.value)) {
                    collection.add(templateNode)
                }
            } else if (YAML.isPair(templateNode)) {
                const i = keyToIndex.get(templateNode.key.value)
                const [collection] = collectionStack
                const targetNode = collection.items?.[i]
                if (targetNode) {
                    _traverseYAMLNode(targetNode, templateIndex, actions, templateNode)
                } else {
                    actions.mergePair?.(collection, templateNode)
                }
            } else if (YAML.isMap(templateNode)) {
                const targetNode = collection.items[templateIndexToIndex.get(templateIndex)]
                actions.mergeCollection?.(targetNode, templateNode)
                if (targetNode) {
                    _traverseYAMLNode(targetNode, templateIndex, actions, templateNode)
                }
            }
        })
    }
    _traverseYAMLCollection.bind(this)

    // 1. Register the entries in the template
    _traverseYAMLNode(source.contents, null, {
        visitMap: (node, index) => _validateWhere(node, templateEntryMap, index)
    })

    // 2. Register the entries in the project
    _traverseYAMLNode(target.contents, null, {
        visitMap: (node, index) => _validateWhere(node, entryMap, index)
    })

    // 3. Apply overwrites to already found entries
    overwrites?.forEach(({ item, withValue }) => {
        const keyPath = Array.isArray(item) && typeof item[0] === 'string' ? item[0] : typeof item === 'string' ? item : item[1]
        const inEntry = Array.isArray(item) && typeof item[0] === 'object' ? item[0] : typeof item === 'object' ? item : item[1]
        const node = inEntry && entryMap.get(inEntry) ? entryMap.get(inEntry).node : collectionStack[collectionStack.length - 1]
        const keys = keyPath.split('.')
        if (!node.getIn(keys)) return
        _getYAMLProperty(node, keys.slice(0, keys.length - 1).join('.'))
            .set(keys[keys.length - 1], withValue)
    })

    // 4. Delete entries from the project (e.g. separate deployer module when adding mtx)
    deletions?.forEach(({ item, relationships, key }) => {
        const entry = entryMap.get(item)
        if (!entry) return
        if (key) entry.collection.deleteIn([entry.index, ...key.split('.')])
        else entry.collection.delete(entry.index)

        relationships?.forEach(relationship => {
            const [allWithinKeyPath, inKeyPath, into] = relationship.allWithin
            const parent = _getYAMLProperty(target, allWithinKeyPath)
            for (const child of parent.items) {
                const grandchild = _getYAMLProperty(child, inKeyPath)
                const i = grandchild?.items?.findIndex(node =>
                    node.get(into) === _getProperty(entry.json, relationship.removeProperty)
                ) ?? - 1
                if (i > -1) grandchild.delete(i)
            }
        })
    })

    // 5. Create missing entries and pairs
    _traverseYAMLNode(target.contents, null, {
        mergePair: (collection, templateNode) => {
            if (YAML.isMap(collection)) {
                collection.add(templateNode)
            }
        },
        mergeCollection: (targetNode, templateNode) => {
            if (YAML.isSeq(templateNode)) {
                let [,parent] = collectionStack
                additions?.filter(addition => {
                        if (typeof addition.in === 'string' || Array.isArray(addition.in) && typeof addition.in[0] === 'string') return true
                        const inEntry = Array.isArray(addition.in) ? entryMap.get(addition.in[0]) : entryMap.get(addition.in)
                        return inEntry?.node === parent
                    })
                    .filter(addition => {
                        const keyPath = typeof addition.in === 'string' ? addition.in : Array.isArray(addition.in) && typeof addition.in[0] === 'string' ? addition.in[0] : addition.in[1]
                        const keys = keyPath.split('.')
                        if (keys.length > collectionStack.length + 1) return false
                        if (keys.length > 1) parent = collectionStack[keys.length]
                        return _getYAMLProperty(parent, keyPath) === targetNode
                    })
                    .filter(item => !entryMap.get(item))
                    .forEach(item => {
                        if (!templateEntryMap.get(item)) throw 'Error: did not find entry in template for ' + inspect(item, { colors, depth:11 }) + ' in ' + from
                        const templateNode = templateEntryMap.get(item).node
                        item.at !== undefined ? targetNode.items.splice(item.at, 0, templateNode) : targetNode.add(templateNode)
                    })

            }
        },
    }, source.contents)

    // 6. Re-register the entries in the project
    _traverseYAMLNode(target.contents, null, {
        visitMap: (node, index) => _validateWhere(node, entryMap, index)
    })

    // 7. Create missing relationships
    _traverseYAMLNode(target.contents, null, {
        mergeCollection: (targetNode, templateNode) => {
          if (YAML.isSeq(templateNode)) {
            const targetJSON = YAML.parse(String(targetNode))
            const relationships = semantics.relationships?.filter(relationship => {
              const [entry, keyPath] = relationship.into
              if (!entryMap.get(entry)) return false
              const existingNode = _getYAMLProperty(entryMap.get(entry).node, keyPath)
              return targetNode === existingNode
            }) ?? []

            for (const { into, insert } of relationships) {
              const intoKey = into[into.length - 1]
              const [insEntry, entryKeyPath] = insert
              const missingPairs = [insEntry]
                .filter(item => {
                  if (!entryMap.get(item)) {
                    const spec = inspect(item, { colors, depth:11 })
                    throw new Error(`Did not find entry in template for ${spec} in ${from}:\n${source}`)
                  }
                  return !targetJSON.some(targetItem =>
                    _getProperty(entryMap.get(item).json, entryKeyPath) === targetItem[intoKey]
                 )
                })
                .map(item => entryMap.get(item).node.get(intoKey))
              for (const pair of missingPairs) {
                targetNode.add({ [intoKey]: pair })
              }
            }
          }
        }
      }, source.contents)

    typeof into === 'string' && await writeYAML(into, target)
    return target
}

/**
 * @param {...(string | Object | YAML.Document)} src The source to merge from.
 * @returns {{
 *   into: (dst: (string | Object | YAML.Document), options?: {project?: Object, semantics?: MergingSemantics}) => Promise<any>,
 *   remove: (items: (string | object)[]) => Promise<void>
 * }} An object with an 'into' method to merge the sources into the specified destination.
 */
module.exports.merge = (...src) => {
  return {
    /**
     * Merges sources into the specified destination.
     * @param {(string | Object | YAML.Document)} dst File path or object to merge into.
     * @param {Object} [options] Merging options.
     * @param {Object} [options.project] Project descriptor used for Mustache replacements.
     * @param {MergingSemantics} [options.semantics] Merging algorithm semantics.
     * @returns {Promise} A promise resolving to the merged object.
     */
    into: (dst, { with: withProject, project, ...semantics } = {}) => {
      if (Object.keys(semantics).length === 0) semantics = undefined
      project ??= withProject // to allow { with: project } for a more fluent API
      const isYAML = typeof dst === 'string' && ['.yaml', '.yml', '.yaml.hbs', '.yml.hbs'].some(ext => dst.endsWith(ext))
      const _merge = isYAML ? _mergeYAML : _mergeJSON
      const target = typeof dst === 'string' ? resolve(cds.root, dst) : dst
      if (typeof src[src.length - 1] === 'string') {
        return _merge(target, join(...src), project, semantics)
      } else {
        return _merge(target, ...src, project, semantics)
      }
    },
    /**
     * Removes items from the specified YAML file.
     * @param {(string | object)[]} items An array of dotted‐path strings or objects to remove.
     * @returns {Promise<void>} A promise that resolves once the file is updated.
     */
    remove: async items => {
      const yaml = await readYAML(resolve(cds.root, ...src))
      for (const item of items) {
        if (typeof item === 'string') {
          const keys = item.split('.')
          let node = yaml
          for (let i = 0; i < keys.length - 1; i++) {
            node = node.get(keys[i])
            if (node === undefined) break
          }
          node?.delete?.(keys[keys.length - 1])
        } else if (item && typeof item === 'object') {
          const stack = [{ node: yaml, obj: item }]
          while (stack.length) {
            const { node, obj } = stack.pop()
            for (const [k, v] of Object.entries(obj)) {
              const child = node.get(k)
              if (YAML.isSeq(child)) {
                child.items = child.items.filter(x => {
                  if (YAML.isMap(x)) {
                    return !Object.entries(v).every(([kk, vv]) => x.get(kk) === vv)
                  }
                  return true
                })
              } else if (YAML.isMap(child)) {
                stack.push({ node: child, obj: v })
              }
            }
          }
        }
        await writeYAML(join(...src), yaml)
      }
    }
  }
}