@cap-js/db-service/lib/cqn4sql.js

CDS base database service

'use strict'

const cds = require('@sap/cds')
cds.infer.target ??= q => q._target || q.target // instanceof cds.entity ? q._target : q.target

const infer = require('./infer')
const { computeColumnsToBeSearched } = require('./search')
const { prettyPrintRef, isCalculatedOnRead, isCalculatedElement, getImplicitAlias, getModelUtils } = require('./utils')

/**
 * For operators of <eqOps>, this is replaced by comparing all leaf elements with null, combined with and.
 * If there are at least two leaf elements and if there are tokens before or after the recognized pattern, we enclose the resulting condition in parens (...)
 */
const eqOps = [['is'], ['='] /* ['=='] */]
/**
 * For operators of <notEqOps>, do the same but use or instead of and.
 * This ensures that not struct == <value> is the same as struct != <value>.
 */
const notEqOps = [['is', 'not'], ['<>'], ['!=']]
/**
 * not supported in comparison w/ struct because of unclear semantics
 */
const notSupportedOps = [['>'], ['<'], ['>='], ['<=']]

const allOps = eqOps.concat(eqOps).concat(notEqOps).concat(notSupportedOps)

const { pseudos } = require('./infer/pseudos')
/**
 * Transforms a CDL style query into SQL-Like CQN:
 *  - transform association paths in `from` to `WHERE exists` subqueries
 *  - transforms columns into their flat representation.
 *      1. Flatten managed associations to their foreign
 *      2. Flatten structures to their leafs
 *      3. Replace join-relevant ref paths (i.e. non-fk accesses in association paths) with the correct join alias
 *  - transforms `expand` columns into special, normalized subqueries
 *  - transform `where` clause.
 *      That is the flattening of all `ref`s and the expansion of `where exists` predicates
 *  - rewrites `from` clause:
 *      Each join relevant association path traversal is translated to a join condition.
 *
 * `cqn4sql` is applied recursively to all queries found in `from`, `columns` and `where`
 *  of a query.
 *
 * @param {object} originalQuery
 * @param {object} model
 * @returns {object} transformedQuery the transformed query
 */
function cqn4sql(originalQuery, model) {
  let inferred = typeof originalQuery === 'string' ? cds.parse.cql(originalQuery) : cds.ql.clone(originalQuery)
  const hasCustomJoins =
    originalQuery.SELECT?.from.args && (!originalQuery.joinTree || originalQuery.joinTree.isInitial)

  if (!hasCustomJoins && inferred.SELECT?.search) {
    // we need an instance of query because the elements of the query are needed for the calculation of the search columns
    if (!inferred.SELECT.elements) Object.setPrototypeOf(inferred, SELECT.class.prototype)
    const searchTerm = getSearchTerm(inferred.SELECT.search, inferred)
    if (searchTerm) {
      // Search target can be a navigation, in that case use _target to get the correct entity
      const { where, having } = transformSearch(searchTerm)
      if (where) inferred.SELECT.where = where
      else if (having) inferred.SELECT.having = having
    }
  }
  // query modifiers can also be defined in from ref leaf infix filter
  // > SELECT from bookshop.Books[order by price] {ID}
  if (inferred.SELECT?.from.ref) {
    for (const [key, val] of Object.entries(inferred.SELECT.from.ref.at(-1))) {
      if (key in { orderBy: 1, groupBy: 1 }) {
        if (inferred.SELECT[key]) inferred.SELECT[key].push(...val)
        else inferred.SELECT[key] = val
      } else if (key === 'limit') {
        // limit defined on the query has precedence
        if (!inferred.SELECT.limit) inferred.SELECT.limit = val
      } else if (key === 'having') {
        if (!inferred.SELECT.having) inferred.SELECT.having = val
        else inferred.SELECT.having.push('and', ...val)
      }
    }
  }
  inferred = infer(inferred, model)
  const { getLocalizedName, isLocalized, getDefinition } = getModelUtils(model, originalQuery) // TODO: pass model to getModelUtils
  // if the query has custom joins we don't want to transform it
  // TODO: move all the way to the top of this function once cds.infer supports joins as well
  //       we need to infer the query even if no transformation will happen because cds.infer can't calculate the target
  if (hasCustomJoins) return originalQuery

  let transformedQuery = cds.ql.clone(inferred)
  const kind = inferred.kind || Object.keys(inferred)[0]

  if (inferred.INSERT || inferred.UPSERT) {
    transformedQuery = transformQueryForInsertUpsert(kind)
  } else {
    const queryProp = inferred[kind]
    const { entity, where } = queryProp
    const from = queryProp.from

    const transformedProp = { __proto__: queryProp } // IMPORTANT: don't lose anything you might not know of

    // Transform the existing where, prepend table aliases, and so on...
    if (where) {
      transformedProp.where = getTransformedTokenStream(where)
    }

    // Transform the from clause: association path steps turn into `WHERE EXISTS` subqueries.
    // The already transformed `where` clause is then glued together with the resulting subqueries.
    const { transformedWhere, transformedFrom } = getTransformedFrom(from || entity, transformedProp.where)
    const queryNeedsJoins = inferred.joinTree && !inferred.joinTree.isInitial

    if (inferred.SELECT) {
      transformedQuery = transformSelectQuery(queryProp, transformedFrom, transformedWhere, transformedQuery)
    } else {
      if (from) {
        transformedProp.from = transformedFrom
      } else if (!queryNeedsJoins) {
        transformedProp.entity = transformedFrom
      }

      if (transformedWhere?.length > 0) {
        transformedProp.where = transformedWhere
      }

      transformedQuery[kind] = transformedProp

      if (inferred.UPDATE?.with) {
        Object.entries(inferred.UPDATE.with).forEach(([key, val]) => {
          const transformed = getTransformedTokenStream([val])
          inferred.UPDATE.with[key] = transformed[0]
        })
      }
    }

    if (queryNeedsJoins) {
      if (inferred.UPDATE || inferred.DELETE) {
        const prop = inferred.UPDATE ? 'UPDATE' : 'DELETE'
        const subquery = {
          SELECT: {
            from: { ...transformedFrom },
            columns: [], // primary keys of the query target will be added later
            where: [...transformedProp.where],
          },
        }
        // The alias of the original query is now the alias for the subquery
        // so that potential references in the where clause to the alias match.
        // Hence, replace the alias of the original query with the next
        // available alias, so that each alias is unique.
        const uniqueSubqueryAlias = getNextAvailableTableAlias(transformedFrom.as)
        transformedFrom.as = uniqueSubqueryAlias

        // calculate the primary keys of the target entity, there is always exactly
        // one query source for UPDATE / DELETE
        const queryTarget = Object.values(inferred.sources)[0].definition
        const primaryKey = { list: [] }
        for (const k of Object.keys(queryTarget.elements)) {
          const e = queryTarget.elements[k]
          if (e.key === true && !e.virtual && e.isAssociation !== true) {
            subquery.SELECT.columns.push({ ref: [e.name] })
            primaryKey.list.push({ ref: [transformedFrom.as, e.name] })
          }
        }

        const transformedSubquery = cqn4sql(subquery, model)

        // replace where condition of original query with the transformed subquery
        // correlate UPDATE / DELETE query with subquery by primary key matches
        transformedQuery[prop].where = [primaryKey, 'in', transformedSubquery]

        if (prop === 'UPDATE') transformedQuery.UPDATE.entity = transformedFrom
        else transformedQuery.DELETE.from = transformedFrom
      } else {
        transformedQuery[kind].from = translateAssocsToJoins(transformedQuery[kind].from)
      }
    }
  }

  return transformedQuery

  function transformSelectQuery(queryProp, transformedFrom, transformedWhere, transformedQuery) {
    const { columns, having, groupBy, orderBy, limit } = queryProp

    // Trivial replacement -> no transformations needed
    if (limit) {
      transformedQuery.SELECT.limit = limit
    }

    transformedQuery.SELECT.from = transformedFrom

    if (transformedWhere?.length > 0) {
      transformedQuery.SELECT.where = transformedWhere
    }

    if (columns) {
      transformedQuery.SELECT.columns = getTransformedColumns(columns)
    } else {
      transformedQuery.SELECT.columns = getColumnsForWildcard(inferred.SELECT?.excluding)
    }

    // Like the WHERE clause, aliases from the SELECT list are not accessible for `group by`/`having` (in most DB's)
    if (having) {
      transformedQuery.SELECT.having = getTransformedTokenStream(having)
    }

    if (groupBy) {
      const transformedGroupBy = getTransformedOrderByGroupBy(groupBy)
      if (transformedGroupBy.length) {
        transformedQuery.SELECT.groupBy = transformedGroupBy
      }
    }

    // Since all the expressions in the SELECT part of the query have been computed,
    // one can reference aliases of the queries columns in the orderBy clause.
    if (orderBy) {
      const transformedOrderBy = getTransformedOrderByGroupBy(orderBy, true)
      if (transformedOrderBy.length) {
        transformedQuery.SELECT.orderBy = transformedOrderBy
      }
    }
    return transformedQuery
  }

  /**
   * Transforms a query object for INSERT or UPSERT operations by modifying the `into` clause.
   *
   * @param {string} kind - The type of operation: "INSERT" or "UPSERT".
   *
   * @returns {object} - The transformed query with updated `into` clause.
   */
  function transformQueryForInsertUpsert(kind) {
    const { as } = transformedQuery[kind].into
    const target = cds.infer.target(inferred) // REVISIT: we should reliably use inferred._target instead
    transformedQuery[kind].into = { ref: [target.name] }
    if (as) transformedQuery[kind].into.as = as
    return transformedQuery
  }

  /**
   * Transforms a search expression into a WHERE or HAVING clause for a SELECT operation, depending on the context of the query.
   * The function decides whether to use a WHERE or HAVING clause based on the presence of aggregated columns in the search criteria.
   *
   * @param {object} searchTerm - The search expression to be applied to the searchable columns within the query source.
   * @param {object} from - The FROM clause of the CQN statement.
   *
   * @returns {Object} - The function returns an object representing the WHERE or HAVING clause of the query.
   *
   * Note: The WHERE clause is used for filtering individual rows before any aggregation occurs.
   * The HAVING clause is utilized for conditions on aggregated data, applied after grouping operations.
   */
  function transformSearch(searchTerm) {
    let prop = 'where'

    // if the query is grouped and the queries columns contain an aggregate function,
    // we must put the search term into the `having` clause, as the search expression
    // is defined on the aggregated result, not on the individual rows
    const usesAggregation =
      inferred.SELECT.groupBy &&
      (searchTerm.args[0].func || searchTerm.args[0].xpr || searchTerm.args[0].list?.some(c => c.func || c.xpr))

    if (usesAggregation) prop = 'having'
    if (inferred.SELECT[prop]) {
      return { [prop]: [asXpr(inferred.SELECT.where), 'and', searchTerm] }
    } else {
      return { [prop]: [searchTerm] }
    }
  }

  /**
   * Rewrites the from clause based on the `query.joinTree`.
   *
   * For each join relevant node in the join tree, the respective join is generated.
   * Each join relevant node in the join tree has an unique table alias which is the query source for the respective
   * path traversals. Hence, all join relevant `ref`s must be rewritten to point to the generated join aliases. However,
   * this is done in the @function getFlatColumnsFor().
   *
   * @returns {CQN.from}
   */
  function translateAssocsToJoins() {
    let from
    /**
     * remember already seen aliases, do not create a join for them again
     */
    const alreadySeen = new Map()
    inferred.joinTree._roots.forEach(r => {
      const args = []
      if (r.queryArtifact.SELECT) args.push({ SELECT: transformSubquery(r.queryArtifact).SELECT, as: r.alias })
      else {
        const id = getLocalizedName(r.queryArtifact)
        args.push({ ref: [r.args ? { id, args: r.args } : id], as: r.alias })
      }
      from = { join: r.join || 'left', args, on: [] }
      r.children.forEach(c => {
        from = joinForBranch(from, c)
        from = { join: c.join || 'left', args: [from], on: [] }
      })
    })
    return from.args.length > 1 ? from : from.args[0]

    function joinForBranch(lhs, node) {
      const nextAssoc = inferred.joinTree.findNextAssoc(node)
      if (!nextAssoc || alreadySeen.has(nextAssoc.$refLink.alias)) return lhs.args.length > 1 ? lhs : lhs.args[0]

      lhs.on.push(
        ...onCondFor(
          nextAssoc.$refLink,
          node.parent.$refLink || /** tree roots do not have $refLink */ {
            alias: node.parent.alias,
            definition: node.parent.queryArtifact,
            target: node.parent.queryArtifact,
          },
          /** flip source and target in on condition */ true,
        ),
      )

      const id = getDefinition(nextAssoc.$refLink.definition.target).name
      const { args } = nextAssoc
      const arg = {
        ref: [args ? { id, args } : id],
        as: nextAssoc.$refLink.alias,
      }

      lhs.args.push(arg)
      alreadySeen.set(nextAssoc.$refLink.alias, true)
      if (nextAssoc.where) {
        const filter = getTransformedTokenStream(nextAssoc.where, nextAssoc.$refLink)
        lhs.on = [
          ...(hasLogicalOr(lhs.on) ? [asXpr(lhs.on)] : lhs.on),
          'and',
          ...(hasLogicalOr(filter) ? [asXpr(filter)] : filter),
        ]
      }
      if (node.children) {
        node.children.forEach(c => {
          lhs = { join: c.join || 'left', args: [lhs], on: [] }
          lhs = joinForBranch(lhs, c)
        })
      }
      return lhs.args.length > 1 ? lhs : lhs.args[0]
    }
  }

  /**
   * Walks over a list of columns (ref's, xpr, subqueries, val), applies flattening on structured types and expands wildcards.
   *
   * @param {object[]} columns
   * @returns {object[]} the transformed representation of the input. Expanded and flattened.
   */
  function getTransformedColumns(columns) {
    const transformedColumns = []
    for (let i = 0; i < columns.length; i++) {
      const col = columns[i]

      if (isCalculatedOnRead(col.$refLinks?.[col.$refLinks.length - 1].definition)) {
        const name = getName(col)
        if (!transformedColumns.some(inserted => getName(inserted) === name)) {
          const calcElement = resolveCalculatedElement(col)
          transformedColumns.push(calcElement)
        }
      } else if (col.expand) {
        if (col.ref?.length > 1 && col.ref[0] === '$self' && !col.$refLinks[0].definition.kind) {
          const dollarSelfReplacement = calculateDollarSelfColumn(col)
          transformedColumns.push(...getTransformedColumns([dollarSelfReplacement]))
          continue
        }
        transformedColumns.push(() => {
          const expandResult = handleExpand(col)
          if (expandResult.length > 1) {
            return expandResult
          } else {
            return expandResult[0]
          }
        })
      } else if (col.inline) {
        handleInline(col)
      } else if (col.ref) {
        if (col.ref.length > 1 && col.ref[0] === '$self' && !col.$refLinks[0].definition.kind) {
          const dollarSelfReplacement = calculateDollarSelfColumn(col)
          transformedColumns.push(...getTransformedColumns([dollarSelfReplacement]))
          continue
        }
        handleRef(col)
      } else if (col === '*') {
        handleWildcard(columns)
      } else if (col.SELECT) {
        handleSubquery(col)
      } else {
        handleDefault(col)
      }
    }
    // subqueries are processed in the end
    for (let i = 0; i < transformedColumns.length; i++) {
      const c = transformedColumns[i]
      if (typeof c === 'function') {
        const res = c() || [] // target of expand / subquery could also be skipped -> no result
        if (res.length !== undefined) {
          transformedColumns.splice(i, 1, ...res)
          i += res.length - 1
        } else {
          const replaceWith = res.as
            ? transformedColumns.findIndex(t => (t.as || t.ref?.[t.ref.length - 1]) === res.as)
            : -1
          if (replaceWith === -1) transformedColumns.splice(i, 1, res)
          else {
            transformedColumns.splice(replaceWith, 1, res)
            transformedColumns.splice(i, 1)
            // When removing an element, the next element moves to the current index
            i--
          }
        }
      }
    }

    if (transformedColumns.length === 0 && columns.length) {
      handleEmptyColumns(columns)
    }

    return transformedColumns

    function handleSubquery(col) {
      transformedColumns.push(() => {
        const res = transformSubquery(col)
        if (col.as) res.as = col.as
        return res
      })
    }

    function handleExpand(col) {
      const { $refLinks } = col
      const res = []
      const last = $refLinks?.[$refLinks.length - 1]
      if (last && !last.skipExpand && last.definition.isAssociation) {
        const expandedSubqueryColumn = expandColumn(col)
        if (!expandedSubqueryColumn) return []
        setElementOnColumns(expandedSubqueryColumn, col.element)
        res.push(expandedSubqueryColumn)
      } else if (!last?.skipExpand) {
        const expandCols = nestedProjectionOnStructure(col, 'expand')
        res.push(...expandCols)
      }
      return res
    }

    function handleInline(col) {
      const inlineCols = nestedProjectionOnStructure(col)
      transformedColumns.push(...inlineCols)
    }

    function handleRef(col) {
      if (pseudos.elements[col.ref[0]] || col.param) {
        transformedColumns.push({ ...col })
        return
      }

      const tableAlias = getTableAlias(col)
      // re-adjust usage of implicit alias in subquery
      if (col.$refLinks[0].definition.kind === 'entity' && col.ref[0] !== tableAlias) {
        col.ref[0] = tableAlias
      }
      const leaf = col.$refLinks[col.$refLinks.length - 1].definition
      if (leaf.virtual === true) return

      let baseName
      if (col.ref.length >= 2) {
        baseName = col.ref
          .map(idOnly)
          .slice(col.ref[0] === tableAlias ? 1 : 0, col.ref.length - 1)
          .join('_')
      }

      let columnAlias = col.as || (col.isJoinRelevant ? col.flatName : null)
      const refNavigation = col.ref.slice(col.$refLinks[0].definition.kind !== 'element' ? 1 : 0).join('_')
      if (!columnAlias && col.flatName && col.flatName !== refNavigation) columnAlias = refNavigation

      if (col.$refLinks.some(link => getDefinition(link.definition.target)?.['@cds.persistence.skip'] === true)) return

      const flatColumns = getFlatColumnsFor(col, { baseName, columnAlias, tableAlias })
      flatColumns.forEach(flatColumn => {
        const name = getName(flatColumn)
        if (!transformedColumns.some(inserted => getName(inserted) === name)) transformedColumns.push(flatColumn)
      })
    }

    function handleWildcard(columns) {
      const wildcardIndex = columns.indexOf('*')
      const ignoreInWildcardExpansion = columns.slice(0, wildcardIndex)
      const { excluding } = inferred.SELECT
      if (excluding) ignoreInWildcardExpansion.push(...excluding)

      const wildcardColumns = getColumnsForWildcard(ignoreInWildcardExpansion, columns.slice(wildcardIndex + 1))
      transformedColumns.push(...wildcardColumns)
    }

    function handleDefault(col) {
      let transformedColumn = getTransformedColumn(col)
      if (col.as) transformedColumn.as = col.as

      const replaceWith = transformedColumns.findIndex(
        t => (t.as || t.ref?.[t.ref.length - 1]) === transformedColumn.as,
      )
      if (replaceWith === -1) transformedColumns.push(transformedColumn)
      else transformedColumns.splice(replaceWith, 1, transformedColumn)

      setElementOnColumns(transformedColumn, inferred.elements[col.as])
    }

    function getTransformedColumn(col) {
      let ret
      if (col.func) {
        ret = {
          func: col.func,
          args: getTransformedFunctionArgs(col.args),
          as: col.func, // may be overwritten by the explicit alias
        }
      }
      if (col.xpr) {
        ret ??= {}
        ret.xpr = getTransformedTokenStream(col.xpr)
      }
      if (ret) {
        if (col.cast) ret.cast = col.cast
        return ret
      }
      return copy(col)
    }

    function handleEmptyColumns(columns) {
      if (columns.some(c => c.$refLinks?.[c.$refLinks.length - 1].definition.type === 'cds.Composition')) return
      throw new Error('Queries must have at least one non-virtual column')
    }
  }

  function resolveCalculatedElement(column, omitAlias = false, baseLink = null) {
    let value

    if (column.$refLinks) {
      const { $refLinks } = column
      value = $refLinks[$refLinks.length - 1].definition.value
      if (column.$refLinks.length > 1) {
        baseLink =
          [...$refLinks].reverse().find($refLink => $refLink.definition.isAssociation) ||
          // if there is no association in the path, the table alias is the base link
          // TA might refer to subquery -> we need to propagate the alias to all paths of the calc element
          column.$refLinks[0]
      }
    } else {
      value = column.value
    }
    const { ref, val, xpr, func } = value

    let res
    if (ref) {
      res = getTransformedTokenStream([value], baseLink)[0]
    } else if (xpr) {
      res = { xpr: getTransformedTokenStream(value.xpr, baseLink) }
    } else if (val !== undefined) {
      res = { val }
    } else if (func) {
      res = { args: getTransformedFunctionArgs(value.args, baseLink), func: value.func }
    }
    if (!omitAlias) res.as = column.as || column.name || column.flatName
    return res
  }

  /**
   * This function resolves a `ref` starting with a `$self`.
   * Such a path targets another element of the query by it's implicit, or explicit alias.
   *
   * A `$self` reference may also target another `$self` path. In this case, this function
   * recursively resolves the tail of the `$self` references (`$selfPath.ref.slice(2)`) onto it's
   * new base.
   *
   * @param {object} col with a ref like `[ '$self', <target column>, <optional further path navigation> ]`
   * @param {boolean} omitAlias if we replace a $self reference in an aggregation or a token stream, we must not add an "as" to the result
   */
  function calculateDollarSelfColumn(col, omitAlias = false) {
    const dummyColumn = buildDummyColumnForDollarSelf({ ...col }, col.$refLinks)

    return dummyColumn

    function buildDummyColumnForDollarSelf(dollarSelfColumn, $refLinks) {
      const { ref, as } = dollarSelfColumn
      const stepToFind = ref[1]
      let referencedColumn = inferred.SELECT.columns.find(
        otherColumn =>
          otherColumn !== dollarSelfColumn &&
          (otherColumn.as
            ? stepToFind === otherColumn.as
            : stepToFind === otherColumn.ref?.[otherColumn.ref.length - 1]),
      )
      if (referencedColumn.ref?.[0] === '$self') {
        referencedColumn = buildDummyColumnForDollarSelf({ ...referencedColumn }, referencedColumn.$refLinks)
      }

      if (referencedColumn.ref) {
        dollarSelfColumn.ref = [...referencedColumn.ref, ...dollarSelfColumn.ref.slice(2)]
        Object.defineProperties(dollarSelfColumn, {
          flatName: {
            value:
              referencedColumn.$refLinks[0].definition.kind === 'entity'
                ? dollarSelfColumn.ref.slice(1).join('_')
                : dollarSelfColumn.ref.join('_'),
          },
          isJoinRelevant: {
            value: referencedColumn.isJoinRelevant,
          },
          $refLinks: {
            value: [...referencedColumn.$refLinks, ...$refLinks.slice(2)],
          },
        })
      } else {
        // target column is `val` or `xpr`, destructure and throw away the ref with the $self
        // eslint-disable-next-line no-unused-vars
        const { xpr, val, ref, as: _as, ...rest } = referencedColumn
        if (xpr) rest.xpr = xpr
        else rest.val = val
        dollarSelfColumn = { ...rest } // reassign dummyColumn without 'ref'
        if (!omitAlias) dollarSelfColumn.as = as
      }
      return dollarSelfColumn.ref?.[0] === '$self'
        ? buildDummyColumnForDollarSelf({ ...dollarSelfColumn }, $refLinks)
        : dollarSelfColumn
    }
  }

  /**
   * Calculates the columns for a nested projection on a structure.
   *
   * @param {object} col
   * @param {'inline'|'expand'} prop the property on which to operate. Default is `inline`.
   * @returns a list of flat columns.
   */
  function nestedProjectionOnStructure(col, prop = 'inline') {
    const res = []

    col[prop].forEach((nestedProjection, i) => {
      let rewrittenColumns = []
      if (nestedProjection === '*') {
        res.push(...expandNestedProjectionWildcard(col, i, prop))
      } else {
        const nameParts = col.as ? [col.as] : [col.ref.map(idOnly).join('_')]
        nameParts.push(nestedProjection.as ? nestedProjection.as : nestedProjection.ref.map(idOnly).join('_'))
        const name = nameParts.join('_')
        if (nestedProjection.ref) {
          const augmentedInlineCol = { ...nestedProjection }
          augmentedInlineCol.ref = col.ref ? [...col.ref, ...nestedProjection.ref] : nestedProjection.ref
          if (
            col.as ||
            nestedProjection.as ||
            nestedProjection.$refLinks[nestedProjection.$refLinks.length - 1].definition.value ||
            nestedProjection.isJoinRelevant
          ) {
            augmentedInlineCol.as = nameParts.join('_')
          }
          Object.defineProperties(augmentedInlineCol, {
            $refLinks: { value: [...nestedProjection.$refLinks], writable: true },
            isJoinRelevant: {
              value: nestedProjection.isJoinRelevant,
              writable: true,
            },
          })
          // if the expand is not anonymous, we must prepend the expand columns path
          // to make sure the full path is resolvable
          if (col.ref) {
            augmentedInlineCol.$refLinks.unshift(...col.$refLinks)
            augmentedInlineCol.isJoinRelevant = augmentedInlineCol.isJoinRelevant || col.isJoinRelevant
          }
          const flatColumns = getTransformedColumns([augmentedInlineCol])
          flatColumns.forEach(flatColumn => {
            const flatColumnName = flatColumn.as || flatColumn.ref[flatColumn.ref.length - 1]
            if (!res.some(c => (c.as || c.ref.slice(1).map(idOnly).join('_')) === flatColumnName)) {
              const rewrittenColumn = { ...flatColumn }
              if (nestedProjection.as) rewrittenColumn.as = flatColumnName
              rewrittenColumns.push(rewrittenColumn)
            }
          })
        } else {
          // func, xpr, val..
          // we need to check if the column was already added
          // in the wildcard expansion
          if (!res.some(c => (c.as || c.ref.slice(1).map(idOnly).join('_')) === name)) {
            const rewrittenColumn = { ...nestedProjection }
            rewrittenColumn.as = name
            rewrittenColumns.push(rewrittenColumn)
          }
        }
      }
      res.push(...rewrittenColumns)
    })

    return res
  }

  /**
   * Expand the wildcard of the given column into all leaf elements.
   * Respect smart wildcard rules and excluding clause.
   *
   * Every column before the wildcardIndex is excluded from the wildcard expansion.
   * Columns after the wildcardIndex overwrite columns within the wildcard expansion in place.
   *
   * @TODO use this also for `expand` wildcards on structures.
   *
   * @param {csn.Column} col
   * @param {integer} wildcardIndex
   * @returns an array of columns which represents the expanded wildcard
   */
  function expandNestedProjectionWildcard(col, wildcardIndex, prop = 'inline') {
    const res = []
    // everything before the wildcard is inserted before the wildcard
    // and ignored from the wildcard expansion
    const exclude = col[prop].slice(0, wildcardIndex)
    // everything after the wildcard, is a potential replacement
    // in the wildcard expansion
    const replace = []

    const baseRef = col.ref || []
    const baseRefLinks = col.$refLinks || []

    // column has no ref, then it is an anonymous expand:
    // select from books { { * } as bar }
    // only possible if there is exactly one query source
    if (!baseRef.length) {
      const [tableAlias, { definition }] = Object.entries(inferred.sources)[0]
      baseRef.push(tableAlias)
      baseRefLinks.push({ definition, source: definition })
    }
    // we need to make the refs absolute
    col[prop].slice(wildcardIndex + 1).forEach(c => {
      const fakeColumn = { ...c }
      if (fakeColumn.ref) {
        fakeColumn.ref = [...baseRef, ...fakeColumn.ref]
        fakeColumn.$refLinks = [...baseRefLinks, ...c.$refLinks]
      }
      replace.push(fakeColumn)
    })
    // respect excluding clause
    if (col.excluding) {
      // fake the ref since excluding only has strings
      col.excluding.forEach(c => {
        const fakeColumn = {
          ref: [...baseRef, c],
        }
        exclude.push(fakeColumn)
      })
    }

    if (baseRefLinks.at(-1).definition.kind === 'entity') {
      res.push(...getColumnsForWildcard(exclude, replace, col.as))
    } else
      res.push(
        ...getFlatColumnsFor(col, { columnAlias: col.as, tableAlias: getTableAlias(col) }, [], {
          exclude,
          replace,
        }),
      )
    return res
  }

  /**
   * This function converts a column with an `expand` property into a subquery.
   *
   * It operates by using the following steps:
   *
   * 1. It creates an intermediate SQL query, selecting `from <effective query source>:...<column>.ref { ...<column>.expand }`.
   *    For example, from the query `SELECT from Authors { books { title } }`, it generates:
   *    - `SELECT from Authors:books as books {title}`
   *
   * 2. It then adds the properties `expand: true` and `one: <expand assoc>.is2one` to the intermediate SQL query.
   *
   * 3. It applies `cqn4sql` to the intermediate query (ensuring the aliases of the outer query are maintained).
   *    For example, `cqn4sql(…)` is used to create the following query:
   *    - `SELECT from Books as books {books.title} where exists ( SELECT 1 from Authors as Authors where Authors.ID = books.author_ID )`
   *
   * 4. It then replaces the `exists <subquery>` with the where condition of the `<subquery>` and correlates it with the effective query source.
   *    For example, this query is created:
   *    - `SELECT from Books as books { books.title } where Authors.ID = books.author_ID`
   *
   * 5. Lastly, it replaces the `expand` column of the original query with the transformed subquery.
   *    For example, the query becomes:
   *    - `SELECT from Authors { (SELECT from Books as books { books.title } where Authors.ID = books.author_ID) as books }`
   *
   * @param {CSN.column} column - The column with the 'expand' property to be transformed into a subquery.
   *
   * @returns {object} Returns a subquery correlated with the enclosing query, with added properties `expand:true` and `one:true|false`.
   */
  function expandColumn(column) {
    let outerAlias
    let subqueryFromRef
    const ref = column.$refLinks[0].definition.kind === 'entity' ? column.ref.slice(1) : column.ref
    const assoc = column.$refLinks.at(-1)
    ensureValidForeignKeys(assoc.definition, column.ref, 'expand')
    if (column.isJoinRelevant) {
      // all n-1 steps of the expand column are already transformed into joins
      // find the last join relevant association. That is the n-1 assoc in the ref path.
      // slice the ref array beginning from the n-1 assoc in the ref and take that as the postfix for the subqueries from ref.
      ;[...column.$refLinks]
        .reverse()
        .slice(1)
        .find((link, i) => {
          if (link.definition.isAssociation) {
            subqueryFromRef = [link.definition.target, ...column.ref.slice(-(i + 1), column.ref.length)]
            // alias of last join relevant association is also the correlation alias for the subquery
            outerAlias = link.alias
            return true
          }
        })
    } else {
      outerAlias = transformedQuery.SELECT.from.as
      const getInnermostTarget = q => (q._target ? getInnermostTarget(q._target) : q)
      subqueryFromRef = [
        ...(transformedQuery.SELECT.from.ref || /* subq in from */ [getInnermostTarget(transformedQuery).name]),
        ...ref,
      ]
    }

    // this is the alias of the column which holds the correlated subquery
    const columnAlias = column.as || ref.map(idOnly).join('_')

    // if there is a group by on the main query, all
    // columns of the expand must be in the groupBy
    if (transformedQuery.SELECT.groupBy) {
      const baseRef = column.$refLinks[0].definition.SELECT || ref

      return _subqueryForGroupBy(column, baseRef, columnAlias)
    }

    // Alias in expand subquery is derived from but not equal to
    // the alias of the column because to account for potential ambiguities
    // the alias cannot be addressed anyways
    const uniqueSubqueryAlias = getNextAvailableTableAlias(getImplicitAlias(columnAlias), inferred.outerQueries)

    // `SELECT from Authors {  books.genre as genreOfBooks { name } } becomes `SELECT from Books:genre as genreOfBooks`
    const from = { ref: subqueryFromRef, as: uniqueSubqueryAlias }
    const subqueryBase = {}
    const queryModifiers = { ...column }
    for (const [key, value] of Object.entries(queryModifiers)) {
      if (key in { limit: 1, orderBy: 1, groupBy: 1, excluding: 1, where: 1, having: 1, count: 1 })
        subqueryBase[key] = value
    }

    const subquery = {
      SELECT: {
        ...subqueryBase,
        from,
        columns: JSON.parse(JSON.stringify(column.expand)),
        expand: true,
        one: column.$refLinks.at(-1).definition.is2one,
      },
    }
    const expanded = transformSubquery(subquery)
    const correlated = _correlate({ ...expanded, as: columnAlias }, outerAlias)
    Object.defineProperty(correlated, 'elements', {
      value: expanded.elements,
      writable: true,
    })
    return correlated

    function _correlate(subq, outer) {
      const subqueryFollowingExists = (a, indexOfExists) => a[indexOfExists + 1]
      let {
        SELECT: { where },
      } = subq
      let recent = where
      let i = where.indexOf('exists')
      while (i !== -1) {
        where = subqueryFollowingExists((recent = where), i).SELECT.where
        i = where.indexOf('exists')
      }
      const existsIndex = recent.indexOf('exists')
      recent.splice(
        existsIndex,
        2,
        ...where.map(x => {
          return replaceAliasWithSubqueryAlias(x)
        }),
      )

      function replaceAliasWithSubqueryAlias(x) {
        const existsSubqueryAlias = recent[existsIndex + 1].SELECT.from.as
        if (existsSubqueryAlias === x.ref?.[0]) return { ref: [outer, ...x.ref.slice(1)] }
        if (x.xpr) x.xpr = x.xpr.map(replaceAliasWithSubqueryAlias)
        if (x.args) x.args = x.args.map(replaceAliasWithSubqueryAlias)
        return x
      }
      return subq
    }

    /**
     * Generates a special subquery for the `expand` of the `column`.
     * All columns in the `expand` must be part of the GROUP BY clause of the main query.
     * If this is the case, the subqueries columns match the corresponding references of the group by.
     * Nested expands are also supported.
     *
     * @param {Object} column - To expand.
     * @param {Array} baseRef - The base reference for the expanded column.
     * @param {string} subqueryAlias - The alias of the `expand` subquery column.
     * @returns {Object} - The subquery object or null if the expand has a wildcard.
     * @throws {Error} - If one of the `ref`s in the `column.expand` is not part of the GROUP BY clause.
     */
    function _subqueryForGroupBy(column, baseRef, subqueryAlias) {
      const groupByLookup = new Map(
        transformedQuery.SELECT.groupBy.map(c => [c.ref && c.ref.map(refWithConditions).join('.'), c]),
      )

      // to be attached to dummy query
      const elements = {}
      const containsWildcard = column.expand.includes('*')
      if (containsWildcard) {
        // expand with wildcard vanishes as expand is part of the group by (OData $apply + $expand)
        return null
      }
      const expandedColumns = column.expand
        .flatMap(expand => {
          if (!expand.ref) return expand
          const fullRef = [...baseRef, ...expand.ref]

          if (expand.expand) {
            const nested = _subqueryForGroupBy(expand, fullRef, expand.as || expand.ref.map(idOnly).join('_'))
            if (nested) {
              setElementOnColumns(nested, expand.element)
              elements[expand.as || expand.ref.map(idOnly).join('_')] = nested
            }
            return nested
          }

          const groupByRef = groupByLookup.get(fullRef.map(refWithConditions).join('.'))
          if (!groupByRef) {
            throw new Error(
              `The expanded column "${fullRef.map(refWithConditions).join('.')}" must be part of the group by clause`,
            )
          }

          const copy = Object.create(groupByRef)
          // always alias for this special case, so that they nested element names match the expected result structure
          // otherwise we'd get `author { <outer>.author_ID }`, but we need `author { <outer>.author_ID as ID }`
          copy.as = expand.as || expand.ref.at(-1)
          const tableAlias = getTableAlias(copy)
          const res = getFlatColumnsFor(copy, { tableAlias })
          res.forEach(c => {
            elements[c.as || c.ref.at(-1)] = c.element
          })
          return res
        })
        .filter(c => c)

      if (expandedColumns.length === 0) {
        return null
      }

      const SELECT = {
        from: null,
        columns: expandedColumns,
      }
      return Object.defineProperties(
        {},
        {
          SELECT: {
            value: Object.defineProperties(SELECT, {
              expand: { value: true, writable: true }, // non-enumerable
              one: { value: column.$refLinks.at(-1).definition.is2one, writable: true }, // non-enumerable
            }),
            enumerable: true,
          },
          as: { value: subqueryAlias, enumerable: true, writable: true },
          elements: { value: elements }, // non-enumerable
        },
      )
    }
  }

  function getTransformedOrderByGroupBy(columns, inOrderBy = false) {
    const res = []
    for (let i = 0; i < columns.length; i++) {
      let col = columns[i]
      if (isCalculatedOnRead(col.$refLinks?.[col.$refLinks.length - 1].definition)) {
        const calcElement = resolveCalculatedElement(col, true)
        res.push(calcElement)
      } else if (pseudos.elements[col.ref?.[0]]) {
        res.push({ ...col })
      } else if (col.ref) {
        if (col.$refLinks.some(link => getDefinition(link.definition.target)?.['@cds.persistence.skip'] === true))
          continue
        if (col.ref.length > 1 && col.ref[0] === '$self' && !col.$refLinks[0].definition.kind) {
          const dollarSelfReplacement = calculateDollarSelfColumn(col)
          res.push(...getTransformedOrderByGroupBy([dollarSelfReplacement], inOrderBy))
          continue
        }
        const { target, definition } = col.$refLinks[0]
        let tableAlias = null
        if (target.SELECT?.columns && inOrderBy) {
          // usually TA is omitted if order by ref is a column
          // if a localized sorting is requested, we add `COLLATE`s
          // later on, which transforms the simple name to an expression
          // --> in an expression, only source elements can be addressed, hence we must add TA
          if (target.SELECT.localized && definition.type === 'cds.String') {
            const referredCol = target.SELECT.columns.find(c => {
              return c.as === col.ref[0] || c.ref?.at(-1) === col.ref[0]
            })
            if (referredCol) {
              // keep sort and nulls properties
              referredCol.sort = col.sort
              referredCol.nulls = col.nulls
              col = referredCol
              if (definition.kind === 'element') {
                tableAlias = getTableAlias(col)
              } else {
                // we must replace the reference with the underlying expression
                const { val, func, args, xpr } = col
                if (val) res.push({ val })
                if (func) res.push({ func, args })
                if (xpr) res.push({ xpr })
                continue
              }
            }
          }
        } else {
          tableAlias = getTableAlias(col) // do not prepend TA if orderBy column addresses element of query
        }
        const leaf = col.$refLinks[col.$refLinks.length - 1].definition
        if (leaf.virtual === true) continue // already in getFlatColumnForElement
        let baseName
        if (col.ref.length >= 2) {
          // leaf might be intermediate structure
          baseName = col.ref
            .map(idOnly)
            .slice(col.ref[0] === tableAlias ? 1 : 0, col.ref.length - 1)
            .join('_')
        }
        const flatColumns = getFlatColumnsFor(col, { baseName, tableAlias })
        /**
         * We can't guarantee that the element order will NOT change in the future.
         * We claim that the element order doesn't matter, hence we can't allow elements
         * in the order by clause which expand to more than one column, as the order impacts
         * the result.
         */
        if (inOrderBy && flatColumns.length > 1)
          throw new Error(`"${getFullName(leaf)}" can't be used in order by as it expands to multiple fields`)
        flatColumns.forEach(fc => {
          if (col.nulls) fc.nulls = col.nulls
          if (col.sort) fc.sort = col.sort
          if (fc.as) delete fc.as
        })
        res.push(...flatColumns)
      } else {
        let transformedColumn
        if (col.SELECT) transformedColumn = transformSubquery(col)
        else if (col.xpr) transformedColumn = { xpr: getTransformedTokenStream(col.xpr) }
        else if (col.func) transformedColumn = { args: getTransformedFunctionArgs(col.args), func: col.func }
        // val
        else transformedColumn = copy(col)
        if (col.sort) transformedColumn.sort = col.sort
        if (col.nulls) transformedColumn.nulls = col.nulls
        res.push(transformedColumn)
      }
    }
    return res
  }

  /**
   * Transforms a subquery.
   *
   * If the current query contains outer queries (is itself a subquery),
   * it appends the current inferred query.
   * Otherwise, it initializes the `outerQueries` array and adds the inferred query.
   * The `outerQueries` property makes sure
   * that the table aliases of the outer queries are accessible within the scope of the subquery.
   * Lastly, it recursively calls cqn4sql on the subquery.
   *
   * @param {object} q - The query to be transformed. This should be a subquery object.
   * @returns {object} - The cqn4sql transformed subquery.
   */
  function transformSubquery(q) {
    if (q.outerQueries) q.outerQueries.push(inferred)
    else {
      const outerQueries = inferred.outerQueries || []
      outerQueries.push(inferred)
      Object.defineProperty(q, 'outerQueries', { value: outerQueries })
    }
    const target = cds.infer.target(inferred) // REVISIT: we should reliably use inferred._target instead
    if (isLocalized(target)) q.SELECT.localized = true
    if (q.SELECT.from.ref && !q.SELECT.from.as) assignUniqueSubqueryAlias()
    return cqn4sql(q, model)

    function assignUniqueSubqueryAlias() {
      if (q.SELECT.from.uniqueSubqueryAlias) return
      const last = q.SELECT.from.ref.at(-1)
      const uniqueSubqueryAlias = inferred.joinTree.addNextAvailableTableAlias(
        getImplicitAlias(last.id || last),
        inferred.outerQueries,
      )
      Object.defineProperty(q.SELECT.from, 'uniqueSubqueryAlias', { value: uniqueSubqueryAlias })
    }
  }

  /**
   * This function converts a wildcard into explicit columns.
   *
   * Based on the query's `$combinedElements` attribute, the function computes the flat column representations
   * and returns them. Additionally, it prepends the respective table alias to each column. Columns specified
   * in the `excluding` clause are ignored during this transformation.
   *
   * Furthermore, foreign keys (FK) for OData CSN and blobs are excluded from the wildcard expansion.
   *
   * @param {array} exclude - An optional list of columns to be excluded during the wildcard expansion.
   * @param {array} replace - An optional list of columns to replace during the wildcard expansion.
   * @param {string} baseName - the explicit alias of the column.
   * Only possible for anonymous expands on implicit table alias: `select from books { { * } as FOO }`
   *
   * @returns {Array} Returns an array of explicit columns derived from the wildcard.
   */
  function getColumnsForWildcard(exclude = [], replace = [], baseName = null) {
    const wildcardColumns = []
    for (const k of Object.keys(inferred.$combinedElements)) {
      if (!exclude.includes(k)) {
        const { index, tableAlias } = inferred.$combinedElements[k][0]
        const element = tableAlias.elements[k]
        // ignore FK for odata csn / ignore blobs from wildcard expansion
        if (isManagedAssocInFlatMode(element) || element.type === 'cds.LargeBinary') continue
        // for wildcard on subquery in from, just reference the elements
        if (tableAlias.SELECT && !element.elements && !element.target) {
          wildcardColumns.push(index ? { ref: [index, k] } : { ref: [k] })
        } else if (isCalculatedOnRead(element)) {
          wildcardColumns.push(resolveCalculatedElement(replace.find(r => r.as === k) || element))
        } else {
          const flatColumns = getFlatColumnsFor(
            element,
            { tableAlias: index, baseName },
            [],
            { exclude, replace },
            true,
          )
          wildcardColumns.push(...flatColumns)
        }
      }
    }
    return wildcardColumns

    /**
     * foreign keys are already part of the elements in a flat model
     * not excluding the associations from the wildcard columns would cause duplicate columns upon foreign key expansion
     * @param {CSN.element} e
     * @returns {boolean} true if the element is a managed association and the model is flat
     */
    function isManagedAssocInFlatMode(e) {
      return (
        e.isAssociation && e.keys && (model.meta.transformation === 'odata' || model.meta.unfolded?.includes('structs'))
      )
    }
  }

  /**
   * Resolve `ref` within `def` and return the element
   *
   * @param {string[]} ref
   * @param {CSN.Artifact} def
   * @returns {CSN.Element}
   */
  function getElementForRef(ref, def) {
    return ref.reduce((prev, res) => {
      return (prev?.elements || prev?.foreignKeys)?.[res] || getDefinition(prev?.target)?.elements[res] // PLEASE REVIEW: should we add the .foreignKey check here for the non-ucsn case?
    }, def)
  }

  /**
   * Recursively expands a structured element into flat columns, representing all leaf paths.
   * This function transforms complex structured elements into simple column representations.
   *
   * For each element, the function checks if it's a structure, an association or a scalar,
   * and proceeds accordingly. If the element is a structure, it recursively fetches flat columns for all sub-elements.
   * If it's an association, it fetches flat columns for it's foreign keys.
   * If it's a scalar, it creates a flat column for it.
   *
   * Columns excluded in a wildcard expansion or replaced by other columns are also handled accordingly.
   *
   * @param {object} column - The structured element which needs to be expanded.
   * @param {{
   *  columnAlias: string
   *  tableAlias: string
   *  baseName: string
   * }} names - configuration object for naming parameters:
   * columnAlias - The explicit alias which the user has defined for the column.
   *                               For instance `{ struct.foo as bar}` will be transformed into
   *                               `{ struct_foo_leaf1 as bar_foo_leaf1, struct_foo_leaf2 as bar_foo_leaf2 }`.
   * tableAlias - The table alias to prepend to the column name. Optional.
   * baseName - The prefixes of the column reference (joined with '_'). Optional.
   * @param {string} columnAlias - The explicit alias which the user has defined for the column.
   *                               For instance `{ struct.foo as bar}` will be transformed into
   *                               `{ struct_foo_leaf1 as bar_foo_leaf1, struct_foo_leaf2 as bar_foo_leaf2 }`.
   * @param {string} tableAlias - The table alias to prepend to the column name. Optional.
   * @param {Array} csnPath - An array containing CSN paths. Optional.
   * @param {Array} exclude - An array of columns to be excluded from the flat structure. Optional.
   * @param {Array} replace - An array of columns to be replaced in the flat structure. Optional.
   *
   * @returns {object[]} Returns an array of flat column(s) for the given element.
   */
  function getFlatColumnsFor(column, names, csnPath = [], excludeAndReplace, isWildcard = false) {
    if (!column) return column
    if (column.val || column.func || column.SELECT) return [column]

    const structsAreUnfoldedAlready = model.meta.unfolded?.includes('str