@sap/cds-dk/lib/init/template/data/as-json.js

Command line client and development toolkit for the SAP Cloud Application Programming Model

const { randomUUID } = require('node:crypto')
const { dirname } = require('node:path')
const cds = require('../../../cds')

let counter = Math.round(Math.random() * 10000) // Default random seed // Non-deterministic behaviour
const MAX_ASSOC_DEPTH = 5

/**
 * Creates JSON test data for the given entities
 *
 * @param {Record<string, object[]>|undefined} data the data with entities and their records to be created
 * @param {object} csn the model
 * @param {Number} numRecords the number of records to be created for each artifact
 * @param {{referenceData? : Record<string, string}} options misc. options
 * @returns {Record<string, object[]>} the filled data
 */
const asJson = module.exports = async (data, csn, numRecords=1, options={}) => {
  let { referenceData, contentType, artifact='entity' } = options

  // if no data is given, use all data
  if (!data) {
    data = csn.all(artifact).reduce((all, e) => { all[e.name] = []; return all }, {})
  }

  // if no reference data is given, scan for it using cds.deploy.resources
  if (!referenceData)  referenceData = await cds.deploy.resources(csn)

  // sort all entities first that are referenced by others, so that dependant entities can be created after them
  const { definitions: defs } = csn
  const names = Object.keys(data).sort((n1, n2) => {
    const n1HasComp = Object.values(defs[n1].elements??[]).some(el => el.type === 'cds.Composition')
    const n2HasComp = Object.values(defs[n2].elements??[]).some(el => el.type === 'cds.Composition')
    if (n1HasComp && !n2HasComp)  return -1
    if (!n1HasComp && n2HasComp)  return 1
    return 0
  })

  // requested entities shall always be created, so remove it from reference data
  Object.entries(referenceData).filter(([, name]) => names.includes(name)).forEach(([file]) => delete referenceData[file])

  addSemantics(csn)

  for (const name of names) {
    if (!data[name] // removed on purpose because records are created inline through compositions
      || data[name].length >= numRecords) { // numRecords already created by following associations
      continue
    }

    let numRecordsToGenerate // numRecords might be reduced if some records were created by following associations
    let records = []
    if (data[name].length > 0) {
      numRecordsToGenerate = numRecords - data[name].length
      records = Array(numRecordsToGenerate).fill({}).map(() => ({}))
    } else {
      numRecordsToGenerate = numRecords
      records = data[name] = Array(numRecordsToGenerate).fill({}).map(() => ({}))
    }
    const artifact = defs[name]

    const idNum = randomNumber()
    for (let i = 0; i < numRecordsToGenerate; i++) {
      fillRecord(records[i], idNum+i, artifact, csn, data, null, 0, referenceData)
    }
    data[name] = data[name].concat(records) // append the new records to the possibly existing and referenced records
    // remove records w/ duplicate keys. This can happen if the key is an enum (and we have less enum values than requested records).
    data[name] = uniqRecords(data[name], artifact)
  }
  for (let [key, d] of Object.entries(data)) {
    if (contentType === 'json' && d.deleteReference && !d.keepReferece) {
      delete data[key]
    }
    delete d.keepReferece
    delete d.deleteReference
    uniqRecords(d, csn.definitions[key])
  }
  return data
}

function fillRecord(record, recordID, artifact, csn, data, parent, depth, referenceData) {
  Object.defineProperty(record, '$id', {value: recordID, enumerable: false})
  if (data[artifact.name]) data[artifact.name].push(record) // add id to data, to be referenced by associations to prevent circular dependencies

  const elems = Object.values(artifact.elements ?? artifact.params ?? {}).sort(asJson.elementsSorter(csn))
  elems.forEach(el => {
    record[el.name] = dataForField(el, artifact, csn, record, data, parent, depth, referenceData)
  })
  return record
}

asJson.elementsSorter = function (csn) {
  return (el1, el2) => {
    // keys first, need to have them ready first for foreign key resolution anyway
    if ( el1.key  && !el2.key) return -1
    if (!el1.key  &&  el2.key) return 1
    // own elements first
    if ( isOwnElement(el1, el1.parent) && !isOwnElement(el2, el2.parent)) return -1
    if (!isOwnElement(el1, el1.parent) &&  isOwnElement(el2, el2.parent)) return 1
    return 0
  }
  function isOwnElement(el, entity) {
    if (entity && entity === el.parent && (!entity.includes || entity.includes.length === 0)) {
      // shortcut if element's entity has no includes
      return true
    }
    // look for elements in entity includes/parents
    for (const inc of entity?.includes||[]) {
      if (csn.definitions[inc]?.elements?.[el.name])  return false
      if (!isOwnElement(el, csn.definitions[inc]))  return false
    }
    return true
  }
}


function dataForField(element, artifact, csn, record, data, parent, depth, referenceData) {
  if (element.virtual || (element['@Core.Computed'] && element.value))
    return
  if (record[element.name] !== undefined)
    return record[element.name] // value is already set by previous run, e.g. CodeList.code sets .name and .descr

  if (element.items?.elements)  // `array of` / `many` with complex type
    return [ fillRecord({}, record['$id'], element.items, csn, data, null, depth+1, referenceData) ]
  else if (element.items?.type) // `array of` / `many` with simple type
    return [ dataForField(element.items, artifact, csn, record, data, null, depth+1, referenceData) ]

  if (element.is_struct) {
    let struct = {}
    for (const el of element.elements) {
      struct[el.name] = dataForField(el, artifact, csn, record, data, null, depth+1, referenceData)
    }
    return struct
  }

  const type = element._type || element.type
  switch (type) {
    case 'cds.Time':
      return '22:59:53'

    case 'cds.Timestamp': {
      const range = {from: new Date('2000-01-01'), to: new Date('2024-01-01')}
      return randomDateInRange(range).toISOString()
    }
    case 'cds.Date': {
      const range = {from: new Date('2000-01-01'), to: new Date('2024-01-01')}
      annotatedDateRange(element, range)
      const generatedDate = randomDateInRange(range)
      const date = generatedDate.toISOString().split('T', 1)[0]
      return date
    }

    case 'cds.DateTime': {
      const range = {from: new Date('2000-01-01'), to: new Date('2024-01-01')}
      annotatedDateRange(element, range)
      const generatedDate = randomDateInRange(range)
      return generatedDate.toISOString()
    }

    case 'cds.hana.CHAR':
    case 'cds.hana.CLOB':
    case 'cds.hana.NCHAR':
    case 'cds.hana.VARCHAR':
    case 'cds.LargeString':
    case 'cds.String': {
      if (element['@data.gen']?.startsWith('faker.')) {
        return fakerValue(element['@data.gen'], csn)
      }
      if (element.enum) {
        const enumName = randomFromList(Object.keys(element.enum))
        const value = element.enum[enumName].val ?? element.enum[enumName].value ?? enumName
        // key of common.CodeList, also fill name and descr with enum name
        if (element.key && artifact.elements.name && artifact.elements.descr)
          record.name = record.descr = enumName
        return value
      }
      if (artifact['@Communication.Contact.email']?.some(mail => mail.address?.['='] === element.name)
        || element['@Communication.IsEmailAddress']
        || element['@cds.on.insert']?.['='] === '$user'
        || element['@cds.on.update']?.['='] === '$user') {
        return randomEmailAddress(element.name, record['$id'], csn)
      }

      let str = `${element.name}-${record['$id']}`

      if (element.key) {
        str = `${artifact.name.split('.').pop()}-${record['$id']}`
        if (element.name === 'locale' && artifact.name.endsWith('texts')) {
          const record = randomFromReferenceData(artifact, csn, referenceData)
          if (record)  return record.locale
          return randomLanguage()
        }
      }
      if (element['@assert.format']) {
        str = randomStrForRegex(element['@assert.format'], csn)
      }

      if (element.length && str.length > element.length) {
        str = str.slice(-element.length) // truncate to length
      }
      return str
    }

    case 'cds.UUID': {
      // create a pseudo-random UUID based on the record ID
      const idStr = '' + record['$id']
      return idStr + randomUUID().slice(idStr.length)
    }

    case 'cds.Boolean':
      return (record['$id'].toString(2).at(-1)) < 0.5

    case 'cds.hana.SMALLINT':
    case 'cds.hana.TINYINT':
    case 'cds.UInt8':
    case 'cds.Int16':
    case 'cds.Int32':
    case 'cds.Int64':
    case 'cds.Integer64':
    case 'cds.Integer': {
      if (element.key)
        return record['$id']
      if (element.enum) {
        const enumName = randomFromList(Object.keys(element.enum))
        const value = element.enum[enumName].val ?? element.enum[enumName].value ?? enumName
        return parseInt(value)
      }
      const range = {from: 0, to: 100}
      annotatedIntRange(element, range)
      return randomIntInRange(range)
    }
    case 'cds.hana.REAL':
    case 'cds.Decimal': {
      const prec  = element.precision ? Math.min(Math.pow(10, element.precision), Number.MAX_SAFE_INTEGER) : 100
      const range = { from: 0, to: prec }
      const scale = element.scale ?? (element.precision ? 0 : 2) // scale is 0 if only precision is given
      annotatedIntRange(element, range)
      return +(randomDecimalInRange(range).toFixed(scale))
    }
    case 'cds.Double': {
      const range = {from: 0, to: 100}
      return randomDecimalInRange(range)
    }
    case 'cds.Composition': {
      const target = csn.definitions[element.target]
      if (!target || artifact.name === target.name || depth > MAX_ASSOC_DEPTH) return

      let targetRecords = [] // in findKeysInOnCondition target records are linked to the entity, we don't want to overwrite any existing data
      if (data[target.name]) data[target.name].deleteReference = true // for json composites references are nested and will be deleted when writing to file

      const targetRecordCount = element.is2many ? 2 : 1 // TODO: make this configurable
      const targetRecordID = randomNumber()
      for (let i = 0; i < targetRecordCount; i++) {
        const newRecord = fillRecord({}, targetRecordID+i, target, csn, data, artifact, depth+1, referenceData)
        if (newRecord) targetRecords.push(newRecord)
      }

      const [ownKey, ownKeyStructured, targetKey] = findKeysInOnCondition(element, target)
      if (ownKey && targetKey) {
        const value = ownKeyStructured ? { [ownKey]: record[ownKey] } : record[ownKey]
        targetRecords.forEach(targetRecord => {
          // Set parent in target as non-enumerable. Json serialization doesn't need it, but csv does.
          Object.defineProperty(targetRecord, targetKey, { value, enumerable: false })
        })
      }
      uniqRecords(targetRecords, target)

      // consistent return data type: array
      return element.is2many ? targetRecords : [targetRecords[0]]
    }
    case 'cds.Association': {
      if (element.is2many)
        return
      const target = csn.definitions[element.target]
      if (!target || depth > MAX_ASSOC_DEPTH)
        return

      let targetRecord = randomFromReferenceData(target, csn, referenceData)
      if (targetRecord) {
        delete data[target.name] // indicate that existing data is used and no new data shall be created
        delete data[target.name + '.texts'] // also delete associated texts entity
      }
      else {
        let targetRecords = data[target.name]
        if (!targetRecords && (element['@mandatory']) || element['@assert.target'])
          targetRecords = data[target.name] = []
        else if (!targetRecords) // target entity is not in the list of entities to be created
          return

        if (targetRecords.length > 0) { // use record generated by following asociation
          targetRecord = targetRecords.at(-1)  // make sure the localized record references the newly generated texts
        } else {
          targetRecord = fillRecord({}, randomNumber(), target, csn, data, artifact, depth+1, referenceData)
          if (targetRecord) {
            targetRecords.push(targetRecord)
            uniqRecords(targetRecords, target)
          }
          if (parent && (parent.name != artifact.name)) data[target.name].keepReferece = true
        }
      }

      // TODO: handle foreign keys correctly
      const assoc = {}
      Object.keys(target.elements)
        .filter(keyName => target.elements[keyName].key)
        .forEach(keyName => assoc[keyName] = targetRecord[keyName])
      return assoc
    }

    case 'cds.hana.BINARY':
    case 'cds.Binary':
    case 'cds.LargeBinary': // TODO add binary
      return
    case 'cds.Vector': // TODO add Vector
      return
    default: {
      console.error(`Unknown type ${element._type} (${element.type}) in '${artifact.name}:${element.name}'`)
      return
    }
  }
}

function randomFromReferenceData (entity, csn, referenceData) {
  const files = Object.entries(referenceData)
    .filter(([, name]) => matchesEntityOrProjection(name, entity, csn))
    .map(([file]) => file)
  if (files.length === 0)  return

  const toCSV = (content) => {
    const records = cds.parse.csv(content)
    if (records?.length > 1) {
      const header = records[0] // 0 is header line
      const randomRecord = records.length === 2 ? records[1] : records[randomIntInRange({from: 1, to: records.length-1} )]
      let result = {}
      for (let i = 0; i < records[0].length; i++) {
        // For csv, which is all strings, convert decimal values to numbers
        // boolean is already covered by CSV parser
        const val = /^cds\.(U?Int|Decimal|Double)/.test(entity.elements[header[i]]?.type) ? +randomRecord[i] : randomRecord[i]
        result[header[i]] = val
      }
      return result
    }
  }
  const toJSON = (records) => {
    if (records.length === 1)
      return records[0]
    else if (records.length > 1)
      return records[randomIntInRange({from: 0, to: records.length-1} )]
  }

  if (files[0].endsWith('.csv')) {
    const content = cds.utils.fs.readFileSync(files[0]).toString()
    return toCSV(content)
  }
  else if (files[0].endsWith('.json')) {
    const content = cds.utils.fs.readFileSync(files[0]).toString()
    const records = JSON.parse(content)
    return toJSON(records)
  }
  else {
    const content = files[0] // for tests, assuming the content is given inline
    try {
      const records = JSON.parse(content)
      return toJSON(records)
    } catch {
      return toCSV(content)
    }
  }
}

/* whether the given name matches the entity or one of its projections */
function matchesEntityOrProjection(name, entity, csn) {
  if (name === entity?.name)  return true
  // follow simle projections
  const p = entity?.query?.SELECT || entity?.projection
  if (p?.from?.ref?.length === 1) {
    if (p.from.ref[0] === name)  return true
    return matchesEntityOrProjection(name, csn.definitions[p.from.ref[0]], csn)
  }
}

function findKeysInOnCondition(assocElement, target) {
  const { on } = assocElement
  let ownKey, targetKey, ownKeyStructured
  if (on?.length === 3 && on[1] === '=') {
    for (let { ref } of [on[0], on[2]]) {
      if (ref.length > 1 && ref[0] === assocElement.name && target.elements[ref[1]]) {
        targetKey = ref[1]
      }
      else if (ref[0] === '$self') {
        ownKey = Object.keys(assocElement.parent.keys)[0]  // TODO composite keys
        ownKeyStructured = true
      }
      else if (assocElement.parent.elements[ref[0]]) {
        ownKey = ref[0]
      }
    }
  }
  return [ownKey, ownKeyStructured, targetKey]
}

function uniqRecords(records, artifact) {
  const keys = Object.keys(artifact.keys||{})
  if (keys.length === 0)  keys.push('$id') // no keys (e.g. in actions) -> use record's $id
  const keySet = new Set()
  for (let i = 0; i < records.length; i++) {
    const key = keys.map(k => records[i][k]).join('|')
    if (keySet.has(key)) {
      // console.debug('Removing', keys, key, 'from', artifact.name)
      records.splice(i, 1)
      i--
    }
    keySet.add(key)
  }
  return records
}

function annotatedDateRange(el, dateRange) {
  if (el['@assert.range']?.length === 2) {
    const dateFrom = new Date(el['@assert.range'][0])
    const dateTo = new Date(el['@assert.range'][1])
    if (dateRange.from < dateFrom) {
      dateRange.from = dateFrom
    }
    if (dateRange.to > dateTo) {
      dateRange.to = dateTo
    }
  }
}

function annotatedIntRange(el, intRange) {
  if (el['@assert.range']?.length === 2) {
    const rangeFrom = el['@assert.range'][0]
    const rangeTo = el['@assert.range'][1]
    if (Number.isInteger(rangeFrom) && intRange.from < rangeFrom) {
      intRange.from = rangeFrom
    }
    if (Number.isInteger(rangeTo) && intRange.to > rangeTo) {
      intRange.to = rangeTo
    }
  }
   //TODO : decimalRange
}

const langList = ['ar', 'cs', 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja', 'ko', 'ms', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'th', 'tr', 'zh']
function randomLanguage() {
  return randomFromList(langList)

}
function randomFromList(list) {
  return list[randomNumber() % list.length]
}

function randomEmailAddress(firstName, num, csn) {
  let addr = fakerValue('faker.internet.email', csn, false)
  if (addr)  return addr
  const provider = randomFromList(['example.com', 'example.org', 'example.net'])
  return `${firstName}.${num.toString(36)}@${provider}`
}

function randomStrForRegex(regexStr, csn) {
  // TODO make 'randexp' a hard dependency when this code is in a plugin
  const randexp = requirePackage('randexp', csn, regexStr)
  if (!randexp)  return `Install package 'randexp' to create data for regex ${regexStr}`
  return randexp.randexp(regexStr)
}

function fakerValue(expr, csn, useFallback=true, ...args) {
  // `en` bundle is way smaller than full faker. TODO use different Faker locales?
  const Faker = requirePackage('@faker-js/faker/locale/en', csn)
  if (!Faker && !useFallback)  return
  if (!Faker)  return `Install package '@faker-js/faker' to create data for ${expr}`

  let resultPart
  let [, category, func] = expr.split('.')
  const cat = Faker.faker[category]
  if (typeof cat !== 'object')  return `Unknown faker category ${category} in ${expr}`
  const funcParts = func.split('::')
  if (funcParts.length === 2) {
    resultPart = funcParts.pop()
    func = funcParts.pop()
  }
  const fn = cat[func]
  if (typeof fn !== 'function')  return `Unknown faker function ${func} in ${expr}`
  if (category === 'internet' && func === 'email') {
    // only allow domains that cannot be registered
    const provider = randomFromList(['example.com', 'example.org', 'example.net'])
    return fn({ provider })
  }
  let res = args ? fn(...args) : fn()
  if (res && resultPart) {
    return res[resultPart]
  }
  return res
}

const packages = {}
function requirePackage(id, csn) {
  try {
    if (!packages[id]) {
      const pkg = resolvePackage(id, csn)
      if (pkg)
        packages[id] = require(pkg)
    }
    return packages[id]
  } catch (err) {
    if (err.code !== 'MODULE_NOT_FOUND')  throw err
  }
}

function resolvePackage(id, csn) {
  try {
    return require.resolve(id, { paths: [...csn.$sources.map(dirname), cds.root, __dirname] })
  } catch (err) {
    if (err.code !== 'MODULE_NOT_FOUND')  throw err

  }
}

// add semantics to well-known entities
function addSemantics(csn) {
  if (resolvePackage('@faker-js/faker', csn)) { // only if faker is installed
    csn.all('entity').forEach(entity => {
      const { elements } = entity
      if (entity.name === 'sap.common.Countries') {
        elements.code['@data.gen'] = 'faker.location.countryCode'
        elements.name['@data.gen'] = elements.descr['@data.gen'] = 'faker.location.country'
      }
      else if (entity.name === 'sap.common.Currencies') {
        elements.code['@data.gen'] = 'faker.finance.currency::code'
        elements.name['@data.gen'] = elements.descr['@data.gen'] = 'faker.finance.currency::name'
        elements.symbol['@data.gen'] = 'faker.finance.currency::symbol'
      }
      else if (entity.name === 'sap.common.Timezones') {
        elements.code['@data.gen'] = 'faker.location.timeZone'
        elements.name['@data.gen'] = elements.descr['@data.gen'] = 'faker.location.timeZone'
      }
    })
  }
}

function randomDateInRange(dateRange) {
  const dateFrom = dateRange.from
  const returnDate = new Date(dateFrom.getUTCFullYear(), dateFrom.getUTCMonth(), dateFrom.getUTCDate())//needed to avoid overwriting the original dateFrom when generating a random Date
  const diffTime = dateRange.to.valueOf() - dateRange.from.valueOf();
  const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
  const step = randomNumber() % diffDays
  returnDate.setUTCDate(returnDate.getUTCDate() + step);
  return returnDate
}

function randomDecimalInRange(decimalRange) {
  const max = decimalRange.to
  const min = decimalRange.from
  const num = min + (randomNumber() % (max - min))
  let dec = randomNumber()
  const divider = Math.pow(10, dec.toString().length)
  dec = dec / divider
  const returnValue = num + dec
  return returnValue
}

function randomIntInRange(intRange) {
  const max = intRange.to
  const min = intRange.from
  return min + (randomNumber() % max)
}

const p = 5651;
const q = 5623;
const M = p * q;
function randomNumber() {
  counter = (counter * counter) % M
  return counter;
}