scimgateway/lib/utils-scim.ts

Using SCIM protocol as a gateway for user provisioning to other endpoints

import dot from 'dot-object'
import fs from 'node:fs'
import path from 'node:path'
import * as utils from './utils.ts'
import scimdefV1Default from './scimdef-v1.json' with { type: 'json' }
import scimdefV2Default from './scimdef-v2.json' with { type: 'json' }
import countries from './countries.json' with { type: 'json' }

type ScimVersion = '1.1' | '2.0' | 1.1 | 2.0
type SCIMBulkOperation = {
  method: string
  path: string
  bulkId?: string
  version?: string
  data?: any
}

// Multi-value attributes are customized from array to object based on type
// except: groups, members and roles
// e.g "emails":[{"value":"bjensen@example.com","type":"work"}] => {"emails": {"work": {"value":"bjensen@example.com","type":"work"}}}
// Cleared values are set as user attributes with blank value ""
// e.g {meta:{attributes:['name.givenName','title']}} => {"name": {"givenName": ""}), "title": ""}

/**
* convert SCIM 1.1 regarding "type converted Object" and blank deleted values, also used by convertedScim20()
*/
export function convertedScim(obj: any, multiValueTypes: string[]): any {
  let err: any = null
  const scimdata: any = structuredClone(obj)
  if (scimdata.schemas) delete scimdata.schemas
  const newMulti: Record<string, any> = {}
  if (!multiValueTypes) multiValueTypes = []

  for (const key in scimdata) {
    if (Array.isArray(scimdata[key]) && (scimdata[key].length > 0)) {
      if (key === 'groups' || key === 'members' || key === 'roles') {
        scimdata[key].forEach(function (element, index) {
          if (element.value) scimdata[key][index].value = decodeURIComponent(element.value)
        })
      } else if (multiValueTypes.includes(key)) { // "type converted object" // groups, roles, member and scim.excludeTypeConvert are not included
        const tmpAddr: any = []
        scimdata[key].forEach(function (element) {
          if (!element.type) element.type = 'undefined' // "none-type"
          if (element.operation && element.operation === 'delete') { // add as delete if same type not included as none delete
            const arr = scimdata[key].filter((obj: Record<string, any>) => obj.type && obj.type === element.type && !obj.operation)
            if (arr.length < 1) {
              if (!newMulti[key]) newMulti[key] = {}
              if (newMulti[key][element.type]) {
                if (['addresses'].includes(key)) { // not checking type, but the others have to be unique
                  for (const i in element) {
                    if (i !== 'type') {
                      if (tmpAddr.includes(i)) {
                        err = new Error(`'type converted object' ${key} - includes more than one element having same ${i}, or ${i} is blank on more than one element - note, setting configuration scim.skipTypeConvert=true will disable this logic/check`)
                      }
                      tmpAddr.push(i)
                    }
                  }
                } else {
                  err = new Error(`'type converted object' ${key} - includes more than one element having same type, or type is blank on more than one element - note, setting configuration scim.skipTypeConvert=true will disable this logic/check`)
                }
              }
              newMulti[key][element.type] = {}
              for (const i in element) {
                newMulti[key][element.type][i] = element[i]
              }
              newMulti[key][element.type].value = '' // delete
            }
          } else {
            if (!newMulti[key]) newMulti[key] = {}
            if (newMulti[key][element.type]) {
              if (['addresses'].includes(key)) { // not checking type, but the others have to be unique
                for (const i in element) {
                  if (i !== 'type') {
                    if (tmpAddr.includes(i)) {
                      err = new Error(`'type converted object' ${key} - includes more than one element having same ${i}, or ${i} is blank on more than one element - note, setting configuration scim.skipTypeConvert=true will disable this logic/check`)
                    }
                    tmpAddr.push(i)
                  }
                }
              } else {
                err = new Error(`'type converted object' ${key} - includes more than one element having same type, or type is blank on more than one element - note, setting configuration scim.skipTypeConvert=true will disable this logic/check`)
              }
            }
            newMulti[key][element.type] = {}
            for (const i in element) {
              newMulti[key][element.type][i] = element[i]
            }
          }
        })
        delete scimdata[key]
      }
    } else if (key === 'active' && typeof scimdata[key] === 'string') {
      const lcase = scimdata.active.toLowerCase()
      if (lcase === 'true') scimdata.active = true
      else if (lcase === 'false') scimdata.active = false
    } else if (key === 'meta') { // cleared attributes e.g { meta: { attributes: [ 'name.givenName', 'title' ] } }
      if (Array.isArray(scimdata.meta.attributes)) {
        scimdata.meta.attributes.forEach((el: string) => {
          let rootKey = ''
          let subKey = ''
          if (el.startsWith('urn:')) { // can't use dot.str on key having dot e.g. urn:ietf:params:scim:schemas:extension:enterprise:2.0:User:department
            const i = el.lastIndexOf(':')
            subKey = el.substring(i + 1)
            if (subKey === 'User' || subKey === 'Group') rootKey = el
            else rootKey = el.substring(0, i)
          }
          if (rootKey) {
            if (!scimdata[rootKey]) scimdata[rootKey] = {}
            dot.str(subKey, '', scimdata[rootKey])
          } else {
            dot.str(el, '', scimdata)
          }
        })
      }
      delete scimdata.meta
    } else { // replace any undefined/null with empty string
      if (typeof scimdata[key] === 'object' && scimdata[key] !== null) {
        for (const k in scimdata[key]) {
          if (typeof scimdata[key][k] === 'object' && scimdata[key][k] !== null) {
            for (const _k in scimdata[key][k]) {
              if (scimdata[key][k][_k] === undefined || scimdata[key][k][_k] === null) {
                scimdata[key][k][_k] = ''
              }
            }
          } else {
            if (scimdata[key][k] === undefined || scimdata[key][k] === null) {
              scimdata[key][k] = ''
            }
          }
        }
      } else if (scimdata[key] === undefined || scimdata[key] === null) {
        scimdata[key] = ''
      }
    }
  }

  for (const key in newMulti) {
    dot.copy(key, key, newMulti, scimdata)
  }

  return [scimdata, err]
}

/**
* convertedScim20 convert SCIM 2.0 patch request to SCIM 1.1 and calls convertedScim() for "type converted Object" and blank deleted values
*
* Scim 2.0:  
* {"schemas":["urn:ietf:params:scim:api:messages:2.0:PatchOp"],"Operations":[{"op":"Replace","path":"name.givenName","value":"Rocky"},{"op":"Remove","path":"name.formatted","value":"Rocky Balboa"},{"op":"Add","path":"emails","value":[{"value":"user@compay.com","type":"work"}]}]}
*
* Scim 1.1  
* {"name":{"givenName":"Rocky","formatted":"Rocky Balboa"},"meta":{"attributes":["name.formatted"]},"emails":[{"value":"user@compay.com","type":"work"}]}
*
* "type converted object" and blank deleted values  
* {"name":{"givenName":"Rocky",formatted:""},"emails":{"work":{"value":"user@company.com","type":"work"}}}
*/
export function convertedScim20(obj: any, multiValueTypes: string[]): any {
  if (!obj.Operations || !Array.isArray(obj.Operations)) return {}
  let scimdata: { [key: string]: any } = { meta: { attributes: [] } } // meta is used for deleted attributes
  const o: any = structuredClone(obj)
  const arrPrimaryDone: any = []
  const primaryOrgType: any = {}

  for (let i = 0; i < o.Operations.length; i++) {
    const element = o.Operations[i]
    let type: any = null
    let typeElement: any = null
    let pathRoot: any = null
    let rePattern: any = /^.*\[(.*) eq (.*)\].*$/
    let arrMatches: any = null
    let primaryValue: any = null

    if (element.op) {
      element.op = element.op.toLowerCase()
      if (element.op === 'delete') element.op = 'remove' // correct none standard
    }

    if (element.path) {
      arrMatches = element.path.match(rePattern)

      if (Array.isArray(arrMatches) && arrMatches.length === 3) { // [type eq "work"]
        if (arrMatches[1] === 'type') type = arrMatches[2].replace(/"/g, '') // work
        else if (arrMatches[1] === 'primary') {
          type = 'primary'
          primaryValue = arrMatches[2].replace(/"/g, '') // True
        }
      }

      rePattern = /^(.*)\[(type|primary) eq .*\]\.(.*)$/ // "path":"addresses[type eq \"work\"].streetAddress" - "path":"roles[primary eq \"True\"].streetAddress"
      arrMatches = element.path.match(rePattern)
      if (Array.isArray(arrMatches)) {
        if (arrMatches.length === 2) {
          pathRoot = arrMatches[1]
        } else if (arrMatches.length === 4) {
          if (type) {
            typeElement = arrMatches[3] // streetAddress

            if (type === 'primary' && !arrPrimaryDone.includes(arrMatches[1])) { // make sure primary is included
              const pObj: any = structuredClone(element)
              pObj.path = pObj.path.substring(0, pObj.path.lastIndexOf('.')) + '.primary'
              pObj.value = primaryValue
              o.Operations.push(pObj)
              arrPrimaryDone.push(arrMatches[1])
              primaryOrgType[arrMatches[1]] = 'primary'
            }
          }
          pathRoot = arrMatches[1]
        }
      } else {
        rePattern = /^(.*)\[type eq .*\]$/ // "path":"addresses[type eq \"work\"]"
        arrMatches = element.path.match(rePattern)
        if (Array.isArray(arrMatches) && arrMatches.length === 2) {
          pathRoot = arrMatches[1]
        }
      }

      rePattern = /^(.*)\[(.*)\](.*)$/
      arrMatches = element.path.match(rePattern)
      if (Array.isArray(arrMatches) && arrMatches.length === 4) {
        pathRoot = arrMatches[1]
        if (arrMatches[3] === '') {
          const a = arrMatches[2].trim().split(' and ') // roles[value eq \"Admins\" and type eq \"Permanent\"]"
          if (a.length > 1) {
            const val: Record<string, any> = {}
            a.forEach(function (el: any) {
              const b = el.split(' eq ')
              if (b.length === 2) {
                val[b[0]] = b[1].replace(/"/g, '')
              }
            })
            if (Object.keys(val).length > 0) {
              element.value = val // {"value":"Admins,"type":"Permanent"}
            }
          } else { // "path":"members[value eq \"bjensen\"]"
            const str = 'value eq'
            if (a[0].startsWith(str)) {
              let val = a[0].substring(str.length + 1).trim()
              val = val.replace(/"/g, '') // "bjensen" =@> bjensen
              element.value = val
              typeElement = 'value'
            }
          }
        } else if (arrMatches[3] === '.value') { // {"path": "emails[type eq \"work\"].value", "value": "new_email@testing.org"}        
          /*
          const arr = arrMatches[2].split(' eq ')
          if (arr.length === 2) {
            if (typeof element.value === 'object' && element.value !== null) {
              element.value[arr[0]] = arr[1].replace(/"/g, '')
            } else {
              type = arr[1].replace(/"/g, '')
              typeElement = 'value'
            }
          }
          */
        }
      }

      if (element.value && Array.isArray(element.value)) {
        element.value.forEach(function (el: any, i: any) { // {"value": [{ "value": "jsmith" }]}
          if (el.value) {
            if (typeof el.value === 'object') { // "value": [{"value": {"id":"c20e145e-5459-4a6c-a074-b942bbd4cfe1","value":"admin","displayName":"Administrator"}}]
              element.value[i] = el.value
            } else if (typeof el.value === 'string' && el.value.substring(0, 1) === '{') { // "value": [{"value":"{\"id\":\"c20e145e-5459-4a6c-a074-b942bbd4cfe1\",\"value\":\"admin\",\"displayName\":\"Administrator\"}"}}]
              try {
                element.value[i] = JSON.parse(el.value)
              } catch (err) { void 0 }
            }
          }
        })
      }

      if (pathRoot) { // pathRoot = emails and path = emails.work.value (we may also have path = pathRoot)
        if (!scimdata[pathRoot]) scimdata[pathRoot] = []
        const index = scimdata[pathRoot].findIndex((el: Record<string, any>) => el.type === type)
        if (index < 0) {
          if (typeof element.value === 'object' && element.value !== null) { // e.g. addresses with no typeElement - value includes object having all attributes
            if (!Array.isArray(element.value)) {
              if (element.op && element.op === 'remove') element.value.operation = 'delete'
              if (!element.value.type && type) element.value.type = type
            }
            scimdata[pathRoot].push(element.value)
          } else {
            const el: { [key: string]: any } = {}
            if (element.op && element.op === 'remove') el.operation = 'delete'
            if (type) el.type = type // members no type
            if (element.value) el[typeElement] = element.value // {"value": "some-value"} or {"steetAddress": "some-address"}
            scimdata[pathRoot].push(el)
          }
        } else {
          if (typeElement === 'value' && scimdata[pathRoot][index].value) { // type exist for value index => duplicate type => push new - duplicates handled by last step confertedScim() if needed
            const el: { [key: string]: any } = {}
            if (element.op && element.op === 'remove') el.operation = 'delete'
            if (type) el.type = type
            el[typeElement] = element.value
            scimdata[pathRoot].push(el)
          } else {
            if (type === 'primary' && typeElement === 'type') { // type=primary, don't change but store and correct to original type later
              primaryOrgType[pathRoot] = element.value
            } else scimdata[pathRoot][index][typeElement] = element.value
            if (element.op && element.op === 'remove') {
              scimdata[pathRoot][index].operation = 'delete'
            }
          }
        }
      } else { // use element.path e.g name.familyName and members
        if (Array.isArray(element.value)) {
          if (element.op === 'replace' && element.value.length === 0) { // members:[]
            scimdata[element.path] = []
          }
          for (let i = 0; i < element.value.length; i++) {
            if (!scimdata[element.path]) scimdata[element.path] = []
            if (element.op && element.op === 'remove') {
              if (typeof element.value[i] === 'object') element.value[i].operation = 'delete'
            }
            scimdata[element.path].push(element.value[i])
          }
        } else { // add to operations loop without path => handled by "no path"
          const obj: { [key: string]: any } = {}
          obj.op = element.op
          obj.value = {}
          obj.value[element.path] = element.value
          o.Operations.push(obj)
        }
      }
    } else { // no path
      for (const key in element.value) {
        if (Array.isArray(element.value[key])) {
          if (element.op === 'replace' && element.value[key].length === 0) { // members:[]
            scimdata[key] = []
          }
          element.value[key].forEach(function (el) {
            if (element.op && element.op === 'remove') el.operation = 'delete'
            if (!scimdata[key]) scimdata[key] = []
            scimdata[key].push(el)
          })
        } else {
          let value = element.value[key]
          if (element?.op === 'remove' || value === undefined || value === null) {
            scimdata.meta.attributes.push(key)
            continue
          }
          if (key.startsWith('urn:')) { // can't use dot.str on key having dot e.g. urn:ietf:params:scim:schemas:extension:enterprise:2.0:User:department
            const i = key.lastIndexOf(':')
            let k = key.substring(i + 1) // User, Group or <parentAttribute>.<childAttribute> - <URN>:<parentAttribute>.<childAttribute> e.g. :User:manager.value
            let rootKey
            if (k === 'User' || k === 'Group') rootKey = key
            else rootKey = key.substring(0, i) // urn:ietf:params:scim:schemas:extension:enterprise:2.0:User
            if (k === 'User' || k === 'Group') { // value is object
              for (const _k in value) {
                if (value[_k] === undefined || value[_k] === null) {
                  scimdata.meta.attributes.push(`${key}:${_k}`)
                  delete value[_k]
                } else if (typeof value[_k] === 'object') { // manager.value
                  if (Object.hasOwn(value[_k], 'value') && (value[_k].value === undefined || value[_k].value === null)) {
                    scimdata.meta.attributes.push(`${key}:${_k}.value`)
                    delete value[_k].value
                  }
                }
              }
              const o: Record<string, any> = {}
              o[rootKey] = value
              scimdata = utils.extendObj(scimdata, o)
            } else {
              if (!scimdata[rootKey]) scimdata[rootKey] = {}
              if (k === 'manager' && typeof value !== 'object') { // fix Azure bug sending manager instead of manager.value
                k = 'manager.value'
              }
              if (!element.op || element.op !== 'remove') { // remove handled by general logic above
                dot.str(k, value, scimdata[rootKey])
              }
            }
          } else {
            if (typeof value === 'object') {
              for (const k in element.value[key]) {
                value = element.value[key][k]
                if ((element.op && element.op === 'remove') || value === null || value === undefined) {
                  scimdata.meta.attributes.push(`${key}.${k}`)
                } else dot.str(`${key}.${k}`, value, scimdata)
              }
            } else dot.str(key, value, scimdata)
          }
        }
      }
    }
  }

  for (const key in primaryOrgType) { // revert back to original type when included
    if (scimdata[key]) {
      const index = scimdata[key].findIndex((el: Record<string, any>) => el.type === 'primary')
      if (index >= 0) {
        if (primaryOrgType[key] === 'primary') delete scimdata[key][index].type // temp have not been changed - remove
        else scimdata[key][index].type = primaryOrgType[key]
      }
    }
  }

  // scimdata now SCIM 1.1 formatted, using convertedScim to get "type converted Object" and blank deleted values
  return convertedScim(scimdata, multiValueTypes)
}

/**
* patchObj updates userObj with the PATCH convertedScim body sent to plugin
*/
export function patchObj(userObj: Record<string, any>, attrObj: Record<string, any>, isMultiValueTypes: (attribute: string) => boolean): any {
  if (typeof userObj !== 'object') return userObj
  if (typeof attrObj !== 'object') return userObj

  for (const key in attrObj) {
    if (Array.isArray(attrObj[key])) { // standard, not using type (e.g roles/groups) or skipTypeConvert=true
      const delArr = attrObj[key].filter(el => el.operation === 'delete')
      const addArr = attrObj[key].filter(el => (!el.operation || el.operation !== 'delete'))
      if (!userObj[key] || !Array.isArray(userObj[key])) userObj[key] = []
      // delete
      userObj[key] = userObj[key].filter((el: Record<string, any>) => {
        const index = delArr.findIndex((e) => {
          let elExist = false
          for (const k in e) {
            if (k === 'primary' || k === 'operation') continue
            if (e[k] !== el[k]) {
              elExist = false
              break
            }
            elExist = true
          }
          return elExist
        })
        if (index >= 0) return false
        else return true
      })
      // add
      addArr.forEach((el) => {
        if (Object.prototype.hasOwnProperty.call(el, 'primary')) {
          if (el.primary === true || (typeof el.primary === 'string' && el.primary.toLowerCase() === 'true')) {
            const index = userObj[key].findIndex((e: Record<string, any>) => e.primary === el.primary)
            if (index >= 0) {
              if (key === 'roles') userObj[key].splice(index, 1) // roles, delete existing role having primary attribute true (new role with primary will be added)
              else userObj[key][index].primary = undefined // remove primary attribute, only one primary
            }
          }
        }
        const index = userObj[key].findIndex((e: Record<string, any>, _index: number) => { // avoid adding existing
          if (el.value && el.value === e.value && el.type === e.type) {
            for (const k in el) {
              e[k] = el[k]
            }
            return true
          }
          let elExist = false
          for (const k in el) {
            if (el[k] !== e[k]) {
              if (k === 'primary') continue
              elExist = false
              break
            }
            elExist = true
          }
          return elExist
        })
        if (index < 0) userObj[key].push(el)
      })
    } else if (isMultiValueTypes(key)) { // "type converted object" logic and original blank type having type "undefined"
      if (!attrObj[key]) delete userObj[key] // blank or null
      for (const el in attrObj[key]) {
        attrObj[key][el].type = el
        if (attrObj[key][el].operation && attrObj[key][el].operation === 'delete') { // delete multivalue
          let type: any = el
          if (type === 'undefined') type = undefined
          if (Array.isArray(userObj[key])) {
            userObj[key] = userObj[key].filter((e: Record<string, any>) => e.type !== type)
            if (userObj[key].length < 1) delete userObj[key]
          }
        } else { // modify/create multivalue
          if (!userObj[key]) userObj[key] = []
          if (attrObj[key][el].primary) { // remove any existing primary attribute, should only have one primary set
            const primVal = attrObj[key][el].primary
            if (primVal === true || (typeof primVal === 'string' && primVal.toLowerCase() === 'true')) {
              const index = userObj[key].findIndex((e: Record<string, any>) => e.primary === primVal)
              if (index >= 0) {
                userObj[key][index].primary = undefined
              }
            }
          }
          const found = userObj[key].find((e: Record<string, any>, i: any) => {
            if (e.type === el || (!e.type && el === 'undefined')) {
              for (const k in attrObj[key][el]) {
                userObj[key][i][k] = attrObj[key][el][k]
                if (k === 'type' && attrObj[key][el][k] === 'undefined') delete userObj[key][i][k] // don't store with type "undefined"
              }
              return true
            } else return false
          })
          if (attrObj[key][el].type && attrObj[key][el].type === 'undefined') delete attrObj[key][el].type // don't store with type "undefined"
          if (!found) userObj[key].push(attrObj[key][el]) // create
        }
      }
    } else {
      // None multi value attribute, blank will be deleted
      if (typeof (attrObj[key]) === 'object' && attrObj[key] !== null) {
        // name.familyName=Bianchi
        if (!userObj[key]) userObj[key] = {} // e.g name object does not exist
        for (const sub in attrObj[key]) {
          if (!userObj[key]) userObj[key] = {}
          if (Object.prototype.hasOwnProperty.call(attrObj[key][sub], 'value')
            && attrObj[key][sub].value === '') delete userObj[key][sub] // object having blank value attribute e.g. {"manager": {"value": "",...}}
          else if (attrObj[key][sub] === '') delete userObj[key][sub]
          else {
            if (!userObj[key]) userObj[key] = {} // may have been deleted by length check below
            userObj[key][sub] = attrObj[key][sub]
          }
          if (Object.keys(userObj[key]).length < 1) delete userObj[key]
        }
      } else {
        if (attrObj[key] === '' || attrObj[key] === null) delete userObj[key]
        else userObj[key] = attrObj[key]
      }
    }
  }
  return userObj
}

// recursiveStrMap is used by endpointMapper() for converting obj according to endpointMap type definition
const recursiveStrMap = function (direction: string, dotMap: any, obj: any, dotPath: any) {
  for (const key in obj) {
    if (obj[key] && obj[key].constructor === Object) recursiveStrMap(direction, dotMap, obj[key], (dotPath ? `${dotPath}.${key}` : key))
    let dotKey = ''
    if (!dotPath) dotKey = key
    else dotKey = `${dotPath}.${key}`
    if (direction === 'outbound') { // outbound
      if (dotMap[`${dotKey}.type`]) {
        const type = dotMap[`${dotKey}.type`].toLowerCase()
        if (type === 'boolean' && obj[key].constructor === String) {
          if ((obj[key]).toLowerCase() === 'true') obj[key] = true
          else if ((obj[key]).toLowerCase() === 'false') obj[key] = false
        } else if (type === 'array') {
          if (!Array.isArray(obj[key])) {
            if (!obj[key]) obj[key] = []
            else obj[key] = obj[key].split(',').map((item: string) => item.trim())
          }
        } else if (dotMap.sAMAccountName) { // Active Directory
          if (dotMap[`${dotKey}.mapTo`].startsWith('addresses.') && dotMap[`${dotKey}.mapTo`].endsWith('.country')) {
            if (countries && Array.isArray(countries)) {
              const arr = countries.filter(el => obj[key] && el.name === obj[key].toUpperCase())
              if (arr.length === 1) { // country name found in countries, include corresponding c (shortname) and countryCode
                obj.c = arr[0]['alpha-2']
                obj.countryCode = arr[0]['country-code']
              }
            }
          }
        }
      }
    } else { // inbound - convert all values to string unless array or boolean
      if (obj[key] === null) delete obj[key] // or set to ''
      else if (obj[key] || obj[key] === false) {
        if (key === 'id') {
          obj[key] = encodeURIComponent(obj[key]) // escaping in case idp don't e.g. Symantec/Broadcom/CA
        }
        if (Array.isArray(obj[key])) { // array
          if (key === 'members' || key === 'groups') {
            for (const el in obj[key]) {
              if (obj[key][el].value) {
                obj[key][el].value = encodeURIComponent(obj[key][el].value) // escaping values because id have also been escaped
              }
            }
          }
        } else if (obj[key].constructor !== Object) {
          if (obj[key].constructor !== Boolean) obj[key] = obj[key].toString() // might have integer that also should be SCIM integer?
        }
      }
    }
  }
}

/**
* SCIM/CustomScim <=> endpoint attribute parsing used by plugins  
* TODO: rewrite and simplify...
* @returns [object/string, err]
*/
export function endpointMapper(direction: string, parseObj: any, mapObj: any) {
  if (direction !== 'inbound' && direction !== 'outbound') {
    const msg = 'Plugin using endpointMapper(direction, parseObj, mapObj) with incorrect direction - direction must be set to \'outbound\' or \'inbound\''
    return [parseObj, new Error(msg)]
  }

  const dotMap = dot.dot(mapObj)
  let str: any
  let isObj = false
  let noneCore = false
  const arrUnsupported: any = []
  const inboundArrCheck: any = []
  const complexArr: any = []
  const complexObj: Record<string, any> = {
    addresses: {},
    emails: {},
    phoneNumbers: {},
    entitlements: {},
    ims: {},
    photos: {},
    // roles: {} using array
  }
  let dotParse: any = null
  const dotNewObj: any = {}

  if (parseObj.constructor === String || parseObj.constructor === Array) str = parseObj // parseObj is attributes list e.g. 'userName,id' or ['userName', 'id']
  else {
    isObj = true
    if (parseObj['@odata.context']) delete parseObj['@odata.context'] // AAD cleanup
    if (parseObj.controls) delete parseObj.controls // Active Directory cleanup
    dotParse = dot.dot(parseObj) // {"name": {"givenName": "myName"}} => {"name.givenName": "myName"}

    // deletion of complex entry => set to blank
    const arrDelete: any = []
    for (const key in dotParse) {
      if (key.endsWith('.operation')) {
        const arr: string[] = key.split('.') // addresses.work.operation
        if (arr.length > 2 && complexObj[arr[0]] && dotParse[key] === 'delete') {
          arrDelete.push(`${arr[0]}.${arr[1]}.`) // addresses.work.
          delete dotParse[key]
        }
      }
    }
    for (let i = 0; i < arrDelete.length; i++) {
      for (const key in dotParse) {
        if (key.startsWith(arrDelete[i])) dotParse[key] = '' // Active Directory: if country included, no logic on country codes cleanup - c (shortname) and countryCode
      }
    }
  }

  switch (direction) {
    case 'outbound':
      if (isObj) { // body (patch/put)
        let foundComplex: string[] = []
        for (let key in dotParse) {
          let found = false
          let arrIndex = 0
          const arr = key.split('.') // multivalue/array - servicePlan.0.value
          const keyOrg = key
          if (arr.length > 1 && arr[arr.length - 1] === 'value') {
            const secondLast = arr.length - 2
            if (!isNaN(parseInt(arr[secondLast]))) { // servicePlan.0.value => servicePlan.0
              for (let i = 0; i < (secondLast); i++) {
                if (i === 0) key = arr[i]
                else key += `.${arr[i]}`
              }
              arrIndex = parseInt(arr[secondLast])
            } else if (arr[secondLast].slice(-1) === ']') { // groups[0].value => groups.value
              const prefix = arr.slice(0, -1).join('.')
              const startPos = prefix.indexOf('[')
              if (startPos > 0) {
                key = prefix.substring(0, startPos) + '.value' // groups.value
                arrIndex = parseInt(prefix.substring(startPos + 1, prefix.length - 1)) // 1
              }
            }
          }
          for (const key2 in dotMap) {
            if (!key2.endsWith('.mapTo')) continue
            const key2Root = key2.split('.').slice(0, -1).join('.') // xx.yy.mapTo => xx.yy
            const isArr = dotMap[`${key2Root}.type`] === 'array'
            const isMulti = key.split('.').length > 1
            const typeInbound = dotMap[`${key2Root}.typeInbound"`]
            if (['complexArray', 'complexObject'].includes(dotMap[`${key2Root}.type`]) || (isArr && !isMulti && !typeInbound)) {
              const tmpKey = key.split('.')[0].split('[')[0]
              if (dotMap[key2] === tmpKey) {
                found = true
                if (foundComplex.includes(tmpKey)) break
                dot.str(key2Root, parseObj[tmpKey], dotNewObj) // copy from original - supports both array and type converted
                foundComplex.push(tmpKey)
                break
              }
            } else if (dotMap[key2].split(',').map((item: string) => item.trim().toLowerCase()).includes(key.toLowerCase())) {
              found = true
              if (dotMap[`${key2Root}.type`] === 'array' && arrIndex >= 0) {
                dotNewObj[`${key2Root}.${arrIndex}`] = dotParse[keyOrg] // servicePlan.0.value => servicePlan.0 and groups[0].value => memberOf.0
              } else dotNewObj[key2Root] = dotParse[key] // {"accountEnabled": {"mapTo": "active"} => str.replace("accountEnabled", "active")
              break
            }
          }
          if (!found) arrUnsupported.push(key)

          // valueMap, must be the last check because using final dotNewObj
          const valueMap = mapObj[key]?.valueMap
          if (valueMap && typeof valueMap === 'object' && Object.keys(valueMap).length > 0 && dotNewObj[key]) {
            const keyFromValue = Object.entries(valueMap).find(([, v]) => v === dotNewObj[key])
            if (keyFromValue && keyFromValue[0]) dotNewObj[key] = keyFromValue[0]
            else arrUnsupported.push(`${key}.valueMap{"n/a":"${dotNewObj[key]}"}`)
          }
        }
      } else { // string (get)
        const resArr: any = []
        let strArr: any = []
        if (Array.isArray(str)) {
          for (let i = 0; i < str.length; i++) {
            strArr = strArr.concat(str[i].split(',').map((item: string) => item.trim())) // supports "id,userName" e.g. {"mapTo": "id,userName"}
          }
        } else strArr = str.split(',').map((item: string) => item.trim())
        for (let i = 0; i < strArr.length; i++) {
          const attr = strArr[i]
          let found = false
          for (const key in mapObj) {
            if (!mapObj[key].mapTo) continue
            const val = mapObj[key].mapTo
            if (val.split(',').map((item: string) => item.trim()).includes(attr)) { // supports { "mapTo": "userName,id" }
              found = true
              if (!resArr.includes(key)) resArr.push(key)
              break
            } else if (val === attr.split('.')[0] && ['complexArray', 'complexObject'].includes(mapObj[key].type)) {
              found = true
              const a = attr.split('.') // entitlements.value
              if (a.length > 0) {
                let tmp = key
                for (let pos = 1; pos < a.length; pos++) {
                  tmp += `.${a[pos]}`
                }
                if (!resArr.includes(tmp)) resArr.push(tmp)
              } else if (!resArr.includes(key)) resArr.push(key)
              break
            } else if (val.split('.')[0] === attr) { // roles.value, manager.managerId
              found = true
              if (!resArr.includes(key)) resArr.push(key)
              break
            } else {
              if (val.startsWith(attr + '.')) { // e.g. emails - complex definition
                if (complexObj[attr]) {
                  found = true
                  if (!resArr.includes(key)) resArr.push(key)
                  // don't break - check for multiple complex definitions
                }
              }
            }
          }
          if (!found) {
            arrUnsupported.push(attr) // comment out? - let caller decide if not to handle unsupported on GET requests (string)
          }
        }
        if (Array.isArray(str)) str = resArr
        else str = resArr.toString()
      }
      break

    case 'inbound':
      if (isObj) {
        let foundComplex: string[] = []
        for (let key in dotParse) {
          if (Array.isArray(dotParse[key]) && dotParse[key].length < 1) continue // avoid including 'value' in empty array if mapTo xx.value
          if (key.startsWith('lastLogon') && !isNaN(dotParse[key])) { // Active Directory date convert e.g. 132340394347050132 => "2020-05-15 20:03:54"
            const ll = new Date(parseInt(dotParse[key], 10) / 10000 - 11644473600000)
            dotParse[key] = ll.getFullYear() + '-'
              + ('00' + (ll.getMonth() + 1)).slice(-2) + '-'
              + ('00' + ll.getDate()).slice(-2) + ' '
              + ('00' + (ll.getHours())).slice(-2) + ':'
              + ('00' + ll.getMinutes()).slice(-2) + ':'
              + ('00' + ll.getSeconds()).slice(-2)
          }

          // first element array gives xxx[0] instead of xxx.0
          let keyArr: any = key.split('.')
          if (keyArr[0].slice(-1) === ']') { // last character=]
            let newStr = keyArr[0]
            newStr = newStr.replace('[', '.')
            newStr = newStr.replace(']', '') // member[0] => member.0
            dotParse[newStr] = dotParse[key]
            key = newStr // will be handled below
          }

          let dotArrIndex = null
          let keyRoot = ''
          keyArr = key.split('.')
          if (keyArr.length > 1) {
            if (!isNaN(keyArr[1])) { // array
              key = keyArr[0] // "proxyAddresses.0" => "proxyAddresses"
              dotArrIndex = keyArr[1]
            } else keyRoot = keyArr[0]
          }

          let mapTo = dotMap[`${key}.mapTo`]
          if (!mapTo) {
            if (keyRoot && dotMap[`${keyRoot}.mapTo`]) { // e.g., type=complex and dotMap is object
              key = keyRoot
              mapTo = dotMap[`${key}.mapTo`]
            } else continue
          }
          if (mapTo.startsWith('urn:')) { // dot workaround for none core (e.g. enterprise and custom schema attributes) having dot in key e.g "2.0": urn:ietf:params:scim:schemas:extension:enterprise:2.0:User.department
            mapTo = mapTo.replace('.', '##') // only first occurence
            noneCore = true
          }

          if (dotMap[`${key}.type`] === 'array') {
            let newStr = mapTo
            if (newStr === 'roles') { // {"mapTo": "roles"} should be {"mapTo": "roles.value"}
              arrUnsupported.push('roles.value')
            }
            let multiValue = true
            if (newStr.indexOf('.value') > 0) newStr = newStr.substring(0, newStr.indexOf('.value')) // multivalue back to ScimGateway - remove .value if defined
            else multiValue = false
            if (dotArrIndex !== null) { // array e.g proxyAddresses.value mapTo proxyAddresses converts proxyAddresses.0 => proxyAddresses.0.value
              if (multiValue) dotNewObj[`${newStr}.${dotArrIndex}.value`] = dotParse[`${key}.${dotArrIndex}`]
              else {
                if (dotMap[`${key}.typeInbound`] && dotMap[`${key}.typeInbound`] === 'string') {
                  if (!dotNewObj[newStr]) dotNewObj[newStr] = dotParse[`${key}.${dotArrIndex}`]
                  else {
                    if (dotMap[`${key}.typeOutboundReverse`]) { // e.g., ldap server not OpenLdap - Active Directory
                      dotNewObj[newStr] = `${dotParse[`${key}.${dotArrIndex}`]},${dotNewObj[newStr]}`
                    } else {
                      dotNewObj[newStr] = `${dotNewObj[newStr]},${dotParse[`${key}.${dotArrIndex}`]}` // OpenLdap - { "isOpenLdap": true }
                    }
                  }
                } else dotNewObj[`${newStr}.${dotArrIndex}`] = dotParse[`${key}.${dotArrIndex}`]
              }
            } else { // type=array but element is not array
              if (multiValue) dotNewObj[`${newStr}.0.value`] = dotParse[key]
              else dotNewObj[newStr] = dotParse[key]
              if (!dotMap[`${key}.typeInbound`] || dotMap[`${key}.typeInbound`] !== 'string') {
                if (!inboundArrCheck.includes(newStr)) inboundArrCheck.push(newStr) // will be checked
              }
            }
          } else if (['complexArray', 'complexObject'].includes(dotMap[`${key}.type`])) { // mapping complex one to one
            if (foundComplex.includes(key)) continue
            dot.str(mapTo, parseObj[key], dotNewObj) // copy from original - supports both array and type converted
            foundComplex.push(key)
          } else { // none array/complex
            const arrMapTo = mapTo.split(',').map((item: string) => item.trim()) // supports {"mapTo": "id,userName"}
            for (let i = 0; i < arrMapTo.length; i++) {
              dotNewObj[arrMapTo[i]] = dotParse[key] // {"active": {"mapTo": "accountEnabled"} => str.replace("accountEnabled", "active")
            }
          }
          if (!['complexArray', 'complexObject'].includes(dotMap[`${key}.type`])) {
            const mapTos = mapTo.split(',').map((item: string) => item.trim()) // 'displayName,addresses.work.postalCode'
            for (let i = 0; i < mapTos.length; i++) {
              const arr = mapTos[i].split('.') // addresses.work.postalCode
              if (arr.length > 2 && complexObj[arr[0]]) {
                complexArr.push(arr[0]) // addresses
              }
            }
          }
          // valueMap, must be the last check because using final dotNewObj
          const valueMap = mapObj[key]?.valueMap
          if (valueMap && typeof valueMap === 'object' && Object.keys(valueMap).length > 0 && dotNewObj[mapTo]) {
            if (valueMap[dotNewObj[mapTo]]) dotNewObj[mapTo] = valueMap[dotNewObj[mapTo]]
            else arrUnsupported.push(`${key}.valueMap{"${dotNewObj[mapTo]}":"n/a"}`)
          }
        }
      } else { // string
        let newStr = ''
        let strArr: any = []
        if (Array.isArray(str)) {
          for (let i = 0; i < str.length; i++) {
            strArr = strArr.concat(str[i].split(',').map((item: string) => item.trim())) // supports "id,userName" e.g. {"mapTo": "id,userName"}
          }
        } else strArr = str.split(',').map((item: string) => item.trim())
        for (let i = 0; i < strArr.length; i++) {
          const attr = strArr[i]
          let found = false
          if (mapObj[attr]) {
            if (mapObj[attr].mapTo) {
              found = true
              const mapTos = mapObj[attr].mapTo.split(',')
              for (let mapTo of mapTos) {
                if (!newStr) newStr = mapTo
                else newStr += `,${mapTo}`
              }
            }
          }
          if (!found) {
            arrUnsupported.push(attr)
          }
        }
        str = newStr
      }
      break

    default:
      str = parseObj
  }

  // error handling (only outbound, not inbound)
  let err: any = null
  const arrErr: string[] = []
  for (let i = 0; i < arrUnsupported.length; i++) {
    const arr = arrUnsupported[i].split('.')
    if (arr.length > 2 && complexObj[arr[0]]) continue // no error on complex
    else if (arr.length === 2 && arr[0].startsWith('roles')) {
      if (arr[1] === 'operation') err = new Error('endpointMapper: roles cannot include operation - telling to be deleted - roles needs proper preprocessing when used by endpointMapper')
      else if (arr[1] !== 'value') continue // no error on roles.display, roles.primary
    }
    arrErr.push(arrUnsupported[i])
  }
  if (!err && arrErr.length > 0) {
    err = new Error(`endpointMapper - no valid map for: ${arrErr.toString()}`)
    if (err.message.includes('valueMap')) err.name = 'invalidValue'
  }

  if (isObj) {
    let newObj = dot.object(dotNewObj) as Record<string, any>// from dot to normal
    if (noneCore) { // revert back dot workaround
      const tmpObj: Record<string, any> = {}
      for (const key in newObj) {
        if (key.startsWith('urn:') && key.includes('##')) {
          const newKey = key.replace('##', '.')
          tmpObj[newKey] = newObj[key]
        } else tmpObj[key] = newObj[key]
      }
      newObj = tmpObj
    }

    if (arrUnsupported.length > 0) { // delete from newObj when not included in map
      for (const i in arrUnsupported) {
        const arr = arrUnsupported[i].split('.') // emails.work.type
        if (!mapObj[arrUnsupported[i]]) dot.delete(arrUnsupported[i], newObj) // delete leaf - check mapObj incase unsupported outbound (scim) match a defined inbound (target) map attribute that should not be deleted
        for (let i = arr.length - 2; i > -1; i--) { // delete above if not empty
          let oStr = arr[0]
          for (let j = 1; j <= i; j++) {
            oStr += `.${arr[j]}`
          }
          const sub = dot.pick(oStr, newObj)
          if (!sub || JSON.stringify(sub) === '{}') {
            dot.delete(oStr, newObj)
          }
        }
      }
    }

    recursiveStrMap(direction, dotMap, newObj, null) // converts according to type defined

    if (direction === 'inbound' && newObj.constructor === Object) { // convert any multivalue object syntax to array
      //
      // map config e.g.:
      // "postalCode": {
      //  "mapTo": "addresses.work.postalCode",
      //  "type": "string"
      // }
      //
      if (complexArr.length > 0) {
        const tmpObj: Record<string, any> = {}
        for (let i = 0; i < complexArr.length; i++) { // e.g. ['emails', 'addresses', 'phoneNumbers', 'ims', 'photos']
          const el = complexArr[i]
          if (newObj[el]) { // { work: { postalCode: '1733' }, work: { streetAddress: 'Roteveien 10' } }
            const tmp: Record<string, any> = {}
            for (const key in newObj[el]) {
              if (newObj[el][key].constructor === Object) { // { postalCode: '1733' }
                if (!tmp[key]) tmp[key] = [{ type: key }]
                const o = tmp[key][0]
                for (const k in newObj[el][key]) { // merge into one object
                  o[k] = newObj[el][key][k]
                }
                tmp[key][0] = o // { addresses: [ { type: 'work', postalCode: '1733', streetAddress: 'Roteveien 10'} ] } - !isNaN because of push
              }
            }
            delete newObj[el]
            tmpObj[el] = []
            for (const key in tmp) {
              tmpObj[el].push(tmp[key][0])
            }
          }
        }
        utils.extendObj(newObj, tmpObj)
      }

      // make sure inboundArrCheck elements are array
      // e.g. AD group "member" could be string if one, and array if more than one
      for (const i in inboundArrCheck) {
        const el = inboundArrCheck[i]
        if (newObj[el] && !Array.isArray(newObj[el])) {
          newObj[el] = [newObj[el]]
        }
      }
    }

    return [newObj, err]
  } else return [str, err]
}

/**
* returns an array of mulitvalue attributes allowing type e.g [emails,addresses,...]  
* objName should be 'User' or 'Group'
*/
export function getMultivalueTypes(objName: string, scimDef: Record<string, any>) { // objName = 'User' or 'Group'
  if (!objName) return []

  const obj = scimDef.Schemas.Resources.find((el: Record<string, any>) => {
    return (el.name === objName)
  })
  if (!obj) return []

  return obj.attributes
    .filter((el: Record<string, any>) => {
      return (el.multiValued === true && el.subAttributes
        && el.subAttributes
          .find(function (subel: Record<string, any>) {
            return (subel.name === 'type')
          })
      )
    })
    .map((obj: Record<string, any>) => obj.name)
}

export function addResources(data: any, startIndex?: string, sortBy?: string, sortOrder?: string) {
  if (!data || JSON.stringify(data) === '{}') data = [] // no user/group found
  const res: { [key: string]: any } = { Resources: [] }
  if (Array.isArray(data)) res.Resources = data
  else if (data.Resources) {
    res.Resources = data.Resources
  } else res.Resources.push(data)

  if (Object.hasOwn(data, 'totalResults')) res.totalResults = data.totalResults
  if (data.startIndex) {
    res.startIndex = data.startIndex
  } else if (startIndex) {
    res.startIndex = parseInt(startIndex, 10)
  } else {
    res.startIndex = 1
  }

  // pagination
  if (!res.totalResults) res.totalResults = res.Resources.length // Specifies the total number of results matching the Consumer query
  res.itemsPerPage = res.Resources.length // Specifies the number of search results returned in a query response page

  if (!res.startIndex || isNaN(res.startIndex)) {
    if (startIndex) res.startIndex = parseInt(startIndex) // The 1-based index of the first result in the current set of search results
    else res.startIndex = 1
  }
  if (res.startIndex > res.totalResults) { // invalid paging request
    res.Resources = []
    res.itemsPerPage = 0
  }

  if (sortBy) res.Resources.sort(utils.sortByKey(sortBy, sortOrder))
  return res
}

export function addSchemasStripAttr(data: Record<string, any>, isScimv2: boolean, type?: string, attributes?: string, excludedAttributes?: string, location?: string) {
  if (!type) {
    if (isScimv2) data.schemas = ['urn:ietf:params:scim:api:messages:2.0:ListResponse']
    else data.schemas = ['urn:scim:schemas:core:1.0']
    return data
  }

  if (data.Resources) {
    if (isScimv2) data.schemas = ['urn:ietf:params:scim:api:messages:2.0:ListResponse']
    else data.schemas = ['urn:scim:schemas:core:1.0']
    for (let i = 0; i < data.Resources.length; i++) {
      utils.getEtag(data.Resources[i])
      data.Resources[i] = utils.stripObj(data.Resources[i], attributes, excludedAttributes)

      if (isScimv2) { // scim v2 add schemas/resourceType on each element
        if (type === 'User') {
          const val = 'urn:ietf:params:scim:schemas:core:2.0:User'
          if (!data.Resources[i].schemas) data.Resources[i].schemas = [val]
          else if (!data.Resources[i].schemas.includes(val)) data.Resources[i].schemas.push(val)
          if (!data.Resources[i].meta) data.Resources[i].meta = {}
          data.Resources[i].meta.resourceType = type
          if (location && data.Resources[i].id) data.Resources[i].meta.location = `${location}/${data.Resources[i].id}`
        } else if (type === 'Group') {
          const val = 'urn:ietf:params:scim:schemas:core:2.0:Group'
          if (!data.Resources[i].schemas) data.Resources[i].schemas = [val]
          else if (!data.Resources[i].schemas.includes(val)) data.Resources[i].schemas.push(val)
          if (!data.Resources[i].meta) data.Resources[i].meta = {}
          data.Resources[i].meta.resourceType = type
        } else if (type === 'Entitlement') {
          const val = 'urn:ietf:params:scim:schemas:custom:2.0:Entitlement'
          if (!data.Resources[i].schemas) data.Resources[i].schemas = [val]
          else if (!data.Resources[i].schemas.includes(val)) data.Resources[i].schemas.push(val)
          if (!data.Resources[i].meta) data.Resources[i].meta = {}
          data.Resources[i].meta.resourceType = type
        }
      }
      if (location && data.Resources[i].id) {
        if (!data.Resources[i].meta) data.Resources[i].meta = {}
        data.Resources[i].meta.location = `${location}/${data.Resources[i].id}`
      }
      for (const key in data.Resources[i]) {
        if (key.startsWith('urn:')) {
          if (key.includes(':1.0')) {
            if (!data.schemas) data.schemas = []
            if (!data.schemas.includes(key)) data.schemas.push(key)
          } else { // scim v2 add none core schemas on each element
            if (!data.Resources[i].schemas) data.Resources[i].schemas = []
            if (!data.Resources[i].schemas.includes(key)) data.Resources[i].schemas.push(key)