@arcane-utils/datastore/dist/index.cjs.map

Datastore helper functions.

{"version":3,"sources":["../src/index.ts","../src/datastore.ts"],"sourcesContent":["export * as googleDatastore from '@google-cloud/datastore'\nexport {\n    INGESTION_EXCLUDE_FROM_INDEXES,\n    PROCESSING_EXCLUDE_FROM_INDEXES,\n    EXPORT_EXCLUDE_FROM_INDEXES_CLASSIC,\n    EXPORT_EXCLUDE_FROM_INDEXES,\n    PROTECTION_EXCLUDE_FROM_INDEXES,\n    DATA_EXPORT_KIND,\n    DATA_INGESTION_KIND,\n    DATA_PROCESSING_KIND,\n    PROTECTION_KIND,\n    DATA_ALERT_KIND,\n    getDatastoreClient,\n    saveEntity,\n    getEntity,\n    getEntityExcludeFromIndexes,\n} from './datastore';\n","import promiseRetry from 'promise-retry'\nimport { cloneDeep } from 'lodash'\nimport { Datastore, Key, Entity } from '@google-cloud/datastore'\n\nimport { NotFoundError } from '@arcane-utils/error'\n\nconst COMMON_EMBEDDED_KEYS = ['execution_info', 'parameters']\nconst INGESTION_EXCLUDE_FROM_INDEXES_CLASSIC = ['parameters', 'name',\n  'is_archived', 'latest_exec',\n  'latest_load', 'execution_info',\n  'headers', 'meta', 'types_override', 'data_pre_preparation_enabled']\nconst INGESTION_EMBEDDED_KEYS = [...COMMON_EMBEDDED_KEYS, 'headers', 'meta', 'product_evolution_info']\nexport const INGESTION_EXCLUDE_FROM_INDEXES = {\n  classic: INGESTION_EXCLUDE_FROM_INDEXES_CLASSIC,\n  embedded: INGESTION_EMBEDDED_KEYS\n}\n\nconst PROCESSING_EXCLUDE_FROM_INDEXES_CLASSIC = ['name', 'parameters', 'sleepTime',\n  'latestExec', 'execution_info']\nconst PROCESSING_EMBEDDED_KEYS = [...COMMON_EMBEDDED_KEYS]\nexport const PROCESSING_EXCLUDE_FROM_INDEXES = {\n  classic: PROCESSING_EXCLUDE_FROM_INDEXES_CLASSIC,\n  embedded: PROCESSING_EMBEDDED_KEYS\n}\n\nexport const EXPORT_EXCLUDE_FROM_INDEXES_CLASSIC = ['parameters', 'name',\n  'export_format', 'feed_setup',\n  'latest_exec', 'execution_info',\n  'dataflow_job_status', 'dataflow_job_id', 'execution_time']\nconst EXPORT_EMBEDDED_KEYS = [...COMMON_EMBEDDED_KEYS, 'feed_setup', 'execution_time']\nexport const EXPORT_EXCLUDE_FROM_INDEXES = {\n  classic: EXPORT_EXCLUDE_FROM_INDEXES_CLASSIC,\n  embedded: EXPORT_EMBEDDED_KEYS\n}\n\nexport const PROTECTION_EXCLUDE_FROM_INDEXES = {\n  classic: [\n    'size_variation_threshold', 'last_modified', 'current_size', 'previous_size', 'notif_name'\n  ]\n}\n\nexport const DATA_EXPORT_KIND = 'smart-feeds-data-exports'\nexport const DATA_INGESTION_KIND = 'smart-feeds-data-ingestions'\nexport const DATA_PROCESSING_KIND = 'smart-feeds-data-processings'\nexport const PROTECTION_KIND = 'protection-file-size'\nexport const DATA_ALERT_KIND = 'alert-file-change'\n\nconst getAllExcludeFromIndexes = (input: any, prefix: any): string[] => {\n  if (Array.isArray(input)) {\n    return [...new Set(\n      input\n        .flatMap(value => getAllExcludeFromIndexes(value, `${prefix}[]`))\n    )]\n  } if (typeof input === 'object' && input !== null) {\n    return [...new Set([prefix, ...Object.entries(input)\n      .flatMap(([key, value]) => getAllExcludeFromIndexes(value, `${prefix}.${key}`))\n    ]\n    )]\n  }\n  return [prefix]\n}\n\n/*\n* Get Datastore client.\n*\n* To be used in Cloud Functions to prevent memory leak when declaring API clients as global declaration.\n*/\nexport const getDatastoreClient = (gcpProject: string, serviceAccount: string) => {\n  return new Datastore({\n    projectId: gcpProject,\n    keyFilename: serviceAccount\n  })\n}\n\n/*\n* Get excludeFromIndexes for an entity by searching all subindexes in embeddedKeysToExcludeFromIndex and combining it with excludeFromIndexes.\n*/\nexport const getEntityExcludeFromIndexes = (\n  entity: Entity,\n  excludeFromIndexes: string[],\n  embeddedKeysToExcludeFromIndex: string[]\n): string[] => {\n  const finalExcludeFromIndexes = excludeFromIndexes\n    .concat(embeddedKeysToExcludeFromIndex\n      .flatMap(key => getAllExcludeFromIndexes(entity[key], key))\n    )\n  return finalExcludeFromIndexes\n}\n\nexport const saveEntity = async (\n  kind: string,\n  entityId: string,\n  updatedProperties: any,\n  datastore: Datastore,\n  excludeFromIndexes: string[] = [],\n  embeddedKeysToExcludeFromIndex: string[] = [],\n  logger: any = console\n): Promise<string | undefined> => {\n  return promiseRetry(async (retry, number) => {\n    const updatedPropertiesCopy = cloneDeep(updatedProperties)\n    logger.info(`Saving updated properties for entity ${entityId}`)\n    const transaction = datastore.transaction()\n    try {\n      await transaction.run()\n      const [newEntity] = await transaction.get({ kind, id: entityId } as Key)\n      if (!newEntity) {\n        return Promise.resolve(`Entity ${entityId} was deleted`)\n      }\n      let updatedEntity = newEntity\n      if (Object.keys(updatedPropertiesCopy).some(key => key === 'parameters')) {\n        updatedEntity.parameters = {\n          ...updatedEntity.parameters,\n          ...updatedPropertiesCopy.parameters\n        }\n        delete updatedPropertiesCopy.parameters\n      }\n\n      if (Object.keys(updatedPropertiesCopy).some(key => key === 'execution_info')) {\n        updatedEntity.execution_info = {\n          ...updatedEntity.execution_info,\n          ...updatedPropertiesCopy.execution_info\n        }\n        delete updatedPropertiesCopy.execution_info\n      }\n\n      updatedEntity = {\n        ...updatedEntity,\n        ...updatedPropertiesCopy\n      }\n      const finalExcludeFromIndexes = getEntityExcludeFromIndexes(\n        updatedEntity,\n        excludeFromIndexes,\n        embeddedKeysToExcludeFromIndex\n      )\n\n      transaction.save({\n        key: { kind, id: updatedEntity.id },\n        excludeFromIndexes: finalExcludeFromIndexes,\n        data: updatedEntity\n      })\n      await transaction.commit()\n      const logMessage = `Entity ${entityId} was correctly updated`\n      logger.info(logMessage)\n      return Promise.resolve(logMessage)\n    } catch (err: any) {\n      logger.info(`Transaction failed because: ${err.message}. Attempt is ${number}. Trying again...`)\n      retry(err)\n    }\n  },\n    {\n      retries: 5,\n      factor: 2,\n      minTimeout: 1000\n    }\n  )\n}\n\nexport const getEntity = async (kind: string, entityId: string, datastore: Datastore, logger: any = console, key: Key | null = null) => {\n  const entityKey = key ?? { kind, id: entityId }\n  return promiseRetry(async (retry) => {\n    let entity\n    try {\n      [entity] = await datastore.get(entityKey as Key)\n    } catch (err) {\n      logger.info(`Error while getting entity ${err}`)\n      retry(err)\n    }\n    if (!entity) {\n      throw new NotFoundError(`No corresponding ${kind} entity id ${entityId}.`)\n    }\n    return entity\n  },\n    {\n      retries: 5,\n      factor: 2,\n      minTimeout: 1000\n    })\n}\n"],"mappings":"6iBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,qBAAAE,EAAA,qBAAAC,EAAA,wBAAAC,EAAA,yBAAAC,EAAA,gCAAAC,EAAA,wCAAAC,EAAA,mCAAAC,EAAA,oCAAAC,EAAA,oCAAAC,EAAA,oBAAAC,EAAA,uBAAAC,EAAA,cAAAC,EAAA,gCAAAC,EAAA,oBAAAC,EAAA,eAAAC,IAAA,eAAAC,EAAAjB,GAAA,IAAAe,EAAiC,wCCAjC,IAAAG,EAAyB,8BACzBC,EAA0B,kBAC1BC,EAAuC,mCAEvCC,EAA8B,+BAExBC,EAAuB,CAAC,iBAAkB,YAAY,EACtDC,EAAyC,CAAC,aAAc,OAC5D,cAAe,cACf,cAAe,iBACf,UAAW,OAAQ,iBAAkB,8BAA8B,EAC/DC,EAA0B,CAAC,GAAGF,EAAsB,UAAW,OAAQ,wBAAwB,EACxFG,EAAiC,CAC5C,QAASF,EACT,SAAUC,CACZ,EAEME,EAA0C,CAAC,OAAQ,aAAc,YACrE,aAAc,gBAAgB,EAC1BC,EAA2B,CAAC,GAAGL,CAAoB,EAC5CM,EAAkC,CAC7C,QAASF,EACT,SAAUC,CACZ,EAEaE,EAAsC,CAAC,aAAc,OAChE,gBAAiB,aACjB,cAAe,iBACf,sBAAuB,kBAAmB,gBAAgB,EACtDC,EAAuB,CAAC,GAAGR,EAAsB,aAAc,gBAAgB,EACxES,EAA8B,CACzC,QAASF,EACT,SAAUC,CACZ,EAEaE,EAAkC,CAC7C,QAAS,CACP,2BAA4B,gBAAiB,eAAgB,gBAAiB,YAChF,CACF,EAEaC,EAAmB,2BACnBC,EAAsB,8BACtBC,EAAuB,+BACvBC,EAAkB,uBAClBC,EAAkB,oBAEzBC,EAA2B,CAACC,EAAYC,IACxC,MAAM,QAAQD,CAAK,EACd,CAAC,GAAG,IAAI,IACbA,EACG,QAAQE,GAASH,EAAyBG,EAAO,GAAGD,CAAM,IAAI,CAAC,CACpE,CAAC,EACG,OAAOD,GAAU,UAAYA,IAAU,KACpC,CAAC,GAAG,IAAI,IAAI,CAACC,EAAQ,GAAG,OAAO,QAAQD,CAAK,EAChD,QAAQ,CAAC,CAACG,EAAKD,CAAK,IAAMH,EAAyBG,EAAO,GAAGD,CAAM,IAAIE,CAAG,EAAE,CAAC,CAChF,CACA,CAAC,EAEI,CAACF,CAAM,EAQHG,EAAqB,CAACC,EAAoBC,IAC9C,IAAI,YAAU,CACnB,UAAWD,EACX,YAAaC,CACf,CAAC,EAMUC,EAA8B,CACzCC,EACAC,EACAC,IAEgCD,EAC7B,OAAOC,EACL,QAAQP,GAAOJ,EAAyBS,EAAOL,CAAG,EAAGA,CAAG,CAAC,CAC5D,EAISQ,EAAa,MACxBC,EACAC,EACAC,EACAC,EACAN,EAA+B,CAAC,EAChCC,EAA2C,CAAC,EAC5CM,EAAc,aAEP,EAAAC,SAAa,MAAOC,EAAOC,IAAW,CAC3C,IAAMC,KAAwB,aAAUN,CAAiB,EACzDE,EAAO,KAAK,wCAAwCH,CAAQ,EAAE,EAC9D,IAAMQ,EAAcN,EAAU,YAAY,EAC1C,GAAI,CACF,MAAMM,EAAY,IAAI,EACtB,GAAM,CAACC,CAAS,EAAI,MAAMD,EAAY,IAAI,CAAE,KAAAT,EAAM,GAAIC,CAAS,CAAQ,EACvE,GAAI,CAACS,EACH,OAAO,QAAQ,QAAQ,UAAUT,CAAQ,cAAc,EAEzD,IAAIU,EAAgBD,EAChB,OAAO,KAAKF,CAAqB,EAAE,KAAKjB,GAAOA,IAAQ,YAAY,IACrEoB,EAAc,WAAa,CACzB,GAAGA,EAAc,WACjB,GAAGH,EAAsB,UAC3B,EACA,OAAOA,EAAsB,YAG3B,OAAO,KAAKA,CAAqB,EAAE,KAAKjB,GAAOA,IAAQ,gBAAgB,IACzEoB,EAAc,eAAiB,CAC7B,GAAGA,EAAc,eACjB,GAAGH,EAAsB,cAC3B,EACA,OAAOA,EAAsB,gBAG/BG,EAAgB,CACd,GAAGA,EACH,GAAGH,CACL,EACA,IAAMI,EAA0BjB,EAC9BgB,EACAd,EACAC,CACF,EAEAW,EAAY,KAAK,CACf,IAAK,CAAE,KAAAT,EAAM,GAAIW,EAAc,EAAG,EAClC,mBAAoBC,EACpB,KAAMD,CACR,CAAC,EACD,MAAMF,EAAY,OAAO,EACzB,IAAMI,EAAa,UAAUZ,CAAQ,yBACrC,OAAAG,EAAO,KAAKS,CAAU,EACf,QAAQ,QAAQA,CAAU,CACnC,OAASC,EAAU,CACjBV,EAAO,KAAK,+BAA+BU,EAAI,OAAO,gBAAgBP,CAAM,mBAAmB,EAC/FD,EAAMQ,CAAG,CACX,CACF,EACE,CACE,QAAS,EACT,OAAQ,EACR,WAAY,GACd,CACF,EAGWC,EAAY,MAAOf,EAAcC,EAAkBE,EAAsBC,EAAc,QAASb,EAAkB,OAAS,CACtI,IAAMyB,EAAYzB,GAAO,CAAE,KAAAS,EAAM,GAAIC,CAAS,EAC9C,SAAO,EAAAI,SAAa,MAAOC,GAAU,CACnC,IAAIV,EACJ,GAAI,CACF,CAACA,CAAM,EAAI,MAAMO,EAAU,IAAIa,CAAgB,CACjD,OAASF,EAAK,CACZV,EAAO,KAAK,8BAA8BU,CAAG,EAAE,EAC/CR,EAAMQ,CAAG,CACX,CACA,GAAI,CAAClB,EACH,MAAM,IAAI,gBAAc,oBAAoBI,CAAI,cAAcC,CAAQ,GAAG,EAE3E,OAAOL,CACT,EACE,CACE,QAAS,EACT,OAAQ,EACR,WAAY,GACd,CAAC,CACL","names":["src_exports","__export","DATA_ALERT_KIND","DATA_EXPORT_KIND","DATA_INGESTION_KIND","DATA_PROCESSING_KIND","EXPORT_EXCLUDE_FROM_INDEXES","EXPORT_EXCLUDE_FROM_INDEXES_CLASSIC","INGESTION_EXCLUDE_FROM_INDEXES","PROCESSING_EXCLUDE_FROM_INDEXES","PROTECTION_EXCLUDE_FROM_INDEXES","PROTECTION_KIND","getDatastoreClient","getEntity","getEntityExcludeFromIndexes","googleDatastore","saveEntity","__toCommonJS","import_promise_retry","import_lodash","import_datastore","import_error","COMMON_EMBEDDED_KEYS","INGESTION_EXCLUDE_FROM_INDEXES_CLASSIC","INGESTION_EMBEDDED_KEYS","INGESTION_EXCLUDE_FROM_INDEXES","PROCESSING_EXCLUDE_FROM_INDEXES_CLASSIC","PROCESSING_EMBEDDED_KEYS","PROCESSING_EXCLUDE_FROM_INDEXES","EXPORT_EXCLUDE_FROM_INDEXES_CLASSIC","EXPORT_EMBEDDED_KEYS","EXPORT_EXCLUDE_FROM_INDEXES","PROTECTION_EXCLUDE_FROM_INDEXES","DATA_EXPORT_KIND","DATA_INGESTION_KIND","DATA_PROCESSING_KIND","PROTECTION_KIND","DATA_ALERT_KIND","getAllExcludeFromIndexes","input","prefix","value","key","getDatastoreClient","gcpProject","serviceAccount","getEntityExcludeFromIndexes","entity","excludeFromIndexes","embeddedKeysToExcludeFromIndex","saveEntity","kind","entityId","updatedProperties","datastore","logger","promiseRetry","retry","number","updatedPropertiesCopy","transaction","newEntity","updatedEntity","finalExcludeFromIndexes","logMessage","err","getEntity","entityKey"]}