@arcane-utils/ingestion/dist/index.cjs.map

Ingestion utilities for Arcane

{"version":3,"sources":["../src/index.ts","../src/ingestion.ts","../src/history.ts","../src/executionHandler.ts"],"sourcesContent":["export {\n    shouldCallProductEvolutionCalculator,\n    createFileName,\n    PREFIX_IMPORT\n} from './ingestion'\n\nexport {\n    storeExecutionDetails,\n    getHistoricalTableName,\n    HISTORICAL_TABLE_SCHEMA\n} from './history'\n\nexport {\n    SkipExecutionError,\n    handleStartExecution\n} from './executionHandler'\n","\nimport { isNil } from 'lodash'\nimport moment from 'moment'\n\nimport { Ingestion } from '@arcane-utils/types'\n\n\n/**\n * Determines if the product evolution information should be called.\n *\n * This function evaluates whether the product evolution information needs to be updated\n * based on the following criteria:\n * - The last calculation date is not the same as today.\n * - The identifier key exists.\n * - The identifier key is not empty.\n *\n * @param {Ingestion} dataIngestion - The ingestion data object containing product evolution information.\n * @returns {boolean} - Returns true if product evolution information should be called, false otherwise.\n */\nexport const shouldCallProductEvolutionCalculator = (\n  dataIngestion: Ingestion\n): boolean => {\n  const today = moment()\n  if (\n    isNil(dataIngestion.product_evolution_info) ||\n    isNil(dataIngestion.product_evolution_info.identifier_key) ||\n    dataIngestion.product_evolution_info.identifier_key.length === 0\n  ) {\n    return false\n  }\n  if (isNil(dataIngestion.product_evolution_info.last_calculation) || isNil(dataIngestion.product_evolution_info.last_calculation.date)) {\n    return true\n  }\n  const lastCalculationDate = moment(dataIngestion.product_evolution_info.last_calculation.date)\n  return !(today.isSame(lastCalculationDate, 'day'))\n}\n\n\n\nexport const PREFIX_IMPORT = 'pf__import__'\n\n/**\n * Builds the datalake import file name for a given ingestion, mirroring the Python\n * convention: `pf__import__<client_id>__<import_id>` with every space replaced by `_`.\n *\n * @param clientId - Id of the client owning the import.\n * @param importId - Id of the import (e.g. the master ingestion id).\n * @returns The normalized import file name.\n */\nexport const createFileName = (clientId: string, importId: number): string =>\n  [`${PREFIX_IMPORT}${clientId}`, importId.toString()].join('__').replace(/ /g, '_')\n","import type winston from 'winston'\nimport { getJobResult, type googleBigquery } from '@arcane-utils/bigquery'\nimport { type IngestionHeader } from '@arcane-utils/types'\nimport { type JobLabels } from '@arcane-utils/job-helper'\n\nexport const getHistoricalTableName = (ingestionId: number): string => `historical_${ingestionId}`\nexport const HISTORICAL_TABLE_SCHEMA = [\n  { name: 'timestamp', type: 'TIMESTAMP' },\n  { name: 'invoker', type: 'STRING' },\n  { name: 'headers', type: 'STRING' },\n  { name: 'additional_info', type: 'STRING' }\n]\n\n/**\n * Creates a historical table if it doesn't exist.\n *\n * @param tableId - The table identifier.\n * @param bigqueryClient - Instance of the BigQuery client.\n * @returns {Promise<void>} - Resolves when the table is created.\n */\nconst createHistoricalTable = async (\n  tableId: string,\n  bigqueryClient: googleBigquery.BigQuery\n): Promise<void> => {\n  const historicalDataset = process.env.HISTORICAL_DATASET\n\n  if (!historicalDataset) {\n    throw new Error('HISTORICAL_DATASET environment variable is not defined')\n  }\n\n  const dataset = bigqueryClient.dataset(historicalDataset)\n  await dataset.createTable(tableId, { schema: HISTORICAL_TABLE_SCHEMA })\n}\n\n/**\n * Stores execution details in a BigQuery table named historical_{ingestion_id}.\n * The table is created if it does not exist.\n * This function handles both changed files (with jobId and jobRows details) and unchanged files (same hash, without job details)\n *\n * @param bigqueryClient - Instance of the BigQuery client.\n * @param ingestionId - Ingestion Id.\n * @param monitoringId - monitoring Id of the execution.\n * @param headers - Array of ingestion headers to be stored. Will stored key values jsonified.\n * @param jobRows - Number of rows processed by the load job. Null for unchanged files.\n * @param jobId - Load job Id to be stored. Empty string for unchanged files.\n * @param fileHash - Hash of the file being processed.\n * @param logger - Instance of the logger.\n * @param labels - Job labels to be stored in the historical table.\n * @param invoker - Optional string to identify the invoker of the execution during manual fetch.\n * @returns {Promise<void>} - Resolves when the operation is complete.\n */\nexport const storeExecutionDetails = async (\n  bigqueryClient: googleBigquery.BigQuery,\n  ingestionId: number,\n  monitoringId: string,\n  headers: IngestionHeader[] | undefined,\n  jobRows: number | null,\n  jobId: string,\n  fileHash: string,\n  logger: winston.Logger,\n  labels: JobLabels,\n  invoker?: string | null\n): Promise<void> => {\n  const tableId = getHistoricalTableName(ingestionId)\n  const fullTableId = `${process.env.GCP_PROJECT}.${process.env.HISTORICAL_DATASET}.${tableId}`\n\n  const row = {\n    timestamp: new Date().toISOString(),\n    invoker: invoker ?? null,\n    headers: (headers ?? []).map(header => header.key).join(','),\n    additional_info: JSON.stringify({\n      monitoringId,\n      jobRows,\n      jobId,\n      fileHash\n    })\n  }\n\n  const query = `\n    INSERT INTO \\`${fullTableId}\\` (timestamp, invoker, headers, additional_info)\n    VALUES (@timestamp, @invoker, @headers, @additional_info)\n  `\n  const options = {\n    query,\n    location: 'EU',\n    labels,\n    params: {\n      ...row\n    },\n    // We need to specify the type for the invoker parameter because it can be explicitly null, \n    // and the BigQuery node library needs to know how to handle it.\n    types: {\n        invoker: 'STRING'\n    }\n  }\n\n  try {\n    const [job] = await bigqueryClient.createQueryJob(options)\n    await getJobResult(\n      job.id!,\n      'storeExecutionDetails',\n      bigqueryClient,\n      logger\n    )\n    logger.info(`Row inserted into ${tableId} successfully.`)\n  } catch (err: any) {\n    if (err.message.includes('Not found')) {\n      logger.info('Historical Table not found. Creating it...')\n      await createHistoricalTable(tableId, bigqueryClient)\n      logger.info(`Table ${tableId} created. Retrying insert...`)\n      const [job] = await bigqueryClient.createQueryJob(options)\n      await getJobResult(\n        job.id!,\n        'storeExecutionDetails',\n        bigqueryClient,\n        logger\n      )\n      logger.info(`Row inserted into ${tableId} after table creation.`)\n    } else {\n      console.error('DML insert failed:', err)\n    }\n  }\n}\n","import { type googleDatastore } from '@arcane-utils/datastore'\nimport { type Logger } from '@arcane-utils/logger'\nimport { type Ingestion, type IngestionType, type MonitoringStatus } from '@arcane-utils/types'\n\n/**\n * Custom error to indicate that an execution should be skipped due to outdated\n * schedule information or concurrent execution.\n */\nexport class SkipExecutionError extends Error {\n  constructor (message: string) {\n    super(message)\n    this.name = 'SkipExecutionError'\n  }\n}\n\ninterface IngestionStartBody {\n  entity_id: string\n  monitoring_id: string\n  schedule_version: string\n  manual_execution?: boolean\n}\n\n/**\n * Determine the delta time in milliseconds to consider for skipping execution\n * based on the ingestion type. This helps to prevent over-retrying for expensive\n * or long-running ingestions.\n * NOTE: For now, all node ingestion have the same delta time, but this can be adjusted in the future if needed so I keep it\n */\nconst getDeltaTimeForIngestionType = (dataIngestionType: IngestionType): number => {\n  // Default delta is 30 minutes in milliseconds\n  let deltaMs = 30 * 60 * 1000\n\n  // Because bigquery ingestion is expensive, we prevent overretrying\n  // Because HTTP ingestion can run during 30 minutes and can be retry 3 times\n  if (dataIngestionType === 'BIG_QUERY' || dataIngestionType === 'HTTP') {\n    deltaMs = 90 * 60 * 1000 // 90 minutes\n  // Because SHOPIFY_API ingestion can run during several minutes and can be retry 5 times (5 bulk exec per token)\n  } else if (dataIngestionType === 'SHOPIFY_API') {\n    deltaMs = 60 * 60 * 1000 // 60 minutes\n  }\n\n  return deltaMs\n}\n\n/**\n * Set the ingestion execution info to 'running' status\n */\nexport const setIngestionExecutionInfoAsStart = async (\n  datastore: googleDatastore.Datastore,\n  ingestion: Ingestion,\n  saveEntity: (entityId: string, updatedProperties: object, _datastore: googleDatastore.Datastore) => Promise<any>\n): Promise<void> => {\n  const ingestionId = ingestion.id.toString()\n  const now = new Date()\n  // Remove microseconds for consistency\n  now.setMilliseconds(0)\n\n  const executionInfo = {\n    ...(ingestion.execution_info ?? {}),\n    status: 'running',\n    timestamp: now,\n    errors: []\n  }\n\n  await saveEntity(ingestionId, { execution_info: executionInfo }, datastore)\n}\n\n/**\n * Handle the start of an execution for an entity.\n * This function validates the schedule version, checks if the entity is already running,\n * updates the execution status, and publishes a monitoring event for every services except datalab.\n * This function's implementation must match our python implementation https://github.com/arcane-run/python-utility/blob/abc088fb3fe95fb38ca613c25baa011626fcb867/src/arcane-ingestion/arcane/ingestion/execution_handler.py#L29-L30\n * @throws {SkipExecutionError} If the schedule version is outdated or if a previous execution\n *         is still running (less than the allowed delta time for the ingestion type).\n */\nexport const handleStartExecution = async (\n  body: IngestionStartBody,\n  entity: Ingestion,\n  datastore: googleDatastore.Datastore,\n  ingestionType: IngestionType,\n  logger: Logger,\n  publishMonitoringStatusCurried: (entityId: number, monitoringId: string, status: MonitoringStatus, childLogger: Logger) => Promise<void>,\n  saveEntity: (entityId: string, updatedProperties: object, _datastore: googleDatastore.Datastore) => Promise<any>,\n  taskRetryCount: number = 0\n): Promise<void> => {\n  const monitoringId = body.monitoring_id\n  const entityId = parseInt(body.entity_id, 10)\n  const scheduleVersion = body.schedule_version\n  const manualExecution = body.manual_execution ?? false\n\n  if (manualExecution) {\n    logger.info(`Manual execution for entity with id ${entityId}. Running without checking schedule version and execution status.`)\n  } else {\n    // Check schedule version\n    const entityScheduleVersion = 'schedule_version' in entity.parameters ? entity.parameters.schedule_version : undefined\n    if (entity['enabled'] === false) {\n      logger.info(`Entity with id ${entityId} is disabled. Skipping execution.`)\n      throw new SkipExecutionError('Entity is disabled')\n    }\n    if (scheduleVersion !== entityScheduleVersion) {\n      logger.info(`Entity with id ${entityId} has its schedule info. Previous version ${scheduleVersion} and ${entityScheduleVersion ?? ''}`)\n      throw new SkipExecutionError(`Schedule info is outdated. Previous version ${scheduleVersion} and current version ${entityScheduleVersion ?? ''}`)\n    }\n\n    // Check if entity is already running\n    if (entity.execution_info != null && 'status' in entity.execution_info) {\n      if (entity.execution_info.status === 'running') {\n        const timestamp = entity.execution_info.timestamp\n        if (timestamp != null) {\n          const timestampDate = timestamp instanceof Date ? timestamp : new Date(timestamp)\n          const elapsedMs = Date.now() - timestampDate.getTime()\n          const deltaMs = getDeltaTimeForIngestionType(ingestionType)\n\n          if (elapsedMs < deltaMs) {\n            if (taskRetryCount > 0) {\n              logger.warn(`Entity with id ${entityId} is already running but this is a Cloud Tasks retry (retry_count=${taskRetryCount}). Allowing retry.`)\n            } else {\n              logger.info(`Entity with id ${entityId} is already running since ${timestampDate.toISOString()}. Skipping execution.`)\n              throw new SkipExecutionError('Previous execution is still running')\n            }\n\n          }\n        }\n      }\n    }\n\n    await setIngestionExecutionInfoAsStart(datastore, entity, saveEntity)\n  }\n\n  // Publish monitoring event if service is not datalab\n  if (entity.service !== 'datalab') {\n    try {\n      await publishMonitoringStatusCurried(entity.id, monitoringId, 'start', logger)\n    } catch (error) {\n      logger.error(`Failed to publish start monitoring message for entity ${entityId}`)\n    }\n  }\n}\n"],"mappings":"6iBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,6BAAAE,EAAA,kBAAAC,EAAA,uBAAAC,EAAA,mBAAAC,EAAA,2BAAAC,EAAA,yBAAAC,EAAA,yCAAAC,EAAA,0BAAAC,IAAA,eAAAC,EAAAV,GCCA,IAAAW,EAAsB,kBACtBC,EAAmB,uBAiBNC,EACXC,GACY,CACZ,IAAMC,KAAQ,EAAAC,SAAO,EACrB,MACE,SAAMF,EAAc,sBAAsB,MAC1C,SAAMA,EAAc,uBAAuB,cAAc,GACzDA,EAAc,uBAAuB,eAAe,SAAW,EAE/D,MAAO,GAET,MAAI,SAAMA,EAAc,uBAAuB,gBAAgB,MAAK,SAAMA,EAAc,uBAAuB,iBAAiB,IAAI,EAClI,MAAO,GAET,IAAMG,KAAsB,EAAAD,SAAOF,EAAc,uBAAuB,iBAAiB,IAAI,EAC7F,MAAO,CAAEC,EAAM,OAAOE,EAAqB,KAAK,CAClD,EAIaC,EAAgB,eAUhBC,EAAiB,CAACC,EAAkBC,IAC/C,CAAC,GAAGH,CAAa,GAAGE,CAAQ,GAAIC,EAAS,SAAS,CAAC,EAAE,KAAK,IAAI,EAAE,QAAQ,KAAM,GAAG,ECjDnF,IAAAC,EAAkD,kCAIrCC,EAA0BC,GAAgC,cAAcA,CAAW,GACnFC,EAA0B,CACrC,CAAE,KAAM,YAAa,KAAM,WAAY,EACvC,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,kBAAmB,KAAM,QAAS,CAC5C,EASMC,EAAwB,MAC5BC,EACAC,IACkB,CAClB,IAAMC,EAAoB,QAAQ,IAAI,mBAEtC,GAAI,CAACA,EACH,MAAM,IAAI,MAAM,wDAAwD,EAI1E,MADgBD,EAAe,QAAQC,CAAiB,EAC1C,YAAYF,EAAS,CAAE,OAAQF,CAAwB,CAAC,CACxE,EAmBaK,EAAwB,MACnCF,EACAJ,EACAO,EACAC,EACAC,EACAC,EACAC,EACAC,EACAC,EACAC,IACkB,CAClB,IAAMX,EAAUJ,EAAuBC,CAAW,EAC5Ce,EAAc,GAAG,QAAQ,IAAI,WAAW,IAAI,QAAQ,IAAI,kBAAkB,IAAIZ,CAAO,GAErFa,EAAM,CACV,UAAW,IAAI,KAAK,EAAE,YAAY,EAClC,QAASF,GAAW,KACpB,SAAUN,GAAW,CAAC,GAAG,IAAIS,GAAUA,EAAO,GAAG,EAAE,KAAK,GAAG,EAC3D,gBAAiB,KAAK,UAAU,CAC9B,aAAAV,EACA,QAAAE,EACA,MAAAC,EACA,SAAAC,CACF,CAAC,CACH,EAMMO,EAAU,CACd,MALY;AAAA,oBACIH,CAAW;AAAA;AAAA,IAK3B,SAAU,KACV,OAAAF,EACA,OAAQ,CACN,GAAGG,CACL,EAGA,MAAO,CACH,QAAS,QACb,CACF,EAEA,GAAI,CACF,GAAM,CAACG,CAAG,EAAI,MAAMf,EAAe,eAAec,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAf,EACAQ,CACF,EACAA,EAAO,KAAK,qBAAqBT,CAAO,gBAAgB,CAC1D,OAASiB,EAAU,CACjB,GAAIA,EAAI,QAAQ,SAAS,WAAW,EAAG,CACrCR,EAAO,KAAK,4CAA4C,EACxD,MAAMV,EAAsBC,EAASC,CAAc,EACnDQ,EAAO,KAAK,SAAST,CAAO,8BAA8B,EAC1D,GAAM,CAACgB,CAAG,EAAI,MAAMf,EAAe,eAAec,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAf,EACAQ,CACF,EACAA,EAAO,KAAK,qBAAqBT,CAAO,wBAAwB,CAClE,MACE,QAAQ,MAAM,qBAAsBiB,CAAG,CAE3C,CACF,EClHO,IAAMC,EAAN,cAAiC,KAAM,CAC5C,YAAaC,EAAiB,CAC5B,MAAMA,CAAO,EACb,KAAK,KAAO,oBACd,CACF,EAeMC,EAAgCC,GAA6C,CAEjF,IAAIC,EAAU,KAId,OAAID,IAAsB,aAAeA,IAAsB,OAC7DC,EAAU,GAAK,GAAK,IAEXD,IAAsB,gBAC/BC,EAAU,GAAK,GAAK,KAGfA,CACT,EAKaC,EAAmC,MAC9CC,EACAC,EACAC,IACkB,CAClB,IAAMC,EAAcF,EAAU,GAAG,SAAS,EACpCG,EAAM,IAAI,KAEhBA,EAAI,gBAAgB,CAAC,EAErB,IAAMC,EAAgB,CACpB,GAAIJ,EAAU,gBAAkB,CAAC,EACjC,OAAQ,UACR,UAAWG,EACX,OAAQ,CAAC,CACX,EAEA,MAAMF,EAAWC,EAAa,CAAE,eAAgBE,CAAc,EAAGL,CAAS,CAC5E,EAUaM,EAAuB,MAClCC,EACAC,EACAR,EACAS,EACAC,EACAC,EACAT,EACAU,EAAyB,IACP,CAClB,IAAMC,EAAeN,EAAK,cACpBO,EAAW,SAASP,EAAK,UAAW,EAAE,EACtCQ,EAAkBR,EAAK,iBAG7B,GAFwBA,EAAK,kBAAoB,GAG/CG,EAAO,KAAK,uCAAuCI,CAAQ,mEAAmE,MACzH,CAEL,IAAME,EAAwB,qBAAsBR,EAAO,WAAaA,EAAO,WAAW,iBAAmB,OAC7G,GAAIA,EAAO,UAAe,GACxB,MAAAE,EAAO,KAAK,kBAAkBI,CAAQ,mCAAmC,EACnE,IAAIpB,EAAmB,oBAAoB,EAEnD,GAAIqB,IAAoBC,EACtB,MAAAN,EAAO,KAAK,kBAAkBI,CAAQ,4CAA4CC,CAAe,QAAQC,GAAyB,EAAE,EAAE,EAChI,IAAItB,EAAmB,+CAA+CqB,CAAe,wBAAwBC,GAAyB,EAAE,EAAE,EAIlJ,GAAIR,EAAO,gBAAkB,MAAQ,WAAYA,EAAO,gBAClDA,EAAO,eAAe,SAAW,UAAW,CAC9C,IAAMS,EAAYT,EAAO,eAAe,UACxC,GAAIS,GAAa,KAAM,CACrB,IAAMC,EAAgBD,aAAqB,KAAOA,EAAY,IAAI,KAAKA,CAAS,EAC1EE,EAAY,KAAK,IAAI,EAAID,EAAc,QAAQ,EAC/CpB,EAAUF,EAA6Ba,CAAa,EAE1D,GAAIU,EAAYrB,EACd,GAAIc,EAAiB,EACnBF,EAAO,KAAK,kBAAkBI,CAAQ,oEAAoEF,CAAc,oBAAoB,MAE5I,OAAAF,EAAO,KAAK,kBAAkBI,CAAQ,6BAA6BI,EAAc,YAAY,CAAC,uBAAuB,EAC/G,IAAIxB,EAAmB,qCAAqC,CAIxE,CACF,CAGF,MAAMK,EAAiCC,EAAWQ,EAAQN,CAAU,CACtE,CAGA,GAAIM,EAAO,UAAY,UACrB,GAAI,CACF,MAAMG,EAA+BH,EAAO,GAAIK,EAAc,QAASH,CAAM,CAC/E,MAAgB,CACdA,EAAO,MAAM,yDAAyDI,CAAQ,EAAE,CAClF,CAEJ","names":["src_exports","__export","HISTORICAL_TABLE_SCHEMA","PREFIX_IMPORT","SkipExecutionError","createFileName","getHistoricalTableName","handleStartExecution","shouldCallProductEvolutionCalculator","storeExecutionDetails","__toCommonJS","import_lodash","import_moment","shouldCallProductEvolutionCalculator","dataIngestion","today","moment","lastCalculationDate","PREFIX_IMPORT","createFileName","clientId","importId","import_bigquery","getHistoricalTableName","ingestionId","HISTORICAL_TABLE_SCHEMA","createHistoricalTable","tableId","bigqueryClient","historicalDataset","storeExecutionDetails","monitoringId","headers","jobRows","jobId","fileHash","logger","labels","invoker","fullTableId","row","header","options","job","err","SkipExecutionError","message","getDeltaTimeForIngestionType","dataIngestionType","deltaMs","setIngestionExecutionInfoAsStart","datastore","ingestion","saveEntity","ingestionId","now","executionInfo","handleStartExecution","body","entity","ingestionType","logger","publishMonitoringStatusCurried","taskRetryCount","monitoringId","entityId","scheduleVersion","entityScheduleVersion","timestamp","timestampDate","elapsedMs"]}