@arcane-utils/ingestion/dist/executionHandler.cjs.map

Ingestion utilities for Arcane

{"version":3,"sources":["../src/executionHandler.ts"],"sourcesContent":["import { type googleDatastore } from '@arcane-utils/datastore'\nimport { type Logger } from '@arcane-utils/logger'\nimport { type Ingestion, type IngestionType, type MonitoringStatus } from '@arcane-utils/types'\n\n/**\n * Custom error to indicate that an execution should be skipped due to outdated\n * schedule information or concurrent execution.\n */\nexport class SkipExecutionError extends Error {\n  constructor (message: string) {\n    super(message)\n    this.name = 'SkipExecutionError'\n  }\n}\n\ninterface IngestionStartBody {\n  entity_id: string\n  monitoring_id: string\n  schedule_version: string\n  manual_execution?: boolean\n}\n\n/**\n * Determine the delta time in milliseconds to consider for skipping execution\n * based on the ingestion type. This helps to prevent over-retrying for expensive\n * or long-running ingestions.\n * NOTE: For now, all node ingestion have the same delta time, but this can be adjusted in the future if needed so I keep it\n */\nconst getDeltaTimeForIngestionType = (dataIngestionType: IngestionType): number => {\n  // Default delta is 30 minutes in milliseconds\n  let deltaMs = 30 * 60 * 1000\n\n  // Because bigquery ingestion is expensive, we prevent overretrying\n  // Because HTTP ingestion can run during 30 minutes and can be retry 3 times\n  if (dataIngestionType === 'BIG_QUERY' || dataIngestionType === 'HTTP') {\n    deltaMs = 90 * 60 * 1000 // 90 minutes\n  // Because SHOPIFY_API ingestion can run during several minutes and can be retry 5 times (5 bulk exec per token)\n  } else if (dataIngestionType === 'SHOPIFY_API') {\n    deltaMs = 60 * 60 * 1000 // 60 minutes\n  }\n\n  return deltaMs\n}\n\n/**\n * Set the ingestion execution info to 'running' status\n */\nexport const setIngestionExecutionInfoAsStart = async (\n  datastore: googleDatastore.Datastore,\n  ingestion: Ingestion,\n  saveEntity: (entityId: string, updatedProperties: object, _datastore: googleDatastore.Datastore) => Promise<any>\n): Promise<void> => {\n  const ingestionId = ingestion.id.toString()\n  const now = new Date()\n  // Remove microseconds for consistency\n  now.setMilliseconds(0)\n\n  const executionInfo = {\n    ...(ingestion.execution_info ?? {}),\n    status: 'running',\n    timestamp: now,\n    errors: []\n  }\n\n  await saveEntity(ingestionId, { execution_info: executionInfo }, datastore)\n}\n\n/**\n * Handle the start of an execution for an entity.\n * This function validates the schedule version, checks if the entity is already running,\n * updates the execution status, and publishes a monitoring event for every services except datalab.\n * This function's implementation must match our python implementation https://github.com/arcane-run/python-utility/blob/abc088fb3fe95fb38ca613c25baa011626fcb867/src/arcane-ingestion/arcane/ingestion/execution_handler.py#L29-L30\n * @throws {SkipExecutionError} If the schedule version is outdated or if a previous execution\n *         is still running (less than the allowed delta time for the ingestion type).\n */\nexport const handleStartExecution = async (\n  body: IngestionStartBody,\n  entity: Ingestion,\n  datastore: googleDatastore.Datastore,\n  ingestionType: IngestionType,\n  logger: Logger,\n  publishMonitoringStatusCurried: (entityId: number, monitoringId: string, status: MonitoringStatus, childLogger: Logger) => Promise<void>,\n  saveEntity: (entityId: string, updatedProperties: object, _datastore: googleDatastore.Datastore) => Promise<any>,\n  taskRetryCount: number = 0\n): Promise<void> => {\n  const monitoringId = body.monitoring_id\n  const entityId = parseInt(body.entity_id, 10)\n  const scheduleVersion = body.schedule_version\n  const manualExecution = body.manual_execution ?? false\n\n  if (manualExecution) {\n    logger.info(`Manual execution for entity with id ${entityId}. Running without checking schedule version and execution status.`)\n  } else {\n    // Check schedule version\n    const entityScheduleVersion = 'schedule_version' in entity.parameters ? entity.parameters.schedule_version : undefined\n    if (entity['enabled'] === false) {\n      logger.info(`Entity with id ${entityId} is disabled. Skipping execution.`)\n      throw new SkipExecutionError('Entity is disabled')\n    }\n    if (scheduleVersion !== entityScheduleVersion) {\n      logger.info(`Entity with id ${entityId} has its schedule info. Previous version ${scheduleVersion} and ${entityScheduleVersion ?? ''}`)\n      throw new SkipExecutionError(`Schedule info is outdated. Previous version ${scheduleVersion} and current version ${entityScheduleVersion ?? ''}`)\n    }\n\n    // Check if entity is already running\n    if (entity.execution_info != null && 'status' in entity.execution_info) {\n      if (entity.execution_info.status === 'running') {\n        const timestamp = entity.execution_info.timestamp\n        if (timestamp != null) {\n          const timestampDate = timestamp instanceof Date ? timestamp : new Date(timestamp)\n          const elapsedMs = Date.now() - timestampDate.getTime()\n          const deltaMs = getDeltaTimeForIngestionType(ingestionType)\n\n          if (elapsedMs < deltaMs) {\n            if (taskRetryCount > 0) {\n              logger.warn(`Entity with id ${entityId} is already running but this is a Cloud Tasks retry (retry_count=${taskRetryCount}). Allowing retry.`)\n            } else {\n              logger.info(`Entity with id ${entityId} is already running since ${timestampDate.toISOString()}. Skipping execution.`)\n              throw new SkipExecutionError('Previous execution is still running')\n            }\n\n          }\n        }\n      }\n    }\n\n    await setIngestionExecutionInfoAsStart(datastore, entity, saveEntity)\n  }\n\n  // Publish monitoring event if service is not datalab\n  if (entity.service !== 'datalab') {\n    try {\n      await publishMonitoringStatusCurried(entity.id, monitoringId, 'start', logger)\n    } catch (error) {\n      logger.error(`Failed to publish start monitoring message for entity ${entityId}`)\n    }\n  }\n}\n"],"mappings":"4ZAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,wBAAAE,EAAA,yBAAAC,EAAA,qCAAAC,IAAA,eAAAC,EAAAL,GAQO,IAAME,EAAN,cAAiC,KAAM,CAC5C,YAAaI,EAAiB,CAC5B,MAAMA,CAAO,EACb,KAAK,KAAO,oBACd,CACF,EAeMC,EAAgCC,GAA6C,CAEjF,IAAIC,EAAU,KAId,OAAID,IAAsB,aAAeA,IAAsB,OAC7DC,EAAU,GAAK,GAAK,IAEXD,IAAsB,gBAC/BC,EAAU,GAAK,GAAK,KAGfA,CACT,EAKaL,EAAmC,MAC9CM,EACAC,EACAC,IACkB,CAClB,IAAMC,EAAcF,EAAU,GAAG,SAAS,EACpCG,EAAM,IAAI,KAEhBA,EAAI,gBAAgB,CAAC,EAErB,IAAMC,EAAgB,CACpB,GAAIJ,EAAU,gBAAkB,CAAC,EACjC,OAAQ,UACR,UAAWG,EACX,OAAQ,CAAC,CACX,EAEA,MAAMF,EAAWC,EAAa,CAAE,eAAgBE,CAAc,EAAGL,CAAS,CAC5E,EAUaP,EAAuB,MAClCa,EACAC,EACAP,EACAQ,EACAC,EACAC,EACAR,EACAS,EAAyB,IACP,CAClB,IAAMC,EAAeN,EAAK,cACpBO,EAAW,SAASP,EAAK,UAAW,EAAE,EACtCQ,EAAkBR,EAAK,iBAG7B,GAFwBA,EAAK,kBAAoB,GAG/CG,EAAO,KAAK,uCAAuCI,CAAQ,mEAAmE,MACzH,CAEL,IAAME,EAAwB,qBAAsBR,EAAO,WAAaA,EAAO,WAAW,iBAAmB,OAC7G,GAAIA,EAAO,UAAe,GACxB,MAAAE,EAAO,KAAK,kBAAkBI,CAAQ,mCAAmC,EACnE,IAAIrB,EAAmB,oBAAoB,EAEnD,GAAIsB,IAAoBC,EACtB,MAAAN,EAAO,KAAK,kBAAkBI,CAAQ,4CAA4CC,CAAe,QAAQC,GAAyB,EAAE,EAAE,EAChI,IAAIvB,EAAmB,+CAA+CsB,CAAe,wBAAwBC,GAAyB,EAAE,EAAE,EAIlJ,GAAIR,EAAO,gBAAkB,MAAQ,WAAYA,EAAO,gBAClDA,EAAO,eAAe,SAAW,UAAW,CAC9C,IAAMS,EAAYT,EAAO,eAAe,UACxC,GAAIS,GAAa,KAAM,CACrB,IAAMC,EAAgBD,aAAqB,KAAOA,EAAY,IAAI,KAAKA,CAAS,EAC1EE,EAAY,KAAK,IAAI,EAAID,EAAc,QAAQ,EAC/ClB,EAAUF,EAA6BW,CAAa,EAE1D,GAAIU,EAAYnB,EACd,GAAIY,EAAiB,EACnBF,EAAO,KAAK,kBAAkBI,CAAQ,oEAAoEF,CAAc,oBAAoB,MAE5I,OAAAF,EAAO,KAAK,kBAAkBI,CAAQ,6BAA6BI,EAAc,YAAY,CAAC,uBAAuB,EAC/G,IAAIzB,EAAmB,qCAAqC,CAIxE,CACF,CAGF,MAAME,EAAiCM,EAAWO,EAAQL,CAAU,CACtE,CAGA,GAAIK,EAAO,UAAY,UACrB,GAAI,CACF,MAAMG,EAA+BH,EAAO,GAAIK,EAAc,QAASH,CAAM,CAC/E,MAAgB,CACdA,EAAO,MAAM,yDAAyDI,CAAQ,EAAE,CAClF,CAEJ","names":["executionHandler_exports","__export","SkipExecutionError","handleStartExecution","setIngestionExecutionInfoAsStart","__toCommonJS","message","getDeltaTimeForIngestionType","dataIngestionType","deltaMs","datastore","ingestion","saveEntity","ingestionId","now","executionInfo","body","entity","ingestionType","logger","publishMonitoringStatusCurried","taskRetryCount","monitoringId","entityId","scheduleVersion","entityScheduleVersion","timestamp","timestampDate","elapsedMs"]}