@arcane-utils/ingestion/dist/index.cjs.map

Ingestion utilities for Arcane

{"version":3,"sources":["../src/index.ts","../src/ingestion.ts","../src/history.ts"],"sourcesContent":["export {\n    shouldCallProductEvolutionCalculator\n} from './ingestion'\n\nexport {\n    storeExecutionDetails,\n    getHistoricalTableName,\n    HISTORICAL_TABLE_SCHEMA\n} from './history'","\nimport { isNil } from 'lodash'\nimport moment from 'moment'\n\nimport { Ingestion } from '@arcane-utils/types'\n\n\n/**\n * Determines if the product evolution information should be called.\n *\n * This function evaluates whether the product evolution information needs to be updated\n * based on the following criteria:\n * - The last calculation date is not the same as today.\n * - The identifier key exists.\n * - The identifier key is not empty.\n *\n * @param {Ingestion} dataIngestion - The ingestion data object containing product evolution information.\n * @returns {boolean} - Returns true if product evolution information should be called, false otherwise.\n */\nexport const shouldCallProductEvolutionCalculator = (\n  dataIngestion: Ingestion\n): boolean => {\n  const today = moment()\n  if (\n    isNil(dataIngestion.product_evolution_info) ||\n    isNil(dataIngestion.product_evolution_info.identifier_key) ||\n    dataIngestion.product_evolution_info.identifier_key.length === 0\n  ) {\n    return false\n  }\n  if (isNil(dataIngestion.product_evolution_info.last_calculation) || isNil(dataIngestion.product_evolution_info.last_calculation.date)) {\n    return true\n  }\n  const lastCalculationDate = moment(dataIngestion.product_evolution_info.last_calculation.date)\n  return !(today.isSame(lastCalculationDate, 'day'))\n}\n","import type winston from 'winston'\nimport { getJobResult, type googleBigquery } from '@arcane-utils/bigquery'\nimport { type IngestionHeader } from '@arcane-utils/types'\n\nexport const getHistoricalTableName = (ingestionId: number): string => `historical_${ingestionId}`\nexport const HISTORICAL_TABLE_SCHEMA = [\n  { name: 'timestamp', type: 'TIMESTAMP' },\n  { name: 'type', type: 'STRING' },\n  { name: 'invoker', type: 'STRING' },\n  { name: 'headers', type: 'STRING' },\n  { name: 'additional_info', type: 'STRING' }\n]\n\n/**\n * Creates a historical table if it doesn't exist.\n *\n * @param tableId - The table identifier.\n * @param bigqueryClient - Instance of the BigQuery client.\n * @returns {Promise<void>} - Resolves when the table is created.\n */\nconst createHistoricalTable = async (\n    tableId: string,\n    bigqueryClient: googleBigquery.BigQuery\n  ): Promise<void> => {\n    const historicalDataset = process.env.HISTORICAL_DATASET\n\n  if (!historicalDataset) {\n    throw new Error('HISTORICAL_DATASET environment variable is not defined')\n  }\n  \n    const dataset = bigqueryClient.dataset(process.env.HISTORICAL_DATASET)\n    await dataset.createTable(tableId, { schema: HISTORICAL_TABLE_SCHEMA })\n  }\n\n/**\n * Stores execution details in a BigQuery table named historical_{ingestion_id}.\n * The table is created if it does not exist.\n * This function handles both changed files (with jobId and jobRows details) and unchanged files (same hash, without job details)\n *\n * @param bigqueryClient - Instance of the BigQuery client.\n * @param ingestionId - Ingestion Id.\n * @param monitoringId - monitoring Id of the execution.\n * @param headers - Array of ingestion headers to be stored. Will stored key values jsonified.\n * @param jobRows - Number of rows processed by the load job. 0 for unchanged files.\n * @param jobId - Load job Id to be stored. Empty string for unchanged files.\n * @param fileHash - Hash of the file being processed.\n * @param logger - Instance of the logger.\n * @returns {Promise<void>} - Resolves when the operation is complete.\n */\nexport const storeExecutionDetails = async (\n  bigqueryClient: googleBigquery.BigQuery,\n  ingestionId: number,\n  monitoringId: string,\n  headers: IngestionHeader[] | undefined,\n  jobRows: number,\n  jobId: string,\n  fileHash: string,\n  logger: winston.Logger\n): Promise<void> => {\n  const tableId = getHistoricalTableName(ingestionId)\n  const fullTableId = `${process.env.GCP_PROJECT}.${process.env.HISTORICAL_DATASET}.${tableId}`\n\n  const row = {\n    timestamp: new Date().toISOString(),\n    type: 'fetch',\n    invoker: '',\n    headers: (headers ?? []).map(header => header.key).join(','),\n    additional_info: JSON.stringify({\n      monitoringId,\n      jobRows,\n      jobId,\n      fileHash\n    })\n  }\n\n  const query = `\n    INSERT INTO \\`${fullTableId}\\` (timestamp, type, invoker, headers, additional_info)\n    VALUES (@timestamp, @type, @invoker, @headers, @additional_info)\n  `\n  const options = {\n    query,\n    location: 'EU',\n    params: {\n      ...row\n    }\n  }\n\n  try {\n    const [job] = await bigqueryClient.createQueryJob(options)\n    await getJobResult(\n      job.id,\n      'storeExecutionDetails',\n      bigqueryClient,\n      logger\n    )\n    logger.info(`Row inserted into ${tableId} successfully.`)\n  } catch (err: any) {\n    if (err.message.includes('Not found')) {\n      logger.info('Historical Table not found. Creating it...')\n      await createHistoricalTable(tableId, bigqueryClient)\n      logger.info(`Table ${tableId} created. Retrying insert...`)\n      const [job] = await bigqueryClient.createQueryJob(options)\n      await getJobResult(\n        job.id,\n        'storeExecutionDetails',\n        bigqueryClient,\n        logger\n      )\n      logger.info(`Row inserted into ${tableId} after table creation.`)\n    } else {\n      console.error('DML insert failed:', err)\n    }\n  }\n}\n"],"mappings":"6iBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,6BAAAE,EAAA,2BAAAC,EAAA,yCAAAC,EAAA,0BAAAC,IAAA,eAAAC,EAAAN,GCCA,IAAAO,EAAsB,kBACtBC,EAAmB,uBAiBNC,EACXC,GACY,CACZ,IAAMC,KAAQ,EAAAC,SAAO,EACrB,MACE,SAAMF,EAAc,sBAAsB,MAC1C,SAAMA,EAAc,uBAAuB,cAAc,GACzDA,EAAc,uBAAuB,eAAe,SAAW,EAE/D,MAAO,GAET,MAAI,SAAMA,EAAc,uBAAuB,gBAAgB,MAAK,SAAMA,EAAc,uBAAuB,iBAAiB,IAAI,EAClI,MAAO,GAET,IAAMG,KAAsB,EAAAD,SAAOF,EAAc,uBAAuB,iBAAiB,IAAI,EAC7F,MAAO,CAAEC,EAAM,OAAOE,EAAqB,KAAK,CAClD,EClCA,IAAAC,EAAkD,kCAGrCC,EAA0BC,GAAgC,cAAcA,CAAW,GACnFC,EAA0B,CACrC,CAAE,KAAM,YAAa,KAAM,WAAY,EACvC,CAAE,KAAM,OAAQ,KAAM,QAAS,EAC/B,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,kBAAmB,KAAM,QAAS,CAC5C,EASMC,EAAwB,MAC1BC,EACAC,IACkB,CAGpB,GAAI,CAFwB,QAAQ,IAAI,mBAGtC,MAAM,IAAI,MAAM,wDAAwD,EAIxE,MADgBA,EAAe,QAAQ,QAAQ,IAAI,kBAAkB,EACvD,YAAYD,EAAS,CAAE,OAAQF,CAAwB,CAAC,CACxE,EAiBWI,EAAwB,MACnCD,EACAJ,EACAM,EACAC,EACAC,EACAC,EACAC,EACAC,IACkB,CAClB,IAAMR,EAAUJ,EAAuBC,CAAW,EAC5CY,EAAc,GAAG,QAAQ,IAAI,WAAW,IAAI,QAAQ,IAAI,kBAAkB,IAAIT,CAAO,GAErFU,EAAM,CACV,UAAW,IAAI,KAAK,EAAE,YAAY,EAClC,KAAM,QACN,QAAS,GACT,SAAUN,GAAW,CAAC,GAAG,IAAIO,GAAUA,EAAO,GAAG,EAAE,KAAK,GAAG,EAC3D,gBAAiB,KAAK,UAAU,CAC9B,aAAAR,EACA,QAAAE,EACA,MAAAC,EACA,SAAAC,CACF,CAAC,CACH,EAMMK,EAAU,CACd,MALY;AAAA,oBACIH,CAAW;AAAA;AAAA,IAK3B,SAAU,KACV,OAAQ,CACN,GAAGC,CACL,CACF,EAEA,GAAI,CACF,GAAM,CAACG,CAAG,EAAI,MAAMZ,EAAe,eAAeW,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAZ,EACAO,CACF,EACAA,EAAO,KAAK,qBAAqBR,CAAO,gBAAgB,CAC1D,OAASc,EAAU,CACjB,GAAIA,EAAI,QAAQ,SAAS,WAAW,EAAG,CACrCN,EAAO,KAAK,4CAA4C,EACxD,MAAMT,EAAsBC,EAASC,CAAc,EACnDO,EAAO,KAAK,SAASR,CAAO,8BAA8B,EAC1D,GAAM,CAACa,CAAG,EAAI,MAAMZ,EAAe,eAAeW,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAZ,EACAO,CACF,EACAA,EAAO,KAAK,qBAAqBR,CAAO,wBAAwB,CAClE,MACE,QAAQ,MAAM,qBAAsBc,CAAG,CAE3C,CACF","names":["src_exports","__export","HISTORICAL_TABLE_SCHEMA","getHistoricalTableName","shouldCallProductEvolutionCalculator","storeExecutionDetails","__toCommonJS","import_lodash","import_moment","shouldCallProductEvolutionCalculator","dataIngestion","today","moment","lastCalculationDate","import_bigquery","getHistoricalTableName","ingestionId","HISTORICAL_TABLE_SCHEMA","createHistoricalTable","tableId","bigqueryClient","storeExecutionDetails","monitoringId","headers","jobRows","jobId","fileHash","logger","fullTableId","row","header","options","job","err"]}