@arcane-utils/ingestion/dist/history.cjs.map

Ingestion utilities for Arcane

{"version":3,"sources":["../src/history.ts"],"sourcesContent":["import type winston from 'winston'\nimport { getJobResult, type googleBigquery } from '@arcane-utils/bigquery'\nimport { type IngestionHeader } from '@arcane-utils/types'\n\nexport const getHistoricalTableName = (ingestionId: number): string => `historical_${ingestionId}`\nexport const HISTORICAL_TABLE_SCHEMA = [\n  { name: 'timestamp', type: 'TIMESTAMP' },\n  { name: 'type', type: 'STRING' },\n  { name: 'invoker', type: 'STRING' },\n  { name: 'headers', type: 'STRING' },\n  { name: 'additional_info', type: 'STRING' }\n]\n\n/**\n * Creates a historical table if it doesn't exist.\n *\n * @param tableId - The table identifier.\n * @param bigqueryClient - Instance of the BigQuery client.\n * @returns {Promise<void>} - Resolves when the table is created.\n */\nconst createHistoricalTable = async (\n    tableId: string,\n    bigqueryClient: googleBigquery.BigQuery\n  ): Promise<void> => {\n    const historicalDataset = process.env.HISTORICAL_DATASET\n\n  if (!historicalDataset) {\n    throw new Error('HISTORICAL_DATASET environment variable is not defined')\n  }\n  \n    const dataset = bigqueryClient.dataset(process.env.HISTORICAL_DATASET)\n    await dataset.createTable(tableId, { schema: HISTORICAL_TABLE_SCHEMA })\n  }\n\n/**\n * Stores execution details in a BigQuery table named historical_{ingestion_id}.\n * The table is created if it does not exist.\n * This function handles both changed files (with jobId and jobRows details) and unchanged files (same hash, without job details)\n *\n * @param bigqueryClient - Instance of the BigQuery client.\n * @param ingestionId - Ingestion Id.\n * @param monitoringId - monitoring Id of the execution.\n * @param headers - Array of ingestion headers to be stored. Will stored key values jsonified.\n * @param jobRows - Number of rows processed by the load job. 0 for unchanged files.\n * @param jobId - Load job Id to be stored. Empty string for unchanged files.\n * @param fileHash - Hash of the file being processed.\n * @param logger - Instance of the logger.\n * @returns {Promise<void>} - Resolves when the operation is complete.\n */\nexport const storeExecutionDetails = async (\n  bigqueryClient: googleBigquery.BigQuery,\n  ingestionId: number,\n  monitoringId: string,\n  headers: IngestionHeader[] | undefined,\n  jobRows: number,\n  jobId: string,\n  fileHash: string,\n  logger: winston.Logger\n): Promise<void> => {\n  const tableId = getHistoricalTableName(ingestionId)\n  const fullTableId = `${process.env.GCP_PROJECT}.${process.env.HISTORICAL_DATASET}.${tableId}`\n\n  const row = {\n    timestamp: new Date().toISOString(),\n    type: 'fetch',\n    invoker: '',\n    headers: (headers ?? []).map(header => header.key).join(','),\n    additional_info: JSON.stringify({\n      monitoringId,\n      jobRows,\n      jobId,\n      fileHash\n    })\n  }\n\n  const query = `\n    INSERT INTO \\`${fullTableId}\\` (timestamp, type, invoker, headers, additional_info)\n    VALUES (@timestamp, @type, @invoker, @headers, @additional_info)\n  `\n  const options = {\n    query,\n    location: 'EU',\n    params: {\n      ...row\n    }\n  }\n\n  try {\n    const [job] = await bigqueryClient.createQueryJob(options)\n    await getJobResult(\n      job.id,\n      'storeExecutionDetails',\n      bigqueryClient,\n      logger\n    )\n    logger.info(`Row inserted into ${tableId} successfully.`)\n  } catch (err: any) {\n    if (err.message.includes('Not found')) {\n      logger.info('Historical Table not found. Creating it...')\n      await createHistoricalTable(tableId, bigqueryClient)\n      logger.info(`Table ${tableId} created. Retrying insert...`)\n      const [job] = await bigqueryClient.createQueryJob(options)\n      await getJobResult(\n        job.id,\n        'storeExecutionDetails',\n        bigqueryClient,\n        logger\n      )\n      logger.info(`Row inserted into ${tableId} after table creation.`)\n    } else {\n      console.error('DML insert failed:', err)\n    }\n  }\n}\n"],"mappings":"4ZAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,6BAAAE,EAAA,2BAAAC,EAAA,0BAAAC,IAAA,eAAAC,EAAAL,GACA,IAAAM,EAAkD,kCAGrCH,EAA0BI,GAAgC,cAAcA,CAAW,GACnFL,EAA0B,CACrC,CAAE,KAAM,YAAa,KAAM,WAAY,EACvC,CAAE,KAAM,OAAQ,KAAM,QAAS,EAC/B,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,kBAAmB,KAAM,QAAS,CAC5C,EASMM,EAAwB,MAC1BC,EACAC,IACkB,CAGpB,GAAI,CAFwB,QAAQ,IAAI,mBAGtC,MAAM,IAAI,MAAM,wDAAwD,EAIxE,MADgBA,EAAe,QAAQ,QAAQ,IAAI,kBAAkB,EACvD,YAAYD,EAAS,CAAE,OAAQP,CAAwB,CAAC,CACxE,EAiBWE,EAAwB,MACnCM,EACAH,EACAI,EACAC,EACAC,EACAC,EACAC,EACAC,IACkB,CAClB,IAAMP,EAAUN,EAAuBI,CAAW,EAC5CU,EAAc,GAAG,QAAQ,IAAI,WAAW,IAAI,QAAQ,IAAI,kBAAkB,IAAIR,CAAO,GAErFS,EAAM,CACV,UAAW,IAAI,KAAK,EAAE,YAAY,EAClC,KAAM,QACN,QAAS,GACT,SAAUN,GAAW,CAAC,GAAG,IAAIO,GAAUA,EAAO,GAAG,EAAE,KAAK,GAAG,EAC3D,gBAAiB,KAAK,UAAU,CAC9B,aAAAR,EACA,QAAAE,EACA,MAAAC,EACA,SAAAC,CACF,CAAC,CACH,EAMMK,EAAU,CACd,MALY;AAAA,oBACIH,CAAW;AAAA;AAAA,IAK3B,SAAU,KACV,OAAQ,CACN,GAAGC,CACL,CACF,EAEA,GAAI,CACF,GAAM,CAACG,CAAG,EAAI,MAAMX,EAAe,eAAeU,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAX,EACAM,CACF,EACAA,EAAO,KAAK,qBAAqBP,CAAO,gBAAgB,CAC1D,OAASa,EAAU,CACjB,GAAIA,EAAI,QAAQ,SAAS,WAAW,EAAG,CACrCN,EAAO,KAAK,4CAA4C,EACxD,MAAMR,EAAsBC,EAASC,CAAc,EACnDM,EAAO,KAAK,SAASP,CAAO,8BAA8B,EAC1D,GAAM,CAACY,CAAG,EAAI,MAAMX,EAAe,eAAeU,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAX,EACAM,CACF,EACAA,EAAO,KAAK,qBAAqBP,CAAO,wBAAwB,CAClE,MACE,QAAQ,MAAM,qBAAsBa,CAAG,CAE3C,CACF","names":["history_exports","__export","HISTORICAL_TABLE_SCHEMA","getHistoricalTableName","storeExecutionDetails","__toCommonJS","import_bigquery","ingestionId","createHistoricalTable","tableId","bigqueryClient","monitoringId","headers","jobRows","jobId","fileHash","logger","fullTableId","row","header","options","job","err"]}