@arcane-utils/ingestion/dist/history.cjs.map

Ingestion utilities for Arcane

{"version":3,"sources":["../src/history.ts"],"sourcesContent":["import type winston from 'winston'\nimport { getJobResult, type googleBigquery } from '@arcane-utils/bigquery'\nimport { type IngestionHeader } from '@arcane-utils/types'\nimport { type JobLabels } from '@arcane-utils/job-helper'\n\nexport const getHistoricalTableName = (ingestionId: number): string => `historical_${ingestionId}`\nexport const HISTORICAL_TABLE_SCHEMA = [\n  { name: 'timestamp', type: 'TIMESTAMP' },\n  { name: 'invoker', type: 'STRING' },\n  { name: 'headers', type: 'STRING' },\n  { name: 'additional_info', type: 'STRING' }\n]\n\n/**\n * Creates a historical table if it doesn't exist.\n *\n * @param tableId - The table identifier.\n * @param bigqueryClient - Instance of the BigQuery client.\n * @returns {Promise<void>} - Resolves when the table is created.\n */\nconst createHistoricalTable = async (\n  tableId: string,\n  bigqueryClient: googleBigquery.BigQuery\n): Promise<void> => {\n  const historicalDataset = process.env.HISTORICAL_DATASET\n\n  if (!historicalDataset) {\n    throw new Error('HISTORICAL_DATASET environment variable is not defined')\n  }\n\n  const dataset = bigqueryClient.dataset(historicalDataset)\n  await dataset.createTable(tableId, { schema: HISTORICAL_TABLE_SCHEMA })\n}\n\n/**\n * Stores execution details in a BigQuery table named historical_{ingestion_id}.\n * The table is created if it does not exist.\n * This function handles both changed files (with jobId and jobRows details) and unchanged files (same hash, without job details)\n *\n * @param bigqueryClient - Instance of the BigQuery client.\n * @param ingestionId - Ingestion Id.\n * @param monitoringId - monitoring Id of the execution.\n * @param headers - Array of ingestion headers to be stored. Will stored key values jsonified.\n * @param jobRows - Number of rows processed by the load job. Null for unchanged files.\n * @param jobId - Load job Id to be stored. Empty string for unchanged files.\n * @param fileHash - Hash of the file being processed.\n * @param logger - Instance of the logger.\n * @param labels - Job labels to be stored in the historical table.\n * @param invoker - Optional string to identify the invoker of the execution during manual fetch.\n * @returns {Promise<void>} - Resolves when the operation is complete.\n */\nexport const storeExecutionDetails = async (\n  bigqueryClient: googleBigquery.BigQuery,\n  ingestionId: number,\n  monitoringId: string,\n  headers: IngestionHeader[] | undefined,\n  jobRows: number | null,\n  jobId: string,\n  fileHash: string,\n  logger: winston.Logger,\n  labels: JobLabels,\n  invoker?: string | null\n): Promise<void> => {\n  const tableId = getHistoricalTableName(ingestionId)\n  const fullTableId = `${process.env.GCP_PROJECT}.${process.env.HISTORICAL_DATASET}.${tableId}`\n\n  const row = {\n    timestamp: new Date().toISOString(),\n    invoker: invoker ?? null,\n    headers: (headers ?? []).map(header => header.key).join(','),\n    additional_info: JSON.stringify({\n      monitoringId,\n      jobRows,\n      jobId,\n      fileHash\n    })\n  }\n\n  const query = `\n    INSERT INTO \\`${fullTableId}\\` (timestamp, invoker, headers, additional_info)\n    VALUES (@timestamp, @invoker, @headers, @additional_info)\n  `\n  const options = {\n    query,\n    location: 'EU',\n    labels,\n    params: {\n      ...row\n    },\n    // We need to specify the type for the invoker parameter because it can be explicitly null, \n    // and the BigQuery node library needs to know how to handle it.\n    types: {\n        invoker: 'STRING'\n    }\n  }\n\n  try {\n    const [job] = await bigqueryClient.createQueryJob(options)\n    await getJobResult(\n      job.id!,\n      'storeExecutionDetails',\n      bigqueryClient,\n      logger\n    )\n    logger.info(`Row inserted into ${tableId} successfully.`)\n  } catch (err: any) {\n    if (err.message.includes('Not found')) {\n      logger.info('Historical Table not found. Creating it...')\n      await createHistoricalTable(tableId, bigqueryClient)\n      logger.info(`Table ${tableId} created. Retrying insert...`)\n      const [job] = await bigqueryClient.createQueryJob(options)\n      await getJobResult(\n        job.id!,\n        'storeExecutionDetails',\n        bigqueryClient,\n        logger\n      )\n      logger.info(`Row inserted into ${tableId} after table creation.`)\n    } else {\n      console.error('DML insert failed:', err)\n    }\n  }\n}\n"],"mappings":"4ZAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,6BAAAE,EAAA,2BAAAC,EAAA,0BAAAC,IAAA,eAAAC,EAAAL,GACA,IAAAM,EAAkD,kCAIrCH,EAA0BI,GAAgC,cAAcA,CAAW,GACnFL,EAA0B,CACrC,CAAE,KAAM,YAAa,KAAM,WAAY,EACvC,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,UAAW,KAAM,QAAS,EAClC,CAAE,KAAM,kBAAmB,KAAM,QAAS,CAC5C,EASMM,EAAwB,MAC5BC,EACAC,IACkB,CAClB,IAAMC,EAAoB,QAAQ,IAAI,mBAEtC,GAAI,CAACA,EACH,MAAM,IAAI,MAAM,wDAAwD,EAI1E,MADgBD,EAAe,QAAQC,CAAiB,EAC1C,YAAYF,EAAS,CAAE,OAAQP,CAAwB,CAAC,CACxE,EAmBaE,EAAwB,MACnCM,EACAH,EACAK,EACAC,EACAC,EACAC,EACAC,EACAC,EACAC,EACAC,IACkB,CAClB,IAAMV,EAAUN,EAAuBI,CAAW,EAC5Ca,EAAc,GAAG,QAAQ,IAAI,WAAW,IAAI,QAAQ,IAAI,kBAAkB,IAAIX,CAAO,GAErFY,EAAM,CACV,UAAW,IAAI,KAAK,EAAE,YAAY,EAClC,QAASF,GAAW,KACpB,SAAUN,GAAW,CAAC,GAAG,IAAIS,GAAUA,EAAO,GAAG,EAAE,KAAK,GAAG,EAC3D,gBAAiB,KAAK,UAAU,CAC9B,aAAAV,EACA,QAAAE,EACA,MAAAC,EACA,SAAAC,CACF,CAAC,CACH,EAMMO,EAAU,CACd,MALY;AAAA,oBACIH,CAAW;AAAA;AAAA,IAK3B,SAAU,KACV,OAAAF,EACA,OAAQ,CACN,GAAGG,CACL,EAGA,MAAO,CACH,QAAS,QACb,CACF,EAEA,GAAI,CACF,GAAM,CAACG,CAAG,EAAI,MAAMd,EAAe,eAAea,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAd,EACAO,CACF,EACAA,EAAO,KAAK,qBAAqBR,CAAO,gBAAgB,CAC1D,OAASgB,EAAU,CACjB,GAAIA,EAAI,QAAQ,SAAS,WAAW,EAAG,CACrCR,EAAO,KAAK,4CAA4C,EACxD,MAAMT,EAAsBC,EAASC,CAAc,EACnDO,EAAO,KAAK,SAASR,CAAO,8BAA8B,EAC1D,GAAM,CAACe,CAAG,EAAI,MAAMd,EAAe,eAAea,CAAO,EACzD,QAAM,gBACJC,EAAI,GACJ,wBACAd,EACAO,CACF,EACAA,EAAO,KAAK,qBAAqBR,CAAO,wBAAwB,CAClE,MACE,QAAQ,MAAM,qBAAsBgB,CAAG,CAE3C,CACF","names":["history_exports","__export","HISTORICAL_TABLE_SCHEMA","getHistoricalTableName","storeExecutionDetails","__toCommonJS","import_bigquery","ingestionId","createHistoricalTable","tableId","bigqueryClient","historicalDataset","monitoringId","headers","jobRows","jobId","fileHash","logger","labels","invoker","fullTableId","row","header","options","job","err"]}