@arcane-utils/ingestion/dist/index.cjs

Ingestion utilities for Arcane

var A=Object.create;var g=Object.defineProperty;var R=Object.getOwnPropertyDescriptor;var L=Object.getOwnPropertyNames;var H=Object.getPrototypeOf,M=Object.prototype.hasOwnProperty;var N=(t,e)=>{for(var o in e)g(t,o,{get:e[o],enumerable:!0})},E=(t,e,o,n)=>{if(e&&typeof e=="object"||typeof e=="function")for(let i of L(e))!M.call(t,i)&&i!==o&&g(t,i,{get:()=>e[i],enumerable:!(n=R(e,i))||n.enumerable});return t};var O=(t,e,o)=>(o=t!=null?A(H(t)):{},E(e||!t||!t.__esModule?g(o,"default",{value:t,enumerable:!0}):o,t)),k=t=>E(g({},"__esModule",{value:!0}),t);var J={};N(J,{HISTORICAL_TABLE_SCHEMA:()=>w,PREFIX_IMPORT:()=>S,SkipExecutionError:()=>l,createFileName:()=>D,getHistoricalTableName:()=>x,handleStartExecution:()=>$,shouldCallProductEvolutionCalculator:()=>b,storeExecutionDetails:()=>P});module.exports=k(J);var p=require("lodash"),h=O(require("moment"),1),b=t=>{let e=(0,h.default)();if((0,p.isNil)(t.product_evolution_info)||(0,p.isNil)(t.product_evolution_info.identifier_key)||t.product_evolution_info.identifier_key.length===0)return!1;if((0,p.isNil)(t.product_evolution_info.last_calculation)||(0,p.isNil)(t.product_evolution_info.last_calculation.date))return!0;let o=(0,h.default)(t.product_evolution_info.last_calculation.date);return!e.isSame(o,"day")},S="pf__import__",D=(t,e)=>[`${S}${t}`,e.toString()].join("__").replace(/ /g,"_");var T=require("@arcane-utils/bigquery"),x=t=>`historical_${t}`,w=[{name:"timestamp",type:"TIMESTAMP"},{name:"invoker",type:"STRING"},{name:"headers",type:"STRING"},{name:"additional_info",type:"STRING"}],C=async(t,e)=>{let o=process.env.HISTORICAL_DATASET;if(!o)throw new Error("HISTORICAL_DATASET environment variable is not defined");await e.dataset(o).createTable(t,{schema:w})},P=async(t,e,o,n,i,d,_,r,y,a)=>{let s=x(e),v=`${process.env.GCP_PROJECT}.${process.env.HISTORICAL_DATASET}.${s}`,u={timestamp:new Date().toISOString(),invoker:a??null,headers:(n??[]).map(c=>c.key).join(","),additional_info:JSON.stringify({monitoringId:o,jobRows:i,jobId:d,fileHash:_})},f={query:`
    INSERT INTO \`${v}\` (timestamp, invoker, headers, additional_info)
    VALUES (@timestamp, @invoker, @headers, @additional_info)
  `,location:"EU",labels:y,params:{...u},types:{invoker:"STRING"}};try{let[c]=await t.createQueryJob(f);await(0,T.getJobResult)(c.id,"storeExecutionDetails",t,r),r.info(`Row inserted into ${s} successfully.`)}catch(c){if(c.message.includes("Not found")){r.info("Historical Table not found. Creating it..."),await C(s,t),r.info(`Table ${s} created. Retrying insert...`);let[I]=await t.createQueryJob(f);await(0,T.getJobResult)(I.id,"storeExecutionDetails",t,r),r.info(`Row inserted into ${s} after table creation.`)}else console.error("DML insert failed:",c)}};var l=class extends Error{constructor(e){super(e),this.name="SkipExecutionError"}},B=t=>{let e=18e5;return t==="BIG_QUERY"||t==="HTTP"?e=90*60*1e3:t==="SHOPIFY_API"&&(e=60*60*1e3),e},F=async(t,e,o)=>{let n=e.id.toString(),i=new Date;i.setMilliseconds(0);let d={...e.execution_info??{},status:"running",timestamp:i,errors:[]};await o(n,{execution_info:d},t)},$=async(t,e,o,n,i,d,_,r=0)=>{let y=t.monitoring_id,a=parseInt(t.entity_id,10),s=t.schedule_version;if(t.manual_execution??!1)i.info(`Manual execution for entity with id ${a}. Running without checking schedule version and execution status.`);else{let u="schedule_version"in e.parameters?e.parameters.schedule_version:void 0;if(e.enabled===!1)throw i.info(`Entity with id ${a} is disabled. Skipping execution.`),new l("Entity is disabled");if(s!==u)throw i.info(`Entity with id ${a} has its schedule info. Previous version ${s} and ${u??""}`),new l(`Schedule info is outdated. Previous version ${s} and current version ${u??""}`);if(e.execution_info!=null&&"status"in e.execution_info&&e.execution_info.status==="running"){let m=e.execution_info.timestamp;if(m!=null){let f=m instanceof Date?m:new Date(m),c=Date.now()-f.getTime(),I=B(n);if(c<I)if(r>0)i.warn(`Entity with id ${a} is already running but this is a Cloud Tasks retry (retry_count=${r}). Allowing retry.`);else throw i.info(`Entity with id ${a} is already running since ${f.toISOString()}. Skipping execution.`),new l("Previous execution is still running")}}await F(o,e,_)}if(e.service!=="datalab")try{await d(e.id,y,"start",i)}catch{i.error(`Failed to publish start monitoring message for entity ${a}`)}};0&&(module.exports={HISTORICAL_TABLE_SCHEMA,PREFIX_IMPORT,SkipExecutionError,createFileName,getHistoricalTableName,handleStartExecution,shouldCallProductEvolutionCalculator,storeExecutionDetails});
//# sourceMappingURL=index.cjs.map