steps-track
Version:
An observability tool built to track, inspect and visualize intermediate steps in a pipeline, allowing user to easily debug or analyze through a dashboard.
128 lines (113 loc) • 15.7 kB
JavaScript
"use strict";var x=Object.create;var u=Object.defineProperty;var E=Object.getOwnPropertyDescriptor;var w=Object.getOwnPropertyNames;var C=Object.getPrototypeOf,$=Object.prototype.hasOwnProperty;var I=(i,t)=>{for(var e in t)u(i,e,{get:t[e],enumerable:!0})},b=(i,t,e,r)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of w(t))!$.call(i,s)&&s!==e&&u(i,s,{get:()=>t[s],enumerable:!(r=E(t,s))||r.enumerable});return i};var v=(i,t,e)=>(e=i!=null?x(C(i)):{},b(t||!i||!i.__esModule?u(e,"default",{value:i,enumerable:!0}):e,i)),R=i=>b(u({},"__esModule",{value:!0}),i);var G={};I(G,{HttpTransport:()=>y,Pipeline:()=>f,Step:()=>p,WithStep:()=>P,generateExecutionGraphQuickchart:()=>g,generateGanttChartGoogle:()=>m,generateGanttChartQuickchart:()=>c});module.exports=R(G);var k=require("stream");var T=v(require("axios")),S="https://quickchart.io";function g(i){let t=`digraph G {${i.map(r=>`${r.descriptor}${r.label?` [label="${r.label}"]`:""};`).join(`
`)}}`;return`${S}/graphviz?graph=${encodeURIComponent(t)}`}async function c(i,t){let{unit:e,minWidth:r,minHeight:s}={unit:"ms",minWidth:500,minHeight:300,...t??{}},a=Math.max(...i.map(n=>n.endTs||0)),o={type:"horizontalBar",data:{labels:i.map(n=>`${n.key} - ${n.endTs?(n.endTs-n.startTs)/(e==="ms"?1:1e3):"N/A"}${e}`),datasets:[{data:i.map(n=>[n.startTs/(e==="ms"?1:1e3),(n.endTs??a)/(e==="ms"?1:1e3)])}]},options:{legend:{display:!1},scales:{xAxes:[{position:"top",ticks:{min:0,max:a/(e==="ms"?1:1e3)}}]}}},l=Math.max(r,i.length*25),d=Math.max(s,i.length*25);try{let n=await T.default.post(`${S}/chart`,{chart:JSON.stringify(o),width:l.toString(),height:d.toString(),format:"png"},{responseType:"arraybuffer",headers:{"Content-Type":"application/json"}});return Buffer.from(n.data)}catch(n){throw console.error("Error generating QuickChart:",n),new Error("Failed to generate chart with QuickChart API")}}function m(i,t){let{minHeight:e}={minHeight:300,...t??{}},r=Math.max(e,i.length*25);return`
<!DOCTYPE html>
<html>
<head>
<script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
<script>
google.charts.load("current", {packages:["gantt"]});
google.charts.setOnLoadCallback(drawChart);
function drawChart() {
try {
var container = document.getElementById('gantt_chart');
var chart = new google.visualization.Gantt(container);
var dataTable = new google.visualization.DataTable();
// Add columns
dataTable.addColumn('string', 'Task ID');
dataTable.addColumn('string', 'Task Name');
dataTable.addColumn('string', 'Resource');
dataTable.addColumn('date', 'Start Date');
dataTable.addColumn('date', 'End Date');
dataTable.addColumn('number', 'Duration');
dataTable.addColumn('number', 'Percent Complete');
dataTable.addColumn('string', 'Dependencies');
var steps = ${JSON.stringify(i)};
// Compute the minimum start time to make timestamps relative
var ganttStartTs = Number.MAX_SAFE_INTEGER;
for (var i = 0; i < steps.length; i++) {
ganttStartTs = Math.min(ganttStartTs, steps[i].startTs);
}
var rows = [];
for (var i = 0; i < steps.length; i++) {
var step = steps[i];
// Create relative start time (milliseconds from the start of the first step)
var relativeStartTs = step.startTs - ganttStartTs;
// For the Gantt chart, use a base date (Jan 1, 1970) and add the relative time
// This ensures the chart displays properly
var startDate = new Date(relativeStartTs);
var percentComplete;
var duration = null;
// Default to Success status
var resource = 'Success';
// Check for error (if there's an error property in your data)
if (step.error) {
resource = 'Error';
percentComplete = 100; // Error is considered complete
if (step.endTs && step.endTs > 0) {
duration = step.endTs - step.startTs;
}
}
// Check if still running
else if (step.endTs === 0 || !step.endTs) {
resource = 'Running';
percentComplete = 50; // In progress
// For running tasks, set some arbitrary duration to show them on the chart
duration = 1000; // 1 second placeholder
} else {
percentComplete = 100; // Completed successfully
duration = step.endTs - step.startTs;
}
rows.push([
step.key, // Task ID
step.key, // Task Name
resource, // Resource (used for coloring)
startDate, // Start Date
null, // End Date (null when using duration)
duration, // Duration
percentComplete, // Percent Complete
null // Dependencies
]);
}
dataTable.addRows(rows);
var options = {
height: ${r},
gantt: {
trackHeight: 30,
barHeight: 20,
labelMaxWidth: 300,
criticalPathEnabled: false,
percentEnabled: false,
palette: [
{
color: '#2e7d32', // Success - dark green
dark: '#1b5e20', // Darker shade
light: '#e8f5e9' // Light shade for hover
},
{
color: '#d32f2f', // Error - red
dark: '#b71c1c',
light: '#ffebee'
},
{
color: '#ed6c02', // Running - orange
dark: '#e65100',
light: '#fff3e0'
}
]
}
};
chart.draw(dataTable, options);
} catch (err) {
console.error('Error creating chart:', err);
document.getElementById('gantt_chart').innerHTML =
'<div style="color: red; padding: 20px;">Error drawing chart: ' + err + '</div>';
}
}
</script>
</head>
<body>
<div id="gantt_chart" style="width: 100%; height: ${r}px;"></div>
</body>
</html>
`}var p=class i{constructor(t,e){this.name=t,this.records={},this.time={startTs:Date.now(),endTs:void 0,timeUsageMs:void 0},e?.key?this.key=e.key:e?.parent?this.key=`${e?.parent?.key}.${this.name.replace(/[\.]/g,"_")}`:this.key=this.name.replace(/[\.]/g,"_"),this.parent=e?.parent??null,this.ctx=this,this.steps=[],this.eventEmitter=e?.eventEmitter??new k.EventEmitter({captureRejections:!0}),this.eventEmitter.listeners("error")?.length===0&&this.eventEmitter.on("error",()=>{})}async track(t){return await this.run(t)}getName(){return this.name}getKey(){return this.key}getStepMeta(){return{name:this.name,key:this.key,time:this.time,records:this.records,result:this.result,error:this.error?this.error.message||this.error.toString()||this.error.name:void 0}}async run(t){this.time.startTs=Date.now();try{return this.eventEmitter.emit("step-start",this.key,this.getStepMeta()),this.result=await t(this.ctx),this.eventEmitter.emit("step-success",this.key,this.result,this.getStepMeta()),this.result}catch(e){throw this.error=e,this.eventEmitter.emit("step-error",this.key,this.error,this.getStepMeta()),e}finally{this.time.endTs=Date.now(),this.time.timeUsageMs=this.time.endTs-this.time.startTs,this.eventEmitter.emit("step-complete",this.key,this.getStepMeta())}}async step(t,e){let r=new i(t,{parent:this,eventEmitter:this.eventEmitter}),s=this.steps.filter(a=>a.key===r.key).length;if(s>0){let a=`${r.key}___${s}`;console.warn(`Step with key "${r.key}" already exists under same parent step. Assigning a new key "${a}" to avoid confusion.`),r.key=a}return this.steps.push(r),await r.run(e)}log(t,e){return console.warn("Step.log() is deprecated, use Step.record() instead"),this.record(t,e)}async record(t,e){return this.records[t]=e,this.eventEmitter.emit("record",t,e,this.getStepMeta()),this.eventEmitter.emit("step-record",this.key,t,e,this.getStepMeta()),this}on(t,e){return this.eventEmitter.on(t,e),this}outputHierarchy(){return this.outputNested()}outputNested(){return{name:this.name,key:this.key,time:this.time,records:this.records,result:this.result,error:this.error?this.error.message||this.error.toString()||this.error.name:void 0,substeps:this.steps.map(t=>t.outputNested())}}outputFlattened(){let t=this.steps.map(e=>e.outputFlattened()).flat();return[{name:this.name,key:this.key,time:this.time,records:this.records,result:this.result,error:this.error?this.error.message||this.error.toString()||this.error.name:void 0},...t]}getRecords(){return this.records}getTimeMeta(){return this.time}getTime(){return this.getTimeMeta()}_executionGraphQuickchart(){let t=r=>{let s={descriptor:`"${r.key}"`,label:`${r.name}${r.time.timeUsageMs?`
${r.time.timeUsageMs}ms`:""}`};if(r.parent){let a={descriptor:`"${r.parent.key}" -> "${r.key}"`};return[s,a,...r.steps.map(o=>t(o)).flat()]}else return[s,...r.steps.map(a=>t(a)).flat()]},e=t(this);return g(e)}static getGanttSpans(t,e){let r=Math.min(...t.map(a=>a.time.startTs));return t.filter(a=>e?e instanceof RegExp?e.test(a.key):Array.isArray(e)?e.includes(a.key):!0:!0).map(a=>({key:a.key,startTs:a.time.startTs-r,endTs:a.time.endTs?a.time.endTs-r:void 0}))}async ganttQuickchart(t){return c(i.getGanttSpans(this.outputFlattened(),t?.filter),t)}static async ganttQuickChart(t,e){return c(i.getGanttSpans(t,e?.filter),e)}ganttGoogleChartHtml(t){return m(i.getGanttSpans(this.outputFlattened(),t?.filter),t)}static ganttGoogleChartHtml(t,e){return m(i.getGanttSpans(t,e?.filter),e)}static flattenNestedStepMetas(t){let e=t.substeps.map(r=>i.flattenNestedStepMetas(r)).flat();return[t,...e]}};var M=require("uuid"),f=class extends p{constructor(t,e){if(super(t),this.runId=e?.runId??(0,M.v4)(),this.autoSave=e?.autoSave??"off",this.autoSave!=="off"){if(!e?.transport)throw new Error("Transport must be provided when autoSave is enabled");this.transport=e.transport}this.autoSave==="real_time"?(this.on("step-start",async(r,s)=>{r===this.getKey()&&await this.transport?.initiateRun(this.outputPipelineMeta()),s&&await this.transport?.initiateStep(this.runId,s)}),this.on("step-complete",async(r,s)=>{s&&(await this.transport?.finishStep(this.runId,s),r===this.getKey()&&await this.transport?.finishRun(this.outputPipelineMeta(),s.error?"failed":"completed"))})):this.autoSave==="finish"&&this.on("step-complete",async(r,s)=>{r===this.getKey()&&await this.transport?.finishRun(this.outputPipelineMeta(),s?.error?"failed":"completed")})}getRunId(){return this.runId}outputPipelineMeta(){return{...this.getStepMeta(),logVersion:1,runId:this.getRunId(),steps:this.outputFlattened()}}};function P(i){return function(t,e,r){let s=r.value;return r.value=function(...a){let o=a[a.length-1];if(!o||!(o instanceof p))throw new Error(`The last arg of method \`${e}\` must be a \`Step\``);return o.step(i,async l=>{a[a.length-1]=l;let d=[...a.slice(0,-1),l,"AAA"];return s.apply(this,d)})},r}}var h=v(require("axios")),y=class{constructor(t){this.eventCache=[];this.timer=null;this.activeFlushes=0;this.baseUrl=t.baseUrl.endsWith("/")?t.baseUrl:`${t.baseUrl}/`,this.headers={"Content-Type":"application/json"},this.batchLogs=t.batchLogs??!1,this.flushInterval=t.flushInterval??3e3,this.maxBatchSize=t.maxBatchSize??100,this.debug=t.debug??!1,this.maxRetries=t.maxRetries??3,this.batchLogs&&this.startFlushTimer()}startFlushTimer(){this.timer!==null&&clearInterval(this.timer),this.timer=setInterval(()=>{this.flushEvents()},this.flushInterval)}stopFlushTimer(){this.timer!==null&&(clearInterval(this.timer),this.timer=null)}async delay(t){return new Promise(e=>setTimeout(e,t))}async sendEventsWithRetry(t){if(t.length===0)return;this.activeFlushes++;let e=0;try{for(;e<=this.maxRetries;)try{let r=t.map(s=>s.type==="initiate-run"?{type:"pipeline",operation:"start",meta:s.pipelineMeta}:s.type==="finish-run"?{type:"pipeline",operation:"finish",meta:s.pipelineMeta,status:s.status}:s.type==="initiate-step"?{type:"step",operation:"start",runId:s.runId,step:s.step}:s.type==="finish-step"?{type:"step",operation:"finish",runId:s.runId,step:s.step}:null).filter(Boolean);await h.default.post(`${this.baseUrl}api/ingestion/batch`,r,{headers:this.headers}),this.debug&&console.log(`[HttpTransport] Successfully sent ${t.length} events`);return}catch(r){if(console.error("Error sending batched events:",r.message),this.debug&&console.error(`[HttpTransport] Failed to send ${t.length} events: ${r.message}`),e>=this.maxRetries||r.message==="Network error during flush"){e>=this.maxRetries&&this.debug&&console.error(`[HttpTransport] Max retries (${this.maxRetries}) exceeded for batch of ${t.length} events. Dropping events.`);break}let s=1e3*Math.pow(2,e);this.debug&&console.log(`[HttpTransport] Scheduling retry in ${s}ms (attempt ${e+1}/${this.maxRetries})`),await this.delay(s),e++,this.debug&&console.log(`[HttpTransport] Retrying batch of ${t.length} events (attempt ${e}/${this.maxRetries})`)}}finally{this.activeFlushes--}}async flushEvents(){if(this.eventCache.length===0)return;let t=[...this.eventCache];this.eventCache=[],this.debug&&console.log(`[HttpTransport] Flushing ${t.length} events to ${this.baseUrl}api/ingestion/batch`),this.sendEventsWithRetry(t).catch(e=>{console.error("[HttpTransport] Unexpected error in flush process:",e.message)})}flushIfCacheFull(){this.batchLogs&&this.eventCache.length>=this.maxBatchSize&&this.flushEvents()}async initiateRun(t){if(this.batchLogs){this.eventCache.push({type:"initiate-run",pipelineMeta:t}),this.flushIfCacheFull();return}try{await h.default.post(`${this.baseUrl}api/ingestion/pipeline/start`,t,{headers:this.headers})}catch(e){throw console.error("Error initiating run:",e.message),new Error(`Failed to initiate run: ${e.message}`)}}async finishRun(t,e){if(this.batchLogs){this.eventCache.push({type:"finish-run",pipelineMeta:t,status:e}),this.flushIfCacheFull();return}try{await h.default.post(`${this.baseUrl}api/ingestion/pipeline/finish`,{pipelineMeta:t,status:e},{headers:this.headers})}catch(r){throw console.error("Error finishing run:",r),new Error(`Failed to finish run: ${r.message}`)}}async initiateStep(t,e){if(this.batchLogs){this.eventCache.push({type:"initiate-step",runId:t,step:e}),this.flushIfCacheFull();return}try{await h.default.post(`${this.baseUrl}api/ingestion/step/start`,{runId:t,step:e},{headers:this.headers})}catch(r){throw console.error("Error initiating step:",r.message),new Error(`Failed to initiate step: ${r.message}`)}}async finishStep(t,e){if(this.batchLogs){this.eventCache.push({type:"finish-step",runId:t,step:e}),this.flushIfCacheFull();return}try{await h.default.post(`${this.baseUrl}api/ingestion/step/finish`,{runId:t,step:e},{headers:this.headers})}catch(r){throw console.error("Error finishing step:",r.message),new Error(`Failed to finish step: ${r.message}`)}}async flushAndStop(){if(this.debug&&console.log("[HttpTransport] Flushing events and stopping timer before shutdown"),this.stopFlushTimer(),await this.flushEvents(),this.activeFlushes>0)for(this.debug&&console.log(`[HttpTransport] Waiting for ${this.activeFlushes} active flushes to complete`);this.activeFlushes>0;)await this.delay(100)}};0&&(module.exports={HttpTransport,Pipeline,Step,WithStep,generateExecutionGraphQuickchart,generateGanttChartGoogle,generateGanttChartQuickchart});