UNPKG

@cadenza.io/core

Version:

This is a framework for building asynchronous graphs and flows of tasks and signals.

1 lines 195 kB
{"version":3,"sources":["../src/utils/tools.ts","../src/engine/SignalBroker.ts","../src/engine/GraphRunner.ts","../src/engine/GraphRun.ts","../src/utils/ColorRandomizer.ts","../src/engine/exporters/vue-flow/VueFlowExportVisitor.ts","../src/engine/exporters/vue-flow/VueFlowExporter.ts","../src/graph/execution/GraphNode.ts","../src/graph/context/GraphContext.ts","../src/graph/iterators/GraphNodeIterator.ts","../src/interfaces/SignalEmitter.ts","../src/utils/promise.ts","../src/graph/definition/GraphRoutine.ts","../src/graph/iterators/TaskIterator.ts","../src/graph/definition/Task.ts","../src/registry/GraphRegistry.ts","../src/graph/definition/DebounceTask.ts","../src/graph/definition/EphemeralTask.ts","../src/interfaces/ExecutionChain.ts","../src/graph/iterators/GraphLayerIterator.ts","../src/interfaces/GraphLayer.ts","../src/graph/execution/SyncGraphLayer.ts","../src/interfaces/GraphBuilder.ts","../src/engine/builders/GraphBreadthFirstBuilder.ts","../src/interfaces/GraphRunStrategy.ts","../src/engine/ThrottleEngine.ts","../src/graph/execution/AsyncGraphLayer.ts","../src/engine/builders/GraphAsyncQueueBuilder.ts","../src/engine/strategy/GraphAsyncRun.ts","../src/engine/strategy/GraphStandardRun.ts","../src/Cadenza.ts","../src/graph/definition/SignalTask.ts","../src/index.ts"],"sourcesContent":["/**\n * Deep clones an input with optional filter.\n * @param input The input to clone.\n * @param filterOut Predicate to skip keys (true = skip).\n * @returns Cloned input.\n * @edge Handles arrays/objects; skips cycles with visited.\n * @edge Primitives returned as-is; functions copied by reference.\n */\nexport function deepCloneFilter<T>(\n input: T,\n filterOut: (key: string) => boolean = () => false,\n): T {\n if (input === null || typeof input !== \"object\") {\n return input;\n }\n\n const visited = new WeakMap<any, any>(); // For cycle detection\n\n const stack: Array<{ source: any; target: any; key?: string }> = [];\n const output = Array.isArray(input) ? [] : {};\n\n stack.push({ source: input, target: output });\n visited.set(input, output);\n\n while (stack.length) {\n const { source, target, key } = stack.pop()!;\n const currentTarget = key !== undefined ? target[key] : target;\n\n for (const [k, value] of Object.entries(source)) {\n if (filterOut(k)) continue;\n\n if (value && typeof value === \"object\") {\n if (visited.has(value)) {\n currentTarget[k] = visited.get(value); // Cycle: link to existing clone\n continue;\n }\n\n const clonedValue = Array.isArray(value) ? [] : {};\n currentTarget[k] = clonedValue;\n visited.set(value, clonedValue);\n\n stack.push({ source: value, target: currentTarget, key: k });\n } else {\n currentTarget[k] = value;\n }\n }\n }\n\n return output as T;\n}\n\nexport function formatTimestamp(timestamp: number) {\n return new Date(timestamp).toISOString();\n}\n","import GraphRunner from \"./GraphRunner\";\nimport { AnyObject } from \"../types/global\";\nimport Task from \"../graph/definition/Task\";\nimport GraphRoutine from \"../graph/definition/GraphRoutine\";\nimport Cadenza from \"../Cadenza\";\nimport { formatTimestamp } from \"../utils/tools\";\n\nexport default class SignalBroker {\n static instance_: SignalBroker;\n\n /**\n * Singleton instance for signal management.\n * @returns The broker instance.\n */\n static get instance(): SignalBroker {\n if (!this.instance_) {\n this.instance_ = new SignalBroker();\n }\n return this.instance_;\n }\n\n debug: boolean = false;\n verbose: boolean = false;\n\n setDebug(value: boolean) {\n this.debug = value;\n }\n\n setVerbose(value: boolean) {\n this.verbose = value;\n }\n\n validateSignalName(signalName: string) {\n if (signalName.length > 100) {\n throw new Error(\n `Signal name must be less than 100 characters: ${signalName}`,\n );\n }\n\n if (signalName.includes(\" \")) {\n throw new Error(`Signal name must not contain spaces: ${signalName}\"`);\n }\n\n if (signalName.includes(\"\\\\\")) {\n throw new Error(\n `Signal name must not contain backslashes: ${signalName}`,\n );\n }\n\n if (/[A-Z]/.test(signalName.split(\":\")[0].split(\".\").slice(1).join(\".\"))) {\n throw new Error(\n `Signal name must not contain uppercase letters in the middle of the signal name. It is only allowed in the first part of the signal name: ${signalName}`,\n );\n }\n }\n\n runner: GraphRunner | undefined;\n metaRunner: GraphRunner | undefined;\n\n public getSignalsTask: Task | undefined;\n\n signalObservers: Map<\n string,\n {\n fn: (\n runner: GraphRunner,\n tasks: (Task | GraphRoutine)[],\n context: AnyObject,\n ) => void;\n tasks: Set<Task | GraphRoutine>;\n }\n > = new Map();\n\n emitStacks: Map<string, Map<string, AnyObject>> = new Map(); // execId -> emitted signals\n\n constructor() {\n this.addSignal(\"meta.signal_broker.added\");\n }\n\n /**\n * Initializes with runners.\n * @param runner Standard runner for user signals.\n * @param metaRunner Meta runner for 'meta.' signals (suppresses further meta-emits).\n */\n bootstrap(runner: GraphRunner, metaRunner: GraphRunner): void {\n this.runner = runner;\n this.metaRunner = metaRunner;\n }\n\n init() {\n Cadenza.createMetaTask(\n \"Execute and clear queued signals\",\n () => {\n for (const [id, signals] of this.emitStacks.entries()) {\n signals.forEach((context, signal) => {\n this.execute(signal, context);\n signals.delete(signal);\n });\n\n this.emitStacks.delete(id);\n }\n return true;\n },\n \"Executes queued signals and clears the stack\",\n )\n .doOn(\"meta.process_signal_queue_requested\")\n .emits(\"meta.signal_broker.queue_empty\");\n\n this.getSignalsTask = Cadenza.createMetaTask(\"Get signals\", (ctx) => {\n return {\n __signals: Array.from(this.signalObservers.keys()),\n ...ctx,\n };\n });\n }\n\n /**\n * Observes a signal with a routine/task.\n * @param signal The signal (e.g., 'domain.action', 'domain.*' for wildcards).\n * @param routineOrTask The observer.\n * @edge Duplicates ignored; supports wildcards for broad listening.\n */\n observe(signal: string, routineOrTask: Task | GraphRoutine): void {\n this.addSignal(signal);\n this.signalObservers.get(signal)!.tasks.add(routineOrTask);\n }\n\n /**\n * Unsubscribes a routine/task from a signal.\n * @param signal The signal.\n * @param routineOrTask The observer.\n * @edge Removes all instances if duplicate; deletes if empty.\n */\n unsubscribe(signal: string, routineOrTask: Task | GraphRoutine): void {\n const obs = this.signalObservers.get(signal);\n if (obs) {\n obs.tasks.delete(routineOrTask);\n if (obs.tasks.size === 0) {\n this.signalObservers.delete(signal);\n }\n }\n }\n\n /**\n * Emits a signal and bubbles to matching wildcards/parents (e.g., 'a.b.action' triggers 'a.b.action', 'a.b.*', 'a.*').\n * @param signal The signal name.\n * @param context The payload.\n * @edge Fire-and-forget; guards against loops per execId (from context.__graphExecId).\n * @edge For distribution, SignalTask can prefix and proxy remote.\n */\n emit(signal: string, context: AnyObject = {}): void {\n const execId = context.__routineExecId || \"global\"; // Assume from metadata\n delete context.__routineExecId;\n\n if (!this.emitStacks.has(execId)) this.emitStacks.set(execId, new Map());\n const stack = this.emitStacks.get(execId)!;\n stack.set(signal, context);\n\n let executed = false;\n try {\n executed = this.execute(signal, context);\n } finally {\n if (executed) stack.delete(signal);\n if (stack.size === 0) this.emitStacks.delete(execId);\n }\n }\n\n execute(signal: string, context: AnyObject): boolean {\n const isMeta = signal.startsWith(\"meta.\");\n const isSubMeta = signal.startsWith(\"sub_meta.\") || context.__isSubMeta;\n const isMetric = context.__signalEmission?.isMetric;\n\n if (!isSubMeta && (!isMeta || this.debug)) {\n const emittedAt = Date.now();\n context.__signalEmission = {\n ...context.__signalEmission,\n signalName: signal,\n emittedAt: formatTimestamp(emittedAt),\n consumed: false,\n consumedBy: null,\n isMeta,\n };\n } else if (isSubMeta) {\n context.__isSubMeta = true;\n delete context.__signalEmission;\n } else {\n delete context.__signalEmission;\n }\n\n if (this.debug && ((!isMetric && !isSubMeta) || this.verbose)) {\n console.log(\n `EMITTING ${signal} to listeners ${this.signalObservers.get(signal)?.tasks.size ?? 0} with context ${this.verbose ? JSON.stringify(context) : JSON.stringify(context).slice(0, 100)}`,\n );\n }\n\n let executed;\n executed = this.executeListener(signal, context); // Exact signal\n\n if (!isSubMeta) {\n const parts = signal\n .slice(0, Math.max(signal.lastIndexOf(\":\"), signal.lastIndexOf(\".\")))\n .split(\".\");\n for (let i = parts.length; i > -1; i--) {\n const parent = parts.slice(0, i).join(\".\");\n executed = executed || this.executeListener(parent + \".*\", context); // Wildcard\n }\n }\n\n return executed;\n }\n\n executeListener(signal: string, context: AnyObject): boolean {\n const obs = this.signalObservers.get(signal);\n const isMeta = signal.startsWith(\"meta\");\n const runner = isMeta ? this.metaRunner : this.runner;\n if (obs && obs.tasks.size && runner) {\n obs.fn(runner, Array.from(obs.tasks), context);\n return true;\n }\n return false;\n }\n\n addSignal(signal: string): void {\n let _signal = signal;\n if (!this.signalObservers.has(_signal)) {\n this.validateSignalName(_signal);\n this.signalObservers.set(_signal, {\n fn: (\n runner: GraphRunner,\n tasks: (Task | GraphRoutine)[],\n context: AnyObject,\n ) => runner.run(tasks, context),\n tasks: new Set(),\n });\n\n const sections = _signal.split(\":\");\n if (sections.length === 2) {\n _signal = sections[0];\n\n if (!this.signalObservers.has(sections[0])) {\n this.signalObservers.set(_signal, {\n fn: (\n runner: GraphRunner,\n tasks: (Task | GraphRoutine)[],\n context: AnyObject,\n ) => runner.run(tasks, context),\n tasks: new Set(),\n });\n } else {\n return;\n }\n }\n\n this.emit(\"meta.signal_broker.added\", { __signalName: _signal });\n }\n }\n\n // TODO schedule signals\n\n /**\n * Lists all observed signals.\n * @returns Array of signals.\n */\n listObservedSignals(): string[] {\n return Array.from(this.signalObservers.keys());\n }\n\n reset() {\n this.emitStacks.clear();\n this.signalObservers.clear();\n }\n}\n","import { v4 as uuid } from \"uuid\";\nimport Task from \"../graph/definition/Task\";\nimport GraphRun from \"./GraphRun\";\nimport GraphNode from \"../graph/execution/GraphNode\";\nimport GraphRunStrategy from \"../interfaces/GraphRunStrategy\";\nimport { AnyObject } from \"../types/global\";\nimport GraphRoutine from \"../graph/definition/GraphRoutine\";\nimport SignalEmitter from \"../interfaces/SignalEmitter\";\nimport Cadenza from \"../Cadenza\";\nimport GraphRegistry from \"../registry/GraphRegistry\";\nimport GraphContext from \"../graph/context/GraphContext\";\nimport { formatTimestamp } from \"../utils/tools\";\n\nexport default class GraphRunner extends SignalEmitter {\n currentRun: GraphRun;\n debug: boolean = false;\n verbose: boolean = false;\n isRunning: boolean = false;\n readonly isMeta: boolean = false;\n\n strategy: GraphRunStrategy;\n\n /**\n * Constructs a runner.\n * @param isMeta Meta flag (default false).\n * @edge Creates 'Start run' meta-task chained to registry gets.\n */\n constructor(isMeta: boolean = false) {\n super(isMeta);\n this.isMeta = isMeta;\n this.strategy = Cadenza.runStrategy.PARALLEL;\n this.currentRun = new GraphRun(this.strategy);\n }\n\n init() {\n if (this.isMeta) return;\n\n Cadenza.createMetaTask(\n \"Start run\",\n this.startRun.bind(this),\n \"Starts a run\",\n ).doAfter(\n GraphRegistry.instance.getTaskByName,\n GraphRegistry.instance.getRoutineByName,\n );\n }\n\n /**\n * Adds tasks/routines to current run.\n * @param tasks Tasks/routines.\n * @param context Context (defaults {}).\n * @edge Flattens routines to tasks; generates routineExecId if not in context.\n * @edge Emits 'meta.runner.added_tasks' with metadata.\n * @edge Empty tasks warns no-op.\n */\n addTasks(\n tasks: Task | GraphRoutine | (Task | GraphRoutine)[],\n context: AnyObject = {},\n ): void {\n let _tasks = Array.isArray(tasks) ? tasks : [tasks];\n if (_tasks.length === 0) {\n console.warn(\"No tasks/routines to add.\");\n return;\n }\n\n let routineName = _tasks.map((t) => t.name).join(\" | \");\n let routineVersion = null;\n let isMeta = _tasks.every((t) => t.isMeta);\n\n const allTasks = _tasks.flatMap((t) => {\n if (t instanceof GraphRoutine) {\n routineName = t.name;\n routineVersion = t.version;\n isMeta = t.isMeta;\n const routineTasks: Task[] = [];\n t.forEachTask((task: Task) => routineTasks.push(task));\n return routineTasks;\n }\n return t;\n });\n\n const isSubMeta =\n allTasks.some((t) => t.isSubMeta) || !!context.__isSubMeta;\n context.__isSubMeta = isSubMeta;\n\n const ctx = new GraphContext(context || {});\n\n const isNewTrace = !!context.__routineExecId;\n const routineExecId = context.__routineExecId ?? uuid();\n context.__routineExecId = routineExecId;\n\n if (!isSubMeta) {\n const executionTraceId =\n context.__metadata?.__executionTraceId ??\n context.__executionTraceId ??\n uuid();\n const contextData = ctx.export();\n if (isNewTrace) {\n this.emitMetrics(\"meta.runner.new_trace\", {\n data: {\n uuid: executionTraceId,\n issuer_type: \"service\", // TODO: Add issuer type\n issuer_id:\n context.__metadata?.__issuerId ?? context.__issuerId ?? null,\n issued_at: formatTimestamp(Date.now()),\n intent: context.__metadata?.__intent ?? context.__intent ?? null,\n context: contextData,\n is_meta: isMeta,\n },\n });\n }\n\n this.emitMetrics(\"meta.runner.added_tasks\", {\n data: {\n uuid: routineExecId,\n name: routineName,\n routineVersion,\n isMeta,\n executionTraceId,\n context: isNewTrace ? contextData.id : contextData,\n previousRoutineExecution: context.__metadata?.__routineExecId ?? null, // TODO: There is a chance this is not added to the database yet...\n created: formatTimestamp(Date.now()),\n },\n });\n }\n\n allTasks.forEach((task) =>\n this.currentRun.addNode(\n new GraphNode(task, ctx, routineExecId, [], this.debug, this.verbose),\n ),\n );\n }\n\n /**\n * Runs tasks/routines.\n * @param tasks Optional tasks/routines.\n * @param context Optional context.\n * @returns Current/last run (Promise if async).\n * @edge If running, returns current; else runs and resets.\n */\n public run(\n tasks?: Task | GraphRoutine | (Task | GraphRoutine)[],\n context?: AnyObject,\n ): GraphRun | Promise<GraphRun> {\n if (tasks) {\n this.addTasks(tasks, context ?? {});\n }\n\n if (this.isRunning) {\n return this.currentRun;\n }\n\n if (this.currentRun) {\n this.isRunning = true;\n const runResult = this.currentRun.run();\n\n if (runResult instanceof Promise) {\n return this.runAsync(runResult);\n }\n }\n\n return this.reset();\n }\n\n async runAsync(run: Promise<void>): Promise<GraphRun> {\n await run;\n return this.reset();\n }\n\n reset(): GraphRun {\n this.isRunning = false;\n\n const lastRun = this.currentRun;\n\n if (!this.debug) {\n this.destroy();\n }\n\n this.currentRun = new GraphRun(this.strategy);\n\n return lastRun;\n }\n\n public setDebug(value: boolean): void {\n this.debug = value;\n }\n\n public setVerbose(value: boolean): void {\n this.verbose = value;\n }\n\n public destroy(): void {\n this.currentRun.destroy();\n }\n\n public setStrategy(strategy: GraphRunStrategy): void {\n this.strategy = strategy;\n if (!this.isRunning) {\n this.currentRun = new GraphRun(this.strategy);\n }\n }\n\n startRun(context: AnyObject): boolean {\n if (context.__task || context.__routine) {\n const routine = context.__task ?? context.__routine;\n this.run(routine, context);\n return true;\n } else {\n context.errored = true;\n context.__error = \"No routine or task defined.\";\n return false;\n }\n }\n}\n","import { v4 as uuid } from \"uuid\";\nimport GraphNode from \"../graph/execution/GraphNode\";\nimport GraphExporter from \"../interfaces/GraphExporter\";\nimport SyncGraphLayer from \"../graph/execution/SyncGraphLayer\";\nimport GraphRunStrategy from \"../interfaces/GraphRunStrategy\";\nimport GraphLayer from \"../interfaces/GraphLayer\";\nimport VueFlowExporter from \"./exporters/vue-flow/VueFlowExporter\";\n\nexport interface RunJson {\n __id: string;\n __label: string;\n __graph: any;\n __data: any;\n}\n\n// A unique execution of the graph\nexport default class GraphRun {\n readonly id: string;\n graph: GraphLayer | undefined;\n // @ts-ignore\n strategy: GraphRunStrategy;\n exporter: GraphExporter | undefined;\n\n constructor(strategy: GraphRunStrategy) {\n this.id = uuid();\n this.strategy = strategy;\n this.strategy.setRunInstance(this);\n this.exporter = new VueFlowExporter();\n }\n\n setGraph(graph: GraphLayer) {\n this.graph = graph;\n }\n\n addNode(node: GraphNode) {\n this.strategy.addNode(node);\n }\n\n // Composite function / Command execution\n run(): void | Promise<void> {\n return this.strategy.run();\n }\n\n // Composite function\n destroy() {\n this.graph?.destroy();\n this.graph = undefined;\n this.exporter = undefined;\n }\n\n // Composite function\n log() {\n console.log(\"vvvvvvvvvvvvvvvvv\");\n console.log(\"GraphRun\");\n console.log(\"vvvvvvvvvvvvvvvvv\");\n this.graph?.log();\n console.log(\"=================\");\n }\n\n // Memento\n export(): RunJson {\n if (this.exporter && this.graph) {\n const data = this.strategy.export();\n return {\n __id: this.id,\n __label: data.__startTime ?? this.id,\n __graph: this.exporter?.exportGraph(this.graph as SyncGraphLayer),\n __data: data,\n };\n }\n\n return {\n __id: this.id,\n __label: this.id,\n __graph: undefined,\n __data: {},\n };\n }\n\n // Export Strategy\n setExporter(exporter: GraphExporter) {\n this.exporter = exporter;\n }\n}\n","export default class ColorRandomizer {\n numberOfSteps: number;\n stepCounter = 0;\n spread: number;\n range: number;\n\n constructor(numberOfSteps: number = 200, spread: number = 30) {\n this.numberOfSteps = numberOfSteps;\n this.spread = spread;\n this.range = Math.floor(numberOfSteps / this.spread);\n }\n\n rainbow(numOfSteps: number, step: number) {\n // This function generates vibrant, \"evenly spaced\" colours (i.e. no clustering). This is ideal for creating easily distinguishable vibrant markers in Google Maps and other apps.\n // Adam Cole, 2011-Sept-14\n // HSV to RBG adapted from: http://mjijackson.com/2008/02/rgb-to-hsl-and-rgb-to-hsv-color-model-conversion-algorithms-in-javascript\n let r, g, b;\n const h = step / numOfSteps;\n const i = ~~(h * 6);\n const f = h * 6 - i;\n const q = 1 - f;\n switch (i % 6) {\n case 0:\n r = 1;\n g = f;\n b = 0;\n break;\n case 1:\n r = q;\n g = 1;\n b = 0;\n break;\n case 2:\n r = 0;\n g = 1;\n b = f;\n break;\n case 3:\n r = 0;\n g = q;\n b = 1;\n break;\n case 4:\n r = f;\n g = 0;\n b = 1;\n break;\n case 5:\n r = 1;\n g = 0;\n b = q;\n break;\n default:\n r = 0;\n g = 0;\n b = 0;\n break;\n }\n // @ts-ignore\n const c =\n \"#\" +\n (\"00\" + (~~(r * 255)).toString(16)).slice(-2) +\n (\"00\" + (~~(g * 255)).toString(16)).slice(-2) +\n (\"00\" + (~~(b * 255)).toString(16)).slice(-2);\n return c;\n }\n\n getRandomColor() {\n this.stepCounter++;\n\n if (this.stepCounter > this.numberOfSteps) {\n this.stepCounter = 1;\n }\n\n const randomStep =\n ((this.stepCounter * this.range) % this.numberOfSteps) -\n this.range +\n Math.floor(this.stepCounter / this.spread);\n\n return this.rainbow(this.numberOfSteps, randomStep);\n }\n}\n","import GraphVisitor from \"../../../interfaces/GraphVisitor\";\nimport SyncGraphLayer from \"../../../graph/execution/SyncGraphLayer\";\nimport GraphNode from \"../../../graph/execution/GraphNode\";\nimport Task from \"../../../graph/definition/Task\";\nimport ColorRandomizer from \"../../../utils/ColorRandomizer\";\n\nexport default class VueFlowExportVisitor implements GraphVisitor {\n nodeCount = 0;\n elements: any[] = [];\n index = 0;\n numberOfLayerNodes = 0;\n contextToColor: { [id: string]: string } = {};\n colorRandomizer = new ColorRandomizer();\n\n visitLayer(layer: SyncGraphLayer): any {\n const snapshot = layer.export();\n\n this.numberOfLayerNodes = snapshot.__numberOfNodes;\n this.index = 0;\n }\n\n visitNode(node: GraphNode): any {\n const snapshot = node.export();\n\n if (!this.contextToColor[snapshot.__context.id]) {\n this.contextToColor[snapshot.__context.id] =\n this.colorRandomizer.getRandomColor();\n }\n\n const color = this.contextToColor[snapshot.__context.id];\n\n this.elements.push({\n id: snapshot.__id.slice(0, 8),\n label: snapshot.__task.__name,\n position: {\n x: snapshot.__task.__layerIndex * 500,\n y: -50 * this.numberOfLayerNodes * 0.5 + (this.index * 60 + 30),\n },\n sourcePosition: \"right\",\n targetPosition: \"left\",\n style: { backgroundColor: `${color}`, width: \"180px\" },\n data: {\n executionTime: snapshot.__executionTime,\n executionStart: snapshot.__executionStart,\n executionEnd: snapshot.__executionEnd,\n description: snapshot.__task.__description,\n functionString: snapshot.__task.__functionString,\n context: snapshot.__context.context,\n layerIndex: snapshot.__task.__layerIndex,\n },\n });\n\n for (const [index, nextNodeId] of snapshot.__nextNodes.entries()) {\n this.elements.push({\n id: `${snapshot.__id.slice(0, 8)}-${index}`,\n source: snapshot.__id.slice(0, 8),\n target: nextNodeId.slice(0, 8),\n });\n }\n\n this.index++;\n this.nodeCount++;\n }\n\n visitTask(task: Task) {\n const snapshot = task.export();\n\n this.elements.push({\n id: snapshot.__id.slice(0, 8),\n label: snapshot.__name,\n position: { x: snapshot.__layerIndex * 300, y: this.index * 50 + 30 },\n sourcePosition: \"right\",\n targetPosition: \"left\",\n data: {\n description: snapshot.__description,\n functionString: snapshot.__functionString,\n layerIndex: snapshot.__layerIndex,\n },\n });\n\n for (const [index, nextTaskId] of snapshot.__nextTasks.entries()) {\n this.elements.push({\n id: `${snapshot.__id.slice(0, 8)}-${index}`,\n source: snapshot.__id.slice(0, 8),\n target: nextTaskId.slice(0, 8),\n });\n }\n\n this.index++;\n this.nodeCount++;\n }\n\n getElements() {\n return this.elements;\n }\n\n getNodeCount() {\n return this.nodeCount;\n }\n}\n","import GraphExporter from \"../../../interfaces/GraphExporter\";\nimport SyncGraphLayer from \"../../../graph/execution/SyncGraphLayer\";\nimport VueFlowExportVisitor from \"./VueFlowExportVisitor\";\nimport Task from \"../../../graph/definition/Task\";\n\nexport default class VueFlowExporter implements GraphExporter {\n exportGraph(graph: SyncGraphLayer): any {\n const exporterVisitor = new VueFlowExportVisitor();\n const layers = graph.getIterator();\n while (layers.hasNext()) {\n const layer = layers.next();\n layer.accept(exporterVisitor);\n }\n\n return {\n elements: exporterVisitor.getElements(),\n numberOfNodes: exporterVisitor.getNodeCount(),\n };\n }\n\n exportStaticGraph(graph: Task[]) {\n const exporterVisitor = new VueFlowExportVisitor();\n\n let prevTask = null;\n for (const task of graph) {\n if (task === prevTask) {\n continue;\n }\n\n const tasks = task.getIterator();\n const exportedTaskNames: string[] = [];\n\n while (tasks.hasNext()) {\n const task = tasks.next();\n if (task && !exportedTaskNames.includes(task.name)) {\n exportedTaskNames.push(task.name);\n task.accept(exporterVisitor);\n }\n }\n\n prevTask = task;\n }\n\n return {\n elements: exporterVisitor.getElements(),\n numberOfNodes: exporterVisitor.getNodeCount(),\n };\n }\n}\n","import { v4 as uuid } from \"uuid\";\nimport Task, { TaskResult } from \"../definition/Task\";\nimport GraphContext from \"../context/GraphContext\";\nimport Graph from \"../../interfaces/Graph\";\nimport GraphVisitor from \"../../interfaces/GraphVisitor\";\nimport GraphNodeIterator from \"../iterators/GraphNodeIterator\";\nimport SignalEmitter from \"../../interfaces/SignalEmitter\";\nimport GraphLayer from \"../../interfaces/GraphLayer\";\nimport { AnyObject } from \"../../types/global\";\nimport { sleep } from \"../../utils/promise\";\nimport { formatTimestamp } from \"../../utils/tools\";\n\nexport default class GraphNode extends SignalEmitter implements Graph {\n id: string;\n routineExecId: string;\n task: Task;\n context: GraphContext;\n layer: GraphLayer | undefined;\n divided: boolean = false;\n splitGroupId: string = \"\";\n processing: boolean = false;\n subgraphComplete: boolean = false;\n graphComplete: boolean = false;\n result: TaskResult = false;\n retryCount: number = 0;\n retryDelay: number = 0;\n retries: number = 0;\n previousNodes: GraphNode[] = [];\n nextNodes: GraphNode[] = [];\n executionTime: number = 0;\n executionStart: number = 0;\n failed: boolean = false;\n errored: boolean = false;\n destroyed: boolean = false;\n debug: boolean = false;\n verbose: boolean = false;\n\n constructor(\n task: Task,\n context: GraphContext,\n routineExecId: string,\n prevNodes: GraphNode[] = [],\n debug: boolean = false,\n verbose: boolean = false,\n ) {\n super(\n (task.isMeta && !debug) ||\n task.isSubMeta ||\n context?.getMetadata()?.__isSubMeta,\n );\n this.id = uuid();\n this.task = task;\n this.context = context;\n this.retryCount = task.retryCount;\n this.retryDelay = task.retryDelay;\n this.previousNodes = prevNodes;\n this.routineExecId = routineExecId;\n this.splitGroupId = routineExecId;\n this.debug = debug;\n this.verbose = verbose;\n }\n\n setDebug(value: boolean) {\n this.debug = value;\n }\n\n public isUnique() {\n return this.task.isUnique;\n }\n\n public isMeta() {\n return this.task.isMeta;\n }\n\n public isProcessed() {\n return this.divided;\n }\n\n public isProcessing() {\n return this.processing;\n }\n\n public subgraphDone() {\n return this.subgraphComplete;\n }\n\n public graphDone() {\n return this.graphComplete;\n }\n\n public isEqualTo(node: GraphNode) {\n return (\n this.sharesTaskWith(node) &&\n this.sharesContextWith(node) &&\n this.isPartOfSameGraph(node)\n );\n }\n\n public isPartOfSameGraph(node: GraphNode) {\n return this.routineExecId === node.routineExecId;\n }\n\n public sharesTaskWith(node: GraphNode) {\n return this.task.name === node.task.name;\n }\n\n public sharesContextWith(node: GraphNode) {\n return this.context.id === node.context.id;\n }\n\n public getLayerIndex() {\n return this.task.layerIndex;\n }\n\n public getConcurrency() {\n return this.task.concurrency;\n }\n\n public getTag() {\n return this.task.getTag(this.context);\n }\n\n public scheduleOn(layer: GraphLayer) {\n let shouldSchedule = true;\n const nodes = layer.getNodesByRoutineExecId(this.routineExecId);\n for (const node of nodes) {\n if (node.isEqualTo(this)) {\n shouldSchedule = false;\n break;\n }\n\n if (node.sharesTaskWith(this) && node.isUnique()) {\n node.consume(this);\n shouldSchedule = false;\n break;\n }\n }\n\n if (shouldSchedule) {\n this.layer = layer;\n layer.add(this);\n\n const context = this.context.getFullContext();\n\n const scheduledAt = Date.now();\n this.emitMetricsWithMetadata(\"meta.node.scheduled\", {\n data: {\n uuid: this.id,\n routineExecutionId: this.routineExecId,\n executionTraceId:\n context.__executionTraceId ??\n context.__metadata?.__executionTraceId,\n context: this.context.export(),\n taskName: this.task.name,\n taskVersion: this.task.version,\n isMeta: this.isMeta(),\n isScheduled: true,\n splitGroupId: this.splitGroupId,\n created: formatTimestamp(scheduledAt),\n },\n });\n\n this.previousNodes.forEach((node) => {\n this.emitMetricsWithMetadata(\"meta.node.mapped\", {\n data: {\n taskExecutionId: this.id,\n previousTaskExecutionId: node.id,\n executionCount: \"increment\",\n },\n filter: {\n taskName: this.task.name,\n taskVersion: this.task.version,\n previousTaskName: node.task.name,\n previousTaskVersion: node.task.version,\n },\n });\n });\n\n if (\n context.__signalEmission?.consumed === false &&\n (!this.isMeta() || this.debug)\n ) {\n this.emitMetricsWithMetadata(\"meta.node.consumed_signal\", {\n data: {\n signalName: context.__signalEmission.signalName,\n taskName: this.task.name,\n taskVersion: this.task.version,\n taskExecutionId: this.id,\n consumedAt: formatTimestamp(scheduledAt),\n },\n });\n\n context.__signalEmission.consumed = true;\n context.__signalEmission.consumedBy = this.id;\n }\n }\n }\n\n public start() {\n if (this.executionStart === 0) {\n this.executionStart = Date.now();\n }\n\n if (this.previousNodes.length === 0) {\n this.emitMetricsWithMetadata(\"meta.node.started_routine_execution\", {\n data: {\n isRunning: true,\n started: formatTimestamp(this.executionStart),\n },\n filter: { uuid: this.routineExecId },\n });\n }\n\n if (\n (this.debug &&\n !this.task.isSubMeta &&\n !this.context.getMetadata().__isSubMeta) ||\n this.verbose\n ) {\n this.log();\n }\n\n this.emitMetricsWithMetadata(\"meta.node.started\", {\n data: {\n isRunning: true,\n started: formatTimestamp(this.executionStart),\n },\n filter: { uuid: this.id },\n });\n\n return this.executionStart;\n }\n\n public end() {\n if (this.executionStart === 0) {\n return 0;\n }\n\n this.processing = false;\n const end = Date.now();\n this.executionTime = end - this.executionStart;\n\n const context = this.context.getFullContext();\n\n if (this.errored || this.failed) {\n this.emitMetricsWithMetadata(\"meta.node.errored\", {\n data: {\n isRunning: false,\n errored: this.errored,\n failed: this.failed,\n errorMessage: context.__error,\n },\n filter: { uuid: this.id },\n });\n }\n\n this.emitMetricsWithMetadata(\"meta.node.ended\", {\n data: {\n isRunning: false,\n isComplete: true,\n resultContext: this.context.export(),\n errored: this.errored,\n failed: this.failed,\n errorMessage: context.__error,\n progress: 1.0,\n ended: formatTimestamp(end),\n },\n filter: { uuid: this.id },\n });\n\n if (this.graphDone()) {\n // TODO Reminder, Service registry should be listening to this event, (updateSelf)\n this.emitMetricsWithMetadata(\n `meta.node.ended_routine_execution:${this.routineExecId}`,\n {\n data: {\n isRunning: false,\n isComplete: true,\n resultContext: this.context.export(),\n progress: 1.0,\n ended: formatTimestamp(end),\n },\n filter: { uuid: this.routineExecId },\n },\n );\n }\n\n return end;\n }\n\n public execute() {\n if (!this.divided && !this.processing) {\n this.processing = true;\n\n const inputValidation = this.task.validateInput(\n this.isMeta() ? this.context.getMetadata() : this.context.getContext(),\n );\n if (inputValidation !== true) {\n this.onError(inputValidation.__validationErrors);\n this.postProcess();\n return this.nextNodes;\n }\n\n this.result = this.work();\n\n if (this.result instanceof Promise) {\n return this.executeAsync();\n }\n\n const nextNodes = this.postProcess();\n if (nextNodes instanceof Promise) {\n return nextNodes;\n }\n\n this.nextNodes = nextNodes;\n }\n\n return this.nextNodes;\n }\n\n async workAsync() {\n try {\n this.result = await this.result;\n } catch (e: unknown) {\n const result = await this.retryAsync(e);\n if (result === e) {\n this.onError(e);\n }\n }\n }\n\n async executeAsync() {\n await this.workAsync();\n const nextNodes = this.postProcess();\n if (nextNodes instanceof Promise) {\n return nextNodes;\n }\n this.nextNodes = nextNodes;\n return this.nextNodes;\n }\n\n work(): TaskResult | Promise<TaskResult> {\n try {\n const result = this.task.execute(\n this.context,\n this.emitWithMetadata.bind(this),\n this.onProgress.bind(this),\n );\n\n if ((result as any).errored || (result as any).failed) {\n return this.retry(result);\n }\n\n return result;\n } catch (e: unknown) {\n const result = this.retry(e);\n return result.then((result) => {\n if (result !== e) {\n return result;\n }\n\n this.onError(e);\n return this.result;\n });\n }\n }\n\n emitWithMetadata(signal: string, ctx: AnyObject) {\n const data = { ...ctx };\n if (!this.task.isHidden) {\n data.__signalEmission = {\n taskName: this.task.name,\n taskVersion: this.task.version,\n taskExecutionId: this.id,\n };\n data.__metadata = {\n __routineExecId: this.routineExecId,\n };\n }\n\n this.emit(signal, data);\n }\n\n emitMetricsWithMetadata(signal: string, ctx: AnyObject) {\n const data = { ...ctx };\n if (!this.task.isHidden) {\n data.__signalEmission = {\n taskName: this.task.name,\n taskVersion: this.task.version,\n taskExecutionId: this.id,\n isMetric: true,\n };\n data.__metadata = {\n __routineExecId: this.routineExecId,\n };\n }\n\n this.emitMetrics(signal, data);\n }\n\n onProgress(progress: number) {\n progress = Math.min(Math.max(0, progress), 1);\n\n this.emitMetricsWithMetadata(\"meta.node.progress\", {\n data: {\n progress,\n },\n filter: {\n uuid: this.id,\n },\n });\n\n this.emitMetricsWithMetadata(\n `meta.node.routine_execution_progress:${this.routineExecId}`,\n {\n data: {\n progress:\n (progress * this.task.progressWeight) /\n (this.layer?.getIdenticalNodes(this).length ?? 1),\n },\n filter: {\n uuid: this.routineExecId,\n },\n },\n );\n }\n\n postProcess() {\n if (typeof this.result === \"string\") {\n this.onError(\n `Returning strings is not allowed. Returned: ${this.result}`,\n );\n }\n\n if (Array.isArray(this.result)) {\n this.onError(`Returning arrays is not allowed. Returned: ${this.result}`);\n }\n\n const nextNodes = this.divide();\n\n if (nextNodes instanceof Promise) {\n return this.postProcessAsync(nextNodes);\n }\n\n this.nextNodes = nextNodes;\n this.finalize();\n return this.nextNodes;\n }\n\n async postProcessAsync(nextNodes: Promise<GraphNode[]>) {\n this.nextNodes = await nextNodes;\n this.finalize();\n return this.nextNodes;\n }\n\n finalize() {\n if (this.nextNodes.length === 0) {\n this.completeSubgraph();\n }\n\n if (this.errored || this.failed) {\n this.task.mapOnFailSignals((signal: string) =>\n this.emitWithMetadata(signal, this.context.getFullContext()),\n );\n } else if (this.result !== undefined && this.result !== false) {\n this.task.mapSignals((signal: string) =>\n this.emitWithMetadata(signal, this.context.getFullContext()),\n );\n }\n\n this.end();\n }\n\n onError(error: unknown, errorData: AnyObject = {}) {\n this.result = {\n ...this.context.getFullContext(),\n __error: `Node error: ${error}`,\n __retries: this.retries,\n error: `Node error: ${error}`,\n returnedValue: this.result,\n ...errorData,\n };\n this.migrate(this.result);\n this.errored = true;\n }\n\n async retry(prevResult?: any): Promise<TaskResult> {\n if (this.retryCount === 0) {\n return prevResult;\n }\n\n await this.delayRetry();\n return this.work();\n }\n\n async retryAsync(prevResult?: any): Promise<TaskResult> {\n if (this.retryCount === 0) {\n return prevResult;\n }\n\n await this.delayRetry();\n this.result = this.work();\n return this.workAsync();\n }\n\n async delayRetry() {\n this.retryCount--;\n this.retries++;\n await sleep(this.retryDelay);\n this.retryDelay *= this.task.retryDelayFactor;\n if (this.retryDelay > this.task.retryDelayMax) {\n this.retryDelay = this.task.retryDelayMax;\n }\n }\n\n divide(): GraphNode[] | Promise<GraphNode[]> {\n const newNodes: GraphNode[] = [];\n\n if (\n (this.result as Generator)?.next &&\n typeof (this.result as Generator).next === \"function\"\n ) {\n const generator = this.result as Generator;\n let current = generator.next();\n if (current instanceof Promise) {\n return this.divideAsync(current);\n }\n\n while (!current.done && current.value !== undefined) {\n const outputValidation = this.task.validateOutput(current.value as any);\n if (outputValidation !== true) {\n this.onError(outputValidation.__validationErrors);\n break;\n } else {\n newNodes.push(...this.generateNewNodes(current.value));\n current = generator.next();\n }\n }\n } else if (this.result !== undefined && !this.errored) {\n newNodes.push(...this.generateNewNodes(this.result));\n\n if (typeof this.result !== \"boolean\") {\n const outputValidation = this.task.validateOutput(this.result as any);\n if (outputValidation !== true) {\n this.onError(outputValidation.__validationErrors);\n }\n\n this.divided = true;\n this.migrate({\n ...this.result,\n ...this.context.getMetadata(),\n __nextNodes: newNodes.map((n) => n.id),\n __retries: this.retries,\n });\n\n return newNodes;\n }\n }\n\n if (this.errored) {\n newNodes.push(\n ...this.task.mapNext(\n (t: Task) =>\n this.clone()\n .split(uuid())\n .differentiate(t)\n .migrate({ ...(this.result as any) }),\n true,\n ),\n );\n }\n\n this.divided = true;\n this.migrate({\n ...this.context.getFullContext(),\n __nextNodes: newNodes.map((n) => n.id),\n __retries: this.retries,\n });\n\n return newNodes;\n }\n\n async divideAsync(\n current: Promise<IteratorResult<any>>,\n ): Promise<GraphNode[]> {\n const nextNodes: GraphNode[] = [];\n const _current = await current;\n\n const outputValidation = this.task.validateOutput(_current.value as any);\n if (outputValidation !== true) {\n this.onError(outputValidation.__validationErrors);\n return nextNodes;\n } else {\n nextNodes.push(...this.generateNewNodes(_current.value));\n }\n\n for await (const result of this.result as AsyncGenerator<any>) {\n const outputValidation = this.task.validateOutput(result);\n if (outputValidation !== true) {\n this.onError(outputValidation.__validationErrors);\n return [];\n } else {\n nextNodes.push(...this.generateNewNodes(result));\n }\n }\n\n this.divided = true;\n\n return nextNodes;\n }\n\n generateNewNodes(result: any) {\n const groupId = uuid();\n const newNodes = [];\n if (typeof result !== \"boolean\") {\n const failed =\n (result.failed !== undefined && result.failed) ||\n result.error !== undefined;\n newNodes.push(\n ...(this.task.mapNext((t: Task) => {\n const context = t.isUnique\n ? {\n joinedContexts: [\n { ...result, taskName: this.task.name, __nodeId: this.id },\n ],\n ...this.context.getMetadata(),\n }\n : { ...result, ...this.context.getMetadata() };\n return this.clone().split(groupId).differentiate(t).migrate(context);\n }, failed) as GraphNode[]),\n );\n\n this.failed = failed;\n } else {\n const shouldContinue = result;\n if (shouldContinue) {\n newNodes.push(\n ...(this.task.mapNext((t: Task) => {\n const newNode = this.clone().split(groupId).differentiate(t);\n if (t.isUnique) {\n newNode.migrate({\n joinedContexts: [\n {\n ...this.context.getContext(),\n taskName: this.task.name,\n __nodeId: this.id,\n },\n ],\n ...this.context.getMetadata(),\n });\n }\n\n return newNode;\n }) as GraphNode[]),\n );\n }\n }\n\n return newNodes;\n }\n\n differentiate(task: Task): GraphNode {\n this.task = task;\n this.retryCount = task.retryCount;\n this.retryDelay = task.retryDelay;\n this.silent =\n (task.isMeta && !this.debug) ||\n task.isSubMeta ||\n this.context?.getMetadata()?.__isSubMeta;\n return this;\n }\n\n migrate(ctx: any): GraphNode {\n this.context = new GraphContext(ctx);\n return this;\n }\n\n split(id: string): GraphNode {\n this.splitGroupId = id;\n return this;\n }\n\n public clone(): GraphNode {\n return new GraphNode(\n this.task,\n this.context,\n this.routineExecId,\n [this],\n this.debug,\n this.verbose,\n );\n }\n\n public consume(node: GraphNode) {\n this.context = this.context.combine(node.context);\n this.previousNodes = this.previousNodes.concat(node.previousNodes);\n node.completeSubgraph();\n node.changeIdentity(this.id);\n node.destroy();\n }\n\n changeIdentity(id: string) {\n this.id = id;\n }\n\n completeSubgraph() {\n for (const node of this.nextNodes) {\n if (!node.subgraphDone()) {\n return;\n }\n }\n\n this.subgraphComplete = true;\n\n if (this.previousNodes.length === 0) {\n this.completeGraph();\n return;\n }\n\n this.previousNodes.forEach((n) => n.completeSubgraph());\n }\n\n completeGraph() {\n this.graphComplete = true;\n this.nextNodes.forEach((n) => n.completeGraph());\n }\n\n public destroy() {\n // @ts-ignore\n this.context = null;\n // @ts-ignore\n this.task = null;\n this.nextNodes = [];\n this.previousNodes.forEach((n) =>\n n.nextNodes.splice(n.nextNodes.indexOf(this), 1),\n );\n this.previousNodes = [];\n this.result = undefined;\n this.layer = undefined;\n this.destroyed = true;\n }\n\n public getIterator() {\n return new GraphNodeIterator(this);\n }\n\n public mapNext(callback: (node: GraphNode) => any) {\n return this.nextNodes.map(callback);\n }\n\n public accept(visitor: GraphVisitor) {\n visitor.visitNode(this);\n }\n\n public export() {\n return {\n __id: this.id,\n __task: this.task.export(),\n __context: this.context.export(),\n __result: this.result,\n __executionTime: this.executionTime,\n __executionStart: this.executionStart,\n __executionEnd: this.executionStart + this.executionTime,\n __nextNodes: this.nextNodes.map((node) => node.id),\n __previousNodes: this.previousNodes.map((node) => node.id),\n __routineExecId: this.routineExecId,\n __isProcessing: this.processing,\n __isMeta: this.isMeta(),\n __graphComplete: this.graphComplete,\n __failed: this.failed,\n __errored: this.errored,\n __isUnique: this.isUnique(),\n __splitGroupId: this.splitGroupId,\n __tag: this.getTag(),\n };\n }\n\n lightExport() {\n return {\n __id: this.id,\n __task: {\n __name: this.task.name,\n __version: this.task.version,\n },\n __context: this.context.export(),\n __executionTime: this.executionTime,\n __executionStart: this.executionStart,\n __nextNodes: this.nextNodes.map((node) => node.id),\n __previousNodes: this.previousNodes.map((node) => node.id),\n __routineExecId: this.routineExecId,\n __isProcessing: this.processing,\n __graphComplete: this.graphComplete,\n __isMeta: this.isMeta(),\n __failed: this.failed,\n __errored: this.errored,\n __isUnique: this.isUnique(),\n __splitGroupId: this.splitGroupId,\n __tag: this.getTag(),\n };\n }\n\n public log() {\n console.log(\n \"Node EXECUTION:\",\n this.task.name,\n JSON.stringify(this.context.getFullContext()),\n );\n }\n}\n","import { v4 as uuid } from \"uuid\";\nimport { deepCloneFilter } from \"../../utils/tools\";\nimport { AnyObject } from \"../../types/global\";\n\nexport default class GraphContext {\n readonly id: string;\n readonly fullContext: AnyObject; // Raw (for internal)\n readonly userData: AnyObject; // Filtered, frozen\n readonly metadata: AnyObject; // __keys, frozen\n\n constructor(context: AnyObject) {\n if (Array.isArray(context)) {\n throw new Error(\"Array contexts not supported\"); // Per clarification\n }\n this.fullContext = context; // Clone once\n this.userData = Object.fromEntries(\n Object.entries(this.fullContext).filter(([key]) => !key.startsWith(\"__\")),\n );\n this.metadata = Object.fromEntries(\n Object.entries(this.fullContext).filter(([key]) => key.startsWith(\"__\")),\n );\n this.id = uuid();\n }\n\n /**\n * Gets frozen user data (read-only, no clone).\n * @returns Frozen user context.\n */\n getContext(): AnyObject {\n return this.userData;\n }\n\n getClonedContext(): AnyObject {\n return deepCloneFilter(this.userData);\n }\n\n /**\n * Gets full raw context (cloned for safety).\n * @returns Cloned full context.\n */\n getFullContext(): AnyObject {\n return this.fullContext;\n }\n\n getClonedFullContext(): AnyObject {\n return deepCloneFilter(this.fullContext);\n }\n\n /**\n * Gets frozen metadata (read-only).\n * @returns Frozen metadata object.\n */\n getMetadata(): AnyObject {\n return this.metadata;\n }\n\n /**\n * Clones this context (new instance).\n * @returns New GraphContext.\n */\n clone(): GraphContext {\n return this.mutate(this.fullContext);\n }\n\n /**\n * Creates new context from data (via registry).\n * @param context New data.\n * @returns New GraphContext.\n */\n mutate(context: AnyObject): GraphContext {\n return new GraphContext(context);\n }\n\n /**\n * Combines with another for uniques (joins userData).\n * @param otherContext The other.\n * @returns New combined GraphContext.\n * @edge Appends other.userData to joinedContexts in userData.\n */\n combine(otherContext: GraphContext): GraphContext {\n const newUser = { ...this.userData };\n newUser.joinedContexts = this.userData.joinedContexts\n ? [...this.userData.joinedContexts]\n : [this.userData];\n\n const otherUser = otherContext.userData;\n if (Array.isArray(otherUser.joinedContexts)) {\n newUser.joinedContexts.push(...otherUser.jo