UNPKG

torotask

Version:

Task queue processing in NodeJS based on BullMQ and Redis

641 lines 32.6 kB
import { isArray, isPlainObject } from 'lodash-es'; import ms from 'ms'; import { TaskJob } from './job.js'; import { DelayedError, WaitingChildrenError } from './step-errors.js'; import { getDateTime } from './utils/get-datetime.js'; import { isControlError } from './utils/is-control-error.js'; import { deserializeError } from './utils/serialize-error.js'; export class StepExecutor { job; parentTask; logger; stepCounts = new Map(); client; constructor(job, parentTask) { this.job = job; this.parentTask = parentTask; this.logger = job.logger || console; this.client = parentTask.group.client; if (typeof this.job.state !== 'object' || this.job.state === null) { this.job.state = {}; } this.job.state.stepState = this.job.state.stepState || {}; } async persistState() { const stateToUpdate = { stepState: this.job.state.stepState, }; await this.job.updateState(stateToUpdate); } /** * Process step result before storing it in the job state. * This method automatically prepares TaskJob instances ready for serialization. * Recursively processes arrays and objects to find and serialize any TaskJob instances. * * @param result The result from a step's core logic * @param stepKind The kind of step that produced this result * @returns A processed version of the result suitable for storage */ async memoizeStepResult(result, stepKind) { // Base case: null or undefined if (result === null || result === undefined) { return result; } // Handle arrays of TaskJob instances (bulk operations) differently if (isArray(result) && result.length > 0 && result[0] instanceof TaskJob) { const jobs = result; // If array is large (e.g., more than 10 elements), use compact representation if (jobs.length > 10) { this.logger.debug({ jobCount: jobs.length, stepKind }, `Using compact bulk job reference for ${jobs.length} jobs`); // Extract common queue name (assuming all jobs are in the same queue) const queueName = jobs[0].queueName; // Create compact reference const bulkRef = { _isBulkJobsReference: true, count: jobs.length, queueName, timestamp: Date.now(), // Store only first few job IDs as a sample sampleJobIds: jobs.slice(0, 3).map(j => j.id), }; return bulkRef; } // For smaller arrays, continue with normal serialization this.logger.debug({ stepKind, arrayLength: result.length }, `Processing array of TaskJob instances`); return Promise.all(result.map(item => this.memoizeStepResult(item, stepKind))); } // Handle TaskJob instances by simplifying them if (result instanceof TaskJob) { const complexJob = result; this.logger.debug({ jobId: result.id, stepKind }, `Automatically processing TaskJob instance for serialization`); const simpleJob = { jobId: complexJob.id, queue: complexJob.queueName, timestamp: complexJob.timestamp, _isMemoizedTaskJob: true, }; return simpleJob; } // Handle arrays by recursively processing each element if (isArray(result)) { this.logger.debug({ stepKind, arrayLength: result.length }, `Processing array for potential TaskJob instances`); return Promise.all(result.map(item => this.memoizeStepResult(item, stepKind))); } // Handle plain objects by recursively processing each property if (isPlainObject(result)) { this.logger.debug({ stepKind, objectKeys: Object.keys(result).length }, `Processing object for potential TaskJob instances`); const processedObject = {}; for (const [key, value] of Object.entries(result)) { processedObject[key] = await this.memoizeStepResult(value, stepKind); } return processedObject; } // For other types (primitives, functions, etc.), just return as is return result; } /** * Checks if the memoized data is a job that needs to be reconstructed. * Recursively processes arrays and objects to find and reconstruct any serialized jobs. * * @param memoizedData The data from the memoized result * @param userStepId The ID of the step for logging purposes * @returns The original data or a reconstructed job */ async reconstructJobIfNeeded(memoizedData, userStepId) { // Base case: null or undefined if (memoizedData === null || memoizedData === undefined) { return memoizedData; } // Handle bulk job references if (memoizedData && memoizedData._isBulkJobsReference) { this.logger.debug({ count: memoizedData.count, queue: memoizedData.queueName, userStepId }, `Detected bulk job reference for step '${userStepId}' with ${memoizedData.count} jobs`); const bulkRef = memoizedData; // Try to find child jobs by parent relationship instead of reconstructing each one try { if (this.client && this.job.id) { // Get all child jobs from the queue that have this job as parent const childJobs = await this.client.getChildJobs(bulkRef.queueName, this.job.id, // Optionally use timestamp for filtering if needed bulkRef.timestamp); if (childJobs.length === bulkRef.count) { this.logger.debug({ count: childJobs.length, userStepId }, `Successfully retrieved all ${childJobs.length} child jobs`); return childJobs; } else { this.logger.warn({ expectedCount: bulkRef.count, actualCount: childJobs.length, userStepId }, `Found different number of child jobs than expected for step '${userStepId}'`); return childJobs; } } else { throw new Error(`Cannot retrieve bulk jobs: client or job ID not available`); } } catch (error) { this.logger.error({ error: error?.message, count: bulkRef.count, userStepId }, `Error retrieving bulk jobs for step '${userStepId}'`); throw deserializeError(error); } } // Check if this is a memoized job by looking for our marker if (memoizedData && memoizedData._isMemoizedTaskJob) { this.logger.debug({ jobId: memoizedData.jobId, userStepId }, `Detected memoized job data for step '${userStepId}'`); const jobData = memoizedData; // Try to reconstruct the job from the queue if possible try { // If we have a parent task and queue info, we could try to get the job from the queue if (this.client && jobData.queue) { const job = await this.client.getJobById(jobData.queue, jobData.jobId); return job; } else { throw new Error(`Cannot reconstruct job: client or queue not available.`); } } catch (error) { this.logger.error({ error: error?.message, jobId: memoizedData.jobId, userStepId }, `Error during job reconstruction attempt`); throw deserializeError(error); } } // Handle arrays by recursively processing each element if (isArray(memoizedData)) { this.logger.debug({ userStepId, arrayLength: memoizedData.length }, `Processing array for potential serialized jobs`); const result = await Promise.all(memoizedData.map(item => this.reconstructJobIfNeeded(item, userStepId))); return result; } // Handle plain objects by recursively processing each property if (isPlainObject(memoizedData)) { this.logger.debug({ userStepId, objectKeys: Object.keys(memoizedData).length }, `Processing object for potential serialized jobs`); const result = {}; for (const [key, value] of Object.entries(memoizedData)) { result[key] = await this.reconstructJobIfNeeded(value, userStepId); } return result; } // Not a job, return original data return memoizedData; } /** * Handles recovery for steps that were waiting for a child job. * Checks the actual child job state and returns result or throws error. */ async _handleChildJobRecovery(memoizedResult, userStepId, internalStepId) { const { childJobId, childQueueName } = memoizedResult; this.logger.debug({ childJobId, childQueueName, userStepId }, `Checking child job state for recovery`); const childJob = await this.client.getJobById(childQueueName, childJobId); if (!childJob) { this.logger.warn({ childJobId, childQueueName }, `Child job no longer exists`); throw new Error(`Child job ${childJobId} no longer exists`); } const state = await childJob.getState(); this.logger.debug({ childJobId, state }, `Child job state`); if (state === 'completed') { // Child was fixed! Return its result and update our state this.logger.info({ childJobId }, `Child job completed, recovering result`); const result = await childJob.getResult(); // Update step state to completed this.job.state.stepState[internalStepId] = { status: 'completed', childJobId, childQueueName, data: result, }; await this.persistState(); return result; } if (state === 'failed') { // Still failed - get the error from the child job this.logger.debug({ childJobId }, `Child job still failed`); const failedReason = childJob.failedReason || 'Child job failed'; throw new Error(failedReason); } // Child is running/waiting/delayed - wait for it this.logger.info({ childJobId, state }, `Child job is ${state}, waiting for completion`); const result = await childJob.waitForResult(); // Update step state to completed this.job.state.stepState[internalStepId] = { status: 'completed', childJobId, childQueueName, data: result, }; await this.persistState(); return result; } /** * Centralized method to execute a step, handling memoization, state persistence, and errors. * @param userStepId User-defined ID for the step. * @param stepKind A string identifier for the kind of step (e.g., 'do', 'sleep'). * @param coreLogic The async function that performs the actual work of the step. * If it initiates a pending state (e.g., sleep, wait), it should: * 1. Update \`this.job.state.stepState\` with the new status (e.g., 'sleeping'). * 2. Call \`await this.persistState()\`. * 3. Throw a \`WorkflowPendingError\` (e.g., \`DelayedError\`). * If it completes successfully, it returns the result. * If it fails with an unexpected error, it throws that error. * @param handleMemoizedState Optional handler for memoized states that are not 'completed' or 'errored'. * Used for re-evaluating pending states (e.g., checking if sleep duration has passed). * If it handles the state, it should return \`{ processed: true, ... }\`. * If the step completes, it must update state and persist. * @param memoizeStepResult Optional function to customize how the step result is serialized before storage. * If not provided, the default memoizeStepResult method will be used. */ async _executeStep(userStepId, stepKind, coreLogic, handleMemoizedState, memoizeStepResult) { const currentCount = this.stepCounts.get(userStepId) || 0; const internalStepId = `${userStepId}_${currentCount}`; this.stepCounts.set(userStepId, currentCount + 1); const memoizedResult = this.job.state.stepState[internalStepId]; if (memoizedResult) { if (memoizedResult.status === 'completed') { this.logger.debug({ internalStepId, data: memoizedResult.data, stepKind }, `Step '${userStepId}' (id: ${internalStepId}) already completed, returning memoized data.`); // Check if the memoized result is a job that needs reconstruction const result = await this.reconstructJobIfNeeded(memoizedResult.data, userStepId); return result; } else if (memoizedResult.status === 'errored') { // Legacy: step previously errored, but we now allow re-execution on retry // (errors are visible in BullMQ dashboard, steps should be idempotent) this.logger.info({ internalStepId, error: memoizedResult.error, stepKind }, `Step '${userStepId}' (id: ${internalStepId}) previously errored, re-executing.`); // Fall through to re-execute the step } else if (memoizedResult.status === 'waiting_for_child' && memoizedResult.childJobId) { // Child job recovery - check if the child has been fixed return await this._handleChildJobRecovery(memoizedResult, userStepId, internalStepId); } else if (handleMemoizedState) { const intermediateOutcome = await handleMemoizedState(memoizedResult, internalStepId); if (intermediateOutcome.processed) { if (intermediateOutcome.errorToThrow) { throw intermediateOutcome.errorToThrow; } return intermediateOutcome.result; } this.logger.warn({ internalStepId, memoizedResult, stepKind }, `Memoized step '${userStepId}' (id: ${internalStepId}) with status '${memoizedResult.status}' not fully handled by intermediate handler, proceeding to core logic.`); } else { this.logger.warn({ internalStepId, memoizedResult, stepKind }, `Memoized step '${userStepId}' (id: ${internalStepId}) with unhandled status '${memoizedResult.status}', proceeding to core logic.`); } } try { const result = await coreLogic(internalStepId); // Process the result for storage in the state - use custom serializer if provided, otherwise use default const processedResult = memoizeStepResult ? await memoizeStepResult(result, stepKind) : await this.memoizeStepResult(result, stepKind); this.job.state.stepState[internalStepId] = { status: 'completed', data: processedResult, }; await this.persistState(); return result; } catch (error) { if (isControlError(error) && error.message.includes(internalStepId)) { throw error; } this.logger.debug({ internalStepId, stepKind, err: error }, `Core logic for step '${userStepId}' (id: ${internalStepId}) threw an unexpected error.`); // Check if we have a waiting_for_child state - preserve it for recovery // For other steps, don't store the error - just let it re-execute on retry // (errors are visible in BullMQ dashboard, and steps should be idempotent) const existingState = this.job.state.stepState[internalStepId]; if (existingState?.status === 'waiting_for_child' && existingState.childJobId) { this.logger.debug({ internalStepId, childJobId: existingState.childJobId }, `Preserving child job reference for recovery`); } // No error storage - step will re-execute on retry throw error; } } async do(userStepId, handler) { return this._executeStep(userStepId, 'do', async (_internalStepId) => { return handler(); }); } async sleep(userStepId, duration) { return this._executeStep(userStepId, 'sleep', async (internalStepId) => { const durationMs = typeof duration === 'number' ? duration : ms(duration); const newSleepUntil = Date.now() + durationMs; this.job.state.stepState[internalStepId] = { status: 'sleeping', sleepUntil: newSleepUntil, }; await this.persistState(); await this.job.moveToDelayed(newSleepUntil, this.job.token); throw new DelayedError(`Step "${internalStepId}" is sleeping.`); }, async (memoizedResult, internalStepId) => { if (memoizedResult.status === 'sleeping') { const sleepUntil = memoizedResult.sleepUntil; if (Date.now() >= sleepUntil) { this.job.state.stepState[internalStepId] = { status: 'completed' }; await this.persistState(); return { processed: true, result: undefined }; } else { await this.job.moveToDelayed(sleepUntil, this.job.token); return { processed: true, errorToThrow: new DelayedError(`Step "${internalStepId}" is sleeping.`), }; } } return { processed: false }; }); } async sleepUntil(userStepId, datetime) { return this._executeStep(userStepId, 'sleepUntil', async (internalStepId) => { this.logger.debug({ internalStepId, userStepId, datetime }, `sleepUntil called for step '${userStepId}' (id: ${internalStepId}).`); const timestampMs = getDateTime(datetime); if (timestampMs <= Date.now()) { this.logger.debug({ internalStepId, userStepId, timestampMs }, `Timestamp for sleepUntil step '${userStepId}' (id: ${internalStepId}) is in the past. Completing immediately.`); this.job.state.stepState[internalStepId] = { status: 'completed' }; await this.persistState(); return; } this.job.state.stepState[internalStepId] = { status: 'sleeping', sleepUntil: timestampMs }; await this.persistState(); await this.job.moveToDelayed(timestampMs, this.job.token); throw new DelayedError(`Step "${internalStepId}" is sleeping until specific time.`); }, async (memoizedResult, internalStepId) => { if (memoizedResult.status === 'sleeping') { const sleepUntilTime = memoizedResult.sleepUntil; if (Date.now() >= sleepUntilTime) { this.job.state.stepState[internalStepId] = { status: 'completed' }; await this.persistState(); return { processed: true, result: undefined }; } else { await this.job.moveToDelayed(sleepUntilTime, this.job.token); return { processed: true, errorToThrow: new DelayedError(`Step "${internalStepId}" is sleeping until specific time.`), }; } } return { processed: false }; }); } async _runGroupTask(taskName, payload, options) { const taskKey = taskName.toString(); const task = this.parentTask?.group.tasks[taskKey]; if (!task) { throw new Error(`Task '${taskKey}' not found in this Task group.`); } return (await task.run(payload, { parent: this.job, ...options })); } async runGroupTask(userStepId, taskName, payload, options) { return this._executeStep(userStepId, 'runGroupTask', async (_internalStepId) => { return this._runGroupTask(taskName, payload, options); }); } async runGroupTaskStateless(taskName, payload, options) { return this._runGroupTask(taskName, payload, options); } async _runGroupTaskAndWait(taskName, payload, options) { const taskKey = taskName.toString(); const task = this.parentTask?.group.tasks[taskKey]; if (!task) { throw new Error(`Task '${taskKey}' not found in this Task group.`); } // Start the job first, then wait for result // This allows callers to store the job reference before waiting const job = await task.run(payload, { parent: this.job, ...options }); const result = await job.waitForResult(); return { job, result }; } /** * Helper to start a group task without waiting - used for recovery scenarios */ async _startGroupTask(taskName, payload, options) { const taskKey = taskName.toString(); const task = this.parentTask?.group.tasks[taskKey]; if (!task) { throw new Error(`Task '${taskKey}' not found in this Task group.`); } return await task.run(payload, { parent: this.job, ...options }); } async runGroupTaskAndWait(userStepId, taskName, payload, options) { return this._executeStep(userStepId, 'runGroupTaskAndWait', async (internalStepId) => { // Start the child job const job = await this._startGroupTask(taskName, payload, options); // Store child job reference BEFORE waiting - critical for recovery this.job.state.stepState[internalStepId] = { status: 'waiting_for_child', childJobId: job.id, childQueueName: job.queueName, }; await this.persistState(); // Wait for result - if it fails, _executeStep will preserve our state return await job.waitForResult(); }); } async runGroupTaskAndWaitStateless(taskName, payload, options) { const { result } = await this._runGroupTaskAndWait(taskName, payload, options); return result; } async _runTask(groupName, taskName, payload, options) { if (!this.parentTask) { throw new Error('Cannot start task: parentTask is not available.'); } const client = this.parentTask.group.client; const groupKey = groupName.toString(); const taskGroup = client.taskGroups[groupKey]; if (!taskGroup) { throw new Error(`Task group '${groupKey}' not found.`); } const taskKey = taskName.toString(); const task = taskGroup.tasks[taskKey]; if (!task) { throw new Error(`Task '${taskKey}' not found in group '${groupKey}'.`); } const taskJob = (await task.run(payload, { parent: this.job, ...options })); return taskJob; } async runTask(userStepId, groupName, taskName, payload, options) { return this._executeStep(userStepId, 'runTask', async (_internalStepId) => { return this._runTask(groupName, taskName, payload, options); }); } async runTaskStateless(groupName, taskName, payload, options) { return this._runTask(groupName, taskName, payload, options); } /** * Helper to start a cross-group task without waiting - used for recovery scenarios */ async _startTask(groupName, taskName, payload, options) { if (!this.parentTask) { throw new Error('Cannot run task: parentTask is not available.'); } const client = this.parentTask.group.client; const groupKey = groupName.toString(); const taskGroup = client.taskGroups[groupKey]; if (!taskGroup) { throw new Error(`Task group '${groupKey}' not found.`); } const taskKey = taskName.toString(); const task = taskGroup.tasks[taskKey]; if (!task) { throw new Error(`Task '${taskKey}' not found in group '${groupKey}'.`); } return await task.run(payload, { parent: this.job, ...options }); } /** * Helper for stateless runTaskAndWait - starts job and waits */ async _runTaskAndWait(groupName, taskName, payload, options) { const job = await this._startTask(groupName, taskName, payload, options); const result = await job.waitForResult(); return { job, result }; } async runTaskAndWait(userStepId, groupName, taskName, payload, options) { return this._executeStep(userStepId, 'runTaskAndWait', async (internalStepId) => { // Start the child job const job = await this._startTask(groupName, taskName, payload, options); // Store child job reference BEFORE waiting - critical for recovery this.job.state.stepState[internalStepId] = { status: 'waiting_for_child', childJobId: job.id, childQueueName: job.queueName, }; await this.persistState(); // Wait for result - if it fails, _executeStep will preserve our state return await job.waitForResult(); }); } async runTaskAndWaitStateless(groupName, taskName, payload, options) { const { result } = await this._runTaskAndWait(groupName, taskName, payload, options); return result; } async _runTasks(groupName, taskName, tasks, options) { if (!this.parentTask) { throw new Error('Cannot start tasks: parentTask is not available.'); } const client = this.parentTask.group.client; const groupKey = groupName.toString(); const taskGroup = client.taskGroups[groupKey]; if (!taskGroup) { throw new Error(`Task group '${groupKey}' not found.`); } const taskKey = taskName.toString(); const task = taskGroup.tasks[taskKey]; if (!task) { throw new Error(`Task '${taskKey}' not found in group '${groupKey}'.`); } const taskJobs = (await task.runMany(tasks, { parent: this.job, ...options })); return taskJobs; } async runTasks(userStepId, groupName, taskName, tasks, options) { return this._executeStep(userStepId, 'runTasks', async (_internalStepId) => { return this._runTasks(groupName, taskName, tasks, options); }); } async runTasksStateless(groupName, taskName, tasks, options) { return this._runTasks(groupName, taskName, tasks, options); } async _runFlow(task, options) { const result = this.client.runFlow(task, { parent: this.job, ...options }); return result; } async runFlow(userStepId, task, options) { return this._executeStep(userStepId, 'runFlow', async (_internalStepId) => { return this._runFlow(task, options); }); } async runFlowStateless(task, options) { return this._runFlow(task, options); } async _runFlows(tasks, options) { const result = this.client.runFlows(tasks, { parent: this.job, ...options }); return result; } async runFlows(userStepId, tasks, options) { return this._executeStep(userStepId, 'runFlow', async (_internalStepId) => { return this._runFlows(tasks, options); }); } async runFlowsStateless(tasks, options) { return this._runFlows(tasks, options); } async waitForChildTasks(userStepId) { return this._executeStep(userStepId, 'waitForChildTasks', async (internalStepId) => { this.logger.debug({ internalStepId, userStepId }, `waitForChildTasks called for step '${userStepId}' (id: ${internalStepId}).`); this.job.state.stepState[internalStepId] = { status: 'waiting_for_children' }; await this.persistState(); const token = this.job.token || ''; const shouldWait = await this.job.moveToWaitingChildren(token); if (shouldWait) { this.logger.debug({ internalStepId, userStepId }, `Step '${userStepId}' (id: ${internalStepId}) successfully moved to waiting for children, throwing WaitingChildrenError.`); // Create and throw the error - BullMQ will recognize it throw new WaitingChildrenError(`Step "${internalStepId}" is waiting for child tasks.`); } else { this.logger.debug({ internalStepId, userStepId }, `Step '${userStepId}' (id: ${internalStepId}) does not need to wait for children. Completing step.`); return []; } }, async (memoizedResult, internalStepId) => { if (memoizedResult.status === 'waiting_for_children') { this.logger.debug({ internalStepId, userStepId, tasks: memoizedResult.childTaskIds }, `Handling intermediate 'waiting_for_children' state for step '${userStepId}' (id: ${internalStepId}). Re-checking.`); const token = this.job.token || ''; const shouldStillWait = await this.job.moveToWaitingChildren(token); if (shouldStillWait) { this.logger.debug({ internalStepId, userStepId }, `Still waiting for children for step '${userStepId}' (id: ${internalStepId}). Re-throwing WaitingChildrenError.`); return { processed: true, errorToThrow: new WaitingChildrenError(`Step "${internalStepId}" is waiting for child tasks.`), }; } else { this.logger.debug({ internalStepId, userStepId }, `Children for step '${userStepId}' (id: ${internalStepId}) are now complete. Marking step as completed.`); this.job.state.stepState[internalStepId] = { status: 'completed', data: [] }; await this.persistState(); return { processed: true, result: [] }; } } return { processed: false }; }); } async _sendEvent(eventName, eventData) { await this.job.taskClient?.sendEvent(eventName, eventData); } async sendEvent(userStepId, eventName, eventData) { return this._executeStep(userStepId, 'sendEvent', async () => { await this._sendEvent(eventName, eventData); }); } async sendEventStateless(eventName, eventData) { await this._sendEvent(eventName, eventData); } /** * Gets the typed result of a completed child task. * Returns undefined if the job hasn't completed yet. * * Use this after `waitForChildTasks()` to retrieve the return value from a job * that was started with `runTask()`. * * @param job The TaskJob returned from runTask() * @returns The typed return value, or undefined if the job hasn't completed * * @example * ```ts * const childJob = await step.runTask('start-child', 'groupName', 'taskName', payload); * await step.waitForChildTasks('wait-for-children'); * const result = await step.getTaskResult(childJob); // string | undefined * ``` */ async getTaskResult(job) { return job.getResult(); } /** * Waits for a child task to complete and returns its typed result. * Use this when you want to wait for a specific child task without using `waitForChildTasks()`. * * @param job The TaskJob returned from runTask() * @returns The typed return value * * @example * ```ts * const childJob = await step.runTask('start-child', 'groupName', 'taskName', payload); * const result = await step.waitForTaskResult(childJob); // Waits and returns string * ``` */ async waitForTaskResult(job) { return job.waitForResult(); } } //# sourceMappingURL=step-executor.js.map