@nocobase/plugin-workflow
Version:
A powerful BPM tool that provides foundational support for business automation, with the capability to extend unlimited triggers and nodes.
559 lines (557 loc) • 21.3 kB
JavaScript
/**
* This file is part of the NocoBase (R) project.
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
* Authors: NocoBase Team.
*
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var Dispatcher_exports = {};
__export(Dispatcher_exports, {
default: () => Dispatcher
});
module.exports = __toCommonJS(Dispatcher_exports);
var import_node_crypto = require("node:crypto");
var import_sequelize = require("sequelize");
var import_constants = require("./constants");
var import_Plugin = require("./Plugin");
var import_utils = require("./utils");
const EXECUTION_ACQUIRE_MAX_ATTEMPTS = 5;
const PENDING_DISPATCH_MAX_ATTEMPTS = 3;
class Dispatcher {
constructor(plugin) {
this.plugin = plugin;
}
ready = false;
executing = null;
saving = null;
pending = [];
events = [];
eventsCount = 0;
get idle() {
return this.ready && !this.executing && !this.saving && !this.pending.length && !this.events.length;
}
onQueueExecution = async (event) => {
const ExecutionRepo = this.plugin.db.getRepository("executions");
const execution = await ExecutionRepo.findOne({
filterByTk: event.executionId
});
if (!execution || execution.dispatched || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
return;
}
this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) received from queue, adding to pending list`);
await this.run({ execution });
};
setReady(ready) {
this.ready = ready;
}
getEventsCount() {
return this.eventsCount;
}
trigger(workflow, context, options = {}) {
const logger = this.plugin.getLogger(workflow.id);
if (!this.ready) {
logger.warn(`app is not ready, event of workflow ${workflow.id} will be ignored`);
logger.debug(`ignored event data:`, context);
return this.handleTriggerFail(workflow, context, options, new Error("app is not ready"));
}
if (!options.force && !options.manually && !workflow.enabled) {
logger.warn(`workflow ${workflow.id} is not enabled, event will be ignored`);
return;
}
const duplicated = this.events.find(([w, c, { eventKey }]) => {
if (eventKey && options.eventKey) {
return eventKey === options.eventKey;
}
});
if (duplicated) {
logger.warn(`event of workflow ${workflow.id} is duplicated (${options.eventKey}), event will be ignored`);
return;
}
if (context == null) {
logger.warn(`workflow ${workflow.id} event data context is null, event will be ignored`);
return this.handleTriggerFail(workflow, context, options, new Error("event context is null"));
}
if (options.manually || this.plugin.isWorkflowSync(workflow)) {
return this.triggerSync(workflow, context, options);
}
const { transaction, ...rest } = options;
this.events.push([workflow, context, rest]);
this.eventsCount = this.events.length;
logger.info(`new event triggered, now events: ${this.events.length}`);
logger.debug(`event data:`, { context });
this.saveEvent();
}
saveEvent() {
if (this.saving) {
return;
}
this.saving = (async () => {
try {
while (this.events.length) {
if (this.executing && this.plugin.db.options.dialect === "sqlite") {
await this.executing;
}
const event = this.events.shift();
this.eventsCount = this.events.length;
if (!event) continue;
const logger = this.plugin.getLogger(event[0].id);
logger.info(`preparing execution for event`);
try {
const execution = await this.createExecution(...event);
if (!execution.dispatched) {
if (this.plugin.serving() && !this.executing && !this.pending.length) {
logger.info(`local pending list is empty, adding execution (${execution.id}) to pending list`);
this.pending.push({ execution });
} else {
logger.info(
`instance is not serving as worker or local pending list is not empty, sending execution (${execution.id}) to queue`
);
try {
await this.plugin.app.eventQueue.publish(this.plugin.channelPendingExecution, {
executionId: execution.id
});
} catch (qErr) {
logger.error(`publishing execution (${execution.id}) to queue failed:`, { error: qErr });
}
}
}
} catch (error) {
logger.error(`failed to create execution:`, { error });
}
}
} finally {
this.saving = null;
if (this.events.length) {
this.saveEvent();
} else {
this.dispatch();
}
}
})();
}
async resume(job) {
let { execution } = job;
if (!execution) {
execution = await job.getExecution();
}
this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) resuming from job (${job.id}) added to pending list`);
await this.run({ execution, job });
}
async start(execution) {
if (execution.status) {
return;
}
this.plugin.getLogger(execution.workflowId).info(`starting deferred execution (${execution.id})`);
await this.run({ execution });
}
async beforeStop() {
this.ready = false;
this.plugin.getLogger("dispatcher").info("app is stopping, draining local queues...");
while (this.saving || this.executing || this.events.length || this.pending.length) {
if (this.saving) {
await this.saving;
}
if (this.executing) {
await this.executing;
}
if (this.events.length && !this.saving) {
this.saveEvent();
}
if (this.pending.length && !this.executing) {
this.dispatch();
}
await new Promise((resolve) => setImmediate(resolve));
}
this.plugin.getLogger("dispatcher").info("local queues drained");
}
dispatch() {
if (!this.ready && !this.pending.length && !this.events.length) {
return;
}
if (this.executing) {
this.plugin.getLogger("dispatcher").warn(`workflow executing is not finished, new dispatching will be ignored`);
return;
}
if (this.events.length) {
this.saveEvent();
return;
}
this.executing = (async () => {
let next = null;
let pending = null;
try {
pending = this.pending.shift() ?? null;
if (pending || this.ready && this.plugin.serving()) {
const execution = await this.prepare((pending == null ? void 0 : pending.execution) ?? null, {
immediate: pending == null ? void 0 : pending.immediate
});
if (execution) {
next = [execution, pending == null ? void 0 : pending.job, pending == null ? void 0 : pending.rerun];
}
if (pending && next) {
this.plugin.getLogger(next[0].workflowId).info(`pending execution (${next[0].id}) ready to process`);
}
} else {
this.plugin.getLogger("dispatcher").warn(
`${import_Plugin.WORKER_JOB_WORKFLOW_PROCESS} is not serving on this instance or app not ready, new dispatching will be ignored`
);
}
if (next) {
try {
await this.process(next[0], next[1], { rerun: next[2] });
} catch (error) {
this.plugin.getLogger(next[0].workflowId).error(`execution (${next[0].id}) process failed`, { error });
if (pending && (0, import_utils.isLockAcquireError)(error)) {
this.pending.unshift({ ...pending, execution: next[0], immediate: true });
}
}
}
} catch (error) {
this.plugin.getLogger("dispatcher").error(`workflow dispatch failed`, { error });
if (pending) {
const dispatchAttempts = (pending.dispatchAttempts ?? 0) + 1;
if (dispatchAttempts < PENDING_DISPATCH_MAX_ATTEMPTS) {
this.pending.push({ ...pending, dispatchAttempts });
} else {
this.plugin.getLogger(pending.execution.workflowId).error(`pending execution (${pending.execution.id}) dispatch failed, local retry limit reached`, {
error,
dispatchAttempts
});
}
}
} finally {
setImmediate(() => {
this.executing = null;
if (next || this.pending.length) {
this.plugin.getLogger("dispatcher").debug(`last process finished, will do another dispatch`);
this.dispatch();
}
});
}
})();
}
async run(pending) {
this.pending.push({
...pending,
immediate: !this.executing && !this.pending.length && !this.saving && !this.events.length
});
this.dispatch();
}
async triggerSync(workflow, context, { deferred, ...options } = {}) {
let execution;
try {
execution = await this.createExecution(workflow, context, options);
} catch (err) {
if (err instanceof Error) {
this.plugin.getLogger(workflow.id).error(`creating execution failed: ${err.message}`, err);
}
return null;
}
try {
const entered = await this.prepare(execution);
if (!entered) {
return null;
}
return this.process(entered, void 0, options);
} catch (err) {
if (err instanceof Error) {
this.plugin.getLogger(execution.workflowId).error(`execution (${execution.id}) error: ${err.message}`, err);
}
}
return null;
}
async validateEvent(workflow, context, options) {
const trigger = this.plugin.triggers.get(workflow.type);
const triggerValid = await trigger.validateEvent(workflow, context, options);
if (!triggerValid) {
return false;
}
const { stack = [] } = options;
let valid = true;
if (stack == null ? void 0 : stack.length) {
const existed = await workflow.countExecutions({
where: {
id: stack
}
});
const limitCount = workflow.options.stackLimit || 1;
if (existed >= limitCount) {
this.plugin.getLogger(workflow.id).warn(
`workflow ${workflow.id} has already been triggered in stacks executions (${stack}), and max call count is ${limitCount}, newly triggering will be skipped.`
);
valid = false;
}
}
return valid;
}
async handleTriggerFail(workflow, context, options, error) {
var _a;
try {
await ((_a = options.onTriggerFail) == null ? void 0 : _a.call(options, workflow, context, options, error));
} catch (triggerFailError) {
this.plugin.getLogger(workflow.id).error(`trigger failure callback failed`, { error: triggerFailError });
}
}
async createExecution(workflow, context, options) {
const { deferred } = options;
let stack = options.stack;
if (options.parentExecutionId && !stack) {
const parentExecution = await this.plugin.db.getRepository("executions").findOne({
filterByTk: options.parentExecutionId
});
stack = parentExecution ? [...parentExecution.stack ?? [], parentExecution.id] : [];
}
let valid;
try {
valid = await this.validateEvent(workflow, context, { ...options, stack });
} catch (error) {
await this.handleTriggerFail(workflow, context, options, error);
throw error;
}
if (!valid) {
const error = new Error("event is not valid");
await this.handleTriggerFail(workflow, context, options, error);
throw error;
}
let execution;
try {
execution = await workflow.createExecution({
context,
key: workflow.key,
eventKey: options.eventKey ?? (0, import_node_crypto.randomUUID)(),
stack,
parentExecutionId: options.parentExecutionId ?? null,
dispatched: deferred ?? false,
status: deferred ? import_constants.EXECUTION_STATUS.STARTED : import_constants.EXECUTION_STATUS.QUEUEING,
manually: options.manually
});
} catch (error) {
await this.handleTriggerFail(workflow, context, options, error);
throw error;
}
this.plugin.getLogger(workflow.id).info(`execution of workflow ${workflow.id} created as ${execution.id}`);
if (!workflow.stats) {
workflow.stats = await workflow.getStats();
}
await workflow.stats.increment("executed");
if (this.plugin.db.options.dialect !== "postgres") {
await workflow.stats.reload();
}
if (!workflow.versionStats) {
workflow.versionStats = await workflow.getVersionStats();
}
await workflow.versionStats.increment("executed");
if (this.plugin.db.options.dialect !== "postgres") {
await workflow.versionStats.reload();
}
execution.workflow = workflow;
return execution;
}
async prepare(input, options = {}) {
const logger = input ? this.plugin.getLogger(input.workflowId) : this.plugin.getLogger("dispatcher");
if (options.transaction) {
try {
return await this.acquireExecution(input, options, options.transaction);
} catch (error) {
if (error instanceof Error) {
logger.error(`entering execution failed: ${error.message}`, { error });
}
return null;
}
}
let result = null;
try {
await this.acquireWithRetry(
async () => {
const tx = await this.plugin.db.sequelize.transaction({
isolationLevel: this.plugin.db.options.dialect === "sqlite" ? void 0 : import_sequelize.Transaction.ISOLATION_LEVELS.READ_COMMITTED
});
try {
const execution = await this.acquireExecution(input, options, tx);
await tx.commit();
result = execution;
} catch (error) {
await tx.rollback();
if (this.isConcurrentAcquireError(error)) {
throw error;
}
if (error instanceof Error) {
logger.error(`entering execution failed: ${error.message}`, { error });
}
result = null;
}
},
{
logger,
conflictMessage: input ? `acquiring pending execution (${input.id}) conflicted with another worker, retrying` : `acquiring execution conflicted with another worker, retrying`,
maxAttemptsMessage: input ? `acquiring pending execution (${input.id}) reached max retry attempts` : `acquiring execution reached max retry attempts, will retry on next dispatch`
}
);
} catch (error) {
if (error instanceof Error) {
logger.error(`acquiring execution failed: ${error.message}`, { error });
}
return null;
}
return result;
}
async acquireExecution(input, options, transaction) {
let execution = input;
if (execution) {
if (!options.immediate || execution.status !== import_constants.EXECUTION_STATUS.QUEUEING) {
await execution.reload({ transaction });
}
} else {
const workflowIds = [...this.plugin.enabledCache.keys()];
if (!workflowIds.length) {
this.plugin.getLogger("dispatcher").debug(`no enabled workflow to process`);
return null;
}
execution = await this.plugin.db.getRepository("executions").findOne({
filter: {
dispatched: false,
status: import_constants.EXECUTION_STATUS.QUEUEING,
startedAt: null,
workflowId: workflowIds
},
sort: "id",
transaction,
lock: transaction.LOCK.UPDATE,
skipLocked: true
});
if (execution) {
this.plugin.getLogger(execution.workflowId).info(`execution (${execution.id}) fetched from db`);
} else {
this.plugin.getLogger("dispatcher").debug(`no execution in db queued to process`);
}
}
if (!execution) {
return null;
}
return this.enter(execution, transaction);
}
async acquireWithRetry(acquire, options) {
for (let attempt = 1; attempt <= EXECUTION_ACQUIRE_MAX_ATTEMPTS; attempt++) {
try {
await acquire();
break;
} catch (error) {
if (!this.isConcurrentAcquireError(error)) {
throw error;
}
options.logger.warn(options.conflictMessage, { error });
if (attempt >= EXECUTION_ACQUIRE_MAX_ATTEMPTS) {
options.logger.warn(options.maxAttemptsMessage);
break;
}
}
}
}
async enter(execution, transaction) {
const workflow = execution.workflow || this.plugin.enabledCache.get(execution.workflowId) || await execution.getWorkflow({ transaction });
if (!workflow) {
this.plugin.getLogger(execution.workflowId).warn(`workflow (${execution.workflowId}) not found for execution`, {
workflowId: execution.workflowId,
executionId: execution.id
});
}
if (execution.status && execution.status !== import_constants.EXECUTION_STATUS.STARTED) {
return null;
}
if (execution.dispatched && execution.status === import_constants.EXECUTION_STATUS.STARTED && execution.startedAt) {
execution.workflow = workflow;
return execution;
}
const values = {
dispatched: true,
status: import_constants.EXECUTION_STATUS.STARTED
};
const where = {
id: execution.id,
status: execution.status ?? null
};
if (!execution.dispatched) {
where.dispatched = false;
}
if (!execution.startedAt) {
const startedAt = /* @__PURE__ */ new Date();
values.startedAt = startedAt;
execution.workflow = workflow;
values.expiresAt = this.plugin.timeoutManager.getExpiresAt(execution, startedAt);
where.startedAt = null;
}
const ExecutionModelClass = this.plugin.db.getModel("executions");
const [affected] = await ExecutionModelClass.update(values, {
where,
transaction
});
if (!affected) {
return null;
}
await execution.reload({ transaction });
execution.workflow = workflow;
return execution;
}
isConcurrentAcquireError(error) {
var _a, _b, _c, _d;
if (!(error instanceof Error)) {
return false;
}
const databaseError = error;
const code = ((_a = databaseError.parent) == null ? void 0 : _a.code) || ((_b = databaseError.original) == null ? void 0 : _b.code);
const errno = ((_c = databaseError.parent) == null ? void 0 : _c.errno) || ((_d = databaseError.original) == null ? void 0 : _d.errno);
return code === "40001" || code === "40P01" || code === "ER_LOCK_DEADLOCK" || errno === 1205 || errno === 1213;
}
async process(execution, job = null, options = {}) {
const { rerun, ...processorOptions } = options;
const logger = this.plugin.getLogger(execution.workflowId);
const run = async () => {
var _a, _b, _c;
if (!execution.dispatched) {
await execution.update({ dispatched: true, status: import_constants.EXECUTION_STATUS.STARTED });
logger.info(`execution (${execution.id}) from pending list updated to started`);
}
this.plugin.timeoutManager.scheduleExecutionTimeout(execution);
const processor = this.plugin.createProcessor(execution, processorOptions);
logger.info(`execution (${execution.id}) ${rerun ? "rerunning" : job ? "resuming" : "starting"}...`);
try {
await (rerun ? processor.rerun(rerun) : job ? processor.resume(job) : processor.start());
logger.info(`execution (${execution.id}) finished with status: ${execution.status}`);
logger.debug(`execution (${execution.id}) details:`, { execution });
if (execution.status && ((_c = (_b = (_a = execution.workflow) == null ? void 0 : _a.options) == null ? void 0 : _b.deleteExecutionOnStatus) == null ? void 0 : _c.includes(execution.status))) {
await execution.destroy();
}
} catch (err) {
if (err instanceof Error) {
logger.error(`execution (${execution.id}) error: ${err.message}`, err);
}
}
return processor;
};
const lock = await this.plugin.app.lockManager.tryAcquire((0, import_utils.getExecutionLockKey)(execution.id), 6e4);
try {
return await lock.runExclusive(run, 6e4);
} catch (error) {
logger.error(`execution (${execution.id}) could not acquire process lock`, { error });
throw error;
}
}
}