@camunda8/sdk
Version:
[](https://www.npmjs.com/package/@camunda8/sdk)
472 lines • 20.7 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ZBWorkerBase = void 0;
const events_1 = require("events");
const chalk_1 = __importDefault(require("chalk"));
const debug_1 = __importDefault(require("debug"));
const typed_duration_1 = require("typed-duration");
const uuid = __importStar(require("uuid"));
const lib_1 = require("../../lib");
const ConnectionStatusEvent_1 = require("./ConnectionStatusEvent");
const GrpcError_1 = require("./GrpcError");
const TypedEmitter_1 = require("./TypedEmitter");
const ZB = __importStar(require("./interfaces-1.0"));
const _1 = require(".");
const debug = (0, debug_1.default)('camunda:worker');
const verbose = (0, debug_1.default)('camunda:worker:verbose');
debug('Loaded ZBWorkerBase');
const MIN_ACTIVE_JOBS_RATIO_BEFORE_ACTIVATING_JOBS = 0.3;
const CapacityEvent = {
Available: 'AVAILABLE',
Empty: 'CAPACITY_EMPTY',
};
class ZBWorkerBase extends TypedEmitter_1.TypedEmitter {
constructor({ grpcClient, id, log, options, taskHandler, taskType, zbClient, inputVariableDto, customHeadersDto, tenantIds, }) {
super();
this.activeJobs = 0;
this.pollCount = 0;
this.cancelWorkflowOnException = false;
this.closing = false;
this.closed = false;
this.id = uuid.v4();
this.stalled = false;
this.connected = true;
this.readied = false;
this.pollMutex = false;
this.backPressureRetryCount = 0;
this.handleStreamEnd = (id) => {
this.jobStream?.removeAllListeners();
this.jobStream = undefined;
this.logger.logDebug(`Deleted job stream [${id}] listeners and job stream reference`);
};
this.handleJobResponse = (res) => {
// If we are closing, don't start working on these jobs. They will have to be timed out by the server.
if (this.closing) {
return;
}
/**
* At this point we know that we have a working connection to the server, so we can reset the backpressure retry count.
* Putting it here means that if we have a lot of connection errors and increment the backpressure count,
* then get a connection, but no jobs are activated, and before any jobs are activated we get another error condition
* then the backoff will start not from 0, but from the level of backoff we were at previously.
*/
this.backPressureRetryCount = 0;
this.activeJobs += res.jobs.length;
Promise.all(res.jobs
.map((job) => (0, _1.parseVariablesAndCustomHeadersToJSON)(job, this.inputVariableDto, this.customHeadersDto)
.then((job) => job)
.catch((e) => {
this.zbClient.failJob({
jobKey: job.key,
errorMessage: `Error parsing variable payload: ${e}`,
retries: job.retries - 1,
retryBackOff: 0,
});
return null;
}))
.filter((f) => !!f)).then((jobs) => this.handleJobs(jobs));
};
options = options || {};
this.tenantIds = tenantIds;
if (!taskType) {
throw new Error('Missing taskType');
}
if (!taskHandler) {
throw new Error('Missing taskHandler');
}
this.inputVariableDto =
inputVariableDto ??
lib_1.LosslessDto;
this.customHeadersDto =
customHeadersDto ??
lib_1.LosslessDto;
this.taskHandler = taskHandler;
this.taskType = taskType;
this.maxJobsToActivate =
options.maxJobsToActivate || ZBWorkerBase.DEFAULT_MAX_ACTIVE_JOBS;
this.activeJobsThresholdForReactivation =
this.maxJobsToActivate * MIN_ACTIVE_JOBS_RATIO_BEFORE_ACTIVATING_JOBS;
this.timeout =
options.timeout || ZBWorkerBase.DEFAULT_JOB_ACTIVATION_TIMEOUT;
this.pollInterval = options.pollInterval;
this.longPoll = options.longPoll;
this.pollInterval = options.pollInterval;
this.id = id || uuid.v4();
// Set options.debug to true to count the number of poll requests for testing
// See the Worker-LongPoll test
this.debugMode = options.debug === true;
this.grpcClient = grpcClient;
const onError = () => {
// options.onConnectionError?.()
if (this.connected) {
this.emit(ConnectionStatusEvent_1.ConnectionStatusEvent.connectionError);
options.onConnectionError?.();
this.connected = false;
this.readied = false;
}
};
this.grpcClient.on(ConnectionStatusEvent_1.ConnectionStatusEvent.connectionError, onError);
const onReady = async () => {
if (!this.readied) {
this.emit(ConnectionStatusEvent_1.ConnectionStatusEvent.ready);
options.onReady?.();
this.readied = true;
this.connected = true;
}
};
this.grpcClient.on(ConnectionStatusEvent_1.ConnectionStatusEvent.unknown, onReady);
this.grpcClient.on(ConnectionStatusEvent_1.ConnectionStatusEvent.ready, onReady);
this.cancelWorkflowOnException = options.failProcessOnException ?? false;
this.zbClient = zbClient;
this.grpcClient.topologySync().catch((e) => {
// Swallow exception to avoid throwing if retries are off
if (e.thisWillNeverHappenYo) {
this.emit(ConnectionStatusEvent_1.ConnectionStatusEvent.unknown);
}
});
this.fetchVariable = options.fetchVariable;
this.logger = log;
this.capacityEmitter = new events_1.EventEmitter();
this.CAMUNDA_JOB_WORKER_MAX_BACKOFF_MS =
options.CAMUNDA_JOB_WORKER_MAX_BACKOFF_MS; // We assert because it is optional in the explicit arguments, but hydrated from env config with a default
this.pollLoop = setInterval(() => this.poll(), typed_duration_1.Duration.milliseconds.from(this.pollInterval));
debug(`Created worker for task type ${taskType} - polling interval ${typed_duration_1.Duration.milliseconds.from(this.pollInterval)}`);
}
/**
* Returns a promise that the worker has stopped accepting tasks and
* has drained all current active tasks. Will reject if you try to call it more than once.
*/
close(timeout) {
if (this.closePromise) {
return this.closePromise;
}
// eslint-disable-next-line no-async-promise-executor
this.closePromise = new Promise(async (resolve) => {
// this.closing prevents the worker from starting work on any new tasks
this.closing = true;
clearInterval(this.pollLoop);
clearTimeout(this.backoffTimeout);
if (this.activeJobs <= 0) {
await this.grpcClient.close(timeout);
this.grpcClient.removeAllListeners();
this.jobStream?.removeAllListeners();
this.jobStream?.cancel?.();
this.jobStream = undefined;
this.logger.logDebug('Cancelled Job Stream');
resolve(null);
}
else {
this.capacityEmitter.once(CapacityEvent.Empty, async () => {
await this.grpcClient.close(timeout);
this.grpcClient.removeAllListeners();
this.emit(ConnectionStatusEvent_1.ConnectionStatusEvent.close);
this.removeAllListeners();
resolve(null);
});
}
});
return this.closePromise;
}
log(msg) {
this.logger.logInfo(msg);
}
debug(msg) {
this.logger.logDebug(msg);
}
error(msg) {
this.logger.logError(msg);
}
drainOne() {
this.activeJobs--;
this.logger.logDebug(`Load: ${this.activeJobs}/${this.maxJobsToActivate}`);
const hasSufficientAvailableCapacityToRequestMoreJobs = this.activeJobs <= this.activeJobsThresholdForReactivation;
if (!this.closing && hasSufficientAvailableCapacityToRequestMoreJobs) {
this.capacityEmitter.emit(CapacityEvent.Available);
}
if (this.closing && this.activeJobs === 0) {
this.capacityEmitter.emit(CapacityEvent.Empty);
}
// If we are closing and hit zero active jobs, resolve the closing promise.
if (this.activeJobs <= 0 && this.closing) {
if (this.closeCallback && !this.closed) {
this.closeCallback();
}
}
}
handleJobs(jobs) {
this.log(jobs.length);
throw new Error('This method must be declared in a class that extends this base');
}
makeCompleteHandlers(thisJob) {
let methodCalled;
/**
* This is a wrapper that allows us to throw an error if a job acknowledgement function is called more than once,
* for these functions should be called once only (and only one should be called, but we don't handle that case).
*/
const errorMsgOnPriorMessageCall = (thisMethod, wrappedFunction) => {
return (...args) => {
if (methodCalled !== undefined) {
// tslint:disable-next-line: no-console
console.log(chalk_1.default.red(`WARNING: Call to ${thisMethod}() after ${methodCalled}() was called.
You should call only one job action method in the worker handler. This is a bug in the ${this.taskType} worker handler.`));
return wrappedFunction(...args);
}
methodCalled = thisMethod;
return wrappedFunction(...args);
};
};
const cancelWorkflow = (job) => () => this.zbClient
.cancelProcessInstance(job.processInstanceKey)
.then(() => ZB.JOB_ACTION_ACKNOWLEDGEMENT);
const failJob = (job) => (conf, retries) => {
const isFailureConfig = (_conf) => typeof _conf === 'object';
const errorMessage = isFailureConfig(conf) ? conf.errorMessage : conf;
const retryBackOff = isFailureConfig(conf) ? conf.retryBackOff ?? 0 : 0;
const _retries = isFailureConfig(conf) ? conf.retries ?? 0 : retries;
return this.failJob({
job,
errorMessage,
retries: _retries,
retryBackOff,
});
};
const succeedJob = (job) => (completedVariables) => this.completeJob(job.key, completedVariables ?? {});
const errorJob = (job) => (e, errorMessage = '') => {
const isErrorJobWithVariables = (s) => typeof s === 'object';
const errorCode = isErrorJobWithVariables(e) ? e.errorCode : e;
errorMessage = isErrorJobWithVariables(e)
? e.errorMessage ?? ''
: errorMessage;
const variables = isErrorJobWithVariables(e) ? e.variables : {};
return this.errorJob({
errorCode,
errorMessage,
job,
variables,
});
};
const fail = failJob(thisJob);
const succeed = succeedJob(thisJob);
return {
cancelWorkflow: cancelWorkflow(thisJob),
complete: errorMsgOnPriorMessageCall('job.complete', succeed),
error: errorMsgOnPriorMessageCall('error', errorJob(thisJob)),
fail: errorMsgOnPriorMessageCall('job.fail', fail),
forward: errorMsgOnPriorMessageCall('job.forward', () => {
this.drainOne();
return ZB.JOB_ACTION_ACKNOWLEDGEMENT;
}),
};
}
failJob({ job, errorMessage, retries, retryBackOff, variables, }) {
return this.zbClient
.failJob({
errorMessage,
jobKey: job.key,
retries: retries ?? job.retries - 1,
retryBackOff: retryBackOff ?? 0,
variables: variables ?? {},
})
.then(() => ZB.JOB_ACTION_ACKNOWLEDGEMENT)
.finally(() => {
this.logger.logDebug(`Failed job ${job.key} - ${errorMessage}`);
this.drainOne();
});
}
completeJob(jobKey, completedVariables = {}) {
return this.zbClient
.completeJob({
jobKey,
variables: completedVariables,
})
.then((res) => {
this.logger.logDebug(`Completed job ${jobKey} for ${this.taskType}`);
return res;
})
.catch((e) => {
this.logger.logDebug(`Completing job ${jobKey} for ${this.taskType} threw ${e.message}`);
throw e;
})
.then(() => ZB.JOB_ACTION_ACKNOWLEDGEMENT)
.finally(() => {
this.drainOne();
});
}
errorJob({ errorCode, errorMessage, job, variables, }) {
return this.zbClient
.throwError({
errorCode,
errorMessage,
jobKey: job.key,
variables,
})
.then(() => this.logger.logDebug(`Errored job ${job.key} - ${errorMessage}`))
.catch((e) => {
this.logger.logError(`Exception while attempting to raise BPMN Error for job ${job.key} - ${errorMessage}`);
this.logger.logError(e);
})
.then(() => {
this.drainOne();
return ZB.JOB_ACTION_ACKNOWLEDGEMENT;
});
}
async poll() {
const pollAlreadyInProgress = this.pollMutex || this.jobStream !== undefined;
const workerIsClosing = !!this.closePromise || this.closing;
const insufficientCapacityAvailable = this.activeJobs > this.activeJobsThresholdForReactivation;
if (pollAlreadyInProgress ||
workerIsClosing ||
insufficientCapacityAvailable) {
verbose('Worker polling blocked', {
pollAlreadyInProgress,
workerIsClosing,
insufficientCapacityAvailable,
});
return;
}
this.pollMutex = true;
debug('Polling...');
this.logger.logDebug('Activating Jobs...');
const id = uuid.v4();
const jobStream = await this.activateJobs(id);
const start = Date.now();
this.logger.logDebug(`Long poll loop. this.longPoll: ${typed_duration_1.Duration.value.of(this.longPoll)}`, id, start);
let doBackoff = false;
if (jobStream.stream) {
this.logger.logDebug(`Stream [${id}] opened...`);
this.jobStream = jobStream.stream;
jobStream.stream.on('error', (error) => {
this.pollMutex = true;
this.logger.logDebug(`Stream [${id}] error after ${(Date.now() - start) / 1000} seconds`, error);
const errorCode = error.code;
// Backoff on
const backoffErrorCodes = [
GrpcError_1.GrpcError.RESOURCE_EXHAUSTED,
GrpcError_1.GrpcError.INTERNAL,
GrpcError_1.GrpcError.UNAUTHENTICATED,
];
doBackoff = backoffErrorCodes.includes(errorCode);
this.emit('streamError', error);
if (doBackoff) {
const backoffDuration = Math.min(this.CAMUNDA_JOB_WORKER_MAX_BACKOFF_MS, 1000 * 2 ** this.backPressureRetryCount++);
this.emit('backoff', backoffDuration);
this.logger.logInfo(`Backing off worker poll due to failure. Next attempt will be made in ${backoffDuration}ms...`);
setTimeout(() => {
this.handleStreamEnd(id);
this.pollMutex = false;
}, backoffDuration);
}
else {
this.handleStreamEnd(id);
this.pollMutex = false;
}
});
// This event happens when the server cancels the call after the deadline
// And when it has completed a response with work
// And also after an error event.
jobStream.stream.on('end', () => {
this.logger.logDebug(`Stream [${id}] ended after ${(Date.now() - start) / 1000} seconds`);
this.handleStreamEnd(id);
});
}
if (jobStream.error) {
const error = jobStream.error.message;
const message = 'Grpc Stream Error: ';
const backoffErrorCodes = [
`${message}${GrpcError_1.GrpcError.RESOURCE_EXHAUSTED}`,
`${message}${GrpcError_1.GrpcError.INTERNAL}`,
`${message}${GrpcError_1.GrpcError.UNAUTHENTICATED}`,
];
doBackoff = backoffErrorCodes.map((e) => error.includes(e)).includes(true);
const backoffDuration = Math.min(this.CAMUNDA_JOB_WORKER_MAX_BACKOFF_MS, 1000 * 2 ** this.backPressureRetryCount++);
this.logger.logError({ id, error });
if (doBackoff) {
this.emit('backoff', backoffDuration);
this.backoffTimeout = setTimeout(() => {
this.handleStreamEnd(id);
this.pollMutex = false;
}, backoffDuration);
}
}
else {
this.pollMutex = false;
}
}
async activateJobs(id) {
if (this.stalled) {
debug('Stalled');
return { stalled: true };
}
if (this.closing) {
debug('Closing');
return {
closing: true,
};
}
if (this.debugMode) {
this.logger.logDebug('Activating Jobs...');
}
debug('Activating Jobs');
let stream;
const amount = this.maxJobsToActivate - this.activeJobs;
const requestTimeout = this.longPoll ?? -1;
const activateJobsRequest = {
maxJobsToActivate: amount,
requestTimeout,
timeout: this.timeout,
type: this.taskType,
worker: this.id,
fetchVariable: this.fetchVariable,
tenantIds: this.tenantIds,
};
this.logger.logDebug(`Requesting ${amount} jobs on [${id}] with requestTimeout ${typed_duration_1.Duration.value.of(requestTimeout)}, job timeout: ${typed_duration_1.Duration.value.of(this.timeout)}`);
debug(`Requesting ${amount} jobs on [${id}] with requestTimeout ${typed_duration_1.Duration.value.of(requestTimeout)}, job timeout: ${typed_duration_1.Duration.value.of(this.timeout)}`);
try {
stream = await this.grpcClient.activateJobsStream(activateJobsRequest);
if (this.debugMode) {
this.pollCount++;
}
}
catch (error) {
return {
error: error,
};
}
if (stream.error) {
debug('Stream error', stream.error);
return { error: stream.error };
}
stream.on('data', this.handleJobResponse);
return { stream };
}
}
exports.ZBWorkerBase = ZBWorkerBase;
ZBWorkerBase.DEFAULT_JOB_ACTIVATION_TIMEOUT = typed_duration_1.Duration.seconds.of(60);
ZBWorkerBase.DEFAULT_MAX_ACTIVE_JOBS = 32;
//# sourceMappingURL=ZBWorkerBase.js.map