@5minds/processcube_engine
Version:
The ProcessCube Engine. Stores and executes BPMNs.
440 lines • 19.8 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = setupSequelize;
exports.getSequelizeInstanceWithoutModels = getSequelizeInstanceWithoutModels;
exports.destroySequelizeInstance = destroySequelizeInstance;
exports.databaseExists = databaseExists;
exports.executeWithRetry = executeWithRetry;
const async_lock_1 = __importDefault(require("async-lock"));
const dayjs_1 = __importDefault(require("dayjs"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const sequelize_typescript_1 = require("sequelize-typescript");
const uuid = __importStar(require("uuid"));
const processcube_engine_sdk_1 = require("@5minds/processcube_engine_sdk");
const Configurator_1 = __importDefault(require("../../Configurator"));
const Environment_1 = require("../../Environment");
const schemas = __importStar(require("../Models/index"));
const defaultMaxRetryDurationInMs = dayjs_1.default.duration('PT1H').asMilliseconds();
const defaultMaxRetryDurationUnit = 's';
const logger = new processcube_engine_sdk_1.Logger('sequelize_connection_manager');
const asyncLocker = new async_lock_1.default({ maxPending: 100000 });
let sequelizeInstance;
let maxDurationString;
let maxRetryDurationInMs;
/**
* Returns a Sequelize connection and loads all models into it.
*
* @returns A ready-to-use Sequelize instance.
*/
async function setupSequelize() {
if (sequelizeInstance != undefined) {
return sequelizeInstance;
}
const config = Configurator_1.default.database();
maxDurationString = config.retry?.maxDuration ?? '60m';
maxRetryDurationInMs = getMaxRetryDurationInMilliseconds(maxDurationString);
sequelizeInstance = await getSequelizeInstanceWithoutModels(config);
sequelizeInstance.addModels([
schemas.CorrelationMetadataModel,
schemas.CorrelationModel,
schemas.DataObjectModel,
schemas.ExternalTaskModel,
schemas.HttpServiceTaskModel,
schemas.ProcessDefinitionModel,
schemas.ProcessDefinitionProcessModelAssociationModel,
schemas.ProcessInstanceModel,
schemas.ProcessInstanceMetadataModel,
schemas.CallActivityInstanceMetaInfoModel,
schemas.CatchEventInstanceMetaInfoModel,
schemas.ThrowEventInstanceMetaInfoModel,
schemas.ManualTaskMetaInfoModel,
schemas.MultiInstanceMetadataModel,
schemas.SubprocessInstancesMetaInfoModel,
schemas.UserTaskMetaInfoModel,
schemas.FlowNodeInstanceModel,
schemas.CronjobModel,
]);
await sequelizeInstance.sync();
return sequelizeInstance;
}
/**
* Returns a Sequelize connection for the given configuration, without loading any models.
*
* @async
* @param config The config to use.
* @param config.host The name of the host where the database is located.
* @param config.port The port by which to connect to the host.
* @param config.dialect The type of database to which to connect (postgres, sqlite, mysql, etc).
* @param config.database The name of the database to connect to.
* @param config.username The username with which to connect to the database.
* @param config.password The password with which to connect to the database.
*
* @returns A ready-to-use Sequelize instance.
*/
async function getSequelizeInstanceWithoutModels(config) {
try {
return await connectToDatabase(config);
}
catch (error) {
error.category = 'setup';
error.message = `Failure in setup: ${error.message}`;
logger.error(error.message, {
err: error,
});
throw error;
}
}
/**
* Destroys the currently active sequelize instance.
* @async
*/
async function destroySequelizeInstance() {
if (!sequelizeInstance) {
logger.trace(`Skipping destroySequelizeInstance, because no sequelize instance is currently initialized.`);
return;
}
const dbName = sequelizeInstance.options.dialect === 'sqlite' ? sequelizeInstance.options.storage : sequelizeInstance.options.database;
logger.info(`Disposing connection to ${sequelizeInstance.options.dialect} database '${dbName}'...`);
try {
await sequelizeInstance.close();
}
catch (error) {
logger.warn(`Cannot close connection to ${sequelizeInstance.options.dialect} database '${dbName}', because it was already disposed.`);
logger.warn(error.message);
}
finally {
sequelizeInstance = undefined;
logger.info('Done.');
}
}
/**
* Returns a boolean whether the given database exists or not.
*
* @async
* @param config The config to use.
* @param config.host The name of the host where the database is located.
* @param config.port The port by which to connect to the host.
* @param config.dialect The type of database to which to connect (postgres, sqlite, mysql, etc).
* @param config.database The name of the database to connect to.
* @param config.username The username with which to connect to the database.
* @param config.password The password with which to connect to the database.
*
* @return A boolean indicating whether the database exists or not.
*/
async function databaseExists(config) {
try {
await connectToDatabase(config);
return true;
}
catch (error) {
const isPostgresError = error.additionalInformation?.error?.original?.code === '3D000';
const isMysqlError = error.additionalInformation?.error?.original?.code === 'ER_BAD_DB_ERROR' || error.additionalInformation?.error?.original?.code === 'ER_DBACCESS_DENIED_ERROR';
const isMssqlError = error.additionalInformation?.error?.original?.code === 'ELOGIN';
const databaseIsNotExisting = isPostgresError || isMysqlError || isMssqlError;
if (databaseIsNotExisting) {
return false;
}
const connectionError = new processcube_engine_sdk_1.InternalServerError(`Could not connect to database: ${error.message}`, 'setup');
connectionError.fatal = true;
connectionError.additionalInformation = {
originalError: error,
};
throw connectionError;
}
}
/*
* Executes the given query function with retry logic.
*
* @async
* @param queryFunction The query function to execute.
* @param infoText A text describing the query function.
* @param retryCounter The current retry counter.
* @param currentRetryDuration The current retry duration.
* @param dbRequestId The ID of the DB request.
* @param maxTimeoutInMs The maximum timeout in ms.
*
* @return The result of the query function.
*/
async function executeWithRetry(queryFunction, infoText, retryCounter = 0, currentRetryDuration = 0, dbRequestId = uuid.v4(), maxTimeoutInMs = 60000) {
const timeoutBaseInMs = 800;
const timeoutMaxBufferInMs = 500;
const timeoutExponent = 1.04;
logger.trace(`Executing ${infoText} DB request with ID ${dbRequestId}.`, {
dbRequestId: dbRequestId,
});
try {
const result = await queryFunction();
return result;
}
catch (error) {
error.category = 'setup';
error.message = `Failure in setup: ${error.message}`;
if (isFatalError(error)) {
logger.error(`Failure in setup: Failed to execute ${infoText}. Encountered a fatal database error. Exiting.`, {
dbRequestId: dbRequestId,
err: error,
});
setTimeout(() => process.exit(1), 2000);
}
if (!isRetryableError(error)) {
logger.error(`Failure in setup: Failed to execute ${infoText}. Error is not retryable, aborting query.`, {
dbRequestId: dbRequestId,
err: error,
});
throw error;
}
if (currentRetryDuration < maxRetryDurationInMs) {
const nextRetryCounter = retryCounter + 1;
const nextTimeoutBase = timeoutBaseInMs ** (timeoutExponent ** nextRetryCounter);
const nextTimeoutBuffer = Math.random() * timeoutMaxBufferInMs;
let nextTimeoutInMs = Math.floor((nextTimeoutBase > maxTimeoutInMs ? maxTimeoutInMs : nextTimeoutBase) + nextTimeoutBuffer);
if (currentRetryDuration + nextTimeoutInMs > maxRetryDurationInMs) {
nextTimeoutInMs = maxRetryDurationInMs - currentRetryDuration;
}
currentRetryDuration += nextTimeoutInMs;
logger.warn(`Failed to execute ${infoText}. Retrying in ${nextTimeoutInMs} ms.`, {
dbRequestId: dbRequestId,
err: error,
});
await new Promise((resolve) => setTimeout(resolve, nextTimeoutInMs));
const newResult = await executeWithRetry(queryFunction, infoText, nextRetryCounter, currentRetryDuration, dbRequestId);
return newResult;
}
logger.error(`Failure in setup: Failed to execute ${infoText}. Maximum retry duration "${maxDurationString}" exceeded. Exiting...`, {
dbRequestId: dbRequestId,
err: error,
});
// If all retries have failed, the database is likely not reachable. The engine should shut down, because it can't work anyway.
// Plus, this will let the user know that something is terribly wrong.
setTimeout(() => process.exit(1), 2000);
}
}
function isFatalError(error) {
// https://code4developers.com/list-of-all-the-error-codes-or-messages-in-sql-server/
// Note that SQL Server uses error numbers instead of codes.
// The "Codes" are assigned by tedious, are a lot fewer in number and a lot less specific.
// We have to match the error number, not the code, or risk crashing the engine for something that isn't actually fatal.
const fatalMsSQlErrorNumbers = [566, 808, 945, 1101, 1105, 5128, 7622, 9901, 40544, 41822];
// https://www.postgresql.org/docs/current/errcodes-appendix.html
const fatalPostgresErrorCodes = ['53000', '53100', '53200', '53400', 'XX001', 'XX002'];
if (sequelizeInstance?.getDialect() === 'mssql') {
return fatalMsSQlErrorNumbers.some((errorNumber) => error.original?.number === errorNumber);
}
else if (sequelizeInstance?.getDialect() === 'postgres') {
return fatalPostgresErrorCodes.some((errorCode) => error.original?.code == errorCode);
}
// https://dev.mysql.com/doc/mysql-errors/5.7/en/server-error-reference.html
// https://sqlite.org/rescode.html
const fatalErrorCodesforOtherDbs = [
/ER_DISK_FULL/i,
/ER_OUTOFMEMORY/i,
/ER_OUT_OF_RESOURCES/i,
/ER_RECORD_FILE_FULL/i,
/ER_ZLIB_Z_MEM_ERROR/i,
/ER_ENGINE_OUT_OF_MEMORY/i,
/ER_CAPACITY_EXCEEDED/i,
/SQLITE_CORRUPT/i,
/SQLITE_FULL/i,
/SQLITE_NOLFS/i,
/HV001/i,
];
return fatalErrorCodesforOtherDbs.some((match) => error.original?.code != null && match.test(error.original?.code));
}
function isRetryableError(error) {
// Copied from https://github.com/dotnet/efcore/blob/main/src/EFCore.SqlServer/Storage/Internal/SqlServerTransientExceptionDetector.cs
const retryableMssqlErrorNumbers = [
49920, 49919, 49918, 41325, 41305, 40613, 40501, 40197, 20041, 17197, 14355, 10936, 10929, 10928, 10922, 10060, 10054, 10053, 9515, 8651, 8645, 8628, 4221, 3966, 3960, 3935,
1807, 1221, 1205, 1204, 1203, 997, 921, 669, 617, 601, 233, 121, 64,
];
// https://www.postgresql.org/docs/current/errcodes-appendix.html
const retryablePostgresErrorCodes = ['40000', '40001', '40003', '40P01', '53300', '08007', '08000'];
let isRetryableDialectSpecificError = false;
if (sequelizeInstance?.getDialect() === 'mssql') {
isRetryableDialectSpecificError = retryableMssqlErrorNumbers.some((errorNumber) => error.original?.number === errorNumber);
}
else if (sequelizeInstance?.getDialect() === 'postgres') {
isRetryableDialectSpecificError = retryablePostgresErrorCodes.some((code) => error.original?.code == code);
}
const retryableErrors = [
/Committing transaction.*?failed with error.*?We are killing its connection as it is now in an undetermined state/i,
/Rolling back transaction.*?failed with error "Client has encountered a connection error and is not queryable"/i,
/The request limit for the database .*has been reached/i,
/EREQUEST/i,
/ESOCKET/i,
/ETIMEDOUT/i,
/EHOSTUNREACH/i,
/ECONNRESET/i,
/ECONNREFUSED/i,
/ETIMEOUT/i,
/ESOCKETTIMEDOUT/i,
/EHOSTUNREACH/i,
/EPIPE/i,
/EAI_AGAIN/i,
/EINVALIDSTATE/i,
/Failed to connect/i,
/^(?!.*Bad).*RequestError.*/i,
/TimeoutError/i,
/Timeout/i,
/ConnectionAcquireTimeoutError/i,
/SequelizeConnectionAcquireTimeoutError/i,
/SequelizeConnectionRefusedError/i,
/SequelizeConnectionTimedOutError/i,
/SequelizeHostNotFoundError/i,
/SequelizeHostNotReachableError/i,
/SequelizeInvalidConnectionError/i,
/SequelizeTimeoutError/i,
/Server shutdown in progress/i,
/socket hang up/i,
/session/i,
/SQL_BUSY/i,
/SQLITE_BUSY/i,
/SequelizeConnectionError/i,
/ConnectionError/i,
/Could not connect (sequence)/i,
/Requests can only be made in the LoggedIn state/i,
/connection error/i,
/terminating connection/i,
/Connection terminated/i,
/Connection lost/i,
/PROTOCOL_CONNECTION_LOST/i,
/EPIPE/i,
/This socket has been ended/i,
/connection is in closed state/i,
/was deadlocked on lock resources with another process and has been chosen as the deadlock victim/i,
/Rerun the transaction/i,
/Failed to cancel request/i,
];
return (isRetryableDialectSpecificError || retryableErrors.some((match) => match.test(error.message) || (error.code != null && match.test(error.code)) || match.test(error.toString())));
}
async function connectToDatabase(config) {
const newConfig = buildConfig(config);
const dbToUse = newConfig.dialect === 'sqlite' ? newConfig.storage : newConfig.database;
logger.trace(`Evaluated connection pool config for ${(0, Environment_1.getBeautifiedThreadName)()}.`, { pool: newConfig.pool });
return asyncLocker.acquire('GetOrCreateConnection', async () => {
if (sequelizeInstance != undefined) {
logger.trace(`Active connection to ${newConfig.dialect} database '${dbToUse}' found.`);
return sequelizeInstance;
}
try {
if (newConfig.dialect === 'sqlite' && path_1.default.isAbsolute(newConfig.storage) && !fs_1.default.existsSync(newConfig.storage)) {
const parsedPath = path_1.default.parse(newConfig.storage);
fs_1.default.mkdirSync(parsedPath.dir, { recursive: true });
fs_1.default.closeSync(fs_1.default.openSync(path_1.default.normalize(newConfig.storage), 'a'));
}
if (typeof newConfig.logging === 'string') {
newConfig.logging = newConfig.logging === 'true';
}
const connection = new sequelize_typescript_1.Sequelize(dbToUse, newConfig.username, newConfig.password, newConfig);
await connection.authenticate();
if (newConfig.schema) {
await connection.createSchema(newConfig.schema, {});
}
logger.info(`Connection to ${newConfig.dialect} database '${dbToUse}' established.`);
return connection;
}
catch (error) {
const errorMessage = `Unable to connect to ${newConfig.dialect} database '${dbToUse}'`;
const connectionError = new processcube_engine_sdk_1.InternalServerError(errorMessage, 'setup');
connectionError.fatal = true;
connectionError.additionalInformation = {
dialect: newConfig.dialect,
database: dbToUse,
error: error,
};
throw connectionError;
}
});
}
function buildConfig(config) {
const newConfig = {
...config,
};
if (newConfig.pool) {
newConfig.pool = {
...config.pool,
};
newConfig.pool.max = (0, Environment_1.calculateMaxPoolSize)(config.pool) || newConfig.pool.max;
delete newConfig.pool.maxForExternalTasks;
delete newConfig.pool.maxForProcessExecution;
delete newConfig.pool.maxForQueryRequests;
}
if (newConfig.dialect != 'sqlite') {
newConfig.pool = {
max: 100,
...newConfig.pool,
};
}
return newConfig;
}
function getMaxRetryDurationInMilliseconds(durationExpression) {
if (durationExpression == null) {
return defaultMaxRetryDurationInMs;
}
// Numbers are interpreted as seconds.
if (typeof durationExpression === 'number') {
return durationExpression * 1000;
}
const durationRegex = /^([0-9]{1,})([sdhm]{0,1})$/g;
const durationRegexMatches = durationRegex.exec(durationExpression);
if (durationRegexMatches == null) {
const errorMsg = `Unable to parse maximum duration expression '${durationExpression}'. Durations must be provided as "[TIME][d|h|m|s]". For example: "30s".`;
logger.error(errorMsg, { maxDurationExpression: durationExpression });
setImmediate(() => {
process.exit(1);
});
}
const time = durationRegexMatches[1];
const unitOfTime = durationRegexMatches[2] === '' ? defaultMaxRetryDurationUnit : durationRegexMatches[2];
const parsedDuration = getDurationBasedOnAbbreviation(time, unitOfTime);
return parsedDuration.asMilliseconds();
}
function getDurationBasedOnAbbreviation(time, abbreviation) {
switch (abbreviation) {
case 'd':
return dayjs_1.default.duration(`P${time}D`);
case 'h':
return dayjs_1.default.duration(`PT${time}H`);
case 'm':
return dayjs_1.default.duration(`PT${time}M`);
case 's':
return dayjs_1.default.duration(`PT${time}S`);
}
}
//# sourceMappingURL=SequelizeConnectionManager.js.map