pg-transactional-outbox
Version:
A PostgreSQL based transactional outbox and inbox pattern implementation to support exactly once message processing (with at least once message delivery).
142 lines • 9.92 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.initializePollingMessageListener = void 0;
const events_1 = __importDefault(require("events"));
const pg_1 = require("pg");
const timers_1 = require("timers");
const error_1 = require("../common/error");
const utils_1 = require("../common/utils");
const create_error_handler_1 = require("../handler/create-error-handler");
const create_message_handler_1 = require("../handler/create-message-handler");
const message_cleanup_1 = require("../message/message-cleanup");
const message_not_found_retry_strategy_1 = require("../strategies/message-not-found-retry-strategy");
const message_processing_db_client_strategy_1 = require("../strategies/message-processing-db-client-strategy");
const message_processing_timeout_strategy_1 = require("../strategies/message-processing-timeout-strategy");
const message_processing_transaction_level_strategy_1 = require("../strategies/message-processing-transaction-level-strategy");
const message_retry_strategy_1 = require("../strategies/message-retry-strategy");
const poisonous_message_retry_strategy_1 = require("../strategies/poisonous-message-retry-strategy");
const config_1 = require("./config");
const next_messages_1 = require("./next-messages");
const batch_size_strategy_1 = require("./strategies/batch-size-strategy");
/**
* Initialize the listener to watch for outbox or inbox table inserts via
* polling the corresponding PostgreSQL database tables.
* @param config The configuration object with required values to connect to the database for polling.
* @param messageHandlers A list of message handlers to handle specific messages or a single general message handler that handles all messages.
* @param logger A logger instance for logging trace up to error logs
* @param strategies Strategies to provide custom logic for handling specific scenarios
* @returns Functions for a clean shutdown.
*/
const initializePollingMessageListener = (config, messageHandlers, logger, strategies) => {
const fullConfig = (0, config_1.applyDefaultPollingListenerConfigValues)(config);
const allStrategies = applyDefaultStrategies(strategies, fullConfig, logger);
const messageHandler = (0, create_message_handler_1.createMessageHandler)(messageHandlers, allStrategies, fullConfig, logger, 'polling');
const errorHandler = (0, create_error_handler_1.createErrorHandler)(messageHandlers, allStrategies, fullConfig, logger);
let pool = undefined;
let cleanupTimeout;
// Start the asynchronous background polling loop
// TODO: implement LISTEN/NOTIFY to immediately run the next interval (when workers are free)
const signal = { stopped: false };
(function start() {
logger.debug(`Start polling for ${fullConfig.outboxOrInbox} messages.`);
if (pool) {
(0, utils_1.justDoIt)(pool.end);
}
pool = new pg_1.Pool(fullConfig.dbListenerConfig);
pool.on('error', (error) => {
logger.error((0, error_1.ensureExtendedError)(error, 'DB_ERROR'), 'PostgreSQL pool error');
});
pool.on('connect', (client) => {
client.removeAllListeners('notice');
client.on('notice', (msg) => {
logger.trace(`raised notice ${msg.message}`);
});
});
clearTimeout(cleanupTimeout);
cleanupTimeout = (0, message_cleanup_1.runScheduledMessageCleanup)(pool, fullConfig, logger);
const applyRestart = (promise) => {
void promise.catch((e) => __awaiter(this, void 0, void 0, function* () {
logger.error((0, error_1.ensureExtendedError)(e, 'LISTENER_STOPPED'), `Error polling for ${fullConfig.outboxOrInbox} messages.`);
if (!signal.stopped) {
yield (0, utils_1.sleep)(1000);
setImmediate(start); // restart
}
}));
};
const getNextBatch = (batchSize) => __awaiter(this, void 0, void 0, function* () {
return processBatch(batchSize, fullConfig, pool, messageHandler, errorHandler, allStrategies, logger);
});
const processingPool = new Set();
applyRestart((0, utils_1.processPool)(processingPool, getNextBatch, allStrategies.batchSizeStrategy, signal, fullConfig.settings.nextMessagesPollingIntervalInMs));
})();
return [
() => __awaiter(void 0, void 0, void 0, function* () {
var _a;
(0, timers_1.clearInterval)(cleanupTimeout);
signal.stopped = true;
yield Promise.all([
(_a = allStrategies.messageProcessingDbClientStrategy) === null || _a === void 0 ? void 0 : _a.shutdown(),
pool === null || pool === void 0 ? void 0 : pool.end(),
]);
}),
];
};
exports.initializePollingMessageListener = initializePollingMessageListener;
const applyDefaultStrategies = (strategies, config, logger) => {
var _a, _b, _c, _d, _e, _f, _g;
return ({
messageProcessingDbClientStrategy: (_a = strategies === null || strategies === void 0 ? void 0 : strategies.messageProcessingDbClientStrategy) !== null && _a !== void 0 ? _a : (0, message_processing_db_client_strategy_1.defaultMessageProcessingDbClientStrategy)(config, logger),
messageProcessingTimeoutStrategy: (_b = strategies === null || strategies === void 0 ? void 0 : strategies.messageProcessingTimeoutStrategy) !== null && _b !== void 0 ? _b : (0, message_processing_timeout_strategy_1.defaultMessageProcessingTimeoutStrategy)(config),
messageProcessingTransactionLevelStrategy: (_c = strategies === null || strategies === void 0 ? void 0 : strategies.messageProcessingTransactionLevelStrategy) !== null && _c !== void 0 ? _c : (0, message_processing_transaction_level_strategy_1.defaultMessageProcessingTransactionLevelStrategy)(),
messageRetryStrategy: (_d = strategies === null || strategies === void 0 ? void 0 : strategies.messageRetryStrategy) !== null && _d !== void 0 ? _d : (0, message_retry_strategy_1.defaultMessageRetryStrategy)(config),
poisonousMessageRetryStrategy: (_e = strategies === null || strategies === void 0 ? void 0 : strategies.poisonousMessageRetryStrategy) !== null && _e !== void 0 ? _e : (0, poisonous_message_retry_strategy_1.defaultPoisonousMessageRetryStrategy)(config),
batchSizeStrategy: (_f = strategies === null || strategies === void 0 ? void 0 : strategies.batchSizeStrategy) !== null && _f !== void 0 ? _f : (0, batch_size_strategy_1.defaultPollingListenerBatchSizeStrategy)(config),
messageNotFoundRetryStrategy: (_g = strategies === null || strategies === void 0 ? void 0 : strategies.messageNotFoundRetryStrategy) !== null && _g !== void 0 ? _g : (0, message_not_found_retry_strategy_1.defaultMessageNotFoundRetryStrategy)(config),
});
};
const processBatch = (batchSize, config, pool, messageHandler, errorHandler, allStrategies, logger) => __awaiter(void 0, void 0, void 0, function* () {
let messages = [];
try {
messages = yield (0, next_messages_1.getNextMessagesBatch)(batchSize, yield (0, utils_1.getClient)(pool, logger), config.settings, logger, config.outboxOrInbox);
// Messages can be processed in parallel
const batchPromises = messages.map((message) => __awaiter(void 0, void 0, void 0, function* () {
const cancellation = new events_1.default();
try {
const messageProcessingTimeoutInMs = allStrategies.messageProcessingTimeoutStrategy(message);
yield (0, utils_1.awaitWithTimeout)(() => __awaiter(void 0, void 0, void 0, function* () {
logger.debug(message, `Executing the message handler for message with ID ${message.id}.`);
yield messageHandler(message, cancellation);
}), messageProcessingTimeoutInMs, `Could not process the message with id ${message.id} within the timeout of ${messageProcessingTimeoutInMs} milliseconds. Please consider to use a background worker for long running tasks to not block the message processing.`);
logger.trace(message, `Finished processing the message with id ${message.id} and type ${message.messageType}.`);
}
catch (e) {
const err = (0, error_1.ensureExtendedError)(e, 'MESSAGE_HANDLING_FAILED', message);
if (err.errorCode === 'TIMEOUT') {
cancellation.emit('timeout', err);
}
logger.warn(err, `Message processing error for the message with id ${message.id} and type ${message.messageType}.`);
yield errorHandler(message, err);
}
}));
return batchPromises;
}
catch (batchError) {
const error = (0, error_1.ensureExtendedError)(batchError, 'BATCH_PROCESSING_ERROR');
Object.assign(error, { messages });
logger.error(error, `Error when working on a batch of ${config.outboxOrInbox} messages.`);
return [];
}
});
//# sourceMappingURL=polling-message-listener.js.map