pg-transactional-outbox
Version:
A PostgreSQL based transactional outbox and inbox pattern implementation to support exactly once message processing (with at least once message delivery).
298 lines • 15 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.__only_for_unit_tests__ = exports.createLogicalReplicationListener = void 0;
const events_1 = require("events");
const pg_1 = require("pg");
const pg_logical_replication_1 = require("pg-logical-replication");
const error_1 = require("../common/error");
const utils_1 = require("../common/utils");
const message_cleanup_1 = require("../message/message-cleanup");
const acknowledge_manager_1 = require("./acknowledge-manager");
/**
* Initiate the outbox or inbox listener to listen for WAL messages.
* @param config The replication connection settings and general replication settings
* @param messageHandler The message handler that handles the outbox/inbox message
* @param errorHandler A handler that can decide if the WAL message should be acknowledged (permanent_error) or not (transient_error which restarts the logical replication listener)
* @param logger A logger instance for logging trace up to error logs
* @param strategies Strategies to provide custom logic for handling specific scenarios
* @returns A function to stop the logical replication listener
*/
const createLogicalReplicationListener = (config, messageHandler, errorHandler, logger, strategies) => {
const { dbListenerConfig, settings, outboxOrInbox } = config;
const plugin = new pg_logical_replication_1.PgoutputPlugin({
protoVersion: 1,
publicationNames: [settings.dbPublication],
});
let client;
let pool;
let restartTimeout;
let cleanupTimeout;
let stopped = false;
// Run the listener in a self restarting background event "loop" until it gets stopped
(function start() {
if (stopped) {
return;
}
logger.debug(`Transactional ${outboxOrInbox} listener starting`);
restartTimeout = undefined;
client = new pg_1.Client(Object.assign(Object.assign({}, dbListenerConfig), { replication: 'database', stop: false }));
pool = new pg_1.Pool(dbListenerConfig);
clearTimeout(cleanupTimeout);
cleanupTimeout = (0, message_cleanup_1.runScheduledMessageCleanup)(pool, config, logger);
const applyRestart = (promise) => {
void promise.catch((e) => __awaiter(this, void 0, void 0, function* () {
const err = (0, error_1.ensureExtendedError)(e, 'LISTENER_STOPPED');
const timeout = yield strategies.listenerRestartStrategy(err, logger, outboxOrInbox);
// On errors stop the DB client and reconnect from a clean state
yield stopClient(client, logger);
if (!stopped && !restartTimeout) {
restartTimeout = setTimeout(start, typeof timeout === 'number' ? timeout : 250);
}
}));
};
// Start background functions to connect the DB client and handle replication data
applyRestart(subscribe(client, plugin, settings.dbReplicationSlot, logger, outboxOrInbox));
applyRestart(handleIncomingData(client, plugin, settings, messageHandler, errorHandler, strategies.concurrencyStrategy, strategies.messageProcessingTimeoutStrategy, logger));
})();
return [
() => __awaiter(void 0, void 0, void 0, function* () {
logger.debug(`Started transactional ${outboxOrInbox} listener cleanup`);
stopped = true;
clearTimeout(restartTimeout);
clearInterval(cleanupTimeout);
yield stopClient(client, logger);
yield (pool === null || pool === void 0 ? void 0 : pool.end());
}),
];
};
exports.createLogicalReplicationListener = createLogicalReplicationListener;
/** Get and map the outbox/inbox message if the WAL log entry is such a message. Otherwise returns undefined. */
const getRelevantMessage = (log, { dbSchema, dbTable }) => log.tag === 'insert' &&
log.relation.schema === dbSchema &&
log.relation.name === dbTable
? mapMessage(log.new)
: undefined;
/** Maps the WAL log entry to an outbox or inbox message */
const mapMessage = (input) => {
if (typeof input !== 'object' || input === null) {
return undefined;
}
if (!('id' in input) ||
typeof input.id !== 'string' ||
!('aggregate_type' in input) ||
typeof input.aggregate_type !== 'string' ||
!('aggregate_id' in input) ||
typeof input.aggregate_id !== 'string' ||
!('message_type' in input) ||
typeof input.message_type !== 'string' ||
!('created_at' in input) ||
!(input.created_at instanceof Date) ||
!('payload' in input) ||
!('metadata' in input)) {
return undefined;
}
const message = {
id: input.id,
aggregateType: input.aggregate_type,
aggregateId: input.aggregate_id,
messageType: input.message_type,
payload: input.payload,
metadata: input.metadata,
createdAt: input.created_at.toISOString(),
};
if ('segment' in input && typeof input.segment === 'string') {
message.segment = input.segment;
}
if ('concurrency' in input &&
(input.concurrency === 'sequential' || input.concurrency === 'parallel')) {
message.concurrency = input.concurrency;
}
if ('locked_until' in input && input.locked_until instanceof Date) {
message.lockedUntil = input.locked_until.toISOString();
}
return message;
};
const subscribe = (client, plugin, slotName, logger, outboxOrInbox, uptoLsn) => __awaiter(void 0, void 0, void 0, function* () {
const lastLsn = uptoLsn || '0/00000000';
yield client.connect();
client.on('error', (e) => {
logger.error((0, error_1.ensureExtendedError)(e, 'DB_ERROR'), `Transactional ${outboxOrInbox} listener DB client error`);
});
client.connection.on('error', (e) => {
logger.error((0, error_1.ensureExtendedError)(e, 'DB_ERROR'), `Transactional ${outboxOrInbox} listener DB connection error`);
});
client.connection.once('replicationStart', () => {
logger.trace(`Transactional ${outboxOrInbox} listener started`);
});
client.on('notice', (msg) => {
logger.trace(`raised notice ${msg.message}`);
});
return plugin.start(client, slotName, lastLsn);
});
const stopClient = (client, logger) => __awaiter(void 0, void 0, void 0, function* () {
var _a;
if (!client) {
return;
}
try {
(_a = client.connection) === null || _a === void 0 ? void 0 : _a.removeAllListeners();
client.removeAllListeners();
yield (0, utils_1.awaitWithTimeout)(() => __awaiter(void 0, void 0, void 0, function* () { return client.end(); }), 1000, `The PostgreSQL client could not be stopped within a reasonable time frame.`);
}
catch (e) {
logger.warn((0, error_1.ensureExtendedError)(e, 'DB_ERROR'), `Stopping the PostgreSQL client gave an error.`);
}
});
// The acknowledge function is based on the https://github.com/kibae/pg-logical-replication library
const acknowledge = (client, lsn, logger) => {
if (!(client === null || client === void 0 ? void 0 : client.connection)) {
logger.warn(`Could not acknowledge message ${lsn} as the client connection was not open.`);
return;
}
const slice = lsn.split('/');
let [upperWAL, lowerWAL] = [
parseInt(slice[0], 16),
parseInt(slice[1], 16),
];
// Timestamp as microseconds since midnight 2000-01-01
const now = Date.now() - 946080000000;
const upperTimestamp = Math.floor(now / 4294967.296);
const lowerTimestamp = Math.floor(now - upperTimestamp * 4294967.296);
if (lowerWAL === 4294967295) {
// [0xff, 0xff, 0xff, 0xff]
upperWAL = upperWAL + 1;
lowerWAL = 0;
}
else {
lowerWAL = lowerWAL + 1;
}
const response = Buffer.alloc(34);
response.fill(0x72); // 'r'
// Last WAL Byte + 1 received and written to disk locally
response.writeUInt32BE(upperWAL, 1);
response.writeUInt32BE(lowerWAL, 5);
// Last WAL Byte + 1 flushed to disk in the standby
response.writeUInt32BE(upperWAL, 9);
response.writeUInt32BE(lowerWAL, 13);
// Last WAL Byte + 1 applied in the standby
response.writeUInt32BE(upperWAL, 17);
response.writeUInt32BE(lowerWAL, 21);
// Timestamp as microseconds since midnight 2000-01-01
response.writeUInt32BE(upperTimestamp, 25);
response.writeUInt32BE(lowerTimestamp, 29);
// If 1, requests server to respond immediately - can be used to verify connectivity
response.writeInt8(0, 33);
client.connection.sendCopyFromChunk(response);
};
const LOG_FLAG = 0x77; // 119 in base 10
const KEEP_ALIVE_FLAG = 0x6b; // 107 in base 10
const handleIncomingData = (client, plugin, config, messageHandler, errorHandler, concurrencyStrategy, messageProcessingTimeoutStrategy, logger) => {
if (!client.connection) {
throw new error_1.TransactionalOutboxInboxError('Client not connected.', 'DB_ERROR');
}
const ack = (lsn) => {
acknowledge(client, lsn, logger);
};
const { startProcessingLSN, finishProcessingLSN } = (0, acknowledge_manager_1.createAcknowledgeManager)(ack, logger);
let stopped = false;
// This function handles the message processing and message error handler.
// It is inlined to stop processing when "stopped" is true.
const handleOutboxInboxMessage = (messageHandler, errorHandler, message, lsn, finishProcessingLSN, logger) => __awaiter(void 0, void 0, void 0, function* () {
const cancellation = new events_1.EventEmitter();
try {
const messageProcessingTimeoutInMs = messageProcessingTimeoutStrategy(message);
yield (0, utils_1.awaitWithTimeout)(() => __awaiter(void 0, void 0, void 0, function* () {
// Need to double check for stopped for concurrency reasons when a message was awaiting the
// `awaitWithTimeout` function and another message threw an error which should stop this message.
if (stopped) {
logger.trace(`Received LSN ${lsn} after the process stopped.`);
return;
}
logger.debug(message, `Executing the message handler for LSN ${lsn}.`);
yield messageHandler(message, cancellation);
finishProcessingLSN(lsn);
}), messageProcessingTimeoutInMs, `Could not process the message with ID ${message.id} and LSN ${lsn} within the timeout of ${messageProcessingTimeoutInMs} milliseconds. Please consider to use a background worker for long running tasks to not block the message processing.`);
logger.trace(message, `Finished processing LSN ${lsn} with message id ${message.id} and type ${message.messageType}.`);
}
catch (e) {
const err = (0, error_1.ensureExtendedError)(e, 'MESSAGE_HANDLING_FAILED');
if (err.errorCode === 'TIMEOUT') {
cancellation.emit('timeout', err);
}
const shouldRetry = yield errorHandler(message, err);
if (!shouldRetry) {
finishProcessingLSN(lsn);
return;
}
throw new error_1.MessageError(`An error occurred while handling the message with ID ${message.id} and LSN ${lsn}`, 'MESSAGE_HANDLING_FAILED', message, err);
}
});
return new Promise((_resolve, reject) => {
// The copyData part is based on the https://github.com/kibae/pg-logical-replication library
client.connection.on('copyData', (_a) => __awaiter(void 0, [_a], void 0, function* ({ chunk: buffer, }) {
try {
if (buffer[0] !== LOG_FLAG && buffer[0] !== KEEP_ALIVE_FLAG) {
logger.warn({ bufferStart: buffer[0] }, 'Unknown message');
return;
}
const lsn = buffer.readUInt32BE(1).toString(16).toUpperCase() +
'/' +
buffer.readUInt32BE(5).toString(16).toUpperCase();
if (stopped) {
logger.trace(`Received LSN ${lsn} after the process stopped.`);
return;
}
if (buffer[0] === LOG_FLAG) {
const xLogData = plugin.parse(buffer.subarray(25));
const message = getRelevantMessage(xLogData, config);
if (!message) {
return;
}
logger.trace(message, `Parsed the message for the LSN ${lsn} with message id ${message.id} and type ${message.messageType}. Potentially waiting for mutex.`);
const release = yield concurrencyStrategy.acquire(message);
startProcessingLSN(lsn); // finish is called in the `handleOutboxInboxMessage` (if it doesn't throw an Error)
logger.trace(message, `Started processing LSN ${lsn} with message id ${message.id} and type ${message.messageType}.`);
try {
yield handleOutboxInboxMessage(messageHandler, errorHandler, message, lsn, finishProcessingLSN, logger);
}
finally {
release();
}
}
else if (buffer[0] === KEEP_ALIVE_FLAG) {
// Primary keep alive message
const shouldRespond = !!buffer.readInt8(17);
if (shouldRespond) {
startProcessingLSN(lsn);
finishProcessingLSN(lsn);
}
}
}
catch (e) {
stopped = true;
client.connection.removeAllListeners();
concurrencyStrategy.cancel();
reject(e);
}
}));
});
};
/**
* This export is _only_ done for unit tests as the createLogicalReplicationListener
* function is otherwise very hard to unit test. Exports work only for jest tests!
*/
exports.__only_for_unit_tests__ = {};
if (process.env.JEST_WORKER_ID) {
exports.__only_for_unit_tests__.getRelevantMessage = getRelevantMessage;
exports.__only_for_unit_tests__.mapMessage = mapMessage;
}
//# sourceMappingURL=logical-replication-listener.js.map