@arturwojnar/hermes-postgresql
Version:
Production-Ready TypeScript Outbox Pattern for PostgreSQL
1,472 lines (1,442 loc) • 51.8 kB
JavaScript
import { Duration, HermesError, CancellationPromise, noop, assertNever, assert, swallow, literalObject } from '@arturwojnar/hermes';
import postgres from 'postgres';
import { setTimeout } from 'node:timers/promises';
import { setInterval as setInterval$1, clearInterval as clearInterval$1 } from 'node:timers';
class AsyncOutboxConsumer {
_params;
_checkInterval;
_getSql;
_started = false;
_isProcessing = false;
_intervalId = null;
constructor(_params) {
this._params = _params;
this._checkInterval = _params.checkInterval || Duration.ofSeconds(15);
this._getSql = _params.getSql;
}
async send(message, options) {
if (!this._getSql()) {
throw new Error('Database connection not established. Call start() first.');
}
const sql = options?.tx || this._getSql();
if (Array.isArray(message)) {
if ('savepoint' in sql) {
for (const m of message) {
await this._publishOne(sql, m);
}
}
else {
await sql.begin(async (sql) => {
for (const m of message) {
await this._publishOne(sql, m);
}
});
}
}
else {
await this._publishOne(sql, message);
}
}
start() {
if (this._started) {
throw new Error(`AsyncOutboxConsumer is already started`);
}
this._started = true;
this._startPolling();
return (() => Promise.resolve(stop()));
}
async stop() {
if (this._intervalId) {
clearInterval(this._intervalId);
this._intervalId = null;
}
this._started = false;
}
async _publishOne(sql, message) {
await sql `
INSERT INTO "asyncOutbox" (
"consumerName",
"messageId",
"messageType",
"data"
) VALUES (
${this._params.consumerName},
${message.messageId},
${message.messageType},
${this._getSql().json(message.message)}
)
`;
}
_startPolling() {
this._intervalId = setInterval(async () => {
try {
await this._processUndeliveredMessages();
}
catch (error) {
}
}, this._checkInterval.ms);
}
async _processUndeliveredMessages() {
if (this._isProcessing) {
return;
}
this._isProcessing = true;
try {
const pendingMessages = await this._getSql() `
SELECT * FROM "asyncOutbox"
WHERE delivered = false
ORDER BY "addedAt" ASC
LIMIT 10
`;
for (const message of pendingMessages) {
try {
await this._params.publish({
position: message.position,
messageId: message.messageId,
messageType: message.messageType,
message: message.data,
redeliveryCount: message.failsCount || 0,
});
await this._getSql() `
UPDATE "asyncOutbox"
SET "delivered" = true,
"sentAt" = NOW()
WHERE "position" = ${message.position}
`;
}
catch (error) {
await this._getSql() `
UPDATE "asyncOutbox"
SET "failsCount" = COALESCE("failsCount", 0) + 1
WHERE "position" = ${message.position}
`;
}
}
}
finally {
this._isProcessing = false;
}
}
}
const createAsyncOutboxConsumer = (params) => new AsyncOutboxConsumer(params);
const PublicationName = `hermes_pub`;
const SlotNamePrefix = `hermes_slot`;
const getSlotName = (consumerName, partitionKey) => `${SlotNamePrefix}_${consumerName}_${partitionKey}`;
var HermesErrorCode;
(function (HermesErrorCode) {
HermesErrorCode["ConsumerAlreadyTaken"] = "ConsumerAlreadyTaken";
})(HermesErrorCode || (HermesErrorCode = {}));
class HermesConsumerAlreadyTakenError extends HermesError {
constructor(params) {
super(HermesErrorCode.ConsumerAlreadyTaken, params, `Consumer ${params.consumerName} with the ${params.partitionKey} has been already taken by another PID.`);
}
}
const createSerializedPublishingQueue = (publish, options) => {
const waitAfterFailedPublish = options?.waitAfterFailedPublish || Duration.ofSeconds(1);
const onFailedPublish = options?.onFailedPublish || (() => Promise.resolve());
const ids = new Set();
const messages = new Array();
let isPublishing = false;
let publishingPromise = CancellationPromise.resolved();
const queue = (messageToPublish) => {
if (ids.has(messageToPublish.transaction.lsn)) {
return messageToPublish;
}
ids.add(messageToPublish.transaction.lsn);
messages.push(messageToPublish);
return messageToPublish;
};
const dequeueOldest = () => {
if (messages.length === 0) {
return;
}
const oldest = messages.shift();
ids.delete(oldest.transaction.lsn);
};
const run = async () => {
if (isPublishing) {
return;
}
isPublishing = true;
publishingPromise = new CancellationPromise();
try {
do {
const result = await _publishOldestMessage();
switch (result) {
case 'published':
continue;
case 'failed':
await onFailedPublish(messages[0].transaction);
if (waitAfterFailedPublish) {
await setTimeout(waitAfterFailedPublish.ms);
}
break;
case 'exhausted':
isPublishing = false;
publishingPromise.resolve();
break;
default:
assertNever(result);
}
} while (isPublishing);
}
finally {
isPublishing = false;
publishingPromise.resolve();
}
};
const _publishOldestMessage = async () => {
if (messages.length === 0) {
return 'exhausted';
}
const oldest = messages[0];
try {
await publish(oldest);
dequeueOldest();
await oldest.acknowledge();
return 'published';
}
catch (error) {
if (messages.length && messages[0].transaction.lsn !== oldest.transaction.lsn) {
ids.add(oldest.transaction.lsn);
messages.unshift(oldest);
}
return 'failed';
}
};
return {
name: () => 'SerializedPublishingQueue',
queue,
run,
size: () => messages.length,
waitUntilIsEmpty: () => publishingPromise,
dispose: noop,
};
};
const createSimpleQueue = () => {
const _queue = [];
const queue = (item) => {
_queue.push(item);
};
const dequeue = () => {
_queue.shift();
};
const remove = (item) => {
const index = _queue.indexOf(item);
if (index > -1) {
_queue.splice(index, 1);
}
};
const head = () => (_queue.length ? _queue[0] : undefined);
const tail = () => (_queue.length ? _queue[_queue.length - 1] : undefined);
const size = () => _queue.length;
return {
queue,
dequeue,
remove,
head,
tail,
size,
};
};
const createAsyncOpsQueue = () => {
const _ops = createSimpleQueue();
let opInProgress = CancellationPromise.resolved();
const queue = (op) => {
_ops.queue(op);
return op;
};
const waitFor = async (op) => {
if (opInProgress.isPending) {
await opInProgress;
}
opInProgress = new CancellationPromise();
try {
await op();
_ops.remove(op);
}
finally {
opInProgress.resolve();
}
};
return { queue, waitFor };
};
const createIntervalResendingStrategy = () => ({ getMessages, publishMessage, isPublishing, interval }) => {
const _iteration = () => {
const failedMessage = getMessages().find((message) => !message.delivered && message.failed);
if (failedMessage) {
publishMessage(failedMessage);
}
const notDeliveredMessage = getMessages().find((message) => !message.delivered && !message.failed);
if (!isPublishing() && notDeliveredMessage) {
console.log('~~!!!!!!!!!!!!!!!!!!!');
publishMessage(notDeliveredMessage);
}
};
const timer = setInterval$1(_iteration, interval.ms);
return () => clearInterval$1(timer);
};
const createNonBlockingPublishingQueue = (publish, options) => {
const onFailedPublish = options?.onFailedPublish || (() => Promise.resolve());
const ids = new Set();
const acknowledgmentQueue = createAsyncOpsQueue();
const messages = new Array();
let publishingPromise = CancellationPromise.resolved();
const getNextMessageThatShouldBeDelivered = () => {
return messages.find((message) => !message.delivered);
};
const getNextTransactionLsnThatShouldBeDelivered = () => {
return getNextMessageThatShouldBeDelivered()?.transaction?.lsn;
};
const getState = (lsn) => {
return messages.find(({ transaction }) => transaction.lsn === lsn);
};
const queue = (messageToPublish) => {
if (ids.has(messageToPublish.transaction.lsn)) {
return messageToPublish;
}
ids.add(messageToPublish.transaction.lsn);
messages.push({ ...messageToPublish, delivered: false, failed: true });
return messageToPublish;
};
const run = async (messageToPublish) => {
if (messages.length > 0 && !publishingPromise.isPending) {
publishingPromise = new CancellationPromise();
}
messageToPublish = messageToPublish || messages[0];
await _publishMessage(messageToPublish);
};
const _removeMessage = (message) => {
if (messages.length === 0) {
return;
}
const index = messages.findIndex(({ transaction }) => transaction.lsn === message.transaction.lsn);
if (index !== -1) {
messages.splice(index, 1);
ids.delete(message.transaction.lsn);
}
};
const _getFirstMessagedThatIsDeliveredButNotAcked = () => {
return messages.find((message) => message.delivered);
};
const _publishMessage = async (message) => {
try {
await publish(message);
if (message.transaction.lsn === getNextTransactionLsnThatShouldBeDelivered()) {
await acknowledgmentQueue.waitFor(acknowledgmentQueue.queue(() => message.acknowledge()));
_removeMessage(message);
let messageToAck;
while ((messageToAck = _getFirstMessagedThatIsDeliveredButNotAcked())) {
const message = messageToAck;
await acknowledgmentQueue.waitFor(acknowledgmentQueue.queue(() => message.acknowledge()));
_removeMessage(message);
}
if (messages.length === 0) {
publishingPromise.resolve();
}
}
else {
const index = messages.findIndex(({ transaction }) => transaction.lsn === message.transaction.lsn);
if (index >= 0 && !messages[index].delivered) {
console.log(`Processed ${messages[index].transaction.lsn}`);
messages[index].delivered = true;
}
}
return 'published';
}
catch (error) {
const state = getState(message.transaction.lsn);
if (state) {
state.failed = true;
}
await onFailedPublish(message.transaction);
}
};
let stopResending;
if (options?.waitAfterFailedPublish?.ms !== 0) {
stopResending = createIntervalResendingStrategy()({
getMessages: () => messages,
publishMessage: _publishMessage,
isPublishing: () => publishingPromise.isPending,
interval: options?.waitAfterFailedPublish || Duration.ofSeconds(30),
});
}
return {
name: () => 'NonBlockingPublishingQueue',
queue,
run,
size: () => messages.length,
waitUntilIsEmpty: () => publishingPromise,
dispose: () => stopResending?.(),
};
};
var _function = {};
var hasRequired_function;
function require_function () {
if (hasRequired_function) return _function;
hasRequired_function = 1;
(function (exports) {
var __spreadArray = (_function && _function.__spreadArray) || function (to, from, pack) {
if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
if (ar || !(i in from)) {
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
ar[i] = from[i];
}
}
return to.concat(ar || Array.prototype.slice.call(from));
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.dual = exports.getEndomorphismMonoid = exports.SK = exports.hole = exports.constVoid = exports.constUndefined = exports.constNull = exports.constFalse = exports.constTrue = exports.unsafeCoerce = exports.apply = exports.getRing = exports.getSemiring = exports.getMonoid = exports.getSemigroup = exports.getBooleanAlgebra = void 0;
exports.identity = identity;
exports.constant = constant;
exports.flip = flip;
exports.flow = flow;
exports.tuple = tuple;
exports.increment = increment;
exports.decrement = decrement;
exports.absurd = absurd;
exports.tupled = tupled;
exports.untupled = untupled;
exports.pipe = pipe;
exports.not = not;
// -------------------------------------------------------------------------------------
// instances
// -------------------------------------------------------------------------------------
/**
* @category instances
* @since 2.10.0
*/
var getBooleanAlgebra = function (B) {
return function () { return ({
meet: function (x, y) { return function (a) { return B.meet(x(a), y(a)); }; },
join: function (x, y) { return function (a) { return B.join(x(a), y(a)); }; },
zero: function () { return B.zero; },
one: function () { return B.one; },
implies: function (x, y) { return function (a) { return B.implies(x(a), y(a)); }; },
not: function (x) { return function (a) { return B.not(x(a)); }; }
}); };
};
exports.getBooleanAlgebra = getBooleanAlgebra;
/**
* Unary functions form a semigroup as long as you can provide a semigroup for the codomain.
*
* @example
* import { Predicate, getSemigroup } from 'fp-ts/function'
* import * as B from 'fp-ts/boolean'
*
* const f: Predicate<number> = (n) => n <= 2
* const g: Predicate<number> = (n) => n >= 0
*
* const S1 = getSemigroup(B.SemigroupAll)<number>()
*
* assert.deepStrictEqual(S1.concat(f, g)(1), true)
* assert.deepStrictEqual(S1.concat(f, g)(3), false)
*
* const S2 = getSemigroup(B.SemigroupAny)<number>()
*
* assert.deepStrictEqual(S2.concat(f, g)(1), true)
* assert.deepStrictEqual(S2.concat(f, g)(3), true)
*
* @category instances
* @since 2.10.0
*/
var getSemigroup = function (S) {
return function () { return ({
concat: function (f, g) { return function (a) { return S.concat(f(a), g(a)); }; }
}); };
};
exports.getSemigroup = getSemigroup;
/**
* Unary functions form a monoid as long as you can provide a monoid for the codomain.
*
* @example
* import { Predicate } from 'fp-ts/Predicate'
* import { getMonoid } from 'fp-ts/function'
* import * as B from 'fp-ts/boolean'
*
* const f: Predicate<number> = (n) => n <= 2
* const g: Predicate<number> = (n) => n >= 0
*
* const M1 = getMonoid(B.MonoidAll)<number>()
*
* assert.deepStrictEqual(M1.concat(f, g)(1), true)
* assert.deepStrictEqual(M1.concat(f, g)(3), false)
*
* const M2 = getMonoid(B.MonoidAny)<number>()
*
* assert.deepStrictEqual(M2.concat(f, g)(1), true)
* assert.deepStrictEqual(M2.concat(f, g)(3), true)
*
* @category instances
* @since 2.10.0
*/
var getMonoid = function (M) {
var getSemigroupM = (0, exports.getSemigroup)(M);
return function () { return ({
concat: getSemigroupM().concat,
empty: function () { return M.empty; }
}); };
};
exports.getMonoid = getMonoid;
/**
* @category instances
* @since 2.10.0
*/
var getSemiring = function (S) { return ({
add: function (f, g) { return function (x) { return S.add(f(x), g(x)); }; },
zero: function () { return S.zero; },
mul: function (f, g) { return function (x) { return S.mul(f(x), g(x)); }; },
one: function () { return S.one; }
}); };
exports.getSemiring = getSemiring;
/**
* @category instances
* @since 2.10.0
*/
var getRing = function (R) {
var S = (0, exports.getSemiring)(R);
return {
add: S.add,
mul: S.mul,
one: S.one,
zero: S.zero,
sub: function (f, g) { return function (x) { return R.sub(f(x), g(x)); }; }
};
};
exports.getRing = getRing;
// -------------------------------------------------------------------------------------
// utils
// -------------------------------------------------------------------------------------
/**
* @since 2.11.0
*/
var apply = function (a) {
return function (f) {
return f(a);
};
};
exports.apply = apply;
/**
* @since 2.0.0
*/
function identity(a) {
return a;
}
/**
* @since 2.0.0
*/
exports.unsafeCoerce = identity;
/**
* @since 2.0.0
*/
function constant(a) {
return function () { return a; };
}
/**
* A thunk that returns always `true`.
*
* @since 2.0.0
*/
exports.constTrue = constant(true);
/**
* A thunk that returns always `false`.
*
* @since 2.0.0
*/
exports.constFalse = constant(false);
/**
* A thunk that returns always `null`.
*
* @since 2.0.0
*/
exports.constNull = constant(null);
/**
* A thunk that returns always `undefined`.
*
* @since 2.0.0
*/
exports.constUndefined = constant(undefined);
/**
* A thunk that returns always `void`.
*
* @since 2.0.0
*/
exports.constVoid = exports.constUndefined;
function flip(f) {
return function () {
var args = [];
for (var _i = 0; _i < arguments.length; _i++) {
args[_i] = arguments[_i];
}
if (args.length > 1) {
return f(args[1], args[0]);
}
return function (a) { return f(a)(args[0]); };
};
}
function flow(ab, bc, cd, de, ef, fg, gh, hi, ij) {
switch (arguments.length) {
case 1:
return ab;
case 2:
return function () {
return bc(ab.apply(this, arguments));
};
case 3:
return function () {
return cd(bc(ab.apply(this, arguments)));
};
case 4:
return function () {
return de(cd(bc(ab.apply(this, arguments))));
};
case 5:
return function () {
return ef(de(cd(bc(ab.apply(this, arguments)))));
};
case 6:
return function () {
return fg(ef(de(cd(bc(ab.apply(this, arguments))))));
};
case 7:
return function () {
return gh(fg(ef(de(cd(bc(ab.apply(this, arguments)))))));
};
case 8:
return function () {
return hi(gh(fg(ef(de(cd(bc(ab.apply(this, arguments))))))));
};
case 9:
return function () {
return ij(hi(gh(fg(ef(de(cd(bc(ab.apply(this, arguments)))))))));
};
}
return;
}
/**
* @since 2.0.0
*/
function tuple() {
var t = [];
for (var _i = 0; _i < arguments.length; _i++) {
t[_i] = arguments[_i];
}
return t;
}
/**
* @since 2.0.0
*/
function increment(n) {
return n + 1;
}
/**
* @since 2.0.0
*/
function decrement(n) {
return n - 1;
}
/**
* @since 2.0.0
*/
function absurd(_) {
throw new Error('Called `absurd` function which should be uncallable');
}
/**
* Creates a tupled version of this function: instead of `n` arguments, it accepts a single tuple argument.
*
* @example
* import { tupled } from 'fp-ts/function'
*
* const add = tupled((x: number, y: number): number => x + y)
*
* assert.strictEqual(add([1, 2]), 3)
*
* @since 2.4.0
*/
function tupled(f) {
return function (a) { return f.apply(void 0, a); };
}
/**
* Inverse function of `tupled`
*
* @since 2.4.0
*/
function untupled(f) {
return function () {
var a = [];
for (var _i = 0; _i < arguments.length; _i++) {
a[_i] = arguments[_i];
}
return f(a);
};
}
function pipe(a, ab, bc, cd, de, ef, fg, gh, hi) {
switch (arguments.length) {
case 1:
return a;
case 2:
return ab(a);
case 3:
return bc(ab(a));
case 4:
return cd(bc(ab(a)));
case 5:
return de(cd(bc(ab(a))));
case 6:
return ef(de(cd(bc(ab(a)))));
case 7:
return fg(ef(de(cd(bc(ab(a))))));
case 8:
return gh(fg(ef(de(cd(bc(ab(a)))))));
case 9:
return hi(gh(fg(ef(de(cd(bc(ab(a))))))));
default: {
var ret = arguments[0];
for (var i = 1; i < arguments.length; i++) {
ret = arguments[i](ret);
}
return ret;
}
}
}
/**
* Type hole simulation
*
* @since 2.7.0
*/
exports.hole = absurd;
/**
* @since 2.11.0
*/
var SK = function (_, b) { return b; };
exports.SK = SK;
/**
* Use `Predicate` module instead.
*
* @category zone of death
* @since 2.0.0
* @deprecated
*/
function not(predicate) {
return function (a) { return !predicate(a); };
}
/**
* Use `Endomorphism` module instead.
*
* @category zone of death
* @since 2.10.0
* @deprecated
*/
var getEndomorphismMonoid = function () { return ({
concat: function (first, second) { return flow(first, second); },
empty: identity
}); };
exports.getEndomorphismMonoid = getEndomorphismMonoid;
/** @internal */
var dual = function (arity, body) {
var isDataFirst = typeof arity === 'number' ? function (args) { return args.length >= arity; } : arity;
return function () {
var args = Array.from(arguments);
if (isDataFirst(arguments)) {
return body.apply(this, args);
}
return function (self) { return body.apply(void 0, __spreadArray([self], args, false)); };
};
};
exports.dual = dual;
} (_function));
return _function;
}
var _functionExports = /*@__PURE__*/ require_function();
const LSN_REGEXP = new RegExp(`^[0-9a-f]+/[0-9a-f]+$`, 'i');
const getUpperAndLowerWAL = (lsn) => {
return lsn.split('/').map((x) => parseInt(x, 16));
};
const convertLsnToBigInt = (lsn) => {
const [upperWal, lowerWal] = getUpperAndLowerWAL(lsn);
return (BigInt(upperWal) << 32n) | BigInt(lowerWal);
};
const convertBigIntToLsn = (lsn) => {
const upperWal = lsn >> 32n;
const lowerWal = lsn & 0xffffffffn;
return `${upperWal.toString(16).toUpperCase()}/${lowerWal.toString(16).toUpperCase()}`;
};
const incrementWAL = (lsn) => {
return convertLsnToBigInt(lsn) + BigInt(1);
};
const constructLsn = (lsn) => {
const upperWal = lsn.readUInt32BE(0).toString(16).toUpperCase();
const lowerWal = lsn.readUInt32BE(4).toString(16).toUpperCase();
return `${upperWal}/${lowerWal}`;
};
const isLsn = (value) => LSN_REGEXP.test(value);
const toLsn = (value) => {
if (isLsn(value)) {
return value;
}
throw new Error(`not LSN ${value}`);
};
var MessageType;
(function (MessageType) {
MessageType["Begin"] = "B";
MessageType["Insert"] = "I";
MessageType["Commit"] = "C";
MessageType["Other"] = "Other";
})(MessageType || (MessageType = {}));
var Bytes;
(function (Bytes) {
Bytes[Bytes["Int8"] = 1] = "Int8";
Bytes[Bytes["Int16"] = 2] = "Int16";
Bytes[Bytes["Int32"] = 4] = "Int32";
Bytes[Bytes["Int64"] = 8] = "Int64";
})(Bytes || (Bytes = {}));
var TopLevelType;
(function (TopLevelType) {
TopLevelType["XLogData"] = "w";
TopLevelType["PrimaryKeepaliveMessage"] = "k";
})(TopLevelType || (TopLevelType = {}));
const TopLevelTypeValues = Object.values(TopLevelType);
const MessageTypeValues = Object.values(MessageType);
const isTopLevelType = (char) => TopLevelTypeValues.includes(char);
const parseTopLevelType = (char) => {
if (!isTopLevelType(char)) {
throw new Error(`INTERNAL_ERROR`);
}
return char;
};
const parseMessageType = (char) => {
if (MessageTypeValues.includes(char)) {
return char;
}
return MessageType.Other;
};
const XLogData_WalRecordStartByteNumber = Bytes.Int8 + Bytes.Int64 + Bytes.Int64 + Bytes.Int64;
const toTimestamp = (value) => new Date(Date.UTC(2000, 0, 1) + Number(value / 1000n));
const toServerSystemClock = (epochMs) => BigInt(epochMs - Date.UTC(2000, 0, 1)) * 1000n;
const offset = (offset = 0) => ({
add: (bytes) => (offset += bytes),
addInt8: () => (offset += Bytes.Int8),
addInt16: () => (offset += Bytes.Int16),
addInt32: () => (offset += Bytes.Int32),
addInt64: () => (offset += Bytes.Int64),
value: () => offset,
});
const processBeginMessage = (data) => {
const pos = offset(Bytes.Int8);
const lsn = constructLsn(data.subarray(1, 9));
const timestamp = toTimestamp(data.readBigInt64BE(pos.addInt64()));
const transactionId = data.readUInt32BE(pos.addInt64());
return {
topLevelType: TopLevelType.XLogData,
messageType: MessageType.Begin,
transactionId,
lsn,
timestamp,
};
};
const processCommitMessage = (data) => {
const pos = offset(Bytes.Int8 + Bytes.Int8);
const commitLsn = constructLsn(data.subarray(pos.value(), pos.addInt64()));
const transactionEndLsn = constructLsn(data.subarray(pos.value(), pos.addInt64()));
const commitTimestamp = toTimestamp(data.readBigInt64BE(pos.value()));
return {
topLevelType: TopLevelType.XLogData,
messageType: MessageType.Commit,
commitLsn,
transactionEndLsn,
commitTimestamp,
};
};
const readIntFn = {
'1': 'readUInt8',
'2': 'readUInt16BE',
'4': 'readUInt32BE',
'8': 'readBigUInt64BE',
};
const readUInt = (buffer, pos) => {
const columnType = String.fromCharCode(buffer.readInt8(pos.value()));
const columnLength = buffer.readUInt32BE(pos.addInt8()).toString();
assert(columnType === 't', 'readUInt.columnType');
assert(['1', '2', '4', '8'].includes(columnLength), 'readUInt.columnLength');
const value = buffer[readIntFn[columnLength]](pos.addInt32());
pos.add(Number(columnLength));
return value;
};
const readBigInt = (buffer, pos) => {
const columnType = String.fromCharCode(buffer.readInt8(pos.value()));
const columnLength = buffer.readUInt32BE(pos.addInt8());
assert(columnType === 't', 'readUInt.columnType');
const strValue = buffer.subarray(pos.addInt32(), pos.value() + columnLength).toString('utf-8');
const value = columnLength > 8 ? BigInt(strValue) : parseInt(strValue, 10);
pos.add(columnLength);
return value;
};
const readText = (buffer, pos) => {
const columnType = String.fromCharCode(buffer.readInt8(pos.value()));
const columnLength = buffer.readUInt32BE(pos.addInt8());
assert(columnType === 't', 'readText.columnType');
const value = buffer.subarray(pos.addInt32(), pos.add(columnLength));
return value.toString('utf-8');
};
const readJsonb = (buffer, pos) => {
const columnType = String.fromCharCode(buffer.readInt8(pos.value()));
const columnLength = buffer.readUInt32BE(pos.addInt8());
assert(columnType === 't', 'readText.readJsonb');
const value = buffer.subarray(pos.addInt32(), pos.add(columnLength));
return value.toString('utf-8');
};
const columnReaders = {
uint: readUInt,
bigint: readBigInt,
text: readText,
jsonb: readJsonb,
};
const processInsertMessage = (columnConfig) => {
const entries = Object.entries(columnConfig);
const readers = entries.map(([columnName, columnType]) => ({
columnName,
reader: columnReaders[columnType],
}));
readers.forEach(({ reader }) => assert(reader, `Unknown column type`));
const readTuple = (pos, tuplesBuffer) => {
return readers.reduce((result, { columnName, reader }) => {
Object.defineProperty(result, columnName, {
value: reader(tuplesBuffer, pos),
writable: false,
enumerable: true,
});
return result;
}, {});
};
return (data) => {
data.readUInt32BE(Bytes.Int8);
String.fromCharCode(data.readInt8(Bytes.Int8 + Bytes.Int32));
const TUPLE_START_BYTE = Bytes.Int8 + Bytes.Int32 + Bytes.Int8;
const tuplesBuffer = data.subarray(TUPLE_START_BYTE);
tuplesBuffer.readInt16BE(0);
const pos = offset(Bytes.Int16);
const result = readTuple(pos, tuplesBuffer);
return {
topLevelType: TopLevelType.XLogData,
messageType: MessageType.Insert,
result,
};
};
};
const keepAliveResult = (shouldPong) => ({
topLevelType: TopLevelType.PrimaryKeepaliveMessage,
messageType: MessageType.Other,
shouldPong,
});
const processPrimaryKeepAliveMessage = (data) => {
if (!data[Bytes.Int8 + Bytes.Int64 + Bytes.Int64]) {
return keepAliveResult(false);
}
return keepAliveResult(true);
};
const onData = (columnConfig) => {
const processInsertMessage$1 = processInsertMessage(columnConfig);
return (message) => {
const topLevelType = parseTopLevelType(String.fromCharCode(message[0]));
switch (topLevelType) {
case TopLevelType.PrimaryKeepaliveMessage:
return processPrimaryKeepAliveMessage(message);
case TopLevelType.XLogData: {
const data = message.subarray(XLogData_WalRecordStartByteNumber);
const symbol = String.fromCharCode(data[0]);
const type = parseMessageType(symbol);
switch (type) {
case MessageType.Begin:
return processBeginMessage(data);
case MessageType.Insert:
return processInsertMessage$1(data);
case MessageType.Commit:
return processCommitMessage(data);
default:
return { topLevelType: TopLevelType.XLogData, messageType: MessageType.Other, symbol };
}
}
default:
assertNever(topLevelType);
}
};
};
const createStandbyStatusUpdate = (lsn) => {
const x = Buffer.alloc(34);
const pos = offset(Bytes.Int8);
const lsnInt = incrementWAL(lsn);
x[0] = 'r'.charCodeAt(0);
x.writeBigUInt64BE(lsnInt, pos.value());
x.writeBigUInt64BE(lsnInt, pos.addInt64());
x.writeBigUInt64BE(lsnInt, pos.addInt64());
x.writeBigInt64BE(toServerSystemClock(Date.now()), pos.addInt64());
x.writeUInt8(0, pos.addInt8());
return x;
};
const sendStandbyStatusUpdate = (stream, getLastAcknowledgedLsn) => () => {
if (!stream) {
return;
}
const statusUpdate = createStandbyStatusUpdate(getLastAcknowledgedLsn());
stream.write(statusUpdate);
};
const emptyTransaction = (lastProcessedLsn) => ({
lsn: lastProcessedLsn,
timestamp: new Date('1970-01-01T00:00:00Z'),
results: [],
transactionId: 0,
});
const addInsert = (transaction, insert) => {
transaction.results = [...transaction.results, insert];
};
const createTransaction = (transactionId, lsn, timestamp) => {
return { transactionId, lsn, timestamp, results: [] };
};
const PSQL_ADMIN_SHUTDOWN = '57P01';
const startLogicalReplication = async (params) => {
const { state, sql } = params;
const onInsert = params.onInsert || noop;
const location = typeof state.lastProcessedLsn === 'undefined' ? '0/00000000' : state.lastProcessedLsn;
let currentTransaction = emptyTransaction(location);
const stream = await sql
.unsafe(`START_REPLICATION SLOT ${state.slotName} LOGICAL ${convertBigIntToLsn(incrementWAL(location))} (proto_version '1', publication_names '${params.state.publication}')`)
.writable();
const curriedOnData = onData(params.columnConfig);
const onData$1 = (message) => curriedOnData(message);
const acknowledgeLastLsn = sendStandbyStatusUpdate(stream, () => state.lastProcessedLsn);
const close = async () => {
const timeout = Duration.ofSeconds(1).ms;
await Promise.all([Promise.race([swallow(() => sql.end({ timeout })), setTimeout(timeout)])]);
};
const storeResult = (result) => {
if (result.messageType === MessageType.Begin) {
currentTransaction = createTransaction(result.transactionId, result.lsn, result.timestamp);
}
else if (result.messageType === MessageType.Insert) {
addInsert(currentTransaction, result.result);
}
else if (result.messageType === MessageType.Commit) ;
return result;
};
const handleResult = (result) => {
if (result.topLevelType === TopLevelType.PrimaryKeepaliveMessage && result.shouldPong) {
acknowledgeLastLsn();
}
else if (result.topLevelType === TopLevelType.XLogData && result.messageType === MessageType.Commit) {
const acknowledge = () => {
state.lastProcessedLsn = currentTransaction.lsn;
currentTransaction = emptyTransaction(state.lastProcessedLsn);
};
onInsert(currentTransaction, acknowledge);
}
};
stream.on('data', (message) => {
_functionExports.pipe(message, onData$1, storeResult, handleResult);
});
stream.on('error', async (error) => {
const pError = error;
if ((pError.code === PSQL_ADMIN_SHUTDOWN || pError.code === 'CONNECTION_CLOSED') &&
pError.query.includes('START_REPLICATION SLOT')) {
console.info('Replication connection closed due to database shutdown');
await swallow(() => sql.end({ timeout: Duration.ofSeconds(1).ms }));
}
else {
console.error(error, 'startLogicalReplication');
}
});
stream.on('close', () => {
close().catch(noop);
});
};
const killReplicationProcesses = async (sql, slotName) => {
await swallow(async () => {
const backendProcesses = await sql.unsafe(`
SELECT pid
FROM pg_stat_replication
WHERE application_name = '${slotName}'
AND state = 'streaming'
`);
for (const { pid } of backendProcesses) {
await sql.unsafe(`SELECT pg_terminate_backend($1)`, [pid]);
}
});
await swallow(async () => {
const idleProcesses = await sql.unsafe(`
SELECT pid
FROM pg_stat_activity
WHERE application_name = '${slotName}'
AND state = 'idle'
`);
for (const { pid } of idleProcesses) {
await sql.unsafe(`SELECT pg_terminate_backend($1)`, [pid]);
}
});
};
const OutboxConsumerStatuses = [
'INITIAL',
'CREATED',
'SUCCESS_PUBLISH',
'FAILED_PUBLISH',
'DELETED',
];
class OutboxConsumerStore {
_sql;
_consumerName;
_partitionKey;
_consumer = null;
constructor(_sql, _consumerName, _partitionKey = 'default') {
this._sql = _sql;
this._consumerName = _consumerName;
this._partitionKey = _partitionKey;
}
async load() {
const [consumer] = (await this._sql `
SELECT "id", "lastProcessedLsn", "status", "failedNextLsn", "nextLsnRedeliveryCount", "createdAt", "lastUpdatedAt" FROM "outboxConsumer"
WHERE "consumerName"=${this._consumerName}
AND "partitionKey"=${this._partitionKey}
`);
this._consumer = consumer;
return consumer;
}
async createOrLoad(data) {
const slotName = getSlotName(this._consumerName, this._partitionKey);
const restartLsnResults = await this._sql `SELECT * FROM pg_replication_slots WHERE slot_name = ${slotName};`;
const restartLsn = restartLsnResults?.[0]?.restart_lsn || '0/00000000';
await this._sql `
INSERT INTO "outboxConsumer" (
"consumerName",
"partitionKey",
"lastProcessedLsn",
"createdAt"
) VALUES (
${data.consumerName},
${data.partitionKey},
${restartLsn},
${data.createdAt}
)
ON CONFLICT ("consumerName", "partitionKey") DO NOTHING;
`;
return await this.load();
}
async update(consumerName, change, tx) {
assert(this._consumer);
const skipKeys = ['id', 'createdAt', 'consumerName', 'partitionKey'];
const keys = Object.keys(change)
.filter(([key]) => !skipKeys.includes(key))
.map((key) => key);
if (keys.length === 0) {
return this._consumer;
}
const sql = tx ? tx : this._sql;
await sql `
UPDATE "outboxConsumer"
SET ${sql(change, ...keys)}
WHERE "consumerName" = ${consumerName};
`;
Object.assign(this._consumer, change);
return this._consumer;
}
get consumer() {
return this._consumer;
}
get consumerName() {
return this._consumerName;
}
get lastProcessedLsn() {
assert(this._consumer);
const { status } = this._consumer;
if (status === 'INITIAL' || status === 'DELETED') {
return toLsn('0/00000000');
}
return toLsn(this._consumer.lastProcessedLsn);
}
get redeliveryCount() {
assert(this._consumer);
const { status } = this._consumer;
if (status === 'FAILED_PUBLISH') {
return this._consumer.nextLsnRedeliveryCount;
}
return 0;
}
}
class OutboxConsumerState {
_store;
constructor(_store) {
this._store = _store;
}
async createOrLoad(partitionKey = 'default') {
return await this._store.createOrLoad(literalObject({
status: 'CREATED',
id: 0,
consumerName: this._store.consumerName,
partitionKey,
createdAt: new Date(),
}));
}
async moveFurther(lastProcessedLsn, tx) {
const { consumer } = this._store;
assert(consumer);
if (consumer.status === 'DELETED' || consumer.status === 'INITIAL') {
return;
}
if (convertLsnToBigInt(lastProcessedLsn) <= convertLsnToBigInt(this._store.lastProcessedLsn)) {
console.error('outbox state invalid op');
return;
}
await this._store.update(this._store.consumerName, literalObject({
status: 'SUCCESS_PUBLISH',
lastUpdatedAt: new Date(),
lastProcessedLsn,
failedNextLsn: null,
nextLsnRedeliveryCount: 0,
}), tx);
}
async reportFailedDelivery(failedNextLsn, tx) {
const { consumer } = this._store;
assert(consumer);
if (consumer.status === 'DELETED' || consumer.status === 'INITIAL') {
return;
}
const nextLsnRedeliveryCount = consumer.status === 'CREATED' || consumer.status === 'SUCCESS_PUBLISH' ? 1 : consumer.nextLsnRedeliveryCount + 1;
await this._store.update(this._store.consumerName, literalObject({
status: 'FAILED_PUBLISH',
lastUpdatedAt: new Date(),
failedNextLsn,
nextLsnRedeliveryCount,
}), tx);
}
get data() {
return this._store.consumer;
}
get lastProcessedLsn() {
assert(this._store);
return this._store.lastProcessedLsn;
}
get redeliveryCount() {
assert(this._store);
return this._store.redeliveryCount;
}
}
const migrate = async (sql, slotName) => {
await sql `
DO $$
BEGIN
IF current_setting('wal_level') != 'logical' THEN
RAISE EXCEPTION 'wal_level must be set to logical';
END IF;
END $$;
`;
const [{ exists }] = await sql `
SELECT EXISTS (
SELECT 1 FROM pg_type t
JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
WHERE t.typname = 'ConsumerStatus'
)
`;
if (!exists) {
const enumValues = OutboxConsumerStatuses.map((status) => `'${status}'`).join(', ');
await sql.unsafe(`
CREATE TYPE "ConsumerStatus" AS ENUM (${enumValues})
`);
}
await sql `
CREATE TABLE IF NOT EXISTS "outbox" (
"position" BIGSERIAL PRIMARY KEY,
"messageId" VARCHAR(250) NOT NULL,
"messageType" VARCHAR(250) NOT NULL,
"partitionKey" VARCHAR(30) DEFAULT 'default' NOT NULL,
"data" JSONB NOT NULL,
"addedAt" TIMESTAMPTZ DEFAULT NOW() NOT NULL,
"createdAt" TIMESTAMPTZ DEFAULT NOW() NOT NULL,
"lsn" VARCHAR(50) NULL,
"sentAt" TIMESTAMPTZ NULL
);
`;
await sql `
CREATE TABLE IF NOT EXISTS "asyncOutbox" (
"position" BIGSERIAL PRIMARY KEY,
"consumerName" VARCHAR(30) NOT NULL,
"messageId" VARCHAR(250) NOT NULL,
"messageType" VARCHAR(250) NOT NULL,
"data" JSONB NOT NULL,
"addedAt" TIMESTAMPTZ DEFAULT NOW() NOT NULL,
"createdAt" TIMESTAMPTZ DEFAULT NOW() NOT NULL,
"failsCount" INTEGER DEFAULT 0,
"sentAt" TIMESTAMPTZ NULL,
"delivered" BOOLEAN DEFAULT FALSE
);
`;
await sql `CREATE INDEX IF NOT EXISTS "asyncOutboxDeliveredIdx" ON "asyncOutbox" ("delivered" ASC);`;
await sql `CREATE INDEX IF NOT EXISTS "asyncOutboxDeliveredWithDateIdx" ON "asyncOutbox" ("delivered" ASC, "addedAt" ASC);`;
await sql `
CREATE TABLE IF NOT EXISTS "outboxConsumer" (
"id" BIGSERIAL PRIMARY KEY,
"consumerName" VARCHAR(30) NOT NULL,
"partitionKey" VARCHAR(30) DEFAULT 'default' NOT NULL,
"status" "ConsumerStatus" DEFAULT 'CREATED' NOT NULL,
"lastProcessedLsn" VARCHAR(20) DEFAULT '0/00000000' NOT NULL,
"failedNextLsn" VARCHAR(20) DEFAULT NULL,
"nextLsnRedeliveryCount" INTEGER DEFAULT 0 NOT NULL,
"createdAt" TIMESTAMPTZ DEFAULT NOW() NOT NULL,
"lastUpdatedAt" TIMESTAMPTZ DEFAULT NOW() NULL
);
`;
await sql `CREATE UNIQUE INDEX IF NOT EXISTS "consumerNameIdx" ON "outboxConsumer" ("consumerName" DESC, "partitionKey");`;
await sql.unsafe(`
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = '${PublicationName}') THEN
CREATE PUBLICATION ${PublicationName} FOR TABLE outbox;
END IF;
END $$;
`);
await sql.unsafe(`
DO $$
DECLARE
slot_created boolean;
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = '${slotName}')
THEN
PERFORM pg_create_logical_replication_slot('${slotName}', 'pgoutput');
slot_created := true;
END IF;
RAISE NOTICE 'Slot created: %', slot_created;
END $$;
`);
};
class OutboxConsumer {
_params;
_createClient;
_state;
_sql = null;
_sendAsync = null;
constructor(_params, _createClient, _state) {
this._params = _params;
this._createClient = _createClient;
this._state = _state;
}
getCreationParams() {
return this._params;
}
getDbConnection() {
assert(this._sql, `A connection hasn't been yet established.`);
return this._sql;
}
async start() {
const { publish, getOptions, consumerName } = this._params;
const partitionKey = this._params.partitionKey || 'default';
const slotName = getSlotName(consumerName, partitionKey);
const onPublish = async ({ transaction, acknowledge }) => {
assert(this._state);
const messages = transaction.results.map((result) => ({
position: result.position,
messageId: result.messageId,
messageType: result.messageType,
lsn: transaction.lsn,
redeliveryCount: this._state?.redeliveryCount || 0,
message: JSON.parse(result.payload),
}));
await publish(messages);
};
const onFailedPublish = async (tx) => {
assert(this._state);
await this._state.reportFailedDelivery(tx.lsn);
};
const createPublishingQueue = this._params.serialization
? createSerializedPublishingQueue
: createNonBlockingPublishingQueue;
const publishingQueue = createPublishingQueue(onPublish, {
onFailedPublish,
waitAfterFailedPublish: this._params.waitAfterFailedPublish || Duration.ofSeconds(30),
});
const sql = (this._sql = this._createClient({
...getOptions(),
}));
const subscribeSql = this._createClient({
...getOptions(),
publications: PublicationName,
transform: { column: {}, value: {}, row: {} },
max: 1,
fetch_types: false,
idle_timeout: undefined,
max_lifetime: null,
connection: {
application_name: slotName,
replication: 'database',
},
onclose: async () => {
},
});
if (!this._state) {
this._state = new OutboxConsumerState(new OutboxConsumerStore(sql, consumerName, partitionKey));
}
await migrate(sql, slotName);
await this._state.createOrLoad(partitionKey);
const replicationState = {
lastProcessedLsn: this._state.lastProcessedLsn,
timestamp: new Date(),
publication: PublicationName,
slotName,
};
try {
await startLogicalReplication({
state: replicationState,
sql: subscribeSql,
columnConfig: {
position: 'bigint',
messageId: 'text',
messageType: 'text',
partitionKey: 'text',
payload: 'jsonb',
},
onInsert: async (transaction, acknowledge) => {
const message = {
transaction,
acknowledge: async () => {
assert(this._state);
acknowledge();
await this._state.moveFurther(transaction.lsn);
},
};
publishingQueue.queue(message);
await publishingQueue.run(message);
},
});
}
catch (e) {
if (e instanceof postgres.PostgresError && (e.routine === 'ReplicationSlotAcquire' || e.code === '55006')) {
throw new HermesConsumerAlreadyTakenError({ consumerName, partitionKey });
}
throw e;
}
let asyncOutboxStop;
if (this._params.asyncOutbox) {
const asyncOutbox = this._params.asyncOutbox(this);
asyncOutboxStop = asyncOutbox.start();
this._sendAsync = async (message, tx) => {
await asyncOutbox.send(message, { tx });
};
}
return async () => {
const timeout = Duration.ofSeconds(1).ms;
await swallow(() => killReplicationProcesses(this._sql, slotName));
await Promise.all([
swallow(() => this._sql?.end({ timeout })),
Promise.race([swallow(() => subscribeSql?.end({ timeout })), setTimeout(timeout)]),
swallow(() => (asyncOutboxStop ? asyncOutboxStop() : Promise.resolve())),
]);
this._state = undefined;
};
}
async queue(message, options) {
assert(this._sql);
const partitionKey = options?.partitionKey || 'default';
const sql = options?.tx || this._sql;
if (Array.isArray(message)) {
if ('savepoint' in sql) {
for (const m of message) {
await this._publishOne(sql, m, partitionKey);
}
}
else {
await sql.begin(async