pg-trx-outbox
Version:
Transactional outbox of Postgres for Node.js with little Event Sourcing
178 lines (177 loc) • 8.22 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Transfer = void 0;
const throw_1 = __importDefault(require("throw"));
const ts_pattern_1 = require("ts-pattern");
const node_util_1 = require("node:util");
const debug_1 = __importDefault(require("debug"));
const app_name_ts_1 = require("./app-name.js");
class Transfer {
logger = (0, debug_1.default)(`pg-trx-outbox:${app_name_ts_1.appName}`);
queue;
options;
pg;
adapter;
es;
constructor(options, pg, adapter, es) {
this.logger.log = console.log.bind(console);
this.options = options;
this.pg = pg;
this.adapter = adapter;
this.es = es;
Promise.resolve().then(() => __importStar(require('p-queue'))).then(({ default: PQueue }) => (this.queue = new PQueue({ concurrency: 1 })));
}
async start() { }
async stop() {
this.queue.clear();
await this.queue.onIdle();
}
async transferMessages() {
if (this.queue.pending + this.queue.size >= 2) {
return;
}
await this.queue.add(async () => {
let messages = [];
const client = await this.pg.getClient();
try {
await client.query('begin');
messages = await this.fetchPgMessages(client);
if (messages.length) {
const results = await this.adapter.send(messages);
const ids = [];
const responses = [];
const errors = [];
const metas = [];
const processed = [];
const attempts = [];
const sinceAt = [];
const errorApproved = [];
for (const [i, resp] of results.entries()) {
const message = messages[i] ?? (0, throw_1.default)(new Error('Message not exists for result'));
ids.push(message.id);
metas.push(resp.meta ?? null);
responses.push(resp.status === 'fulfilled'
? typeof resp.value === 'string' || Array.isArray(resp.value)
? { r: resp.value }
: resp.value
: null);
errors.push(resp.status === 'rejected' ? this.normalizeError(resp.reason) : message.error);
const needRetry = resp.status === 'rejected' &&
this.options.outboxOptions?.retryError?.(resp.reason) &&
message.attempts < (this.options.outboxOptions?.retryMaxAttempts ?? 5);
processed.push(!needRetry);
attempts.push(message.attempts + (needRetry ? 1 : 0));
sinceAt.push(needRetry ? new Date(Date.now() + (this.options.outboxOptions?.retryDelay ?? 5) * 1000) : message.since_at);
errorApproved.push(resp.status === 'rejected' && resp.reason.isApproved ? true : false);
}
await this.updateToProcessed(client, ids, responses, errors, metas, processed, attempts, sinceAt, errorApproved);
this.es.setLastEventId(messages.at(-1)?.id ?? '0');
}
}
catch (e) {
if (e.code !== '55P03') {
if (messages.length) {
await this.updateToProcessed(client, messages.map(r => r.id), messages.map(() => null), messages.map(() => this.normalizeError(e)), messages.map(() => null), messages.map(() => true), messages.map(m => m.attempts), messages.map(m => m.since_at), messages.map(() => false));
}
throw e;
}
}
finally {
await client.query('commit');
client.release();
}
await this.adapter.onHandled(messages);
});
}
normalizeError(error) {
return (0, ts_pattern_1.match)(error)
.with({ stack: ts_pattern_1.P.string }, ({ stack }) => stack)
.with(ts_pattern_1.P.string, err => err)
.otherwise(err => (0, node_util_1.inspect)(err));
}
async fetchPgMessages(client) {
const limit = this.options.outboxOptions?.limit ?? 50;
const lastEventId = this.es.getLastEventId();
this.logger('fetching of messages, limit %d, from event id %d', limit, lastEventId);
const resp = await client.query(`
select
id,
topic,
key,
value,
context_id,
error,
attempts,
since_at
from pg_trx_outbox${this.options.outboxOptions?.partition == null ? '' : `_${this.options.outboxOptions?.partition}`}
where (is_event = false and processed = false and (since_at is null or now() > since_at)
or is_event = true and id > $2)
${this.options.outboxOptions?.topicFilter?.length ? 'and topic = any($3)' : ''}
order by id
limit $1
for update
`, [
limit,
lastEventId,
...(this.options.outboxOptions?.topicFilter?.length ? [this.options.outboxOptions?.topicFilter] : []),
]);
this.logger('received messages with ids %o', resp.rows.map(r => r.id));
return resp.rows;
}
async updateToProcessed(client, ids, responses, errors, meta, done, attempts, sinceAt, errorApproved) {
await client.query(`
with info as (
select *
from unnest($1::bigint[], $2::jsonb[], $3::text[], $4::jsonb[], $5::boolean[], $6::smallint[], $7::timestamptz[], $8::boolean[])
x(id, resp, err, meta, processed, attempts, since_at, error_approved))
update pg_trx_outbox${this.options.outboxOptions?.partition == null ? '' : `_${this.options.outboxOptions?.partition}`} p
set
processed = (select processed from info where info.id = p.id limit 1),
updated_at = now(),
response = (select resp from info where info.id = p.id limit 1),
error = (select err from info where info.id = p.id limit 1),
meta = (select meta from info where info.id = p.id limit 1),
attempts = (select attempts from info where info.id = p.id limit 1),
since_at = (select since_at from info where info.id = p.id limit 1),
error_approved = (select error_approved from info where info.id = p.id limit 1)
where p.id = any($1)
`, [ids, responses, errors, meta, done, attempts, sinceAt, errorApproved]);
}
}
exports.Transfer = Transfer;