@arturwojnar/hermes-postgresql
Version:
Production-Ready TypeScript Outbox Pattern for PostgreSQL
244 lines • 9.42 kB
JavaScript
import { assertNever, noop } from '@arturwojnar/hermes';
import { assert } from 'console';
import { TransactionManager } from './TransactionManager.js';
import { constructLsn, convertLsnToBigInt } from './lsn.js';
var Bytes;
(function (Bytes) {
Bytes[Bytes["Int8"] = 1] = "Int8";
Bytes[Bytes["Int16"] = 2] = "Int16";
Bytes[Bytes["Int32"] = 4] = "Int32";
Bytes[Bytes["Int64"] = 8] = "Int64";
})(Bytes || (Bytes = {}));
var TopLevelType;
(function (TopLevelType) {
TopLevelType["XLogData"] = "w";
TopLevelType["PrimaryKeepaliveMessage"] = "k";
})(TopLevelType || (TopLevelType = {}));
var MessageType;
(function (MessageType) {
MessageType["Begin"] = "B";
MessageType["Insert"] = "I";
MessageType["Commit"] = "C";
MessageType["Other"] = "Other";
})(MessageType || (MessageType = {}));
const TopLevelTypeValues = Object.values(TopLevelType);
const MessageTypeValues = Object.values(MessageType);
const XLogData_WalRecordStartByteNumber = Bytes.Int8 + Bytes.Int64 + Bytes.Int64 + Bytes.Int64;
const isTopLevelType = (char) => TopLevelTypeValues.includes(char);
const parseTopLevelType = (char) => {
if (!isTopLevelType(char)) {
throw new Error(`INTERNAL_ERROR`);
}
return char;
};
const parseMessageType = (char) => {
if (MessageTypeValues.includes(char)) {
return char;
}
return MessageType.Other;
};
const keepAliveResult = (shouldPong) => ({
topLevelType: TopLevelType.PrimaryKeepaliveMessage,
messageType: MessageType.Other,
shouldPong,
});
const offset = (offset = 0) => ({
add: (bytes) => (offset += bytes),
addInt8: () => (offset += Bytes.Int8),
addInt16: () => (offset += Bytes.Int16),
addInt32: () => (offset += Bytes.Int32),
addInt64: () => (offset += Bytes.Int64),
value: () => offset,
});
const startLogicalReplication = async (state, sql, publish) => {
const transactionManager = new TransactionManager(publish, state.lastProcessedLsn);
transactionManager.on('error', ({ transaction, error }) => {
console.error('Failed to process transaction:', transaction.transactionId, error);
});
let currentTransaction = {
lsn: state.lastProcessedLsn,
timestamp: new Date(),
results: [],
transactionId: 0,
};
const location = typeof state.lastProcessedLsn === 'undefined' ? '0/00000000' : state.lastProcessedLsn.toString();
const stream = await sql
.unsafe(`START_REPLICATION SLOT outbox_slot LOGICAL ${location} (proto_version '1', publication_names 'outbox_pub')`)
.writable();
const close = async () => {
if (stream) {
await new Promise((r) => (stream.once('close', r), stream.end()));
}
return sql.end();
};
const storeResult = async (result) => {
if (result.messageType === MessageType.Begin) {
currentTransaction = result.transaction;
transactionManager.beginTransaction(result.transaction);
console.log(result);
}
else if (result.messageType === MessageType.Insert) {
console.log(result);
currentTransaction.results = [...currentTransaction.results, result.result];
transactionManager.addInsert(result.result);
}
else if (result.messageType === MessageType.Commit) {
console.log(result);
await transactionManager.commitTransaction();
}
return result;
};
const handleResult = (result) => {
if (result.topLevelType === TopLevelType.PrimaryKeepaliveMessage && result.shouldPong) {
sendStandbyStatusUpdate();
}
else if (result.topLevelType === TopLevelType.XLogData && result.messageType === MessageType.Commit) {
}
};
const createStandbyStatusUpdate = (lastProcessedLsn) => {
const messageSize = 1 + 4 * 8 + 1;
const message = Buffer.alloc(messageSize);
let offset = 0;
message.write('r', offset);
offset += 1;
const lsnBigInt = convertLsnToBigInt(lastProcessedLsn);
for (let i = 0; i < 3; i++) {
message.writeBigInt64BE(lsnBigInt, offset);
offset += 8;
}
const systemClock = toServerSystemClock(Date.now());
message.writeBigInt64BE(systemClock, offset);
offset += 8;
message.writeUInt8(0, offset);
return message;
};
const sendStandbyStatusUpdate = () => {
if (!stream)
return;
const statusUpdate = createStandbyStatusUpdate(transactionManager.getLastAcknowledgedLsn());
stream.write(statusUpdate);
};
stream.on('data', async (message) => {
handleResult(await storeResult(onData(message)));
});
stream.on('error', onError);
stream.on('close', () => {
close().catch(noop);
});
};
const onData = (message) => {
const topLevelType = parseTopLevelType(String.fromCharCode(message[0]));
switch (topLevelType) {
case TopLevelType.PrimaryKeepaliveMessage:
return processPrimaryKeepAliveMessage(message);
case TopLevelType.XLogData: {
const data = message.subarray(XLogData_WalRecordStartByteNumber);
const symbol = String.fromCharCode(data[0]);
const type = parseMessageType(symbol);
switch (type) {
case MessageType.Begin:
return processBeginMessage(data);
case MessageType.Insert:
return processInsertMessage(data);
case MessageType.Commit:
return processCommitMessage(data);
default:
return { topLevelType: TopLevelType.XLogData, messageType: MessageType.Other, symbol };
}
}
default:
assertNever(topLevelType);
}
};
const onError = (error) => {
console.error(error);
};
const parse = (data) => {
};
const processCommitMessage = (data) => {
const pos = offset(Bytes.Int8 + Bytes.Int8 + Bytes.Int64 + Bytes.Int64);
const commitTimestamp = toTimestamp(data.readBigInt64BE(pos.value()));
return {
topLevelType: TopLevelType.XLogData,
messageType: MessageType.Commit,
commitTimestamp,
};
};
const processInsertMessage = (data) => {
const messageId = String.fromCharCode(data.readInt8(0));
const relationId = data.readInt32BE(Bytes.Int8);
const newMessageId = String.fromCharCode(data.readInt8(Bytes.Int8 + Bytes.Int32));
const TUPLE_START_BYTE = Bytes.Int8 + Bytes.Int32 + Bytes.Int8;
const tuplesBuffer = data.subarray(TUPLE_START_BYTE);
const columnsCount = tuplesBuffer.readInt16BE(0);
const pos = offset(Bytes.Int16);
const position = readUInt(tuplesBuffer, pos);
const eventType = readText(tuplesBuffer, pos);
const payload = readJsonb(tuplesBuffer, pos);
console.log(position);
console.log(eventType);
console.log(payload);
const columns = {
position: 1,
};
return {
topLevelType: TopLevelType.XLogData,
messageType: MessageType.Insert,
transactionId: 0,
result: {
position,
eventType,
payload,
},
};
};
const readIntFn = {
'1': 'readUInt8',
'2': 'readUInt16BE',
'4': 'readUInt32BE',
'8': 'readBigUInt64BE',
};
const readUInt = (buffer, pos) => {
const columnType = String.fromCharCode(buffer.readInt8(pos.value()));
const columnLength = buffer.readInt32BE(pos.addInt8()).toString();
assert(columnType === 't', 'readUInt.columnType');
assert(['1', '2', '4', '8'].includes(columnLength), 'readUInt.columnLength');
const value = buffer[readIntFn[columnLength]](pos.addInt32());
pos.add(Number(columnLength));
return value;
};
const readText = (buffer, pos) => {
const columnType = String.fromCharCode(buffer.readInt8(pos.value()));
const columnLength = buffer.readInt32BE(pos.addInt8());
assert(columnType === 't', 'readText.columnType');
const value = buffer.subarray(pos.addInt32(), pos.add(columnLength));
return value.toString('utf-8');
};
const readJsonb = (buffer, pos) => {
const columnType = String.fromCharCode(buffer.readInt8(pos.value()));
const columnLength = buffer.readInt32BE(pos.addInt8());
assert(columnType === 't', 'readText.readJsonb');
const value = buffer.subarray(pos.addInt32(), pos.add(columnLength));
return value.toString('utf-8');
};
const processBeginMessage = (data) => {
const pos = offset(Bytes.Int8);
const lsn = constructLsn(data.readInt32BE(pos.value()), data.readInt32BE(pos.addInt32()));
const timestamp = toTimestamp(data.readBigInt64BE(pos.addInt32()));
const transactionId = data.readInt32BE(pos.addInt64());
return {
topLevelType: TopLevelType.XLogData,
messageType: MessageType.Begin,
transaction: { transactionId, lsn, timestamp, results: [] },
};
};
const processPrimaryKeepAliveMessage = (data) => {
if (!data[Bytes.Int8 + Bytes.Int64 + Bytes.Int64]) {
return keepAliveResult(false);
}
return keepAliveResult(true);
};
const toTimestamp = (value) => new Date(Date.UTC(2000, 0, 1) + Number(value / 1000n));
const toServerSystemClock = (epochMs) => BigInt(epochMs - Date.UTC(2000, 0, 1)) * 1000n;
export { startLogicalReplication };
//# sourceMappingURL=logicalReplicationStream.js.map