@eang/core
Version:
eang - model driven enterprise event processing
304 lines • 12.6 kB
JavaScript
import { jetstream, jetstreamManager } from '@nats-io/jetstream';
import { connectable, Subject } from 'rxjs';
import { connect } from '@nats-io/transport-node';
import { parseMsg } from './functions.js';
import { createHash } from 'crypto';
export class NatsStreamService {
// public readonly tenants = {} as TenantStreams
consumerData = {};
nc;
js;
jsm;
// private eventSubj = new Subject<any>()
// private event$ = connectable(this.eventSubj.asObservable().pipe(), {
// connector: () => new Subject(),
// resetOnDisconnect: false
// })
constructor() { }
async initialize(opts) {
this.nc = await connect(opts);
this.jsm = await jetstreamManager(this.nc);
this.js = jetstream(this.nc);
// Log out infos about existing jet streams and consumers
// const streams = await this.jsm?.streams.list().next()
// streams?.forEach(async (si, i) => {
// this.logger.debug(`nss: ${i}. stream: ${JSON.stringify(si.config)}`)
// const consumers = await this.jsm?.consumers.list(si.config.name).next()
// consumers?.forEach((ci, i) => {
// this.logger.debug(`nss: ${i}. consumer: ${JSON.stringify(ci.config)}`)
// })
// })
// this.logger.trace('nss: Finished initializing stream!')
return { nc: this.nc, js: this.js, jsm: this.jsm };
}
async ensureStream(streamName, subjects) {
const existingStreams = await this.jsm.streams.list().next();
const existingStream = existingStreams.find((si) => si.config.name === streamName);
if (existingStream) {
return { created: false, streamInfo: existingStream };
}
else {
const newStream = await this.jsm?.streams.add({
name: streamName,
subjects: subjects
});
return { created: true, streamInfo: newStream };
}
}
async deleteStream(streamName) {
const deleted = await this.jsm.streams.delete(streamName);
return deleted;
}
async ensureConsumer(streamName, consumerName, filter_subjects = [`eang.>`], opts) {
let existingConsumerInfo = undefined;
let consumerConfig = {
ack_policy: 'explicit',
filter_subjects
};
if (consumerName !== undefined) {
consumerConfig.durable_name = consumerName;
}
if (opts) {
consumerConfig = { ...consumerConfig, ...opts };
}
if (consumerName) {
const consumers = await this.jsm?.consumers.list(streamName).next();
existingConsumerInfo = consumers?.find((ci) => ci.config.durable_name === consumerName);
if (existingConsumerInfo) {
// Check if filter subjects differ
const existingFilterSubjects = existingConsumerInfo?.config.filter_subjects || [];
const filterSubjectsDiffer = JSON.stringify(existingFilterSubjects.sort()) !==
JSON.stringify(filter_subjects.sort());
if (filterSubjectsDiffer) {
// Update the consumer with new filter subjects
await this.jsm?.consumers.update(streamName, consumerName, consumerConfig);
existingConsumerInfo =
(await this.jsm?.consumers.info(streamName, consumerName)) || undefined;
}
const consumer = await this.js?.consumers.get(streamName, consumerName);
if (!consumer || !existingConsumerInfo) {
throw new Error('Failed to get consumer or consumer info');
}
return { consumer, consumerInfo: existingConsumerInfo };
}
else {
const consumerInfo = await this.jsm?.consumers.add(streamName, consumerConfig);
const consumer = await this.js?.consumers.get(streamName, consumerName);
if (!consumer || !consumerInfo) {
throw new Error('Failed to create consumer or get consumer info');
}
return { consumer, consumerInfo };
}
}
else {
const consumer = await this.js?.consumers.get(streamName, consumerConfig);
if (!consumer) {
throw new Error('Failed to get consumer');
}
const consumerInfo = await consumer.info();
return { consumer, consumerInfo };
}
}
async ensureConsumerStream(consumerName, subscription, opts) {
let consumerData = undefined;
if (consumerName) {
consumerData = this.consumerData[consumerName];
if (consumerData) {
return consumerData;
}
}
// add a new durable consumer per tenant
if (!this.jsm) {
throw new Error('JetStreamManager is not initialized');
}
const { consumer, consumerInfo } = await this.ensureConsumer('eang', consumerName, subscription, opts);
const eventSubj = new Subject();
const event$ = connectable(eventSubj.asObservable().pipe(), {
connector: () => new Subject(),
resetOnDisconnect: false
});
event$.connect();
if (!consumerInfo) {
throw new Error('Consumer info is undefined');
}
consumerName = consumerInfo.name;
consumerData = {
consumer,
consumerInfo,
consumerName,
eventSubj,
event$,
startToConsume: async () => {
const messages = await consumer.consume({ max_messages: 1 });
for await (const m of messages) {
const eangEvent = parseMsg(m);
eventSubj.next(eangEvent);
// m.ack()
}
}
};
if (!consumerData) {
throw new Error('Consumer data is undefined');
}
this.consumerData[consumerName] = consumerData;
return consumerData;
}
async deleteConsumerStream(consumerName) {
try {
if (this.consumerData[consumerName]) {
this.consumerData[consumerName].eventSubj.complete();
delete this.consumerData[consumerName];
}
const deleted = await this.jsm.consumers.delete('eang', consumerName);
return deleted;
}
catch (e) {
if (e.message === 'consumer not found') {
return true;
}
console.error(e);
return false;
}
}
async startToConsume(consumerName) {
const consumerData = this.consumerData[consumerName];
if (!consumerData) {
throw new Error(`Consumer '${consumerName}' not found. Ensure the consumer is created first.`);
}
const messages = await consumerData.consumer.consume({ max_messages: 1 });
for await (const m of messages) {
const eangEvent = parseMsg(m);
consumerData.eventSubj.next(eangEvent);
m.ack();
}
}
async publish(events, options) {
const pubAcks = [];
for (const e of events) {
if (!e.tenant) {
e.tenant = options.tenant;
}
if (!e.organizationalUnit && options.organizationalUnit !== undefined) {
e.organizationalUnit = options.organizationalUnit;
}
if (!e.organizationalUnit) {
e.organizationalUnit = e.tenant;
}
if (!e.user) {
e.user = options.user;
}
const { subject, payload, opts } = getEangMsgFromEvent(e, options?.jetStreamOptions);
const pAck = await this.js.publish(subject, payload, opts);
pubAcks.push(pAck);
}
return pubAcks;
}
}
/**
* Escapes only NATS-specific special characters: . * > and space
* Uses a single pass for maximum efficiency
*/
export function escapeSubjectToken(token) {
if (token === undefined || token === null) {
return '_'; // Use underscore for undefined/null values
}
return String(token).replace(/[.*> ]/g, (char) => {
switch (char) {
case '.':
return '%2E';
case '*':
return '%2A';
case '>':
return '%3E';
case ' ':
return '%20';
default:
return char;
}
});
}
/**
* Unescapes a subject token by URL-decoding it
*/
export function unescapeSubjectToken(token) {
if (token === '_') {
return ''; // Return empty string for undefined/null placeholder
}
return decodeURIComponent(token);
}
function getEangMsgFromEvent(event, options) {
const { entity, tenant, organizationalUnit, user, context } = event;
let subject;
let payload;
function optimizePayload(key, value) {
if ((key === 'tags' || key === 'attributes') &&
typeof value === 'object' &&
value &&
Object.keys(value).length === 0) {
return undefined;
}
if (key === 'key' || key === 'typeOf' || key === 'entityType') {
return undefined;
}
if (key === 'fromObjKey' ||
key === 'fromObjTypeOf' ||
key === 'toObjKey' ||
key === 'toObjTypeOf') {
return undefined;
}
return value;
}
if (entity.entityType === 'cnx') {
subject = `eang.cnx.${escapeSubjectToken(tenant)}.${escapeSubjectToken(organizationalUnit)}.${escapeSubjectToken(user)}.${escapeSubjectToken(entity.typeOf)}.${escapeSubjectToken(entity.instanceOf)}.${escapeSubjectToken(entity.key)}.${escapeSubjectToken(event.eventType)}.${escapeSubjectToken(entity.fromObjTypeOf)}.${escapeSubjectToken(entity.fromObjInstanceOf)}.${escapeSubjectToken(entity.fromObjKey)}.${escapeSubjectToken(entity.toObjTypeOf)}.${escapeSubjectToken(entity.toObjInstanceOf)}.${escapeSubjectToken(entity.toObjKey)}`;
// TODO: handle content-type... default is JSON and does not require header
payload = JSON.stringify({ cnx: entity, context }, optimizePayload);
}
else if (entity.entityType === 'obj') {
subject = `eang.obj.${escapeSubjectToken(tenant)}.${escapeSubjectToken(organizationalUnit)}.${escapeSubjectToken(user)}.${escapeSubjectToken(entity.typeOf)}.${escapeSubjectToken(entity.instanceOf)}.${escapeSubjectToken(entity.childOf)}.${escapeSubjectToken(entity.key)}.${escapeSubjectToken(event.eventType)}`;
payload = JSON.stringify({ obj: entity, context }, optimizePayload);
}
else {
throw new Error('Invalid object type');
}
let msgId;
if (event.eventType === 'update') {
const hash = calculateUpdateHash(entity.updatedAt, context);
msgId = `${entity.id}.${hash}.update`;
}
else {
msgId = `${entity.id}.${event.eventType}`;
}
const opts = { msgID: msgId };
if (options) {
Object.assign(opts, options);
}
return { subject, payload, opts };
}
/**
* Calculates a hash from the updatedAt timestamp and changed values in the update context
* @param updatedAt - The timestamp when the entity was updated
* @param context - The update context containing the diff of changes
* @returns A short hash string (first 8 characters of SHA256)
*/
function calculateUpdateHash(updatedAt, context) {
const hash = createHash('sha256');
// Round updatedAt to nearest second (remove milliseconds) to prevent
// generating different hashes for rapid updates within the same second
if (updatedAt !== undefined) {
const roundedTimestamp = Math.floor(updatedAt / 1000) * 1000;
hash.update(roundedTimestamp.toString());
}
// Add changed values from the context
if (context && context.diff && Array.isArray(context.diff)) {
const updateContext = context;
// Sort the diff array by field name for consistent hashing
const sortedDiff = [...updateContext.diff].sort((a, b) => a.field.localeCompare(b.field));
for (const change of sortedDiff) {
hash.update(change.field);
hash.update(JSON.stringify(change.newValue));
}
}
// Return first 8 characters of the hex digest for a shorter hash
return hash.digest('hex').substring(0, 8);
}
//# sourceMappingURL=NatsStreamingService.js.map