@hotmeshio/hotmesh
Version:
Permanent-Memory Workflows & AI Agents
281 lines (280 loc) • 10.9 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.QuorumService = void 0;
const enums_1 = require("../../modules/enums");
const utils_1 = require("../../modules/utils");
const compiler_1 = require("../compiler");
const hotmesh_1 = require("../../types/hotmesh");
const factory_1 = require("../sub/factory");
const factory_2 = require("../store/factory");
class QuorumService {
/**
* @private
*/
constructor() {
this.profiles = [];
this.cacheMode = 'cache';
this.untilVersion = null;
this.quorum = null;
this.callbacks = [];
}
/**
* @private
*/
static async init(namespace, appId, guid, config, engine, logger) {
if (config.engine) {
const instance = new QuorumService();
instance.verifyQuorumFields(config);
instance.namespace = namespace;
instance.appId = appId;
instance.guid = guid;
instance.logger = logger;
instance.engine = engine;
await instance.initStoreChannel(config.engine.store);
await instance.initSubChannel(config.engine.sub, config.engine.pub ?? config.engine.store);
//general quorum subscription
await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId);
//app-specific quorum subscription (used for pubsub one-time request/response)
await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid);
instance.engine.processWebHooks();
instance.engine.processTimeHooks();
return instance;
}
}
/**
* @private
*/
verifyQuorumFields(config) {
if (!(0, utils_1.identifyProvider)(config.engine.store) ||
!(0, utils_1.identifyProvider)(config.engine.sub)) {
throw new Error('quorum config must include `store` and `sub` fields.');
}
}
/**
* @private
*/
async initStoreChannel(store) {
this.store = await factory_2.StoreServiceFactory.init(store, this.namespace, this.appId, this.logger);
}
/**
* @private
*/
async initSubChannel(sub, store) {
this.subscribe = await factory_1.SubServiceFactory.init(sub, store, this.namespace, this.appId, this.guid, this.logger);
}
/**
* @private
*/
subscriptionHandler() {
const self = this;
return async (topic, message) => {
self.logger.debug('quorum-event-received', { topic, type: message.type });
if (message.type === 'activate') {
self.engine.setCacheMode(message.cache_mode, message.until_version);
}
else if (message.type === 'ping') {
self.sayPong(self.appId, self.guid, message.originator, message.details);
}
else if (message.type === 'pong' && self.guid === message.originator) {
self.quorum = self.quorum + 1;
if (message.profile) {
self.profiles.push(message.profile);
}
}
else if (message.type === 'throttle') {
self.engine.throttle(message.throttle);
}
else if (message.type === 'work') {
self.engine.processWebHooks();
}
else if (message.type === 'job') {
let jobOutput = message.job;
// If _ref is true, payload was too large - fetch full job data via getState
if (message._ref && message.job?.metadata) {
try {
jobOutput = await self.engine.getState(message.job.metadata.tpc, message.job.metadata.jid);
self.logger.debug('quorum-job-ref-resolved', {
jid: message.job.metadata.jid,
});
}
catch (err) {
self.logger.error('quorum-job-ref-error', {
jid: message.job.metadata.jid,
error: err,
});
return; // Can't route without job data
}
}
self.engine.routeToSubscribers(message.topic, jobOutput);
}
else if (message.type === 'cron') {
self.engine.processTimeHooks();
}
else if (message.type === 'rollcall') {
self.doRollCall(message);
}
//if there are any callbacks, call them
if (self.callbacks.length > 0) {
self.callbacks.forEach((cb) => cb(topic, message));
}
};
}
/**
* @private
*/
async sayPong(appId, guid, originator, details = false) {
let profile;
if (details) {
const stream = this.engine.store.mintKey(hotmesh_1.KeyType.STREAMS, {
appId: this.appId,
});
profile = {
engine_id: this.guid,
namespace: this.namespace,
app_id: this.appId,
stream,
counts: this.engine.router.counts,
timestamp: (0, utils_1.formatISODate)(new Date()),
inited: this.engine.inited,
throttle: this.engine.router.throttle,
reclaimDelay: this.engine.router.reclaimDelay,
reclaimCount: this.engine.router.reclaimCount,
system: await (0, utils_1.getSystemHealth)(),
};
}
this.subscribe.publish(hotmesh_1.KeyType.QUORUM, {
type: 'pong',
guid,
originator,
profile,
}, appId);
}
/**
* A quorum-wide command to request a quorum count.
* @private
*/
async requestQuorum(delay = enums_1.HMSH_QUORUM_DELAY_MS, details = false) {
const quorum = this.quorum;
this.quorum = 0;
this.profiles.length = 0;
await this.subscribe.publish(hotmesh_1.KeyType.QUORUM, {
type: 'ping',
originator: this.guid,
details,
}, this.appId);
await (0, utils_1.sleepFor)(delay);
return quorum;
}
/**
* @private
*/
async doRollCall(message) {
let iteration = 0;
const max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
if (this.rollCallInterval)
clearTimeout(this.rollCallInterval);
const base = message.interval / 2;
const amount = base + Math.ceil(Math.random() * base);
do {
await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
await this.sayPong(this.appId, this.guid, null, true);
if (!message.interval)
return;
const { promise, timerId } = (0, utils_1.XSleepFor)(amount * 1000);
this.rollCallInterval = timerId;
await promise;
} while (this.rollCallInterval && iteration++ < max - 1);
}
/**
* @private
*/
cancelRollCall() {
if (this.rollCallInterval) {
clearTimeout(this.rollCallInterval);
delete this.rollCallInterval;
}
}
/**
* @private
*/
stop() {
this.cancelRollCall();
}
/**
* @private
*/
async pub(quorumMessage) {
return await this.subscribe.publish(hotmesh_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
}
/**
* @private
*/
async sub(callback) {
this.callbacks.push(callback);
}
/**
* @private
*/
async unsub(callback) {
this.callbacks = this.callbacks.filter((cb) => cb !== callback);
}
/**
* @private
*/
async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
await this.requestQuorum(delay, true);
const stream_depths = await this.engine.stream.getStreamDepths(this.profiles);
this.profiles.forEach(async (profile, index) => {
//if nothing in the table, the depth will be 0
//todo: separate table for every worker stream?
profile.stream_depth = stream_depths?.[index]?.depth ?? 0;
});
return this.profiles;
}
/**
* @private
*/
async activate(version, delay = enums_1.HMSH_QUORUM_DELAY_MS, count = 0) {
version = version.toString();
const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
if (!canActivate) {
//another engine is already activating the app version
this.logger.debug('quorum-activation-awaiting', { version });
await (0, utils_1.sleepFor)(delay * 6);
const app = await this.store.getApp(this.appId, true);
return app?.active == true && app?.version === version;
}
const config = await this.engine.getVID();
await this.requestQuorum(delay);
const q1 = await this.requestQuorum(delay);
const q2 = await this.requestQuorum(delay);
const q3 = await this.requestQuorum(delay);
if (q1 && q1 === q2 && q2 === q3) {
this.logger.info('quorum-rollcall-succeeded', { q1, q2, q3 });
this.subscribe.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
await new Promise((resolve) => setTimeout(resolve, delay));
await this.store.releaseScoutRole('activate');
//confirm we received the activation message
if (this.engine.untilVersion === version) {
this.logger.info('quorum-activation-succeeded', { version });
const { id } = config;
const compiler = new compiler_1.CompilerService(this.store, this.engine.stream, this.logger);
return await compiler.activate(id, version);
}
else {
this.logger.error('quorum-activation-error', { version });
throw new Error(`UntilVersion Not Received. Version ${version} not activated`);
}
}
else {
this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
this.store.releaseScoutRole('activate');
if (count < enums_1.HMSH_ACTIVATION_MAX_RETRY) {
//increase the delay (give the quorum time to respond) and try again
return await this.activate(version, delay * 2, count + 1);
}
throw new Error(`Quorum not reached. Version ${version} not activated.`);
}
}
}
exports.QuorumService = QuorumService;