@hotmeshio/hotmesh
Version:
Permanent-Memory Workflows & AI Agents
253 lines (252 loc) • 11.3 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.TaskService = void 0;
const enums_1 = require("../../modules/enums");
const utils_1 = require("../../modules/utils");
const pipe_1 = require("../pipe");
const hotmesh_1 = require("../../types/hotmesh");
const key_1 = require("../../modules/key");
class TaskService {
constructor(store, logger) {
this.cleanupTimeout = null;
this.isScout = false;
this.errorCount = 0;
this.logger = logger;
this.store = store;
}
async processWebHooks(hookEventCallback) {
const workItemKey = await this.store.getActiveTaskQueue();
if (workItemKey) {
const [topic, sourceKey, scrub, ...sdata] = workItemKey.split(key_1.WEBSEP);
const data = JSON.parse(sdata.join(key_1.WEBSEP));
const destinationKey = `${sourceKey}:processed`;
const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
if (jobId) {
//todo: don't use 'id', make configurable using hook rule
await hookEventCallback(topic, { ...data, id: jobId });
}
else {
await this.store.deleteProcessedTaskQueue(workItemKey, sourceKey, destinationKey, scrub === 'true');
}
setImmediate(() => this.processWebHooks(hookEventCallback));
}
}
async enqueueWorkItems(keys) {
await this.store.addTaskQueues(keys);
}
async registerJobForCleanup(jobId, inSeconds = enums_1.HMSH_EXPIRE_DURATION, options) {
if (inSeconds > 0) {
await this.store.expireJob(jobId, inSeconds);
// const fromNow = Date.now() + inSeconds * 1000;
// const fidelityMS = HMSH_FIDELITY_SECONDS * 1000;
// const timeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
// await this.store.registerDependenciesForCleanup(jobId, timeSlot, options);
}
}
async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, dad, transaction) {
const fromNow = Date.now() + inSeconds * 1000;
const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, transaction);
}
/**
* Should this engine instance play the role of 'scout' on behalf
* of the entire quorum? The scout role is responsible for processing
* task lists on behalf of the collective.
*/
async shouldScout() {
const wasScout = this.isScout;
const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
if (isScout) {
if (!wasScout) {
setTimeout(() => {
this.isScout = false;
}, enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000);
}
return true;
}
return false;
}
/**
* Callback handler that takes an item from a work list and
* processes according to its type
*/
async processTimeHooks(timeEventCallback, listKey) {
if (await this.shouldScout()) {
try {
const workListTask = await this.store.getNextTask(listKey);
if (Array.isArray(workListTask)) {
const [listKey, target, gId, activityId, type] = workListTask;
if (type === 'child') {
//continue; this child is listed here for convenience, but
// will be expired by an origin ancestor and is listed there
}
else if (type === 'delist') {
//delist the signalKey (target)
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
appId: this.store.appId,
});
await this.store.delistSignalKey(key, target);
}
else {
//awaken/expire/interrupt
await timeEventCallback(target, gId, activityId, type);
}
await (0, utils_1.sleepFor)(0);
this.errorCount = 0;
this.processTimeHooks(timeEventCallback, listKey);
}
else if (workListTask) {
//a worklist was just emptied; try again immediately
await (0, utils_1.sleepFor)(0);
this.errorCount = 0;
this.processTimeHooks(timeEventCallback);
}
else {
//no worklists exist; sleep before checking
const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
this.cleanupTimeout = sleep.timerId;
await sleep.promise;
this.errorCount = 0;
this.processTimeHooks(timeEventCallback);
}
}
catch (err) {
//most common reasons: deleted job not found; container stopping; test stopping
this.logger.warn('task-process-timehooks-error', err);
await (0, utils_1.sleepFor)(1000 * this.errorCount++);
if (this.errorCount < 5) {
this.processTimeHooks(timeEventCallback);
}
}
}
else {
//didn't get the scout role; try again in 'one-ish' minutes
const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
this.cleanupTimeout = sleep.timerId;
await sleep.promise;
this.processTimeHooks(timeEventCallback);
}
}
cancelCleanup() {
if (this.cleanupTimeout !== undefined) {
clearTimeout(this.cleanupTimeout);
this.cleanupTimeout = undefined;
}
}
async getHookRule(topic) {
const rules = await this.store.getHookRules();
return rules?.[topic]?.[0];
}
async registerWebHook(topic, context, dad, expire, transaction) {
const hookRule = await this.getHookRule(topic);
if (hookRule) {
const mapExpression = hookRule.conditions.match[0].expected;
const resolved = pipe_1.Pipe.resolve(mapExpression, context);
const jobId = context.metadata.jid;
const gId = context.metadata.gid;
const activityId = hookRule.to;
//composite keys are used to fully describe the task target
const compositeJobKey = [activityId, dad, gId, jobId].join(key_1.WEBSEP);
const hook = {
topic,
resolved,
jobId: compositeJobKey,
expire,
};
await this.store.setHookSignal(hook, transaction);
return jobId;
}
else {
throw new Error('signaler.registerWebHook:error: hook rule not found');
}
}
async processWebHookSignal(topic, data) {
const hookRule = await this.getHookRule(topic);
if (hookRule) {
//NOTE: both formats are supported by the mapping engine:
// `$self.hook.data` OR `$hook.data`
const context = { $self: { hook: { data } }, $hook: { data } };
const mapExpression = hookRule.conditions.match[0].actual;
const resolved = pipe_1.Pipe.resolve(mapExpression, context);
const hookSignalId = await this.store.getHookSignal(topic, resolved);
if (!hookSignalId) {
//messages can be double-processed; not an issue; return `undefined`
//users can also provide a bogus topic; not an issue; return `undefined`
return undefined;
}
//`aid` is part of composite key, but the hook `topic` is its public interface;
// this means that a new version of the graph can be deployed and the
// topic can be re-mapped to a different activity id. Outside callers
// can adhere to the unchanged contract (calling the same topic),
// while the internal system can be updated in real-time as necessary.
const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
}
else {
throw new Error('signal-not-found');
}
}
async deleteWebHookSignal(topic, data) {
const hookRule = await this.getHookRule(topic);
if (hookRule) {
//NOTE: both formats are supported by the mapping engine:
// `$self.hook.data` OR `$hook.data`
const context = { $self: { hook: { data } }, $hook: { data } };
const mapExpression = hookRule.conditions.match[0].actual;
const resolved = pipe_1.Pipe.resolve(mapExpression, context);
return await this.store.deleteHookSignal(topic, resolved);
}
else {
throw new Error('signaler.process:error: hook rule not found');
}
}
/**
* Enhanced processTimeHooks that uses notifications for PostgreSQL stores
*/
async processTimeHooksWithNotifications(timeEventCallback) {
// Check if the store supports notifications
if (this.isPostgresStore() && this.supportsNotifications()) {
try {
this.logger.info('task-using-notification-mode', {
appId: this.store.appId,
message: 'Time scout using PostgreSQL LISTEN/NOTIFY mode for efficient task processing',
});
// Use the PostgreSQL store's notification-based approach
await this.store.startTimeScoutWithNotifications(timeEventCallback);
}
catch (error) {
this.logger.warn('task-notifications-fallback', {
appId: this.store.appId,
error: error.message,
fallbackTo: 'polling',
message: 'Notification mode failed - falling back to traditional polling',
});
// Fall back to regular polling
await this.processTimeHooks(timeEventCallback);
}
}
else {
this.logger.info('task-using-polling-mode', {
appId: this.store.appId,
storeType: this.store.constructor.name,
message: 'Time scout using traditional polling mode (notifications not available)',
});
// Use regular polling for non-PostgreSQL stores
await this.processTimeHooks(timeEventCallback);
}
}
/**
* Check if this is a PostgreSQL store
*/
isPostgresStore() {
return this.store.constructor.name === 'PostgresStoreService';
}
/**
* Check if the store supports notifications
*/
supportsNotifications() {
return (typeof this.store.startTimeScoutWithNotifications === 'function');
}
}
exports.TaskService = TaskService;