@hotmeshio/hotmesh
Version:
Permanent-Memory Workflows & AI Agents
772 lines (771 loc) • 28.1 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.EngineService = void 0;
const key_1 = require("../../modules/key");
const enums_1 = require("../../modules/enums");
const utils_1 = require("../../modules/utils");
const activities_1 = __importDefault(require("../activities"));
const compiler_1 = require("../compiler");
const exporter_1 = require("../exporter");
const reporter_1 = require("../reporter");
const router_1 = require("../router");
const serializer_1 = require("../serializer");
const factory_1 = require("../search/factory");
const factory_2 = require("../store/factory");
const factory_3 = require("../stream/factory");
const factory_4 = require("../sub/factory");
const task_1 = require("../task");
const stream_1 = require("../../types/stream");
class EngineService {
/**
* @private
*/
constructor() {
this.cacheMode = 'cache';
this.untilVersion = null;
this.jobCallbacks = {};
this.reporting = false;
this.jobId = 1;
}
/**
* @private
*/
static async init(namespace, appId, guid, config, logger) {
if (config.engine) {
const instance = new EngineService();
instance.verifyEngineFields(config);
instance.namespace = namespace;
instance.appId = appId;
instance.guid = guid;
instance.logger = logger;
await instance.initSearchChannel(config.engine.store);
await instance.initStoreChannel(config.engine.store);
//NOTE: if `pub` is present, use it; otherwise, use `store`
await instance.initSubChannel(config.engine.sub, config.engine.pub ?? config.engine.store);
await instance.initStreamChannel(config.engine.stream, config.engine.store);
instance.router = await instance.initRouter(config);
const streamName = instance.store.mintKey(key_1.KeyType.STREAMS, {
appId: instance.appId,
});
instance.router.consumeMessages(streamName, 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
instance.taskService = new task_1.TaskService(instance.store, logger);
instance.exporter = new exporter_1.ExporterService(instance.appId, instance.store, logger);
instance.inited = (0, utils_1.formatISODate)(new Date());
return instance;
}
}
/**
* @private
*/
verifyEngineFields(config) {
if (!(0, utils_1.identifyProvider)(config.engine.store) ||
!(0, utils_1.identifyProvider)(config.engine.stream) ||
!(0, utils_1.identifyProvider)(config.engine.sub)) {
throw new Error('engine must include `store`, `stream`, and `sub` fields.');
}
}
/**
* @private
*/
async initSearchChannel(search, store) {
this.search = await factory_1.SearchServiceFactory.init(search, store, this.namespace, this.appId, this.logger);
}
/**
* @private
*/
async initStoreChannel(store) {
this.store = await factory_2.StoreServiceFactory.init(store, this.namespace, this.appId, this.logger);
}
/**
* @private
*/
async initSubChannel(sub, store) {
this.subscribe = await factory_4.SubServiceFactory.init(sub, store, this.namespace, this.appId, this.guid, this.logger);
}
/**
* @private
*/
async initStreamChannel(stream, store) {
this.stream = await factory_3.StreamServiceFactory.init(stream, store, this.namespace, this.appId, this.logger);
}
/**
* @private
*/
async initRouter(config) {
const throttle = await this.store.getThrottleRate(':');
return new router_1.Router({
namespace: this.namespace,
appId: this.appId,
guid: this.guid,
role: stream_1.StreamRole.ENGINE,
reclaimDelay: config.engine.reclaimDelay,
reclaimCount: config.engine.reclaimCount,
throttle,
readonly: config.engine.readonly,
}, this.stream, this.logger);
}
/**
* resolves the distributed executable version using a delay
* to allow deployment race conditions to resolve
* @private
*/
async fetchAndVerifyVID(vid, count = 0) {
if (isNaN(Number(vid.version))) {
const app = await this.store.getApp(vid.id, true);
if (!isNaN(Number(app.version))) {
if (!this.apps)
this.apps = {};
this.apps[vid.id] = app;
return { id: vid.id, version: app.version };
}
else if (count < 10) {
await (0, utils_1.sleepFor)(enums_1.HMSH_QUORUM_DELAY_MS * 2);
return await this.fetchAndVerifyVID(vid, count + 1);
}
else {
this.logger.error('engine-vid-resolution-error', {
id: vid.id,
guid: this.guid,
});
}
}
return vid;
}
async getVID(vid) {
if (this.cacheMode === 'nocache') {
const app = await this.store.getApp(this.appId, true);
if (app.version.toString() === this.untilVersion.toString()) {
//new version is deployed; OK to cache again
if (!this.apps)
this.apps = {};
this.apps[this.appId] = app;
this.setCacheMode('cache', app.version.toString());
}
return { id: this.appId, version: app.version };
}
else if (!this.apps && vid) {
this.apps = {};
this.apps[this.appId] = vid;
return vid;
}
else {
return await this.fetchAndVerifyVID({
id: this.appId,
version: this.apps?.[this.appId].version,
});
}
}
/**
* @private
*/
setCacheMode(cacheMode, untilVersion) {
this.logger.info(`engine-executable-cache`, {
mode: cacheMode,
[cacheMode === 'cache' ? 'target' : 'until']: untilVersion,
});
this.cacheMode = cacheMode;
this.untilVersion = untilVersion;
}
/**
* @private
*/
async routeToSubscribers(topic, message) {
const jobCallback = this.jobCallbacks[message.metadata.jid];
if (jobCallback) {
this.delistJobCallback(message.metadata.jid);
jobCallback(topic, message);
}
}
/**
* @private
*/
async processWebHooks() {
this.taskService.processWebHooks(this.hook.bind(this));
}
/**
* @private
*/
async processTimeHooks() {
this.taskService.processTimeHooks(this.hookTime.bind(this));
}
/**
* @private
*/
async throttle(delayInMillis) {
try {
this.router?.setThrottle(delayInMillis);
}
catch (e) {
this.logger.error('engine-throttle-error', { error: e });
}
}
// ************* METADATA/MODEL METHODS *************
/**
* @private
*/
async initActivity(topic, data = {}, context) {
const [activityId, schema] = await this.getSchema(topic);
const ActivityHandler = activities_1.default[schema.type];
if (ActivityHandler) {
const utc = (0, utils_1.formatISODate)(new Date());
const metadata = {
aid: activityId,
atp: schema.type,
stp: schema.subtype,
ac: utc,
au: utc,
};
const hook = null;
return new ActivityHandler(schema, data, metadata, hook, this, context);
}
else {
throw new Error(`activity type ${schema.type} not found`);
}
}
async getSchema(topic) {
const app = (await this.store.getApp(this.appId));
if (!app) {
throw new Error(`no app found for id ${this.appId}`);
}
if (this.isPrivate(topic)) {
//private subscriptions use the schema id (.activityId)
const activityId = topic.substring(1);
const schema = await this.store.getSchema(activityId, await this.getVID(app));
return [activityId, schema];
}
else {
//public subscriptions use a topic (a.b.c) that is associated with a schema id
const activityId = await this.store.getSubscription(topic, await this.getVID(app));
if (activityId) {
const schema = await this.store.getSchema(activityId, await this.getVID(app));
return [activityId, schema];
}
}
throw new Error(`no subscription found for topic ${topic} in app ${this.appId} for app version ${app.version}`);
}
/**
* @private
*/
async getSettings() {
return await this.store.getSettings();
}
/**
* @private
*/
isPrivate(topic) {
return topic.startsWith('.');
}
// ************* COMPILER METHODS *************
/**
* @private
*/
async plan(pathOrYAML) {
const compiler = new compiler_1.CompilerService(this.store, this.stream, this.logger);
return await compiler.plan(pathOrYAML);
}
/**
* @private
*/
async deploy(pathOrYAML) {
const compiler = new compiler_1.CompilerService(this.store, this.stream, this.logger);
return await compiler.deploy(pathOrYAML);
}
// ************* REPORTER METHODS *************
/**
* @private
*/
async getStats(topic, query) {
const { id, version } = await this.getVID();
const reporter = new reporter_1.ReporterService({ id, version }, this.store, this.logger);
const resolvedQuery = await this.resolveQuery(topic, query);
return await reporter.getStats(resolvedQuery);
}
/**
* @private
*/
async getIds(topic, query, queryFacets = []) {
const { id, version } = await this.getVID();
const reporter = new reporter_1.ReporterService({ id, version }, this.store, this.logger);
const resolvedQuery = await this.resolveQuery(topic, query);
return await reporter.getIds(resolvedQuery, queryFacets);
}
/**
* @private
*/
async resolveQuery(topic, query) {
const trigger = (await this.initActivity(topic, query.data));
await trigger.getState();
return {
end: query.end,
start: query.start,
range: query.range,
granularity: trigger.resolveGranularity(),
key: trigger.resolveJobKey(trigger.createInputContext()),
sparse: query.sparse,
};
}
// ****************** STREAM RE-ENTRY POINT *****************
/**
* @private
*/
async processStreamMessage(streamData) {
this.logger.debug('engine-process', {
jid: streamData.metadata.jid,
gid: streamData.metadata.gid,
dad: streamData.metadata.dad,
aid: streamData.metadata.aid,
status: streamData.status || stream_1.StreamStatus.SUCCESS,
code: streamData.code || 200,
type: streamData.type,
});
const context = {
metadata: {
guid: streamData.metadata.guid,
jid: streamData.metadata.jid,
gid: streamData.metadata.gid,
dad: streamData.metadata.dad,
aid: streamData.metadata.aid,
},
data: streamData.data,
};
if (streamData.type === stream_1.StreamDataType.TIMEHOOK) {
//TIMEHOOK AWAKEN
const activityHandler = (await this.initActivity(`.${streamData.metadata.aid}`, context.data, context));
await activityHandler.processTimeHookEvent(streamData.metadata.jid);
}
else if (streamData.type === stream_1.StreamDataType.WEBHOOK) {
//WEBHOOK AWAKEN (SIGNAL IN)
const activityHandler = (await this.initActivity(`.${streamData.metadata.aid}`, context.data, context));
await activityHandler.processWebHookEvent(streamData.status, streamData.code);
}
else if (streamData.type === stream_1.StreamDataType.TRANSITION) {
//TRANSITION (ADJACENT ACTIVITY)
const activityHandler = (await this.initActivity(`.${streamData.metadata.aid}`, context.data, context)); //todo: `as Activity` (type is more generic)
await activityHandler.process();
}
else if (streamData.type === stream_1.StreamDataType.AWAIT) {
//TRIGGER JOB
context.metadata = {
...context.metadata,
pj: streamData.metadata.jid,
pg: streamData.metadata.gid,
pd: streamData.metadata.dad,
pa: streamData.metadata.aid,
px: streamData.metadata.await === false,
trc: streamData.metadata.trc,
spn: streamData.metadata.spn,
};
const activityHandler = (await this.initActivity(streamData.metadata.topic, streamData.data, context));
await activityHandler.process();
}
else if (streamData.type === stream_1.StreamDataType.RESULT) {
//AWAIT RESULT
const activityHandler = (await this.initActivity(`.${context.metadata.aid}`, streamData.data, context));
await activityHandler.processEvent(streamData.status, streamData.code);
}
else {
//WORKER RESULT
const activityHandler = (await this.initActivity(`.${streamData.metadata.aid}`, streamData.data, context));
await activityHandler.processEvent(streamData.status, streamData.code, 'output');
}
this.logger.debug('engine-process-end', {
jid: streamData.metadata.jid,
gid: streamData.metadata.gid,
aid: streamData.metadata.aid,
});
}
// ***************** `AWAIT` ACTIVITY RETURN RESPONSE ****************
/**
* @private
*/
async execAdjacentParent(context, jobOutput, emit = false) {
if (this.hasParentJob(context)) {
//errors are stringified `StreamError` objects
const error = this.resolveError(jobOutput.metadata);
const spn = context['$self']?.output?.metadata?.l2s ||
context['$self']?.output?.metadata?.l1s;
const streamData = {
metadata: {
guid: (0, utils_1.guid)(),
jid: context.metadata.pj,
gid: context.metadata.pg,
dad: context.metadata.pd,
aid: context.metadata.pa,
trc: context.metadata.trc,
spn,
},
type: stream_1.StreamDataType.RESULT,
data: jobOutput.data,
};
if (error && error.code) {
streamData.status = stream_1.StreamStatus.ERROR;
streamData.data = error;
streamData.code = error.code;
streamData.stack = error.stack;
}
else if (emit) {
streamData.status = stream_1.StreamStatus.PENDING;
streamData.code = enums_1.HMSH_CODE_PENDING;
}
else {
streamData.status = stream_1.StreamStatus.SUCCESS;
streamData.code = enums_1.HMSH_CODE_SUCCESS;
}
return (await this.router?.publishMessage(null, streamData));
}
}
/**
* @private
*/
hasParentJob(context, checkSevered = false) {
if (checkSevered) {
return Boolean(context.metadata.pj && context.metadata.pa && !context.metadata.px);
}
return Boolean(context.metadata.pj && context.metadata.pa);
}
/**
* @private
*/
resolveError(metadata) {
if (metadata && metadata.err) {
return JSON.parse(metadata.err);
}
}
// ****************** `INTERRUPT` ACTIVE JOBS *****************
/**
* @private
*/
async interrupt(topic, jobId, options = {}) {
//immediately interrupt the job, going directly to the data source
await this.store.interrupt(topic, jobId, options);
//now that the job is interrupted, we can clean up
const context = (await this.getState(topic, jobId));
const completionOpts = {
interrupt: options.descend,
expire: options.expire,
};
return (await this.runJobCompletionTasks(context, completionOpts));
}
// ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
/**
* @private
*/
async scrub(jobId) {
//todo: do not allow scrubbing of non-existent or actively running job
await this.store.scrub(jobId);
}
// ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
/**
* @private
*/
async hook(topic, data, status = stream_1.StreamStatus.SUCCESS, code = 200) {
const hookRule = await this.taskService.getHookRule(topic);
const [aid] = await this.getSchema(`.${hookRule.to}`);
const streamData = {
type: stream_1.StreamDataType.WEBHOOK,
status,
code,
metadata: {
guid: (0, utils_1.guid)(),
aid,
topic,
},
data,
};
return (await this.router?.publishMessage(null, streamData));
}
/**
* @private
*/
async hookTime(jobId, gId, topicOrActivity, type) {
if (type === 'interrupt' || type === 'expire') {
return await this.interrupt(topicOrActivity, jobId, {
suppress: true,
expire: 1,
});
}
const [aid, ...dimensions] = topicOrActivity.split(',');
const dad = `,${dimensions.join(',')}`;
const streamData = {
type: stream_1.StreamDataType.TIMEHOOK,
metadata: {
guid: (0, utils_1.guid)(),
jid: jobId,
gid: gId,
dad,
aid,
},
data: { timestamp: Date.now() },
};
await this.router?.publishMessage(null, streamData);
}
/**
* @private
*/
async hookAll(hookTopic, data, keyResolver, queryFacets = []) {
const config = await this.getVID();
const hookRule = await this.taskService.getHookRule(hookTopic);
if (hookRule) {
const subscriptionTopic = await (0, utils_1.getSubscriptionTopic)(hookRule.to, this.store, config);
const resolvedQuery = await this.resolveQuery(subscriptionTopic, keyResolver);
const reporter = new reporter_1.ReporterService(config, this.store, this.logger);
const workItems = await reporter.getWorkItems(resolvedQuery, queryFacets);
if (workItems.length) {
const taskService = new task_1.TaskService(this.store, this.logger);
await taskService.enqueueWorkItems(workItems.map((workItem) => [
hookTopic,
workItem,
keyResolver.scrub || false,
JSON.stringify(data),
].join(key_1.VALSEP)));
this.subscribe.publish(key_1.KeyType.QUORUM, { type: 'work', originator: this.guid }, this.appId);
}
return workItems;
}
else {
throw new Error(`unable to find hook rule for topic ${hookTopic}`);
}
}
// ********************** PUB/SUB ENTRY POINT **********************
/**
* @private
*/
async pub(topic, data, context, extended) {
const activityHandler = await this.initActivity(topic, data, context);
if (activityHandler) {
return await activityHandler.process(extended);
}
else {
throw new Error(`unable to process activity for topic ${topic}`);
}
}
/**
* @private
*/
async sub(topic, callback) {
const subscriptionCallback = async (topic, message) => {
let jobOutput = message.job;
// If _ref is true, payload was too large - fetch full job data via getState
if (message._ref && message.job?.metadata) {
jobOutput = await this.getState(message.job.metadata.tpc, message.job.metadata.jid);
}
callback(message.topic, jobOutput);
};
return await this.subscribe.subscribe(key_1.KeyType.QUORUM, subscriptionCallback, this.appId, topic);
}
/**
* @private
*/
async unsub(topic) {
return await this.subscribe.unsubscribe(key_1.KeyType.QUORUM, this.appId, topic);
}
/**
* @private
*/
async psub(wild, callback) {
const subscriptionCallback = async (topic, message) => {
let jobOutput = message.job;
// If _ref is true, payload was too large - fetch full job data via getState
if (message._ref && message.job?.metadata) {
jobOutput = await this.getState(message.job.metadata.tpc, message.job.metadata.jid);
}
callback(message.topic, jobOutput);
};
return await this.subscribe.psubscribe(key_1.KeyType.QUORUM, subscriptionCallback, this.appId, wild);
}
/**
* @private
*/
async punsub(wild) {
return await this.subscribe.punsubscribe(key_1.KeyType.QUORUM, this.appId, wild);
}
/**
* @private
*/
async pubsub(topic, data, context, timeout = enums_1.HMSH_OTT_WAIT_TIME) {
context = {
metadata: {
ngn: this.guid,
trc: context?.metadata?.trc,
spn: context?.metadata?.spn,
},
};
const jobId = await this.pub(topic, data, context);
return new Promise((resolve, reject) => {
this.registerJobCallback(jobId, (topic, output) => {
if (output.metadata.err) {
const error = JSON.parse(output.metadata.err);
reject({
error,
job_id: output.metadata.jid,
});
}
else {
resolve(output);
}
});
setTimeout(() => {
//note: job is still active (the subscriber timed out)
this.delistJobCallback(jobId);
reject({
code: enums_1.HMSH_CODE_TIMEOUT,
message: 'timeout',
job_id: jobId,
});
}, timeout);
});
}
/**
* @private
*/
async pubOneTimeSubs(context, jobOutput, emit = false) {
//todo: subscriber should query for the job...only publish minimum context needed
if (this.hasOneTimeSubscription(context)) {
const message = {
type: 'job',
topic: context.metadata.jid,
job: (0, utils_1.restoreHierarchy)(jobOutput),
};
this.subscribe.publish(key_1.KeyType.QUORUM, message, this.appId, context.metadata.ngn);
}
}
/**
* @private
*/
async getPublishesTopic(context) {
const config = await this.getVID();
const activityId = context.metadata.aid || context['$self']?.output?.metadata?.aid;
const schema = await this.store.getSchema(activityId, config);
return schema.publishes;
}
/**
* @private
*/
async pubPermSubs(context, jobOutput, emit = false) {
const topic = await this.getPublishesTopic(context);
if (topic) {
const message = {
type: 'job',
topic,
job: (0, utils_1.restoreHierarchy)(jobOutput),
};
this.subscribe.publish(key_1.KeyType.QUORUM, message, this.appId, `${topic}.${context.metadata.jid}`);
}
}
/**
* @private
*/
async add(streamData) {
return (await this.router?.publishMessage(null, streamData));
}
/**
* @private
*/
registerJobCallback(jobId, jobCallback) {
this.jobCallbacks[jobId] = jobCallback;
}
/**
* @private
*/
delistJobCallback(jobId) {
delete this.jobCallbacks[jobId];
}
/**
* @private
*/
hasOneTimeSubscription(context) {
return Boolean(context.metadata.ngn);
}
// ********** JOB COMPLETION/CLEANUP (AND JOB EMIT) ***********
/**
* @private
*/
async runJobCompletionTasks(context, options = {}) {
//'emit' indicates the job is still active
const isAwait = this.hasParentJob(context, true);
const isOneTimeSub = this.hasOneTimeSubscription(context);
const topic = await this.getPublishesTopic(context);
let msgId;
if (isAwait || isOneTimeSub || topic) {
const jobOutput = await this.getState(context.metadata.tpc, context.metadata.jid);
msgId = await this.execAdjacentParent(context, jobOutput, options.emit);
this.pubOneTimeSubs(context, jobOutput, options.emit);
this.pubPermSubs(context, jobOutput, options.emit);
}
if (!options.emit) {
this.taskService.registerJobForCleanup(context.metadata.jid, this.resolveExpires(context, options), options);
}
return msgId;
}
/**
* Job hash expiration is typically reliant on the metadata field
* if the activity concludes normally. However, if the job is `interrupted`,
* it will be expired immediately.
* @private
*/
resolveExpires(context, options) {
return options.expire ?? context.metadata.expire ?? enums_1.HMSH_EXPIRE_JOB_SECONDS;
}
// ****** GET JOB STATE/COLLATION STATUS BY ID *********
/**
* @private
*/
async export(jobId) {
return await this.exporter.export(jobId);
}
/**
* @private
*/
async getRaw(jobId) {
return await this.store.getRaw(jobId);
}
/**
* @private
*/
async getStatus(jobId) {
const { id: appId } = await this.getVID();
return await this.store.getStatus(jobId, appId);
}
/**
* @private
*/
async getState(topic, jobId) {
const jobSymbols = await this.store.getSymbols(`$${topic}`);
const consumes = {
[`$${topic}`]: Object.keys(jobSymbols),
};
//job data exists at the 'zero' dimension; pass an empty object
const dIds = {};
const output = await this.store.getState(jobId, consumes, dIds);
if (!output) {
throw new Error(`not found ${jobId}`);
}
const [state, status] = output;
const stateTree = (0, utils_1.restoreHierarchy)(state);
if (status && stateTree.metadata) {
stateTree.metadata.js = status;
}
return stateTree;
}
/**
* @private
*/
async getQueryState(jobId, fields) {
return await this.store.getQueryState(jobId, fields);
}
/**
* @private
* @deprecated
*/
async compress(terms) {
const existingSymbols = await this.store.getSymbolValues();
const startIndex = Object.keys(existingSymbols).length;
const maxIndex = Math.pow(52, 2) - 1;
const newSymbols = serializer_1.SerializerService.filterSymVals(startIndex, maxIndex, existingSymbols, new Set(terms));
return await this.store.addSymbolValues(newSymbols);
}
}
exports.EngineService = EngineService;