@hotmeshio/hotmesh
Version:
Serverless Workflow
480 lines (479 loc) • 15.2 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.HotMesh = void 0;
const key_1 = require("../../modules/key");
const utils_1 = require("../../modules/utils");
const factory_1 = require("../connector/factory");
const engine_1 = require("../engine");
const logger_1 = require("../logger");
const quorum_1 = require("../quorum");
const router_1 = require("../router");
const worker_1 = require("../worker");
const enums_1 = require("../../modules/enums");
/**
* This example shows the full lifecycle of a HotMesh engine instance,
* including: initialization, deployment, activation and execution.
*
* Engine routers are self-managing, but subscribe to the 'quorum' channel
* to establish consensus as necessary for distributed processing.
* Completed workflows are always soft-deleted with a configurable
* retention period.
*
* @example
* ```typescript
* import { Client as Postgres } from 'pg';
* import { HotMesh } from '@hotmeshio/hotmesh';
*
* const hotMesh = await HotMesh.init({
* appId: 'abc',
* engine: {
* connection: {
* class: Postgres,
* options: {
* connectionString: 'postgresql://usr:pwd@localhost:5432/db',
* }
* }
* }
* });
*
* await hotMesh.deploy(`
* app:
* id: abc
* version: '1'
* graphs:
* - subscribes: abc.test
* activities:
* t1:
* type: trigger
* `);
*
* await hotMesh.activate('1');
*
* await hotMesh.pubsub('abc.test');
*
* await HotMesh.stop();
* ```
*/
class HotMesh {
/**
* @private
*/
verifyAndSetNamespace(namespace) {
if (!namespace) {
this.namespace = key_1.HMNS;
}
else if (!namespace.match(/^[A-Za-z0-9-]+$/)) {
throw new Error(`config.namespace [${namespace}] is invalid`);
}
else {
this.namespace = namespace;
}
}
/**
* @private
*/
verifyAndSetAppId(appId) {
if (!appId?.match(/^[A-Za-z0-9-]+$/)) {
throw new Error(`config.appId [${appId}] is invalid`);
}
else if (appId === 'a') {
throw new Error(`config.appId [${appId}] is reserved`);
}
else {
this.appId = appId;
}
}
/**
* Instance initializer. Workers are configured
* similarly to the engine, but as an array with
* multiple worker objects.
*
* @example
* ```typescript
* const config: HotMeshConfig = {
* appId: 'myapp',
* engine: {
* connection: {
* class: Postgres,
* options: {
* connectionString: 'postgresql://usr:pwd@localhost:5432/db',
* }
* }
* },
* workers [...]
* };
* const hotMesh = await HotMesh.init(config);
* ```
*/
static async init(config) {
const instance = new HotMesh();
instance.guid = config.guid ?? (0, utils_1.guid)();
instance.verifyAndSetNamespace(config.namespace);
instance.verifyAndSetAppId(config.appId);
instance.logger = new logger_1.LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
await instance.initEngine(config, instance.logger);
await instance.initQuorum(config, instance.engine, instance.logger);
await instance.doWork(config, instance.logger);
return instance;
}
/**
* returns a guid using the same core guid
* generator used by the HotMesh (nanoid)
*/
static guid() {
return (0, utils_1.guid)();
}
/**
* @private
*/
async initEngine(config, logger) {
if (config.engine) {
//connections that are 'readonly' transfer
//this property directly to the engine,
//and ALWAYS take precendence.
if (config.engine.connection.readonly) {
config.engine.readonly = true;
}
await factory_1.ConnectorService.initClients(config.engine);
this.engine = await engine_1.EngineService.init(this.namespace, this.appId, this.guid, config, logger);
}
}
/**
* @private
*/
async initQuorum(config, engine, logger) {
if (engine) {
this.quorum = await quorum_1.QuorumService.init(this.namespace, this.appId, this.guid, config, engine, logger);
}
}
/**
* @private
*/
constructor() {
/**
* @private
*/
this.engine = null;
/**
* @private
*/
this.quorum = null;
/**
* @private
*/
this.workers = [];
}
/**
* @private
*/
async doWork(config, logger) {
this.workers = await worker_1.WorkerService.init(this.namespace, this.appId, this.guid, config, logger);
}
// ************* PUB/SUB METHODS *************
/**
* Starts a workflow
* @example
* ```typescript
* await hotMesh.pub('a.b.c', { key: 'value' });
* ```
*/
async pub(topic, data = {}, context, extended) {
return await this.engine?.pub(topic, data, context, extended);
}
/**
* Subscribe (listen) to all output and interim emissions of a single
* workflow topic. NOTE: Postgres does not support patterned
* unsubscription, so this method is not supported for Postgres.
*
* @example
* ```typescript
* await hotMesh.psub('a.b.c', (topic, message) => {
* console.log(message);
* });
* ```
*/
async sub(topic, callback) {
return await this.engine?.sub(topic, callback);
}
/**
* Stop listening in on a single workflow topic
*/
async unsub(topic) {
return await this.engine?.unsub(topic);
}
/**
* Listen to all output and interim emissions of a workflow topic
* matching a wildcard pattern.
* @example
* ```typescript
* await hotMesh.psub('a.b.c*', (topic, message) => {
* console.log(message);
* });
* ```
*/
async psub(wild, callback) {
return await this.engine?.psub(wild, callback);
}
/**
* Patterned unsubscribe. NOTE: Postgres does not support patterned
* unsubscription, so this method is not supported for Postgres.
*/
async punsub(wild) {
return await this.engine?.punsub(wild);
}
/**
* Starts a workflow and awaits the response
* @example
* ```typescript
* await hotMesh.pubsub('a.b.c', { key: 'value' });
* ```
*/
async pubsub(topic, data = {}, context, timeout) {
return await this.engine?.pubsub(topic, data, context, timeout);
}
/**
* Add a transition message to the workstream, resuming leg 2 of a paused
* reentrant activity (e.g., await, worker, hook)
*/
async add(streamData) {
return (await this.engine.add(streamData));
}
// ************* QUORUM METHODS *************
/**
* Request a roll call from the quorum (engine and workers)
*/
async rollCall(delay) {
return await this.quorum?.rollCall(delay);
}
/**
* Sends a throttle message to the quorum (engine and/or workers)
* to limit the rate of processing. Pass `-1` to throttle indefinitely.
* The value must be a non-negative integer and not exceed `MAX_DELAY` ms.
*
* When throttling is set, the quorum will pause for the specified time
* before processing the next message. Target specific engines and
* workers by passing a `guid` and/or `topic`. Pass no arguments to
* throttle the entire quorum.
*
* In this example, all processing has been paused indefinitely for
* the entire quorum. This is equivalent to an emergency stop.
*
* HotMesh is a stateless sequence engine, so the throttle can be adjusted up
* and down with no loss of data.
*
*
* @example
* ```typescript
* await hotMesh.throttle({ throttle: -1 });
* ```
*/
async throttle(options) {
let throttle;
if (options.throttle === -1) {
throttle = enums_1.MAX_DELAY;
}
else {
throttle = options.throttle;
}
if (!Number.isInteger(throttle) || throttle < 0 || throttle > enums_1.MAX_DELAY) {
throw new Error(`Throttle must be a non-negative integer and not exceed ${enums_1.MAX_DELAY} ms; send -1 to throttle indefinitely`);
}
const throttleMessage = {
type: 'throttle',
throttle: throttle,
};
if (options.guid) {
throttleMessage.guid = options.guid;
}
if (options.topic !== undefined) {
throttleMessage.topic = options.topic;
}
await this.engine.store.setThrottleRate(throttleMessage);
return await this.quorum?.pub(throttleMessage);
}
/**
* Publish a message to the quorum (engine and/or workers)
*/
async pubQuorum(quorumMessage) {
return await this.quorum?.pub(quorumMessage);
}
/**
* Subscribe to quorum events (engine and workers)
*/
async subQuorum(callback) {
return await this.quorum?.sub(callback);
}
/**
* Unsubscribe from quorum events (engine and workers)
*/
async unsubQuorum(callback) {
return await this.quorum?.unsub(callback);
}
// ************* LIFECYCLE METHODS *************
/**
* Preview changes and provide an analysis of risk
* prior to deployment
* @private
*/
async plan(path) {
return await this.engine?.plan(path);
}
/**
* When the app YAML descriptor file is ready, the `deploy` function can be called.
* This function is responsible for merging all referenced YAML source
* files and writing the JSON output to the file system and to the provider backend. It
* is also possible to embed the YAML in-line as a string.
*
* *The version will not be active until activation is explicitly called.*
*/
async deploy(pathOrYAML) {
return await this.engine?.deploy(pathOrYAML);
}
/**
* Once the app YAML file is deployed to the provider backend, the `activate` function can be
* called to enable it for the entire quorum at the same moment.
*
* The approach is to establish the coordinated health of the system through series
* of call/response exchanges. Once it is established that the quorum is healthy,
* the quorum is instructed to run their engine in `no-cache` mode, ensuring
* that the provider backend is consulted for the active app version each time a
* call is processed. This ensures that all engines are running the same version
* of the app, switching over at the same moment and then enabling `cache` mode
* to improve performance.
*
* *Add a delay for the quorum to reach consensus if traffic is busy, but
* also consider throttling traffic flow to an acceptable level.*
*/
async activate(version, delay) {
return await this.quorum?.activate(version, delay);
}
/**
* Returns the job state as a JSON object, useful
* for understanding dependency chains
*/
async export(jobId) {
return await this.engine?.export(jobId);
}
/**
* Returns all data (HGETALL) for a job.
*/
async getRaw(jobId) {
return await this.engine?.getRaw(jobId);
}
/**
* Reporter-related method to get the status of a job
* @private
*/
async getStats(topic, query) {
return await this.engine?.getStats(topic, query);
}
/**
* Returns the status of a job. This is a numeric
* semaphore value that indicates the job's state.
* Any non-positive value indicates a completed job.
* Jobs with a value of `-1` are pending and will
* automatically be scrubbed after a set period.
* Jobs a value around -1billion have been interrupted
* and will be scrubbed after a set period. Jobs with
* a value of 0 completed normally. Jobs with a
* positive value are still running.
*/
async getStatus(jobId) {
return this.engine?.getStatus(jobId);
}
/**
* Returns the job state (data and metadata) for a job.
*/
async getState(topic, jobId) {
return this.engine?.getState(topic, jobId);
}
/**
* Returns searchable/queryable data for a job. In this
* example a literal field is also searched (the colon
* is used to track job status and is a reserved field;
* it can be read but not written).
*
* @example
* ```typescript
* const fields = ['fred', 'barney', '":"'];
* const queryState = await hotMesh.getQueryState('123', fields);
* //returns { fred: 'flintstone', barney: 'rubble', ':': '1' }
* ```
*/
async getQueryState(jobId, fields) {
return await this.engine?.getQueryState(jobId, fields);
}
/**
* @private
*/
async getIds(topic, query, queryFacets = []) {
return await this.engine?.getIds(topic, query, queryFacets);
}
/**
* @private
*/
async resolveQuery(topic, query) {
return await this.engine?.resolveQuery(topic, query);
}
/**
* Interrupt an active job
*/
async interrupt(topic, jobId, options = {}) {
return await this.engine?.interrupt(topic, jobId, options);
}
/**
* Immediately deletes (DEL) a completed job from the system.
*
* *Scrubbed jobs must be complete with a non-positive `status` value*
*/
async scrub(jobId) {
await this.engine?.scrub(jobId);
}
/**
* Re/entry point for an active job. This is used to resume a paused job
* and close the reentry point or leave it open for subsequent reentry.
* Because `hooks` are public entry points, they include a `topic`
* which is established in the app YAML file.
*
* When this method is called, a hook rule will be located to establish
* the exact activity and activity dimension for reentry.
*/
async hook(topic, data, status, code) {
return await this.engine?.hook(topic, data, status, code);
}
/**
* @private
*/
async hookAll(hookTopic, data, query, queryFacets = []) {
return await this.engine?.hookAll(hookTopic, data, query, queryFacets);
}
/**
* Stop all points of presence, workers and engines
*/
static async stop() {
if (!this.disconnecting) {
this.disconnecting = true;
await router_1.Router.stopConsuming();
await factory_1.ConnectorService.disconnectAll();
}
}
/**
* Stop this point of presence, workers and engines
*/
stop() {
this.engine?.taskService.cancelCleanup();
this.quorum?.stop();
this.workers?.forEach((worker) => {
worker.stop();
});
}
/**
* @private
* @deprecated
*/
async compress(terms) {
return await this.engine?.compress(terms);
}
}
exports.HotMesh = HotMesh;
HotMesh.disconnecting = false;