@hotmeshio/hotmesh
Version:
Serverless Workflow
1,209 lines • 51 kB
JavaScript
"use strict";
var _a;
Object.defineProperty(exports, "__esModule", { value: true });
exports.MeshData = void 0;
const utils_1 = require("../../modules/utils");
const meshflow_1 = require("../meshflow");
const hotmesh_1 = require("../hotmesh");
const hotmesh_2 = require("../../types/hotmesh");
const enums_1 = require("../../modules/enums");
/**
* The `MeshData` service extends the `MeshFlow` service.
* It serves to unify both data record and
* transactional workflow principles into a single
* *Operational Data Layer*. Deployments with a 'search'
* provider configured (e.g.,Redis FT.SEARCH) can deliver
* both OLTP (transactions) and OLAP (analytics)
* with no additional infrastructure.
*
* The following example depicts the full end-to-end
* lifecycle of a `MeshData` app, including the
* connection of a worker function, the execution of
* a remote function, the retrieval of data,
* the creation of a search index, and the execution
* of a full-text search query.
*
* @example
* ```typescript
* import { MeshData, Types } from '@hotmeshio/hotmesh';
* import * as Redis from 'redis';
*
* //1) Define a search schema
* const schema = {
* schema: {
* id: { type: 'TAG', sortable: true },
* plan: { type: 'TAG', sortable: true },
* active: { type: 'TEXT', sortable: false },
* },
* index: 'user',
* prefix: ['user'], //index items with keys starting with 'user'
* } as unknown as Types.WorkflowSearchOptions;
*
* //2) Initialize MeshData and Redis
* const meshData = new MeshData(
* {
* class: Redis,
* options: { url: 'redis://:key_admin@redis:6379' },
* },
* schema,
* );
*
* //3) Connect a 'user' worker function
* await meshData.connect({
* entity: 'user',
* target: async function(userID: string): Promise<string> {
* //used the `search` extension to add searchable data
* const search = await MeshData.workflow.search();
* await search.set('active', 'yes');
* return `Welcome, ${userID}.`;
* },
* options: { namespace: 'meshdata' },
* });
*
* const userID = 'someTestUser123';
*
* //4) Call the 'user' worker function; include search data
* const response = await meshData.exec({
* entity: 'user',
* args: [userID],
* options: {
* ttl: 'infinity',
* id: userID,
* search: {
* data: { id: userID, plan: 'pro' }
* },
* namespace: 'meshdata',
* },
* });
*
* //5) Read data by field name
* const data = await meshData.get(
* 'user',
* userID,
* {
* fields: ['plan', 'id', 'active'],
* namespace: 'meshdata'
* },
* );
*
* //6) Create a search index
* await meshData.createSearchIndex('user', { namespace: 'meshdata' }, schema);
*
* //7) Perform Full Text Search on the indexed dataset
* const results = await meshData.findWhere('user', {
* query: [{ field: 'id', is: '=', value: userID }],
* limit: { start: 0, size: 100 },
* return: ['plan', 'id', 'active']
* });
*
* //8) Shutdown MeshData
* await MeshData.shutdown();
* ```
*
*/
class MeshData {
/**
* Instances a new `MeshData` service.
* @param {ProviderConfig|ProvidersConfig} connection - the connection class and options
* @param {WorkflowSearchOptions} search - the search options for JSON-based configuration of the backend search module (e.g., Redis FT.Search)
* @example
* // Example 1) Instantiate MeshData with `ioredis`
* import Redis from 'ioredis';
*
* const meshData = new MeshData({
* class: Redis,
* options: {
* host: 'localhost',
* port: 6379,
* password: 'shhh123',
* db: 0,
* }});
*
* // Example 2) Instantiate MeshData with `redis`
* import * as Redis from 'redis';
*
* const meshData = new MeshData({
* class: Redis,
* options: {
* url: 'redis://:shhh123@localhost:6379'
* }});
*
* // Instantiate MeshData with `postgres`
* //...
*/
constructor(connection, search) {
/**
* unused; allows wrapped functions to be stringified
* so their source can be shared on the network for
* remote analysis. this is useful for targeting which
* version of a function is being executed.
* @private
*/
this.connectionSignatures = {};
/**
* Cached local instances (map) of HotMesh organized by namespace
* @private
*/
this.instances = new Map();
/**
* Exposes the the service mesh control plane through the
* mesh 'events' (pub/sub) system. This is useful for
* monitoring and managing the operational data layer.
*/
this.mesh = {
/**
* subscribes to the mesh control plane
* @param {QuorumMessageCallback} callback - the callback function
* @param {SubscriptionOptions} options - connection options
* @returns {Promise<void>}
*/
sub: async (callback, options = {}) => {
const hotMesh = await this.getHotMesh(options.namespace || 'meshflow');
const callbackWrapper = (topic, message) => {
if (message.type === 'pong' && !message.originator) {
if (message.profile?.worker_topic) {
const [entity] = message.profile.worker_topic.split('-');
if (entity) {
message.profile.entity = message.entity = entity;
if (this.connectionSignatures[entity]) {
message.profile.signature = this.connectionSignatures[entity];
}
}
}
}
else if (message?.topic) {
const [entity] = message.topic.split('-');
if (entity) {
message.entity = entity;
}
}
callback(topic, message);
};
await hotMesh.quorum?.sub(callbackWrapper);
},
/**
* publishes a message to the mesh control plane
* @param {QuorumMessage} message - the message payload
* @param {SubscriptionOptions} options - connection options
* @returns {Promise<void>}
*/
pub: async (message, options = {}) => {
const hotMesh = await this.getHotMesh(options.namespace || 'meshflow');
await hotMesh.quorum?.pub(message);
},
/**
* unsubscribes from the mesh control plane
* @param {QuorumMessageCallback} callback - the callback function
* @param {SubscriptionOptions} options - connection options
* @returns {Promise<void>}
*/
unsub: async (callback, options = {}) => {
const hotMesh = await this.getHotMesh(options.namespace || 'meshflow');
await hotMesh.quorum?.unsub(callback);
},
};
this.connection = connection;
if (search) {
this.search = search;
}
}
/**
* @private
*/
validate(entity) {
if (entity.includes(':') || entity.includes('$') || entity.includes(' ')) {
throw "Invalid string [':','$',' ' not allowed]";
}
}
/**
* @private
*/
async getConnection() {
return this.connection;
}
/**
* Return a MeshFlow client
* @private
*/
getClient() {
return new meshflow_1.MeshFlow.Client({
connection: this.connection,
});
}
/**
* @private
*/
safeKey(key) {
return `_${key}`;
}
/**
* @private
* todo: move to `utils` (might already be there);
* also might be better as specialized provider utils
*/
arrayToHash(input) {
const max = input.length;
const hashes = [];
for (let i = 1; i < max; i++) {
const fields = input[i];
if (Array.isArray(fields)) {
const hash = {};
const hashId = input[i - 1];
for (let j = 0; j < fields.length; j += 2) {
const fieldKey = fields[j].replace(/^_/, '');
const fieldValue = fields[j + 1];
hash[fieldKey] = fieldValue;
}
if (typeof hashId === 'string') {
hash['$'] = hashId;
}
hashes.push(hash);
}
}
return hashes;
}
/**
* serialize using the HotMesh `toString` format
* @private
*/
toString(value) {
switch (typeof value) {
case 'string':
break;
case 'boolean':
value = value ? '/t' : '/f';
break;
case 'number':
value = '/d' + value.toString();
break;
case 'undefined':
return undefined;
case 'object':
if (value === null) {
value = '/n';
}
else {
value = '/s' + JSON.stringify(value);
}
break;
}
return value;
}
/**
* returns an entity-namespaced guid
* @param {string|null} entity - entity namespace
* @param {string} [id] - workflow id (allowed to be namespaced)
* @returns {string}
* @private
*/
static mintGuid(entity, id) {
if (!id && !entity) {
throw 'Invalid arguments [entity and id are both null]';
}
else if (!id) {
id = hotmesh_1.HotMesh.guid();
}
else if (entity) {
entity = `${entity}-`;
}
else {
entity = '';
}
return `${entity}${id}`;
}
/**
* Returns a HotMesh client
* @param {string} [namespace='meshflow'] - the namespace for the client
* @returns {Promise<HotMesh>}
*/
async getHotMesh(namespace = 'meshflow') {
//try to reuse an existing client
let hotMesh = await this.instances.get(namespace);
if (!hotMesh) {
//expanded config always takes precedence over concise config
hotMesh = hotmesh_1.HotMesh.init({
appId: namespace,
engine: { connection: this.connection },
});
this.instances.set(namespace, hotMesh);
hotMesh = await hotMesh;
this.instances.set(namespace, hotMesh);
}
return hotMesh;
}
/**
* Returns the HASH key given an `entity` name and workflow/job. The
* item identified by this key is a HASH record with multidimensional process
* data interleaved with the function state data.
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} workflowId - the workflow/job id
* @param {string} [namespace='meshflow'] - the namespace for the client
* @returns {Promise<string>}
* @example
* // mint a key
* const key = await meshData.mintKey('greeting', 'jsmith123');
*
* // returns 'hmsh:meshflow:j:greeting-jsmith123'
*/
async mintKey(entity, workflowId, namespace) {
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, namespace);
const store = handle.hotMesh.engine?.store;
return store?.mintKey(hotmesh_2.KeyType.JOB_STATE, {
jobId: workflowId,
appId: handle.hotMesh.engine?.appId,
});
}
/**
* Connects a function to the operational data layer.
*
* @template T The expected return type of the target function.
*
* @param {object} connection - The options for connecting a function.
* @param {string} connection.entity - The global entity identifier for the function (e.g, 'user', 'order', 'product').
* @param {(...args: any[]) => T} connection.target - Function to connect, returns type T.
* @param {ConnectOptions} connection.options={} - Extended connection options (e.g., ttl, taskQueue). A
* ttl of 'infinity' will cache the function indefinitely.
*
* @returns {Promise<boolean>} True if connection is successfully established.
*
* @example
* // Instantiate MeshData with Redis configuration.
* const meshData = new MeshData({
* class: Redis,
* options: { host: 'localhost', port: 6379 }
* });
*
* // Define and connect a function with the 'greeting' entity.
* // The function will be cached indefinitely (infinite TTL).
* await meshData.connect({
* entity: 'greeting',
* target: (email, user) => `Hello, ${user.first}.`,
* options: { ttl: 'infinity' }
* });
*/
async connect({ entity, target, options = {}, }) {
this.validate(entity);
this.connectionSignatures[entity] = target.toString();
const targetFunction = {
[entity]: async (...args) => {
const { callOptions } = this.bindCallOptions(args);
const result = (await target.apply(target, args));
//increase status by 1, set 'done' flag and emit the 'job done' signal
await this.pauseForTTL(result, callOptions);
return result;
},
};
await meshflow_1.MeshFlow.Worker.create({
namespace: options.namespace,
options: options.options,
connection: await this.getConnection(),
taskQueue: options.taskQueue ?? entity,
workflow: targetFunction,
search: options.search,
});
return true;
}
/**
* During remote execution, an argument is injected (the last argument)
* this is then used by the 'connect' function to determine if the call
* is a hook or a exec call. If it is an exec, the connected function has
* precedence and can say that all calls are cached indefinitely.
*
* @param {any[]} args
* @param {StringAnyType} callOptions
* @returns {StringAnyType}
* @private
*/
bindCallOptions(args, callOptions = {}) {
if (args.length) {
const lastArg = args[args.length - 1];
if (lastArg instanceof Object && lastArg?.$type === 'exec') {
//override the caller and force indefinite caching
callOptions = args.pop();
}
else if (lastArg instanceof Object && lastArg.$type === 'hook') {
callOptions = args.pop();
//hooks may not affect `ttl` (it is set at invocation)
delete callOptions.ttl;
}
}
return { callOptions };
}
/**
* Sleeps/WaitsForSignal to keep the function open
* and remain part of the operational data layer
*
* @template T The expected return type of the remote function
*
* @param {string} result - the result to emit before going to sleep
* @param {CallOptions} options - call options
* @private
*/
async pauseForTTL(result, options) {
if (options?.ttl && options.$type === 'exec') {
//exit early if the outer function wrapper has already run
const { counter, replay, workflowDimension, workflowId } = MeshData.workflow.getContext();
const prefix = options.ttl === 'infinity' ? 'wait' : 'sleep';
if (`-${prefix}${workflowDimension}-${counter + 1}-` in replay) {
return;
}
//manually set job state since leaving open/running
await new Promise((resolve) => setImmediate(resolve));
options.$guid = options.$guid ?? workflowId;
const hotMesh = await MeshData.workflow.getHotMesh();
const jobKey = hotMesh.engine?.store?.mintKey(hotmesh_2.KeyType.JOB_STATE, {
jobId: options.$guid,
appId: hotMesh.engine?.appId,
});
const jobResponse = { aAa: '/t', aBa: this.toString(result) };
await hotMesh.engine?.search.setFields(jobKey, jobResponse);
}
}
/**
* Publishes the job result, because pausing the job (in support of
* the 'ttl' option) interrupts the response.
*
* @template T The expected return type of the remote function
*
* @param {string} result - the result to emit before going to sleep
* @param {HotMesh} hotMesh - call options
* @param {CallOptions} options - call options
*
* @returns {Promise<void>}
* @private
*/
async publishDone(result, hotMesh, options) {
await hotMesh.engine?.subscribe?.publish(hotmesh_2.KeyType.QUORUM, {
type: 'job',
topic: `${hotMesh.engine.appId}.executed`,
job: {
metadata: {
tpc: `${hotMesh.engine.appId}.execute`,
app: hotMesh.engine.appId,
vrs: '1',
jid: options.$guid,
aid: 't1',
ts: '0',
js: 0,
},
data: {
done: true,
response: result,
workflowId: options.$guid, //aCa
},
},
}, hotMesh.engine.appId, `${hotMesh.engine.appId}.executed.${options.$guid}`);
}
/**
* Flushes a function with a `ttl` of 'infinity'. These entities were
* created by a connect method that was configured with a
* `ttl` of 'infinity'. It can take several seconds for the function
* to be removed from the cache as it might be actively orchestrating
* sub-workflows.
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - The workflow/job id
* @param {string} [namespace='meshflow'] - the namespace for the client
*
* @example
* // Flush a function
* await meshData.flush('greeting', 'jsmith123');
*/
async flush(entity, id, namespace) {
const workflowId = MeshData.mintGuid(entity, id);
//resolve the system signal (this forces the main wrapper function to end)
await this.getClient().workflow.signal(`flush-${workflowId}`, {}, namespace);
await (0, utils_1.sleepFor)(1000);
//other activities may still be running; call `interrupt` to stop all threads
await this.interrupt(entity, id, {
descend: true,
suppress: true,
expire: 1,
}, namespace);
}
/**
* Interrupts a job by its entity and id. It is best not to call this
* method directly for entries with a ttl of `infinity` (call `flush` instead).
* For those entities that are cached for a specified duration (e.g., '15 minutes'),
* this method will interrupt the job and start the cascaded cleanup/expire/delete.
* As jobs are asynchronous, there is no way to stop descendant flows immediately.
* Use an `expire` option to keep the interrupted job in the cache for a specified
* duration before it is fully removed.
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - The workflow/job id
* @param {JobInterruptOptions} [options={}] - call options
* @param {string} [namespace='meshflow'] - the namespace for the client
*
* @example
* // Interrupt a function
* await meshData.interrupt('greeting', 'jsmith123');
*/
async interrupt(entity, id, options = {}, namespace) {
const workflowId = MeshData.mintGuid(entity, id);
try {
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, namespace);
const hotMesh = handle.hotMesh;
await hotMesh.interrupt(`${hotMesh.appId}.execute`, workflowId, options);
}
catch (e) {
//no-op; interrup throws an error
}
}
/**
* Signals a Hook Function or Main Function to awaken that
* is paused and registered to awaken upon receiving the signal
* matching @guid.
*
* @param {string} guid - The global identifier for the signal
* @param {StringAnyType} payload - The payload to send with the signal
* @param {string} [namespace='meshflow'] - the namespace for the client
* @returns {Promise<string>} - the signal id
* @example
* // Signal a function with a payload
* await meshData.signal('signal123', { message: 'hi!' });
*
* // returns '123456732345-0' (stream message receipt)
*/
async signal(guid, payload, namespace) {
return await this.getClient().workflow.signal(guid, payload, namespace);
}
/**
* Sends a signal to the backend Service Mesh (workers and engines)
* to announce their presence, including message counts, target
* functions, topics, etc. This is useful for establishing
* the network profile and overall message throughput
* of the operational data layer as a unified quorum.
* @param {RollCallOptions} options
* @returns {Promise<QuorumProfile[]>}
*/
async rollCall(options = {}) {
return (await this.getHotMesh(options.namespace || 'meshflow')).rollCall(options.delay || 1000);
}
/**
* Throttles a worker or engine in the backend Service Mesh, using either
* a 'guid' to target a specific worker or engine, or a 'topic' to target
* a group of worker(s) connected to that topic. The throttle value is
* specified in milliseconds and will cause the target(s) to delay consuming
* the next message by this amount. By default, the value is set to `0`.
* @param {ThrottleOptions} options
* @returns {Promise<boolean>}
*
* @example
* // Throttle a worker or engine
* await meshData.throttle({ guid: '1234567890', throttle: 10_000 });
*/
async throttle(options) {
return (await this.getHotMesh(options.namespace || 'meshflow')).throttle(options);
}
/**
* Similar to `exec`, except it augments the workflow state without creating a new job.
*
* @param {object} input - The input parameters for hooking a function.
* @param {string} input.entity - The target entity name (e.g., 'user', 'order', 'product').
* @param {string} input.id - The target execution/workflow/job id.
* @param {string} input.hookEntity - The hook entity name (e.g, 'user.notification').
* @param {any[]} input.hookArgs - The arguments for the hook function; must be JSON serializable.
* @param {HookOptions} input.options={} - Extended hook options (taskQueue, namespace, etc).
* @returns {Promise<string>} The signal id.
*
* @example
* // Hook a function
* const signalId = await meshData.hook({
* entity: 'greeting',
* id: 'jsmith123',
* hookEntity: 'greeting.newsletter',
* hookArgs: ['xxxx@xxxxx'],
* options: {}
* });
*/
async hook({ entity, id, hookEntity, hookArgs, options = {}, }) {
const workflowId = MeshData.mintGuid(entity, id);
this.validate(workflowId);
const args = [
...hookArgs,
{ ...options, $guid: workflowId, $type: 'hook' },
];
return await this.getClient().workflow.hook({
namespace: options.namespace,
args,
taskQueue: options.taskQueue ?? hookEntity,
workflowName: hookEntity,
workflowId: options.workflowId ?? workflowId,
config: options.config ?? undefined,
});
}
/**
* Executes a remote function by its global entity identifier with specified arguments.
* If options.ttl is infinity, the function will be cached indefinitely and can only be
* removed by calling `flush`. During this time, the function will remain active and
* its state can be augmented by calling `set`, `incr`, `del`, etc OR by calling a
* transactional 'hook' function.
*
* @template T The expected return type of the remote function.
*
* @param {object} input - The execution parameters.
* @param {string} input.entity - The function entity name (e.g., 'user', 'order', 'user.bill').
* @param {any[]} input.args - The arguments for the remote function.
* @param {CallOptions} input.options={} - Extended configuration options for execution (e.g, taskQueue).
*
* @returns {Promise<T>} A promise that resolves with the result of the remote function execution. If
* the input options include `await: false`, the promise will resolve with the
* workflow ID (string) instead of the result. Make sure to pass string as the
* return type if you are using `await: false`.
*
* @example
* // Invoke a remote function with arguments and options
* const response = await meshData.exec({
* entity: 'greeting',
* args: ['jsmith@hotmesh', { first: 'Jan' }],
* options: { ttl: '15 minutes', id: 'jsmith123' }
* });
*/
async exec({ entity, args = [], options = {} }) {
const workflowId = MeshData.mintGuid(options.prefix ?? entity, options.id);
this.validate(workflowId);
const client = this.getClient();
try {
//check the cache
const handle = await client.workflow.getHandle(entity, entity, workflowId, options.namespace);
const state = await handle.hotMesh.getState(`${handle.hotMesh.appId}.execute`, handle.workflowId);
if (state?.data?.done) {
return state.data.response;
}
//202 `pending`; await the result
return (await handle.result());
}
catch (e) {
//create, since not found; then await the result
const optionsClone = { ...options };
let seconds;
if (optionsClone.ttl) {
//setting ttl requires the workflow to remain open
if (optionsClone.signalIn !== false) {
optionsClone.signalIn = true; //explicit 'true' forces open
}
if (optionsClone.ttl === 'infinity') {
delete optionsClone.ttl;
seconds = enums_1.MAX_DELAY;
}
else {
seconds = (0, utils_1.s)(optionsClone.ttl);
}
}
delete optionsClone.search;
delete optionsClone.config;
const handle = await client.workflow.start({
args: [...args, { ...optionsClone, $guid: workflowId, $type: 'exec' }],
taskQueue: options.taskQueue ?? entity,
workflowName: entity,
workflowId: options.workflowId ?? workflowId,
config: options.config ?? undefined,
search: options.search,
workflowTrace: options.workflowTrace,
workflowSpan: options.workflowSpan,
namespace: options.namespace,
await: options.await,
marker: options.marker,
pending: options.pending,
expire: seconds ?? options.expire,
//`persistent` flag keeps the job alive and accepting signals
persistent: options.signalIn == false ? undefined : seconds && true,
signalIn: optionsClone.signalIn,
});
if (options.await === false) {
return handle.workflowId;
}
return (await handle.result());
}
}
/**
* Retrieves the job profile for the function execution, including metadata such as
* execution status and result.
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - identifier for the job
* @param {CallOptions} [options={}] - Configuration options for the execution,
* including custom IDs, time-to-live (TTL) settings, etc.
* Defaults to an empty object if not provided.
*
* @returns {Promise<JobOutput>} A promise that resolves with the job's output, which
* includes metadata about the job's execution status. The
* structure of `JobOutput` should contain all relevant
* information such as execution result, status, and any
* error messages if the job failed.
*
* @example
* // Retrieve information about a remote function's execution by job ID
* const jobInfoById = await meshData.info('greeting', 'job-12345');
*
* // Response: JobOutput
* {
* metadata: {
* tpc: 'meshflow.execute',
* app: 'meshflow',
* vrs: '1',
* jid: 'greeting-jsmith123',
* aid: 't1',
* ts: '0',
* jc: '20240208014803.980',
* ju: '20240208065017.762',
* js: 0
* },
* data: {
* done: true,
* response: 'Hello, Jan. Your email is [jsmith@hotmesh.com].',
* workflowId: 'greeting-jsmith123'
* }
* }
*/
async info(entity, id, options = {}) {
const workflowId = MeshData.mintGuid(options.prefix ?? entity, id);
this.validate(workflowId);
const handle = await this.getClient().workflow.getHandle(options.taskQueue ?? entity, entity, workflowId, options.namespace);
return await handle.hotMesh.getState(`${handle.hotMesh.appId}.execute`, handle.workflowId);
}
/**
* Exports the job profile for the function execution, including
* all state, process, and timeline data. The information in the export
* is sufficient to capture the full state of the function in the moment
* and over time.
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - The workflow/job id
* @param {ExportOptions} [options={}] - Configuration options for the export
* @param {string} [namespace='meshflow'] - the namespace for the client
*
* @example
* // Export a function
* await meshData.export('greeting', 'jsmith123');
*/
async export(entity, id, options, namespace) {
const workflowId = MeshData.mintGuid(entity, id);
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, namespace);
return await handle.export(options);
}
/**
* Returns the remote function state. this is different than the function response
* returned by the `exec` method which represents the return value from the
* function at the moment it completed. Instead, function state represents
* mutable shared state that can be set:
* 1) when the record is first created (provide `options.search.data` to `exec`)
* 2) during function execution ((await MeshData.workflow.search()).set(...))
* 3) during hook execution ((await MeshData.workflow.search()).set(...))
* 4) via the meshData SDK (`meshData.set(...)`)
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - the job id
* @param {CallOptions} [options={}] - call options
*
* @returns {Promise<StringAnyType>} - the function state
*
* @example
* // get the state of a function
* const state = await meshData.get('greeting', 'jsmith123', { fields: ['fred', 'barney'] });
*
* // returns { fred: 'flintstone', barney: 'rubble' }
*/
async get(entity, id, options = {}) {
const workflowId = MeshData.mintGuid(options.prefix ?? entity, id);
this.validate(workflowId);
let prefixedFields = [];
if (Array.isArray(options.fields)) {
prefixedFields = options.fields.map((field) => `_${field}`);
}
else if (this.search?.schema) {
prefixedFields = Object.entries(this.search.schema).map(([key, value]) => {
return 'fieldName' in value ? value.fieldName.toString() : `_${key}`;
});
}
else {
return await this.all(entity, id, options);
}
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, options.namespace);
const search = handle.hotMesh.engine?.search;
const jobKey = await this.mintKey(entity, workflowId, options.namespace);
const vals = await search?.getFields(jobKey, prefixedFields);
const result = prefixedFields.reduce((obj, field, index) => {
obj[field.substring(1)] = vals?.[index];
return obj;
}, {});
return result;
}
/**
* Returns the remote function state for all fields. NOTE:
* `all` can be less efficient than calling `get` as it returns all
* fields (HGETALL), not just the ones requested (HMGET). Depending
* upon the duration of the workflow, this could represent a large
* amount of process/history data.
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - the workflow/job id
* @param {CallOptions} [options={}] - call options
*
* @returns {Promise<StringAnyType>} - the function state
*
* @example
* // get the state of the job (this is not the response...this is job state)
* const state = await meshData.all('greeting', 'jsmith123');
*
* // returns { fred: 'flintstone', barney: 'rubble', ... }
*/
async all(entity, id, options = {}) {
const rawResponse = await this.raw(entity, id, options);
const responseObj = {};
for (const key in rawResponse) {
if (key.startsWith('_')) {
responseObj[key.substring(1)] = rawResponse[key];
}
}
return responseObj;
}
/**
* Returns all fields in the HASH record:
*
* 1) `:`: workflow status (a semaphore where `0` is complete)
* 2) `_*`: function state (name/value pairs are prefixed with `_`)
* 3) `-*`: workflow cycle state (cycles are prefixed with `-`)
* 4) `[a-zA-Z]{3}`: mutable workflow job state
* 5) `[a-zA-Z]{3}[,\d]+`: immutable workflow activity state
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - the workflow/job id
* @param {CallOptions} [options={}] - call options
*
* @returns {Promise<StringAnyType>} - the function state
*
* @example
* // get the state of a function
* const state = await meshData.raw('greeting', 'jsmith123');
*
* // returns { : '0', _barney: 'rubble', aBa: 'Hello, John Doe. Your email is [jsmith@hotmesh].', ... }
*/
async raw(entity, id, options = {}) {
const workflowId = MeshData.mintGuid(options.prefix ?? entity, id);
this.validate(workflowId);
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, options.namespace);
const search = handle.hotMesh.engine?.search;
const jobKey = await this.mintKey(entity, workflowId, options.namespace);
return await search?.getAllFields(jobKey);
}
/**
* Sets the remote function state. this is different than the function response
* returned by the exec method which represents the return value from the
* function at the moment it completed. Instead, function state represents
* mutable shared state that can be set
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - the job id
* @param {CallOptions} [options={}] - call options
*
* @returns {Promise<number>} - count. The number inserted (Postgres) / updated(Redis)
* @example
* // set the state of a function
* const count = await meshData.set('greeting', 'jsmith123', { search: { data: { fred: 'flintstone', barney: 'rubble' } } });
*/
async set(entity, id, options = {}) {
const workflowId = MeshData.mintGuid(options.prefix ?? entity, id);
this.validate(workflowId);
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, options.namespace);
const search = handle.hotMesh.engine?.search;
const jobId = await this.mintKey(entity, workflowId, options.namespace);
const safeArgs = {};
for (const key in options.search?.data) {
safeArgs[this.safeKey(key)] = options.search?.data[key].toString();
}
return await search?.setFields(jobId, safeArgs);
}
/**
* Increments a field in the remote function state.
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - the job id
* @param {string} field - the field name
* @param {number} amount - the amount to increment
* @param {CallOptions} [options={}] - call options
*
* @returns {Promise<number>} - the new value
* @example
* // increment a field in the function state
* const count = await meshData.incr('greeting', 'jsmith123', 'counter', 1);
*/
async incr(entity, id, field, amount, options = {}) {
const workflowId = MeshData.mintGuid(options.prefix ?? entity, id);
this.validate(workflowId);
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, options.namespace);
const search = handle.hotMesh.engine?.search;
const jobId = await this.mintKey(entity, workflowId, options.namespace);
return await search?.incrementFieldByFloat(jobId, this.safeKey(field), amount);
}
/**
* Deletes one or more fields from the remote function state.
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {string} id - the job id
* @param {CallOptions} [options={}] - call options
*
* @returns {Promise<number>} - the count of fields deleted
* @example
* // remove two hash fields from the function state
* const count = await meshData.del('greeting', 'jsmith123', { fields: ['fred', 'barney'] });
*/
async del(entity, id, options) {
const workflowId = MeshData.mintGuid(options.prefix ?? entity, id);
this.validate(workflowId);
if (!Array.isArray(options.fields)) {
throw 'Invalid arguments [options.fields is not an array]';
}
const prefixedFields = options.fields.map((field) => `_${field}`);
const handle = await this.getClient().workflow.getHandle(entity, entity, workflowId, options.namespace);
const search = handle.hotMesh.engine?.search;
const jobKey = await this.mintKey(entity, workflowId, options.namespace);
const count = await search?.deleteFields(jobKey, prefixedFields);
return Number(count);
}
/**
* For those implementations without a search backend, this quasi-equivalent
* method is provided with a cursor for rudimentary pagination.
* @param {FindJobsOptions} [options]
* @returns {Promise<[string, string[]]>}
* @example
* // find jobs
* const [cursor, jobs] = await meshData.findJobs({ match: 'greeting*' });
*
* // returns [ '0', [ 'hmsh:meshflow:j:greeting-jsmith123', 'hmsh:meshflow:j:greeting-jdoe456' ] ]
*/
async findJobs(options = {}) {
const hotMesh = await this.getHotMesh(options.namespace);
return (await hotMesh.engine?.store?.findJobs(options.match, options.limit, options.batch, options.cursor));
}
/**
* Executes the search query; optionally specify other commands
* @example '@_quantity:[89 89]'
* @example '@_quantity:[89 89] @_name:"John"'
* @example 'FT.search my-index @_quantity:[89 89]'
* @param {FindOptions} options
* @param {any[]} args
* @returns {Promise<string[] | [number] | Array<number, string | number | string[]>>}
*/
async find(entity, options, ...args) {
return await this.getClient().workflow.search(options.taskQueue ?? entity, options.workflowName ?? entity, options.namespace || 'meshflow', options.index ?? options.search?.index ?? this.search.index ?? '', ...args); //[count, [id, fields[]], [id, fields[]], [id, fields[]], ...]]
}
/**
* Provides a JSON abstraction for the backend search engine
* (e.g, `count`, `query`, `return`, `limit`)
* NOTE: If the type is TAG for an entity, `.`, `@`, and `-` must be escaped.
*
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {FindWhereOptions} options - find options (the query). A custom search schema may be provided.
* @returns {Promise<SearchResults | number>} Returns a number if `count` is true, otherwise a SearchResults object.
* @example
* const results = await meshData.findWhere('greeting', {
* query: [
* { field: 'name', is: '=', value: 'John' },
* { field: 'age', is: '>', value: 2 },
* { field: 'quantity', is: '[]', value: [89, 89] }
* ],
* count: false,
* limit: { start: 0, size: 10 },
* return: ['name', 'quantity']
* });
*
* // returns { count: 1, query: 'FT.SEARCH my-index @_name:"John" @_age:[2 +inf] @_quantity:[89 89] LIMIT 0 10', data: [ { name: 'John', quantity: '89' } ] }
*/
async findWhere(entity, options) {
const targetSearch = options.options?.search ?? this.search;
const args = [
this.generateSearchQuery(options.query, targetSearch),
];
if (options.count) {
args.push('LIMIT', '0', '0');
}
else {
//limit which hash fields to return
args.push('RETURN');
args.push(((options.return?.length ?? 0) + 1).toString());
args.push('$');
options.return?.forEach((returnField) => {
if (returnField.startsWith('"')) {
//request a literal hash value
args.push(returnField.slice(1, -1));
}
else {
//request a search hash value
args.push(`_${returnField}`);
}
});
//paginate
if (options.limit) {
args.push('LIMIT', options.limit.start.toString(), options.limit.size.toString());
}
}
const FTResults = await this.find(entity, options.options ?? {}, ...args);
const count = FTResults[0];
const sargs = `FT.SEARCH ${options.options?.index ?? targetSearch?.index} ${args.join(' ')}`;
if (options.count) {
//always return number format if count is requested
return !isNaN(count) || count > 0 ? count : 0;
}
else if (count === 0) {
return { count, query: sargs, data: [] };
}
const hashes = this.arrayToHash(FTResults);
return { count, query: sargs, data: hashes };
}
/**
* Generates a search query from a FindWhereQuery array
* @param {FindWhereQuery[] | string} [query]
* @param {WorkflowSearchOptions} [search]
* @returns {string}
* @private
*/
generateSearchQuery(query, search) {
if (!Array.isArray(query) || query.length === 0) {
return typeof query === 'string' ? query : '*';
}
const queryString = query
.map((q) => {
const { field, is, value, type } = q;
let prefixedFieldName;
//insert the underscore prefix if requested field in query is not a literal
if (search?.schema && field in search.schema) {
if ('fieldName' in search.schema[field]) {
prefixedFieldName = `@${search.schema[field].fieldName}`;
}
else {
prefixedFieldName = `@_${field}`;
}
}
else {
prefixedFieldName = `@${field}`;
}
const fieldType = search?.schema?.[field]?.type ?? type ?? 'TEXT';
switch (fieldType) {
case 'TAG':
return `${prefixedFieldName}:{${value}}`;
case 'TEXT':
return `${prefixedFieldName}:"${value}"`;
case 'NUMERIC':
let range = '';
if (is.startsWith('=')) {
//equal
range = `[${value} ${value}]`;
}
else if (is.startsWith('<')) {
//less than or equal
range = `[-inf ${value}]`;
}
else if (is.startsWith('>')) {
//greater than or equal
range = `[${value} +inf]`;
}
else if (is === '[]') {
//between
range = `[${value[0]} ${value[1]}]`;
}
return `${prefixedFieldName}:${range}`;
default:
return '';
}
})
.join(' ');
return queryString;
}
/**
* Creates a search index for the specified search backend (e.g., FT.search).
* @param {string} entity - the entity name (e.g, 'user', 'order', 'product')
* @param {CallOptions} [options={}] - call options
* @param {WorkflowSearchOptions} [searchOptions] - search options
* @returns {Promise<string>} - the search index name
* @example
* // create a search index for the 'greeting' entity. pass in search options.
* const index = await meshData.createSearchIndex('greeting', {}, { prefix: 'greeting', ... });
*
* // creates a search index for the 'greeting' entity, using the default search options.
* const index = await meshData.createSearchIndex('greeting');
*/
async createSearchIndex(entity, options = {}, searchOptions) {
const workflowTopic = `${options.taskQueue ?? entity}-${entity}`;
const hotMeshClient = await this.getClient().getHotMeshClient(workflowTopic, options.namespace);
return await meshflow_1.MeshFlow.Search.configureSearchIndex(hotMeshClient, searchOptions ?? this.search);
}
/**
* Lists all search indexes in the operational data layer when the
* targeted search backend is configured/enabled.
* @returns {Promise<string[]>}
* @example
* // list all search indexes
* const indexes = await meshData.listSearchIndexes();
*
* // returns ['greeting', 'user', 'order', 'product']
*/
async listSearchIndexes() {
const hotMeshClient = await this.getHotMesh();
return await meshflow_1.MeshFlow.Search.listSearchIndexes(hotMeshClient);
}
/**
* shut down MeshData (typically on sigint or sigterm)
*/
static async shutdown() {
await meshflow_1.MeshFlow.shutdown();
}
}
exports.MeshData = MeshData;
_a = MeshData;
/**
* Provides a set of static extensions that can be invoked by
* your linked workflow functions during their execution.
* @example
*
* function greet (email: string, user: { first: string}) {
* //persist the user's email and newsletter preferences
* const search = await MeshData.workflow.search();
* await search.set('email', email, 'newsletter', 'yes');
*
* //hook a function to send a newsletter
* await MeshData.workflow.hook({
* entity: 'user.newsletter',
* args: [email]
* });
*
* return `Hello, ${user.first}. Your email is [${email}].`;
* }
*/
MeshData.workflow = {
sleep: meshflow_1.MeshFlow.workflow.sleepFor,
sleepFor: meshflow_1.MeshFlow.workflow.sleepFor,
signal: meshflow_1.MeshFlow.workflow.signal,
hook: meshflow_1.MeshFlow.workflow.hook,
waitForSignal: meshflow_1.MeshFlow.workflow.waitFor,
waitFor: meshflow_1.MeshFlow.workflow.waitFor,
getHotMesh: meshflow_1.MeshFlow.workflow.getHotMesh,
random: meshflow_1.MeshFlow.workflow.random,
search: meshflow_1.MeshFlow.workflow.search,
getContext: meshflow_1.MeshFlow.workflow.getContext,
/**
* Interrupts a job by its entity and id.
*/
interrupt: async (entity, id, options = {}) => {
const jobId = MeshData.mintGuid(entity, id);
await meshflow_1.MeshFlow.workflow.interrupt(jobId, options);
},
/**
* Starts a new, subordinated workflow/job execution. NOTE: The child workflow's
* lifecycle is bound to the parent workflow, and it will be terminated/scrubbed
* when the parent workflow is terminated/scrubbed.
*
* @template T The expected return type of the target function.
*/
execChild: async (options = {}) => {
const pluckOptions = {
...options,
args: [...options.args, { $type: 'exec' }],
};
return meshflow_1.MeshFlow.workflow.execChild(pluckOptions);
},
/**
* Starts a new, subordinated workflow/job execution. NOTE: The child workflow's
* lifecycle is bound to the parent workflow, and it will be terminated/scrubbed
* when the parent workflow is terminated/scrubbed.
*
* @template T The