@godspeedsystems/core/dist/core/functionLoader.js

> 4th Generation Declarative Microservice Framework

/*
* You are allowed to study this software for learning and local * development purposes only. Any other use without explicit permission by Mindgrep, is prohibited.
* © 2022 Mindgrep Technologies Pvt Ltd
*/ "use strict";
Object.defineProperty(exports, "__esModule", {
    value: true
});
function _export(target, all) {
    for(var name in all)Object.defineProperty(target, name, {
        enumerable: true,
        get: all[name]
    });
}
_export(exports, {
    createGSFunction: function() {
        return createGSFunction;
    },
    default: function() {
        return loadFunctions;
    }
});
const _interfaces = require("./interfaces");
const _utils = require("./utils");
const _yamlLoader = /*#__PURE__*/ _interop_require_default(require("./yamlLoader"));
const _codeLoader = /*#__PURE__*/ _interop_require_default(require("./codeLoader"));
const _logger = require("../logger");
const _path = /*#__PURE__*/ _interop_require_default(require("path"));
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
    try {
        var info = gen[key](arg);
        var value = info.value;
    } catch (error) {
        reject(error);
        return;
    }
    if (info.done) {
        resolve(value);
    } else {
        Promise.resolve(value).then(_next, _throw);
    }
}
function _async_to_generator(fn) {
    return function() {
        var self = this, args = arguments;
        return new Promise(function(resolve, reject) {
            var gen = fn.apply(self, args);
            function _next(value) {
                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
            }
            function _throw(err) {
                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
            }
            _next(undefined);
        });
    };
}
function _define_property(obj, key, value) {
    if (key in obj) {
        Object.defineProperty(obj, key, {
            value: value,
            enumerable: true,
            configurable: true,
            writable: true
        });
    } else {
        obj[key] = value;
    }
    return obj;
}
function _interop_require_default(obj) {
    return obj && obj.__esModule ? obj : {
        default: obj
    };
}
function _object_spread(target) {
    for(var i = 1; i < arguments.length; i++){
        var source = arguments[i] != null ? arguments[i] : {};
        var ownKeys = Object.keys(source);
        if (typeof Object.getOwnPropertySymbols === "function") {
            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
            }));
        }
        ownKeys.forEach(function(key) {
            _define_property(target, key, source[key]);
        });
    }
    return target;
}
/*
    Two reasons to keep this module level variable
    1. To check dangling else or elif tasks, set lastIfFn to the ifFn when encountered, 
    and later check when loading elif and else tasks, whether `lastIfFn` is set or not.
    2. An com.gs.GSIFFunction has optional else_fn which is either com.gs.elif or com.gs.else function.  
*/ let lastIfFn;
function createGSFunction(json, workflows, nativeFunctions, onError, location) {
    var _taskJson_fn, _taskJson_on_error;
    const childLogger = _logger.logger.child(location);
    // initializeChildLogger(location);
    // logger.setBindings(null);
    if (Array.isArray(json)) {
        json = {
            tasks: json,
            fn: 'com.gs.sequential',
            workflow_name: json === null || json === void 0 ? void 0 : json.workflow_name
        };
    } else if (!json.fn) {
        json.fn = 'com.gs.sequential';
    }
    childLogger.debug('Starting to parse and load GSFunction id: %s name: %s', json.id, json.workflow_name);
    //First lets handle core framework control flow functions
    //If this workflow is none of that then we will handle that after this switch block.
    let tasks;
    switch(json.fn){
        case 'com.gs.sequential':
            tasks = json.tasks.map((taskJson)=>{
                taskJson.workflow_name = json.workflow_name;
                const taskLocation = {
                    parent: location,
                    workflow_name: json.workflow_name,
                    task_id: taskJson.id
                };
                return createGSFunction(taskJson, workflows, nativeFunctions, onError, taskLocation);
            });
            tasks = tasks.filter(Boolean);
            return new _interfaces.GSSeriesFunction(json, workflows, nativeFunctions, undefined, tasks, false, undefined, location);
        // case 'com.gs.dynamic_fn':
        //     tasks =
        //         (json as WorkflowJSON)
        //             .tasks
        //             .map((taskJson: TaskJSON) => {
        //                 taskJson.workflow_name = json.workflow_name;
        //                 return createGSFunction(taskJson, workflows, nativeFunctions, onError);
        //             });
        //     tasks = tasks.filter(Boolean);
        //     return new GSDynamicFunction(json, workflows, nativeFunctions, undefined, tasks, false);
        case 'com.gs.parallel':
            tasks = json.tasks.map((taskJson)=>{
                taskJson.workflow_name = json.workflow_name;
                const taskLocation = {
                    parent: location,
                    workflow_name: json.workflow_name,
                    task_id: taskJson.id
                };
                return createGSFunction(taskJson, workflows, nativeFunctions, onError, taskLocation);
            });
            tasks = tasks.filter(Boolean); //filter out falsy values from tasks
            json.isParallel = true;
            return new _interfaces.GSParallelFunction(json, workflows, nativeFunctions, undefined, tasks, false, undefined, location);
        case 'com.gs.switch':
            {
                const switchWorkflowJSON = json;
                let args = [
                    switchWorkflowJSON.value
                ];
                let cases = {};
                for(let c in switchWorkflowJSON.cases){
                    switchWorkflowJSON.cases[c].workflow_name = switchWorkflowJSON.workflow_name;
                    //@ts-ignore
                    const taskLocation = {
                        parent: location,
                        case_workflow_name: switchWorkflowJSON.workflow_name,
                        case: c,
                        case_task_id: switchWorkflowJSON.cases[c].id
                    };
                    cases[c] = createGSFunction(switchWorkflowJSON.cases[c], workflows, nativeFunctions, onError, taskLocation);
                }
                if (switchWorkflowJSON.defaults) {
                    switchWorkflowJSON.defaults.workflow_name = switchWorkflowJSON.workflow_name;
                    //@ts-ignore
                    const taskLocation = {
                        parent: location,
                        case_default_task_id: switchWorkflowJSON.defaults.id
                    };
                    cases.default = createGSFunction(switchWorkflowJSON.defaults, workflows, nativeFunctions, onError, taskLocation);
                }
                args.push(cases);
                // logger.debug('loading switch workflow cases %o', switchWorkflowJSON.cases);
                return new _interfaces.GSSwitchFunction(json, workflows, nativeFunctions, undefined, args, false, location);
            }
        case 'com.gs.if':
            {
                const ifWorkflowJSON = json;
                let args = [
                    ifWorkflowJSON.condition
                ];
                tasks = ifWorkflowJSON.tasks.map((taskJson)=>{
                    taskJson.workflow_name = ifWorkflowJSON.workflow_name;
                    const taskLocation = {
                        parent: location,
                        if_task_workflow_name: taskJson.workflow_name,
                        if_task_task_id: taskJson.id
                    };
                    return createGSFunction(taskJson, workflows, nativeFunctions, onError, taskLocation);
                });
                tasks = tasks.filter(Boolean);
                /* 
                Create a series function which will be called 
                by the GSIFFunction if condition evaluates to true.
                Omit the `condition` from if task and make a series 
                function with its child tasks
             */ const tasksJSON = _object_spread({}, ifWorkflowJSON);
                if ('condition' in tasksJSON) {
                    delete tasksJSON.condition;
                }
                const tasksGSSeriesFunction = new _interfaces.GSSeriesFunction(tasksJSON, workflows, nativeFunctions, undefined, tasks, false, undefined, location);
                args.push(tasksGSSeriesFunction);
                const ifFunction = new _interfaces.GSIFFunction(json, workflows, nativeFunctions, undefined, args, false, location);
                // update the lastIfFn state to check later for dangling elif or else.
                lastIfFn = ifFunction;
                return ifFunction;
            }
        case 'com.gs.elif':
            {
                const elifWorkflowJSON = json;
                let args = [
                    elifWorkflowJSON.condition
                ];
                tasks = elifWorkflowJSON.tasks.map((taskJson)=>{
                    taskJson.workflow_name = elifWorkflowJSON.workflow_name;
                    const taskLocation = {
                        parent: location,
                        workflow_name: taskJson.workflow_name,
                        task_id: taskJson.id
                    };
                    return createGSFunction(taskJson, workflows, nativeFunctions, onError, taskLocation);
                });
                tasks = tasks.filter(Boolean);
                /* 
                Create a series function which will be called 
                by the GSIFFunction if condition evaluates to true.
                Omit the `condition` from if task and make a series 
                function with its child tasks
             */ const tasksJSON = _object_spread({}, elifWorkflowJSON);
                if ('condition' in tasksJSON) {
                    delete tasksJSON.condition;
                }
                let tasksGSSeriesFunction = new _interfaces.GSSeriesFunction(tasksJSON, workflows, nativeFunctions, undefined, tasks, false, undefined, location);
                args.push(tasksGSSeriesFunction);
                let elifFunction = new _interfaces.GSIFFunction(json, workflows, nativeFunctions, undefined, args, false, location);
                if (!lastIfFn) {
                    childLogger.error(`If is missing before elif ${json.id}.`);
                    process.exit(1);
                } else {
                    lastIfFn.else_fn = elifFunction;
                }
                lastIfFn = elifFunction;
                return null;
            }
        case 'com.gs.else':
            {
                tasks = json.tasks.map((taskJson)=>{
                    taskJson.workflow_name = json.workflow_name;
                    const taskLocation = {
                        parent: location,
                        else_task_workflow_name: taskJson.workflow_name,
                        else_task_task_id: taskJson.id
                    };
                    return createGSFunction(taskJson, workflows, nativeFunctions, onError, taskLocation);
                });
                tasks = tasks.filter(Boolean);
                let elseFunction = new _interfaces.GSSeriesFunction(json, workflows, nativeFunctions, undefined, tasks, false, undefined, location);
                if (!lastIfFn) {
                    childLogger.error(`If task is missing before else task ${json.id}.`);
                    process.exit(1);
                } else {
                    lastIfFn.else_fn = elseFunction;
                }
                // Reset the state to initial state, to handle next if/else/elseif flow.
                lastIfFn = null;
                return null;
            }
        case 'com.gs.each_parallel':
            {
                var _json_on_error;
                let args = [
                    json.value
                ];
                let tasks = json.tasks.map((taskJson)=>{
                    taskJson.workflow_name = json.workflow_name;
                    taskJson.isEachParallel = true;
                    const taskLocation = {
                        parent: location,
                        each_parallel_workflow_name: taskJson.workflow_name,
                        each_parallel_task_id: taskJson.id
                    };
                    return createGSFunction(taskJson, workflows, nativeFunctions, onError, taskLocation);
                });
                tasks = tasks.filter(Boolean);
                //Get the tasks to do in every loop
                let loopTasks = new _interfaces.GSSeriesFunction(json, workflows, nativeFunctions, undefined, tasks, false, undefined, location);
                args.push(loopTasks);
                if (json === null || json === void 0 ? void 0 : (_json_on_error = json.on_error) === null || _json_on_error === void 0 ? void 0 : _json_on_error.tasks) {
                    json.on_error.tasks.workflow_name = json.workflow_name;
                    //@ts-ignore
                    const taskLocation = {
                        parent: location,
                        workflow_name: json.workflow_name,
                        task_id: json.on_error.tasks.id,
                        section: 'on_error'
                    };
                    json.on_error.tasks = createGSFunction(json.on_error.tasks, workflows, nativeFunctions, null, taskLocation);
                }
                childLogger.debug('loading each parallel workflow %o', json.tasks);
                return new _interfaces.GSEachParallelFunction(json, workflows, nativeFunctions, undefined, args, false, location);
            }
        case 'com.gs.each_sequential':
            {
                var _json_on_error1;
                let args = [
                    json.value
                ];
                let tasks = json.tasks.map((taskJson)=>{
                    taskJson.workflow_name = json.workflow_name;
                    const taskLocation = {
                        parent: location,
                        each_sequential_workflow_name: taskJson.workflow_name,
                        each_sequential_task_id: taskJson.id
                    };
                    return createGSFunction(taskJson, workflows, nativeFunctions, onError, taskLocation);
                });
                tasks = tasks.filter(Boolean);
                let task = new _interfaces.GSSeriesFunction(json, workflows, nativeFunctions, undefined, tasks, false, undefined, location);
                args.push(task);
                if (json === null || json === void 0 ? void 0 : (_json_on_error1 = json.on_error) === null || _json_on_error1 === void 0 ? void 0 : _json_on_error1.tasks) {
                    json.on_error.tasks.workflow_name = json.workflow_name;
                    //@ts-ignore
                    const taskLocation = {
                        parent: location,
                        on_error_workflow_name: json.on_error.tasks.workflow_name,
                        on_error_task_id: json.on_error.tasks.id,
                        section: 'on_error'
                    };
                    json.on_error.tasks = createGSFunction(json.on_error.tasks, workflows, nativeFunctions, null, taskLocation);
                }
                childLogger.debug('loading each sequential workflow %o', json.tasks);
                return new _interfaces.GSEachSeriesFunction(json, workflows, nativeFunctions, undefined, args, false, location);
            }
    }
    /*  
        This was not any of the core framework control functions.
        This must be a `TaskJSON` which is either 
        1. A developer written function (native JS/TS or yaml) or 
        2. A datasource function (also native JS/TS function)
    */ let subwf = false;
    let fn;
    let fnScript;
    const taskJson = json;
    if (((_taskJson_fn = taskJson.fn) === null || _taskJson_fn === void 0 ? void 0 : _taskJson_fn.match(/<(.*?)%/)) && taskJson.fn.includes('%>')) {
        //@ts-ignore
        const taskLocation = _object_spread({}, location, {
            workflow_name: taskJson.fn,
            task_id: taskJson.fn.id
        });
        fnScript = (0, _utils.compileScript)(taskJson.fn, taskLocation);
    } else {
        // Load the fn for this GSFunction
        childLogger.debug('Loading fn %s, which is either the datasource or a JS/TS/YAML workflow', taskJson.fn);
        /*      
            First check if it's a native function (developer written or datasource)
            but, special handling for datasource function, because
            while using datasource fn, starts with datasource.{datasourceName} followed by . 
            followed by the function or nested functions to be invoked
            For ex. in com.gs.prisma tasks, it is in this format, `datasource.{datasourceName}.{entityName}.{method}`
            where as, datasource clients are registered as `datasource.{datasourceName}`
            So we want to extract the datasource.{datasourceName} part
        */ let fnName = String(taskJson.fn).startsWith('datasource.') ? String(taskJson.fn).split('.').splice(0, 2).join('.') : taskJson.fn;
        fn = nativeFunctions[fnName]; //Either a datasource or dev written JS/TS function 
        if (!fn) {
            // If not a native function, it should be a developer written Workflow as function
            const existingWorkflowData = workflows[fnName];
            if (!existingWorkflowData) {
                childLogger.fatal(`Function specified by name ${fnName} not found in src/functions. Please ensure a function by this path exists.`);
                process.exit(1);
            }
            subwf = true;
            if (!(existingWorkflowData instanceof _interfaces.GSFunction)) {
                existingWorkflowData.workflow_name = fnName;
                const taskLocation = _object_spread({}, location, {
                    workflow_name: existingWorkflowData.workflow_name,
                    task_id: existingWorkflowData.id
                });
                fn = workflows[fnName] = createGSFunction(existingWorkflowData, workflows, nativeFunctions, onError, taskLocation);
            } else {
                fn = existingWorkflowData;
            }
        }
    }
    if (taskJson === null || taskJson === void 0 ? void 0 : (_taskJson_on_error = taskJson.on_error) === null || _taskJson_on_error === void 0 ? void 0 : _taskJson_on_error.tasks) {
        taskJson.on_error.tasks.workflow_name = taskJson.workflow_name;
        //@ts-ignore
        const taskLocation = _object_spread({}, location, {
            on_error_task_id: taskJson.on_error.tasks.id
        });
        taskJson.on_error.tasks = createGSFunction(taskJson.on_error.tasks, workflows, nativeFunctions, null, taskLocation);
    } else if (taskJson === null || taskJson === void 0 ? void 0 : taskJson.on_error) {
    // do nothing
    } else if (onError) {
        taskJson.on_error = onError;
    }
    if (taskJson.authz) {
        taskJson.authz.workflow_name = json.workflow_name;
        //@ts-ignore
        const taskLocation = _object_spread({}, location, {
            workflow_name: taskJson.authz.workflow_name,
            task_id: taskJson.authz.id
        });
        taskJson.authz = createGSFunction(taskJson.authz, workflows, nativeFunctions, onError, taskLocation);
    }
    return new _interfaces.GSFunction(taskJson, workflows, nativeFunctions, fn, taskJson.args, subwf, fnScript, location);
}
function loadFunctions(datasources, pathString) {
    return _loadFunctions.apply(this, arguments);
}
function _loadFunctions() {
    _loadFunctions = _async_to_generator(function*(datasources, pathString) {
        // framework defined js/ts functions
        let frameworkFunctions = yield (0, _codeLoader.default)(_path.default.resolve(__dirname, '../functions'));
        // project defined yaml worlflows
        let yamlWorkflows = yield (0, _yamlLoader.default)(pathString);
        // project defined js/ts functions
        let nativeMicroserviceFunctions = yield (0, _codeLoader.default)(pathString);
        let loadFnStatus;
        const childLogger = _logger.logger.child({
            section: 'loading_functions'
        });
        childLogger.debug('JS/TS functions found in src/functions %s', Object.keys(nativeMicroserviceFunctions));
        childLogger.debug('Yaml Workflows found in src/functions %s', Object.keys(yamlWorkflows));
        // logger.debug('Framework defined  functions %s', Object.keys(frameworkFunctions));
        childLogger.debug('Datasources found in src/datasources %o', Object.keys(datasources));
        let _datasourceFunctions = Object.keys(datasources).reduce((acc, dsName)=>{
            // dsName, eg., httpbin, mongo, prostgres, salesforce
            acc[`datasource.${dsName}`] = /*#__PURE__*/ function() {
                var _ref = _async_to_generator(function*(ctx, args) {
                    return datasources[dsName].execute(ctx, args);
                });
                return function(ctx, args) {
                    return _ref.apply(this, arguments);
                };
            }();
            return acc;
        }, {});
        const nativeFunctions = _object_spread({}, frameworkFunctions, _datasourceFunctions, nativeMicroserviceFunctions);
        childLogger.debug('Creating workflows: %s', Object.keys(yamlWorkflows));
        for(let f in yamlWorkflows){
            var _yamlWorkflows_f_yaml;
            if (!yamlWorkflows[f]) {
                childLogger.fatal({
                    fn: f
                }, `Found empty yaml workflow ${f}. Exiting.`);
                process.exit(1);
            }
            //Since a yaml workflow loading recursively loads called workflows by calling createGSFunction on them,
            //which in turn sets yaml key in them, when a workflow's turn in this for loop comes,
            //it may already have been loaded before. 
            //When a workflow is loaded, it has yaml key set in it
            if (!((_yamlWorkflows_f_yaml = yamlWorkflows[f].yaml) === null || _yamlWorkflows_f_yaml === void 0 ? void 0 : _yamlWorkflows_f_yaml.tasks) && !yamlWorkflows[f].tasks) {
                childLogger.fatal({
                    fn: f
                }, `Did not find tasks in yaml workflow ${f}. Exiting.`);
                process.exit(1);
            }
            if (!(yamlWorkflows[f] instanceof _interfaces.GSFunction)) {
                var _yamlWorkflows_f_on_error;
                yamlWorkflows[f].workflow_name = f;
                if ((_yamlWorkflows_f_on_error = yamlWorkflows[f].on_error) === null || _yamlWorkflows_f_on_error === void 0 ? void 0 : _yamlWorkflows_f_on_error.tasks) {
                    yamlWorkflows[f].on_error.tasks.workflow_name = f;
                    childLogger.debug({
                        fn: f
                    }, "Start to load on error tasks for YAML workflow %s", f);
                    try {
                        const taskLocation = {
                            workflow_name: f,
                            task_id: yamlWorkflows[f].on_error.tasks.id,
                            section: 'on_error.tasks'
                        };
                        yamlWorkflows[f].on_error.tasks = createGSFunction(yamlWorkflows[f].on_error.tasks, yamlWorkflows, nativeFunctions, null, taskLocation);
                    } catch (err) {
                        _logger.logger.fatal("Error in loading on error tasks for YAML workflow %s %o", f, err);
                        process.exit(1);
                    }
                    _logger.logger.debug("Loaded on error tasks for YAML workflow %s", f);
                }
                _logger.logger.debug("Starting to load YAML workflow %s", f);
                try {
                    const taskLocation = {
                        workflow_name: f,
                        task_id: yamlWorkflows[f].id
                    };
                    yamlWorkflows[f] = createGSFunction(yamlWorkflows[f], yamlWorkflows, nativeFunctions, yamlWorkflows[f].on_error, taskLocation);
                } catch (err) {
                    _logger.logger.fatal("Error in loading YAML workflow %s %s %o", f, err.message, err);
                    process.exit(1);
                }
                _logger.logger.debug("Loaded YAML workflow %s", f);
            }
        // Need discussion whether to evaluate the fn script on load time and skipping evaluation on runtime in some cases we may evaluate the script on runtime
        // const workflowYaml = yamlWorkflows[f].yaml ?? yamlWorkflows[f]
        // const checkDS = checkDatasource(workflowYaml, datasources,{ workflow_name: f, task_id: yamlWorkflows[f].id});
        // if (!checkDS.success) {
        //     childLogger.fatal({fn: f}, `Error in loading datasource for function ${f} . Error message: ${checkDS.message}. Exiting.`);
        //     process.exit(1);
        // }
        }
        loadFnStatus = {
            success: true,
            nativeFunctions,
            functions: _object_spread({}, yamlWorkflows, nativeMicroserviceFunctions)
        };
        _logger.logger.debug('Loaded YAML workflows: %o', Object.keys(yamlWorkflows));
        _logger.logger.info('Loaded all YAML workflows');
        _logger.logger.info('Loaded JS workflows %o', Object.keys(nativeMicroserviceFunctions));
        _logger.logger.info('Loaded JS/TS workflows');
        return loadFnStatus;
    });
    return _loadFunctions.apply(this, arguments);
}