shelloid
Version:
Open source IoT-ready real-time big data web application platform that integrates Node.js and Clojure.
987 lines (937 loc) • 24.4 kB
JavaScript
/*
Copyright (c) Shelloid Systems LLP. All rights reserved.
The use and distribution terms for this software are covered by the
GNU Lesser General Public License 3.0 (https://www.gnu.org/licenses/lgpl.html)
which can be found in the file LICENSE at the root of this distribution.
By using this software in any fashion, you are agreeing to be bound by
the terms of this license.
You must not remove this notice, or any other, from this software.
*/
var utils = lib_require("utils");
var assert = require("assert");
var md5 = require('MD5');
var util = require("util");
var jsep = require("jsep");
var routines = require("./routines.js");
module.exports = function(){
if(!sh.appCtx.stream){
sh.appCtx.stream = {defs: {}, types: {}, transports:{}};
}
for(var k in routines){
extInfo.services.stream.routines[k] = routines[k];
}
return extInfo;
}
var extInfo =
{
annotations:[
{name: "stream", process: streamAnnotation, raw: true}],
hooks:[
{type: "load", name: "streams", handler: loadStreamDef},
{type: "init", name: "streams", handler: initStreams,
priority: sh.hookPriorities.late},
],
services: {
stream: {
addTransport: addTypeScopedTransport,
routines:{}
}
},
stream: {routines:{}}
};
function initStreams(done){
if(!sh.appCtx.stream || !sh.appCtx.stream.defs){
done();
return;
}
var streamDefs = sh.appCtx.stream.defs;
/*
var db;
if(sh.appCtx.config.stream.db){
db = sh.db(sh.appCtx.config.stream.db);
db.collection("streams");
}
*/
var urls = Object.keys(streamDefs);
var barrier = utils.countingBarrier(urls.length, done);
var streamInit = function(streamDef, err, transport){
streamDef.transport = transport;
if(streamDef.init){
streamDef.init.fn(streamDef,
barrier.countDown.bind(null, barrier));
}else{
barrier.countDown();
}
}
for(var i=0;i<urls.length;i++){
var streamDef = streamDefs[urls[i]];
sh.info("Processing stream def: " + streamDef.url + ". SDL: " + streamDef.sdl);
loadTransport(streamDef.type,
streamInit.bind(null, streamDef));
}
}
function loadTransport(type, callback){
var transport = sh.appCtx.stream.transports[type];
if(transport){
callback(null, transport);
return;
}
var transportModPath = sh.services.stream.transportMods[type];
if(!transportModPath){
sh.error("Stream transport for: " + url +
"(type: " + type + ") not supported");
sh.hasErrors = true;
callback("not supported", null);
}else{
var transport = require(transportModPath);
transport.info.type = type;
transport.init(function(){
sh.appCtx.stream.transports[type] = transport; callback(null, transport);
});
}
}
function addTypeScopedTransport(type, srcId, transportDesc, cb){
var typeInfo = sh.appCtx.stream.types[type];
if(!typeInfo){
sh.warn("Adding transport to empty type. Creating: " + type);
typeInfo = {defs:[]};
sh.appCtx.stream.types[type] = typeInfo;
}
loadTransport(type, function(err, transport){
if(!transport){
sh.error("Couldn't load transport for: " + type);
cb ? cb("load error", null) : null;
return;
}
var scopes = transport.info && transport.info.scopes;
if(scopes && !scopes.contains("type")){
sh.error("The transport for " + type + " does not support type scope");
cb ? cb("scope not supported", null) : null;
}else{
createTransportInstance(transport, typeInfo, srcId, transportDesc, cb);
}
});
}
function addDefScopedTransport(srcId, transportDesc, cb){
var streamDef = this;
var transport = this.transport;
var scopes = transport.info && transport.info.scopes;
if(!srcId){
srdId = "";
}
if(scopes && !scopes.contains("def")){
sh.error("The transport for " + type + " does not support def scope");
cb ? cb("scope not supported", null) : null;
}else{
createTransportInstance(transport, streamDef, srcId, transportDesc, cb);
}
}
function addInstanceScopedTransport(srcId, transportDesc, cb){
var streamInstance = this;
var transport = this.def.transport;
var scopes = transport.info && transport.info.scopes;
if(!srcId){
srdId = "";
}
if(scopes && !scopes.contains("instance")){
sh.error("The transport for " + type + " does not support def scope");
cb ? cb("scope not supported", null): null;
}else{
createTransportInstance(transport, streamInstance, srcId, transportDesc, cb);
}
}
function createTransportInstance(transport, scopeObj, srcId, transportDesc, cb){
transport.create(scopeObj, srcId, transportDesc,
function(err, transportInstance){
if(err){
sh.error("Couldn't create transport instance for " + transport.info.type + " src id: " + srcId);
cb ? cb(err, null) : null;
return;
}
transportInstance.scopeObj = scopeObj;
sh.info("Transport instance created for: " + transport.info.type + " src-id: " + srcId);
transportInstance.onData(function(err, data, srcId){
if(err){
sh.error("Stream data error: " + err +
" for stream type: " + transport.info.type);
return;
}
processStreamData(scopeObj, data, srcId);
});
cb ? cb(null, transportInstance): null;
}
);
}
function parseSdl(sdl, src){
sdl = sdl.trim();
var select = "select:";
var from = "from:";
var where = "where:";
var sample = "sample:";
var let_ = "let:";
var res = {};
var lines = sdl.split(";");//sdl.match(/[^\r\n]+/g);
if(lines.length <= 2){
return null;
}
for(var i=0;i<lines.length;i++){
lines[i] = lines[i].trim();
}
var letFields = [];
if(lines[0].startsWith(let_)){
var s = lines[0].substr(let_.length).trim();
letFields = parseSelectClause(s, "let clause in " + src);
for(var i=0;i<letFields.length;i++){
if(!letFields[i].routineName ||
!letFields[i].aliasName){
sh.error("Invalid name in let clause entry: " + letFields[i].name + " at " + src);
sh.hasErrors = true;
}else{
letFields[i].qName = letFields[i].name;
letFields[i].name = letFields[i].aliasName;
}
}
lines.shift();
}
res.let = letFields;
var selectFields = [];
if(lines[0].startsWith(select)){
var s = lines[0].substr(select.length).trim();
selectFields = parseSelectClause(s, "select clause in " + src);
}
selectFields = selectFields.sort(function(a,b){
return b.name.length - a.name.length;
});
res.select = selectFields;
if(lines[1].startsWith(from)){
res.from = lines[1].substr(from.length).trim();
var v = res.from;
var fields = v.split(".");
res.type = fields[0];
if(fields.length > 1){
res.sensor = fields[1];
}
}
if(lines.length >= 3 && lines[2].startsWith(where)){
var s = lines[2].substr(where.length).trim();
res.where = parseWhereExpr(s, src);
}
var extraStart = 3;
if(lines[3] && lines[3].startsWith(sample)){
extraStart = 4;
try{
var s = lines[3].substr(sample.length).trim();
res.sample = parseInt(s);
}catch(err){
sh.error("Sample field is not a number in: " + sdl + " defaulting to 3000");
}
}
if(!res.sample){
res.sample = 3000;
}
res.extras ={};
for(var j=extraStart;j<lines.length;j++){
var k = lines[j].indexOf(":");
if(k > 0){
var key = lines[j].substring(0,k).trim();
var value = lines[j].substring(k+1).trim();
res.extras[key] = value;
}
}
res.routines = {};
res.aliases = {};
res.routineNames = [];
//let routine names must come first and in sequence.
for(var i=0;i<res.let.length;i++){
var e = res.let[i];
if(e.routineName){
res.routines[e.name] = e;
res.routineNames.push(e.name);
}
}
for(var i=0;i<res.where.names.length;i++){
var e = res.where.names[i];
if(e.routineName){
res.routines[e.name] = e;
if(!res.routineNames.contains(e.name)){
res.routineNames.push(e.name);
}
}
}
for(var i=0;i<res.select.length;i++){
var e = res.select[i];
if(e.routineName){
res.routines[e.name] = e;
if(!res.routineNames.contains(e.name)){
res.routineNames.push(e.name);
}
if(e.aliasName){
res.aliases[e.aliasName] = e.name;
}
}
}
//console.log("Parsed SDL for: " + src + " : " +
// JSON.stringify(res, null, " "));
return res;
}
function processStreamData(scopeObj, data, srcId){
buildIndex(data);
if(scopeObj.defs){//type scoped
var defs = scopeObj.defs;
for(var i=0;i<defs.length;i++){
var def = defs[i];
for(var id in def.instances){
execQuery(def.instances[id], data, srcId);
}
}
}else
if(scopeObj.instances){//def scoped
var def = scopeObj;
for(var id in def.instances){
execQuery(def.instances[id], data, srcId);
}
}else//instance scoped
{
execQuery(scopeObj, data, srcId);
}
}
function buildIndex(data){
data.$index = {};
for(var k in data.$data){
var rows = data.$data[k];
var meta = data.$meta && data.$meta[k];
if(util.isArray(rows) && meta && meta.unique){
var key = meta.unique;
var index = data.$index[k] = {key: key, map:{}};
for(var i=0;i<rows.length;i++){
var keyField = rows[i][key];
index.map[keyField] = i;
}
}
}
}
function execQuery(streamInstance, data, srcId){
var res = {};
var psdl = streamInstance.psdl;
var sensorData = data.$data[psdl.sensor];
if(!sensorData){
sh.error("Couldn't find sensor input : " +
psdl.sensor + " in the stream " +
streamInstance.def.url);
return;
}
srcId = srcId || streamInstance.def.constants.noSrcId;
var srcCache = getSrcCache(streamInstance, srcId);
evalRoutines(streamInstance, psdl.routineNames,
sensorData, srcCache,
function(r0){
srcCache.preData = r0;
if(data.$index){
srcCache.preIndex = data.$index[psdl.sensor];
}
execQuery0(streamInstance, r0, srcId);
}
);
}
function execQuery0(streamInstance, sensorData, srcId){
var res;
if(util.isArray(sensorData)){
var r = [];
for(var i=0;i<sensorData.length;i++){
var r0 = evalQuery(streamInstance, sensorData[i]);
if(r0){
r.push(r0);
}else if(r0 === false){//error
sh.error("Stream query evaluation has errors. Aborting further processing.");
return;
}
}
if(r.length > 0){
res = r;
}
}else{
res = evalQuery(streamInstance, sensorData);
}
processQueryResult(streamInstance, res, srcId);
}
function evalRoutines(streamInstance, names, data, srcCache, cb){
var i = 0;
var streamRoutines = sh.services.stream.routines;
var ccsRoutines = sh.services.ccs.routines;
var next = function(err, res){
if(i < names.length){
var r = streamInstance.psdl.routines[names[i++]];
var method = streamRoutines[r.routineName];
var isLocal = method ? true : false;
if(!method && sh.appCtx.config.ccs.enable){
method = ccsRoutines[r.routineName];
}
if(!method){
sh.error("Routine: " + r.routineName +
" not found. Stream: " + streamInstance.def.url);
}else{
var params = r.paramValues.slice();
params.push(r.name);
params.push(next);
if(isLocal){
params.push(srcCache);
}
params.unshift(res);
sh.info("Invoking routine: " + r.name + " for stream: " + streamInstance.def.url);
method.apply(null, params);
}
}
else{
cb(res);
}
}
next(null, data);
}
function getSrcCache(streamInstance, srcId){
var srcCache = streamInstance.cache[srcId];
if(!srcCache){
srcCache = {
data: {},
dataJson: JSON.stringify({}),
dataTag : (new Date()).getTime()
};
streamInstance.cache[srcId] = srcCache;
}
return srcCache;
}
function processQueryResult(streamInstance, res, srcId){
var edgeTriggered = streamInstance.def.
mod.annotations.stream.edgeTriggered;
var srcCache = getSrcCache(streamInstance, srcId);
srcCache.lastProcessedTs = new Date();
if(!res && !edgeTriggered){
return;
}
res = res || {};
var dataJson = JSON.stringify(res);
if(srcCache.dataJson === dataJson){
if(edgeTriggered){
return;
}
}else{
srcCache.dataJson = dataJson;
srcCache.data = res;
srcCache.dataTs = srcCache.lastProcessedTs;
srcCache.dataTag++;
}
streamInstance.def.mod.fn(res, streamInstance, srcId);
for(var k in streamInstance.pendingRequests){
var pendingReq = streamInstance.pendingRequests[k];
var done = function(pending, status){
if(status === true){
pending.send(res);
}else{
pending.abort(status);
}
}
if(pendingReq.src === srcId){
if(streamInstance.def.auth){
streamInstance.def.auth.fn(
pendingReq.req, streamInstance, srcId,
done.bind(null, pendingReq));
}else{
done(pendingReq, true);
}
}
}
}
function evalQuery(streamInstance, dataRec, callback){
var psdl = streamInstance.psdl;
var where = psdl.where.text;
for(var i=0;i<psdl.where.names.length;i++){
var n = psdl.where.names[i].name;
var v = dataRec[n];
if(v === undefined){
var orig = psdl.aliases[n];
if(orig){
v = dataRec[orig];
}
}
if(v === undefined){
sh.error("Undefined name: " + n + " in the stream query where clause: " + streamInstance.def.url);
return false;
}
where = where.replace(n,v);
}
try
{
eval("var t = " + where);
if(t){
var res = {};
for(var i=0;i<psdl.select.length;i++){
var t = psdl.select[i].aliasName;
var n = psdl.select[i].name;
t = t || n;
var v = dataRec[n];
if(v === undefined){
sh.error("Undefined name: " + n + " in the stream query select clause: "
+ streamInstance.def.url);
return false;
}
res[t] = v;
}
return res;
}
}catch(e){
sh.error("Could not evaluate where clause " + e.stack);
return false;
}
return null;
}
var streamConstants = {noSrcId: "$nosrc"};
function loadStreamDef(loader, mod){
var appCtx = sh.appCtx;
var sdl = mod.annotations.stream && mod.annotations.stream.sdl;
var init = mod.annotations.stream && mod.annotations.stream.init;
var auth = mod.annotations.stream && mod.annotations.stream.auth;
if(sdl){
var streamDef = {
url: mod.url,
mod: mod,
sdl : sdl,
instances: {},
newInstance: newStreamInstance,
addTransport: addDefScopedTransport,
constants: streamConstants
};
var psdl = parseSdl(sdl, streamDef.url);
streamDef.type = psdl.type;
streamDef.psdl = psdl;
var typeInfo = appCtx.stream.types[psdl.type];
if(!typeInfo){
typeInfo = {defs:[], transports:[]};
appCtx.stream.types[psdl.type] = typeInfo;
}
typeInfo.defs.push(streamDef);
if(init){
if(utils.isFunction(init)){
init = {fn: init};//create a trivial module obj.
}else{
init = mod.coll[init];
}
streamDef.init = init;
}
if(auth){
if(utils.isFunction(auth)){
auth = {fn: auth};//create a trivial module obj.
}else{
auth = mod.coll[auth];
}
streamDef.auth = auth;
}
appCtx.stream.defs[mod.url] = streamDef;
addRoute(streamDef);
}
}
function newStreamInstance(params, name){
var streamDef = this;
var sdl = streamDef.sdl;
for(var k in params){
if(params.hasOwnProperty(k)){
sdl = sdl.replace(k, params[k]);
}
}
var id = md5(sdl);
var instance = this.instances[id];
if(!instance){
sh.info("New stream instance created for: " + streamDef.url + " id: " + id);
instance = {
params: params,
sdl: sdl,
names: {},
def: streamDef,
id: id,
cache: {},//srcid: {data, dataJson, dataTag}
lastReqId: 0,
pendingRequests : {},
addTransport: addInstanceScopedTransport
};
instance.psdl = parseSdl(sdl, streamDef.url);
this.instances[id] = instance;
this.releaseInstance = releaseStreamInstance;
}
if(!name){
name = "default";
}
instance.names[name] = true;
return instance;
}
function releaseStreamInstance(name){
if(!name){
name = "default";
}
delete this.names[name];
var names = Object.keys(this.names);
if(names.length == 0){
delete this.def.instances[this.id];
}
}
function streamAnnotation(annotations, keyFields, value){
assert(keyFields[0] === "stream");
if(keyFields.length < 2){
return true;
}
if(!annotations.stream){
annotations.stream = {};
}
switch(keyFields[1]){
case "sdl":
//var v = value.replace(/\s+/g, ' ');
annotations.stream.sdl = value;
break;
case "init":
annotations.stream.init = value.trim();
break;
case "auth":
annotations.stream.auth = value.trim();
break;
case "edgeTriggered":
var v = value.trim();
v = ((v === "true") || (v == ""))? true : false;
annotations.stream.edgeTriggered = v;
break;
default:
sh.error("Don't know how to process stream annotation subtype: " + keyFields[1]);
break;
}
}
function addRoute(streamDef){
if(streamDef.mod.annotations.noroute){
return;
}
var annotations = {
path : sh.appCtx.config.stream.routeBase +
streamDef.url,
method: "all"
};
annotations = utils.merge(streamDef.mod.annotations, annotations);
var route = {
annotations: annotations,
fn: streamRouteHandler.bind(null, streamDef),
fnName: "streamRouteHandler()",
relPath: "_internal",
type: "stream"
};
sh.appCtx.routes.push(route);
}
function streamRouteHandler(streamDef, req, res){
var id = req.query.id;
var tag = req.query.tag;
var src = req.query.src;
if(!src){
if(streamDef.transport.info.auth === "user"){
src = req.user.id;
}else{
src = streamDef.constants.noSrcId;
}
}
var instance;
if(id){
instance = streamDef.instances[id];
}else{
instance = utils.firstChild(streamDef.instances);
}
var srcCache;
if(src){
srcCache = instance.cache[src];
}
if(!srcCache){
res.status(503).end("Data unavailable for the request");
return;
}
var abort = function(status){
var msg = status;
if(!num(status)){
status = 401;
}
res.status(status).end(msg + ". Resource cannot be Accessed");
}
if((!tag || tag != srcCache.dataTag) && srcCache.data){
var done =function(status){
if(status === true){
res.send({data:srcCache.data,
tag: srcCache.dataTag});
}else{
abort(status);
}
}
if(instance.def.auth){
instance.def.auth.fn(req, instance, src, done);
}else{
done(true);
}
return;
}
var reqId = ++srcCache.lastReqId;
var release = function(){
delete instance.pendingRequests[this.reqId];
}
var send = function(data){
if(!data){
data = {};
}
if((!tag || tag != srcCache.dataTag) && data){
res.send({data: data, tag: srcCache.dataTag});
this.release();
}
}
instance.pendingRequests[reqId] =
{src:src, req: req, res: res,
reqId: reqId, release: release,
send: send, abort: abort};
}
function parseWhereExpr(exprText, src){
var expr = {
srcText: exprText,
text: exprText.replace(/\s*\(\s*/g, '(')
.replace(/\s*\)\s*/g, ') ')
.replace(/\s*,\s*/g, ', '),
names : []
};
try{
if(exprText.indexOf('"') >= 0){
sh.error("Strings must be single-quoted in SDL expression in " + src);
sh.hasErrors =true;
}
var ast = jsep(exprText);
postProcessAst(ast, expr, src);
expr.names = expr.names.sort(function(a,b){
return b.name.length - a.name.length;
});
}catch(err){
sh.error("Error parsing where expression at: " + src
+ err.stack);
sh.hasErrors = true;
}
return expr;
}
function postProcessAst(ast, expr, src){
if(ast.type === "Identifier"){
expr.names.push({name: ast.name});
}else
if(ast.type === "CallExpression"){
var r = {routineName: ast.callee.name, params:[],
paramValues:[]};
var n = r.routineName + "(";
for(var i=0;i<ast.arguments.length;i++){
var g = ast.arguments[i];
if(g.type == "Identifier"){
r.params.push({name: g.name});
r.paramValues.push(g.name);
n += g.name;
}else
if(g.type === "Literal"){
r.params.push({value: g.value});
r.paramValues.push(g.value);
if(utils.isString(g.value)){
n += "'" + g.value + "'";
}else{
n += g.value;
}
}else{
sh.error("Unsupported call expression in " + src);
sh.hasErrors = true;
}
}
n += ")";
r.name = n;
expr.names.push(r);
}
if(ast.left){
postProcessAst(ast.left, expr, src);
}
if(ast.right){
postProcessAst(ast.right, expr, src);
}
if(ast.argument){//for unary expr
postProcessAst(ast.argument, expr, src);
}
}
function parseSelectClause(cls, src){
var res = [];
var e = {name:""};
const PARSE_NAME = 0, PARSE_PARAMS = 1, PARSE_AS = 2;
var state = PARSE_NAME;
var openParen = 0;
var skip = 0;
var err;
var i =0;
var line = 0;
for(i=0;i<cls.length;i += (skip+1)){
skip = 0;
var c = cls[i];
if(c == "\n"){
line++;
}
if(utils.idChar(c)){
if(state == PARSE_NAME){
e.name += c;
}else if(state == PARSE_PARAMS){
if(e.params.length == 0){
e.params[0] = {srcText: c};
}else{
e.params[e.params.length-1].srcText += c;
}
}else
if(state == PARSE_AS){
e.aliasName += c;
}
}else
if(c === '(')
{
state = PARSE_PARAMS;
openParen++;
if(openParen > 1){
break;
}
e.params = [];
e.paramValues = [];
e.routineName = e.name;
}else
if(c === ')'){
openParen--;
if(openParen < 0){
break;
}
if(e.params.length > 0){
var p = e.params[e.params.length-1];
postProcessParam(p, e, src);
}
postProcessRoutineCall(e);
skip = 0;
var j = i+1;
while((j < cls.length - 1) &&
utils.isWhitespace(cls[j])){
skip++; j++;
}
if(cls[j].toLowerCase() === 'a' &&
cls[j+1].toLowerCase() === 's'){
skip+=2;
state = PARSE_AS;
e.aliasName = "";
}else{
state = PARSE_NAME;
}
}else
if(c === ','){
if(state === PARSE_NAME){
res.push(e);
e = {name:""};
}else
if(state === PARSE_AS){
res.push(e);
e = {name:""};
state = PARSE_NAME;
}
else if(state === PARSE_PARAMS){
if(e.params.length > 0){
var p = e.params[e.params.length-1];
postProcessParam(p, e, src);
}
e.params.push({srcText:""});
}
}else
if(!utils.isWhitespace(c)){
err = "Invalid character : " + c;
break;
}
}
if(e.name !== ""){
res.push(e);
}
if(openParen > 1){
err = "Nested function calls not allowed."
}else
if(openParen !== 0){
err = "Unmatched parenthesis.";
}
if(err){
sh.error(err + " at char: " + i + "line: " + line
+ " at " + src);
sh.hasErrors = true;
}
return res;
}
function postProcessRoutineCall(e){
var n = e.routineName + "(";
for(var i=0;i<e.params.length;i++){
n += (e.params[i].name || e.params[i].text);
}
n += ")";
e.name = n;
}
function postProcessParam(p, entry, src){
var t = p.srcText.trim();
var d = t[0];
var e = t[t.length-1];
if(d == '"' || d == "'"){
if(d != e){
sh.error("Invalid string literal at " + src);
sh.hasErrors = true;
}else{
t = t.substr(1, t.length - 2);
}
}
const builtins = ["true", "false", "null"];
if(utils.idChar(t[0]) && !builtins.contains(t)){
p.name = t;
entry.paramValues.push(t);
}else{
p.text = t;
var code = "var v = " + t + ";";
try{
eval(code);
p.value = v;
entry.paramValues.push(v);
}catch(err){
sh.error("Error evaluating literal: " + p.text + " at " + src);
sh.hasErrors = true;
}
}
}
//ATTIC:
/*if(db){
db.save(function(){
return {_id: streamInfo.url,
sdl: streamInfo.sdl,
params: streamInfo.params};
});
}*/
/*if(db){
db.save(function(){
return {_id: streamInfo.url,
sdl: streamInfo.sdl,
params: streamInfo.params};
});
}*/
/*
if(db){
db
.success(function(res){
done();
})
.error(function(err){
sh.error("Error creating streams");
})
.execute();
}else{
done();
}*/
/*
if(db){
db
.collection("streamAccessTokens")
.save(rec)
.success(function(res){
console.log("Access token updated for " + type);
})
.error(function(err){
console.log("Error updating Nest access token for " + type);
})
.execute();
}*/