joola.io.engine
Version:
joola.io's Framework Engine
289 lines (232 loc) • 8.35 kB
JavaScript
/**
* joola.io
*
* Copyright Joola Smart Solutions, Ltd. <info@joo.la>
*
* Licensed under GNU General Public License 3.0 or later.
* Some rights reserved. See LICENSE, AUTHORS.
*
* @license GPL-3.0+ <http://spdx.org/licenses/GPL-3.0+>
*/
var
utils = require('../shared/utils'),
connector = require('../connectors/connector'),
_datasources = require('../objects/datasources'),
_datatables = require('../objects/datatables'),
caching = require('../caching/manager.js'),
ce = require('cloneextend'),
async = require('async');
require('date-utils');
require('../shared/utils');
var parseConfiguration = function (next) {
joola.logger.debug('Parsing configuration files...');
joola.config.integration = require(process.env.JOOLA_CONFIG_INTEGRATION).configData;
joola.config.auth = require(process.env.JOOLA_CONFIG_AUTH).configData;
joola.config.content = require(process.env.JOOLA_CONFIG_CONTENT).configData;
joola.config.server = require(process.env.JOOLA_CONFIG_SERVER).configData;
var integration = joola.config.integration;
var datasources = integration.datasources;
var checks = [];
datasources.forEach(function (ds) {
if (!ds.id)
throw 'JSON Config Validation Error: ' + ds.name + ' has no id';
var query = connector.createQuery();
query.sql = 'SELECT 1 as sanity_check;';
query.datasource = ds;
_.each(ds.datatables, function (dt) {
dt.datasourceid = ds.id;
});
var check = function (callback) {
joola.logger.debug('Checking data source [' + ds.id + ']...');
connector.executeQuery(query, function (query, rows, fields, error) {
if (error) {
joola.logger.error('Check for data source [' + ds.id + '] failed: ' + error);
throw error;
}
else {
joola.logger.debug('Check for data source [' + ds.id + '] completed.')
}
callback();
});
};
checks.push(check);
});
fork(checks, function () {
next();
})
};
var watchForConfigChanges = function (api, next) {
next();
};
var fetchEndDates = function (api, callback) {
joola.logger.debug('Fetching end dates...');
var datasources = _datasources.list(true);
var async_calls = [];
datasources.forEach(function (ds) {
if (ds.enddate) {
if (ds.enddate.type == 'query') {
var query = connector.createQuery();
query.datasource = ds;
query.sql = ds.enddate.query;
joola.logger.debug('Fetching end dates for datasource - ' + ds.id);
var call = function (callback) {
connector.executeQuery(query, function (query, rows, fields, error) {
if (error)
throw error;
var row = rows.rows[0];
var col = rows.fields[0].name;
var timestamp;
try {
timestamp = new Date(row[col]); //.fixDate(true, true);
}
catch (ex) {
//no data maybe?
timestamp = new Date();
}
timestamp.setMilliseconds(timestamp.getMilliseconds() - 1);
joola.logger.debug('... [' + ds.id + '] ' + utils.formatDate(timestamp, 'yyyy-mm-dd hh:nn:ss'));
ds.enddate.value = timestamp;
callback();
});
};
async_calls.push(call);
}
}
});
fork(async_calls, function () {
joola.logger.debug('...Datasource end dates fetched.');
/*
joola.config.content.system.enddate = function () {
var _date = ce.clone(datasources[0].enddate.value);
_date.addDays(-1);
return datasources[0].enddate.value;
};*/
callback();
})
};
var buildInitialCache = function (callback) {
var range;
//monitor which datatables need interval based caching
var found = false;
var datatables = _datatables.list();
var calls = [];
var cacheTable = function (args, callback) {
var query = args.query;
var dt = args.dt;
query.enddate = ce.clone(args.enddate);
query.startdate = ce.clone(args.startdate);
caching.cacheTable(dt, ce.clone(query), function () {
callback();
});
};
datatables.forEach(function (dt) {
if (dt.caching) {
if (dt.caching.method == 'persist' || dt.caching.method == 'forward') {
found = true;
joola.logger.debug('Found [' + dt.id + '] for cache management [' + dt.caching.method + ']...');
var step = dt.caching.step;
//first fetch the base line, and then setup the time for the regular forward fetches
var ds = _datasources.get(dt.datasourceid);
var query = connector.createQuery();
dt.datasource = ds;
dt.query = _datatables.basequery(dt);
var basequery = dt.query;
query.sql = basequery.sql;
query.datasource = ds;
var enddate = ce.clone(ds.enddate.value);
var startdate = ce.clone(enddate);
startdate.setMilliseconds(startdate.getMilliseconds() + 1);
startdate.setMinutes(startdate.getMinutes() - parseFloat(range ? range : dt.caching.baseline) * 24 * 60);
var _baseline_start = ce.clone(startdate);
query.enddate = enddate;
query.startdate = startdate;
var firstRun = true;
if (_baseline_start.dateDiff(enddate) > step) {
while (startdate >= _baseline_start) {
if (firstRun) {
startdate = ce.clone(enddate);
startdate.setMilliseconds(startdate.getMilliseconds() + 1);
firstRun = false;
}
//startdate.addDays(-1 * parseFloat((range ? range : dt.caching.step)));
startdate.setMinutes(startdate.getMinutes() - parseFloat(range ? range : dt.caching.step) * 24 * 60);
calls.push({query: query, dt: dt, startdate: ce.clone(startdate), enddate: ce.clone(enddate)});
enddate = ce.clone(startdate);
enddate.setMilliseconds(enddate.getMilliseconds() - 1);
}
}
else {
calls.push({query: query, dt: dt, startdate: ce.clone(startdate), enddate: ce.clone(enddate)});
}
}
}
});
async.mapSeries(calls, cacheTable, function (err, results) {
joola.logger.debug('All mapped async finished');
callback();
});
if (!found)
callback();
};
var setupEagerCache = function (next) {
var tables = _datatables.list();
_.each(tables, function (dt) {
dt.datasource = _datasources.get(dt.datasourceid);
if (dt.caching && dt.caching.eager && dt.caching.eager.interval && dt.caching.eager.interval > 0) {
joola.logger.debug('Setting up eagerCache for [' + dt.id + '] task with interval: ' + dt.caching.eager.interval);
var tFunction = function () {
caching.eagerCache(dt, function () {
joola.logger.debug('Re-setting eagerCache for [' + dt.id + '] task with interval: ' + dt.caching.eager.interval);
setTimeout(tFunction, dt.caching.eager.interval)
});
};
setTimeout(tFunction, dt.caching.eager.interval)
}
});
next();
};
/**
* Main entry point from the API server.
* @param {Object} api passed by the api server
* @param {Function} next callback function to run when finished
*/
exports._joola = function (api, next) {
//global.joola = {};
joola.ts_start = new Date();
joola.state = 'starting';
//joola.config = {};
//joola.joola.logger = joola.logger;
//joola.cache = api.cache;
parseConfiguration(function () {
require('../../lib/shared/redis').redis(joola, function () {
require('../../lib/shared/cache').cache(joola, function () {
watchForConfigChanges(api, function () {
fetchEndDates(api, function () {
buildInitialCache(function () {
setupEagerCache(function () {
});
});
joola.logger.info('Spinning up server...');
next();
});
});
});
});
});
/*
var net = require("net"),
repl = require("repl");
var connections = 0;
net.createServer(function (socket) {
connections += 1;
repl.start({
prompt: "node via TCP socket> ",
input: socket,
output: socket
}).on('exit', function () {
socket.end();
});
}).listen(4211);*/
};
joola.logger.info('_Joola Initialized!');
exports.parseConfiguration = parseConfiguration;