joola.io.engine
Version:
joola.io's Framework Engine
596 lines (528 loc) • 21.8 kB
JavaScript
/**
* joola.io
*
* Copyright Joola Smart Solutions, Ltd. <info@joo.la>
*
* Licensed under GNU General Public License 3.0 or later.
* Some rights reserved. See LICENSE, AUTHORS.
*
* @license GPL-3.0+ <http://spdx.org/licenses/GPL-3.0+>
*/
var
_datasources = require('../objects/datasources'),
_datatables = require('../objects/datatables'),
connector = require('../connectors/connector'),
utils = require('../shared/utils'),
mysql = require('./mysql'),
mongo = require('./mongo'),
zlib = require('zlib'),
path = require('path');
var createConnection = function (datatable) {
var connection = null;
switch (datatable.caching.system.type) {
case 'mysql':
joola.logger.debug('open connection to mysql ' + datatable.datasource.name);
connection = mysql;
break;
case 'mongo':
joola.logger.debug('open connection to mongo ' + datatable.datasource.name);
connection = mongo;
break;
case 'mssql':
joola.logger.debug('open connection to mssql ' + datatable.datasource.name);
break;
default:
break;
}
return connection;
};
var exports = {};
exports.flushCache = function (datatable, callback) {
if (datatable.caching) {
var db = createConnection(datatable);
db.flushCache(datatable, callback);
}
else
return callback();
};
exports.minCacheDate = function (datatable, callback) {
var db = createConnection(datatable);
db.minCacheDate(datatable, callback);
};
exports.maxCacheDate = function (datatable, callback) {
var db = createConnection(datatable);
db.maxCacheDate(datatable, callback);
};
exports.lock = function (key, rangeKey, query, callback) {
var startTimer = new Date().getTime();
var _self = this;
var lockKey = query.lockKey;
var lockRangeKey = function (key, value, callback) {
joola.cache.save('locks', key.clean(), value, joola.config.integration.cache.lockTimeout, function (err) {
joola.cache.load('locks', key.clean(), function (err, _value) {
if (_value && _value.lockKey == value.lockKey) {
joola.logger.silly('RangeKey [' + 'joola:cachedTable:' + key.clean() + ':' + value.lockKey + '][tout:' + joola.config.integration.cache.lockTimeout + '] locked!');
return callback(true);
}
else {
joola.logger.error('Locked by another node [' + key + ']');
return callback(false);
//throw new Error("Locked by another node");
}
})
});
};
joola.cache.load('locks', key.clean(), function (err, value) {
if (value) {
if (value.lockKey != null && value.lockKey != lockKey) {
if (value.nextLock != lockKey) {
joola.logger.silly('Setting nextLock [' + lockKey + ']');
value.nextLock = lockKey;
//cache.save('joola:cachedTable:lock:' + key.clean(), value, joola.config.cache.lockTimeout, function (err) {
//});
}
setTimeout(function () {
if (new Date() - startTimer % 100 == 0)
joola.logger.silly('Waiting on cache for table [' + lockKey + ']');
if (new Date() - startTimer > joola.config.integration.cache.waitOnCache) {
joola.logger.debug('Timeout while waiting for cache [' + lockKey + ']');
return callback(false);
}
else
_self.lock(key, rangeKey, query, callback);
}, joola.config.integration.cache.waitInterval);
}
else if (value.lockKey == lockKey) {
return callback(true, lockKey);
}
else if (value.lockKey == null) {
value = {};
value.lockKey = lockKey;
value.nextLock = null;
lockRangeKey(key, value, function (success) {
if (success) {
return callback(true, lockKey);
}
else
return callback(false);
});
}
else {
return callback(false);
}
}
else {
value = {};
value.lockKey = lockKey;
value.nextLock = null;
lockRangeKey(key, value, function (success) {
if (success)
return callback(true, lockKey);
else
return callback(false);
});
}
});
};
exports.unlock = function (key, rangeKey, lockKey, callback) {
var unlockRangeKey = function (key, value, callback) {
joola.cache.save('locks', key.clean(), value, function () {
joola.logger.silly('RangeKey [' + 'joola:cachedTable:' + key.clean() + ':' + lockKey + '] unlocked!');
return callback();
});
};
joola.cache.load('locks', key.clean(), function (error, value) {
if (value) {
if (value.lockKey == lockKey) {
value.lockKey = value.nextLock;
value.nextLock = null;
unlockRangeKey(key, value, function () {
return callback();
});
}
else {
return callback();
}
}
else {
return callback();
}
});
};
exports.cacheTable = function (datatable, query, callback) {
var _self = this;
/*
if (query._query.type == 'distinct') {
joola.logger.error('DISTINCT');
callback();
return;
}*/
var ds = datatable.datasource;
//datatable.query = _datatables.basequery(datatable);
query.lockKey = require('node-uuid').v4();
query.startdate = query.startdate.fixDate(true, true);
query.enddate = query.enddate.fixDate(true, true);
_self.lock(datatable.id, '', query, function (success, lockKey) {
if (!success) {
joola.logger.error('Failed to obtain lock on [' + lockKey + '].');
//throw 'Failed to obtain lock on [' + datatable.id + '][' + query[0].cache.range.key + '].';
return callback();
}
_self.check(datatable, query, true, function () {
if (false && query.cache.state == 'match') {
joola.logger.silly('Table [' + datatable.id + '] for range [' + utils.formatDate(query.startdate, 'yyyy-mm-dd hh:nn:ss.fff') + ']-[' + utils.formatDate(query.enddate, 'yyyy-mm-dd hh:nn:ss.fff') + '] already available.');
_self.unlock(datatable.id, '', lockKey, function () {
return callback(null);
});
return;
}
if (false && query.cache.state == 'extendleft') {
query.extension = 'left';
query.enddate = ce.clone(query.cache.range.startdate);
query.enddate.setMilliseconds(query.enddate.getMilliseconds() - 1);
joola.logger.debug('Table [' + datatable.id + '] can be extended to the left, new enddate: ' + query.enddate);
}
else if (false && query.cache.state == 'extendright') {
query.extension = 'right';
query.startdate = ce.clone(query.cache.range.enddate);
query.startdate.setMilliseconds(query.startdate.getMilliseconds() + 1);
joola.logger.debug('Table [' + datatable.id + '] can be extended to the right, new startdate: ' + query.startdate);
}
else if (false && query.cache.state == 'extendboth') {
joola.logger.debug('Table [' + datatable.id + '] can be extended in both directions.');
query.extension = 'both';
var _basequery = ce.clone(query);
var _query = ce.clone(query);
_query.enddate = ce.clone(_basequery.cache.range.startdate);
_query.enddate.setMilliseconds(_basequery.enddate.getMilliseconds() - 1);
query = [];
query.push(ce.clone(_query));
_query = ce.clone(_basequery);
_query.startdate = ce.clone(_basequery.cache.range.enddate);
_query.startdate.setMilliseconds(_basequery.startdate.getMilliseconds() + 1);
query.push(ce.clone(_query));
}
else {
joola.logger.debug('Table [' + datatable.id + '] check/extend check found nothing.');
}
if (!Array.isArray(query)) {
var _query = ce.clone(query);
query = [];
query.push(_query);
}
query.forEach(function (q) {
q.startdate = utils.formatDate(q.startdate, 'yyyy-mm-dd hh:nn:ss.fff', true);
q.enddate = utils.formatDate(q.enddate, 'yyyy-mm-dd hh:nn:ss.fff', true);
q.sql = q.sql.replace('%STARTDATE%', q.startdate);
q.sql = q.sql.replace('%ENDDATE%', q.enddate);
q.datatable = datatable;
q.datasource = ds;
q.limit = q.datasource.limit;
q.lockKey = require('node-uuid').v4();
//_self.lock(datatable.id, query[0].cache.range.key, q, function (success, lockKey) {
var calls = [];
if (q.startdate > q.enddate) {
joola.logger.debug('Avoiding query due to mismatching start/end dates');
}
else {
var call = function (callback) {
joola.logger.info('Executing source query [' + datatable.id + '] for range [' + q.startdate + ']-[' + q.enddate + '][lim:' + q.limit + ']...');
connector.executeQuery(q, function (query, rows, fields, err) {
if (err) {
return callback(err);
}
var data = {};
data.rows = rows.rows;
data.fields = rows.fields;
if (data.rows.length == 0) {
joola.logger.debug('Nothing to do here.');
return callback(null);
}
else {
/* if (data.rows.length > 100000) {
var limitCalls = [];
var iCount = Math.floor((data.rows.length / 100000) + 1);
query.balanced = true;
joola.logger.debug('Balanced query for cache data on table [' + datatable.id + '] has finished (' + data.rows.length + '), saving to cache store...');
joola.logger.debug('Setting up calls for rows chunk [' + iCount.toString() + ' chunks]...');
for (var i = 0; i < iCount; i++) {
joola.logger.debug('Setting up call for rows chunk [' + i.toString() + ']...');
var limitCall = function (callback) {
var _data = ce.clone(data);
_data.rows.splice(100000, data.rows.length);
joola.logger.debug('Processing data on table [' + datatable.id + '], length: ' + _data.rows.length + '<-' + data.rows.length + '), saving to cache store...');
if (_data.rows.length < 100000) {
query.balanced = false;
}
_self.saveData(datatable, ce.clone(query), _data, function () {
_data = null;
data.rows.splice(0, 100000);
return callback(null);
});
};
limitCalls.push(limitCall);
}
forkSeries(limitCalls, function () {
return callback(null);
})
}
else {*/
joola.logger.debug('Query for cache data on table [' + datatable.id + '] has finished (' + data.rows.length + '), saving to cache store...');
if (data.rows.length > 0) {
_self.saveData(datatable, query, data, function (datatable, data, err) {
return callback(err);
});
}
// }
}
});
};
calls.push(call);
}
fork(calls, function (err) {
_self.unlock(datatable.id, query[0].cache.range.key, lockKey, function () {
return callback(err);
});
});
});
});
});
};
exports.check = function (datatable, query, wait, callback) {
var _self = this;
joola.logger.debug('Checking table [' + datatable.id + '] for range [' + utils.formatDate(query.startdate, 'yyyy-mm-dd hh:nn:ss.fff') + ']-[' + utils.formatDate(query.enddate, 'yyyy-mm-dd hh:nn:ss.fff') + '].');// with resolution [' + query.resolution + ']')
query.cache = {};
joola.cache.load('tables', datatable.id.clean(), function (error, value) {
if (value) {
_.each(value.ranges, function (range) {
range.startdate = new Date(range.startdate);
range.enddate = new Date(range.enddate);
joola.logger.debug('Found cached table [' + datatable.id + '] with range [' + utils.formatDate(range.startdate, 'yyyy-mm-dd hh:nn:ss.fff') + '] - [' + utils.formatDate(range.enddate, 'yyyy-mm-dd hh:nn:ss.fff') + ']');
//TODO: Better handle finding ranges, currently keeps extending one range.
//we have 4 possibilities
//the range is already cached
if (range.startdate <= query.startdate && range.enddate >= query.enddate) {
query.cache.range = range;
query.cache.state = 'match';
return callback();
}
//both sides extension
else if (query.startdate <= range.startdate && query.enddate >= range.enddate) {
query.cache.range = range;
query.cache.state = 'extendboth';
}
//we can extend the table to the left
else if (query.startdate < range.startdate && range.enddate >= query.enddate) {
//query.cache.key = range.key;
query.cache.range = range;
query.cache.state = 'extendleft';
}
//we can extend the table to the right
else if (range.startdate <= query.startdate && range.enddate < query.enddate) {
//query.cache.key = range.key;
query.cache.range = range;
query.cache.state = 'extendright';
}
});
if (query.cache.state != 'match') {
if (!query.cache.range) {
query.cache.range = {};
query.cache.range.key = utils.formatDate(query.startdate, 'yyyy-mm-dd hh:nn:ss.fff').clean() + '_' + utils.formatDate(query.enddate, 'yyyy-mm-dd hh:nn:ss.fff').clean();
query.cache.range.startdate = query.startdate;
query.cache.range.enddate = query.enddate;
}
return callback();
}
}
else {
query.cache = {};
query.cache.range = {};
query.cache.range.key = utils.formatDate(query.startdate, 'yyyy-mm-dd hh:nn:ss.fff').clean() + '_' + utils.formatDate(query.enddate, 'yyyy-mm-dd hh:nn:ss.fff').clean();
query.cache.range.startdate = query.startdate;
query.cache.range.enddate = query.enddate;
if (wait)
query.cache.range.state = 'building';
value = {ranges: [query.cache.range]};
joola.cache.save('tables', datatable.id.clean(), value, function (error, value) {
return callback();
});
}
});
};
exports.saveData = function (datatable, query, data, callback) {
joola.logger.debug('Saving cache data for [' + datatable.id + ']:[ext:' + query.extension + ']...');
var db = createConnection(datatable);
db.saveData(query, datatable, data, function (error, firstDate, lastDate) {
if (error) {
return callback(datatable, data, error);
}
else if (data) {
data = data.rows;
joola.logger.debug('Cache engine finished storing [' + datatable.id + '], found [' + (data ? data.length : 0) + '] documents, [' + firstDate + ']:[' + lastDate + '].');
//db.maxNotHandledCacheDate(datatable, function (date) {
if (data && data.length > 0 && typeof(firstDate) != 'undefined' && firstDate != null && typeof(lastDate) != 'undefined' && lastDate != null) {
var key = 'joola:cachedTable:' + datatable.id.clean();
var save = function (key, value, range) {
if (!query.balanced) {
joola.cache.save('tables', datatable.id.clean(), value, null, function () {
joola.logger.debug('Cache key saved for [' + datatable.id.clean() + '] with range [' + utils.formatDate(range.startdate, 'yyyy-mm-dd hh:nn:ss.fff') + ']-[' + utils.formatDate(range.enddate, 'yyyy-mm-dd hh:nn:ss') + '].');
return callback(datatable, data, error);
});
}
else
return callback(datatable, data, error);
};
joola.cache.load('tables', datatable.id.clean(), function (error, value) {
if (value && query.cache.range.key) {
var _range = _.find(value.ranges, function (range) {
return range.key == query.cache.range.key;
});
if (!_range) {
//we have a new range.
_range = {};
value.ranges.push(_range);
}
else {
// firstDate = (new Date(_range.startdate) < firstDate ? new Date(_range.startdate) : firstDate);
// lastDate = (new Date(_range.enddate) > lastDate ? new Date(_range.enddate) : lastDate);
}
_range.extension = query.extension;
switch (query.extension) {
case null:
_range.startdate = firstDate//.fixDate(true);
_range.enddate = lastDate//.fixDate(true);
break;
case 'left':
_range.startdate = firstDate//.fixDate(true);
if (!_range.enddate)
_range.enddate = new Date(lastDate);
break;
case 'right':
_range.enddate = lastDate//.fixDate(true);
break;
case 'both':
_range.startdate = query.startdate//.fixDate(true);
_range.enddate = query.enddate//.fixDate(true);
break;
default:
_range.startdate = firstDate;//.fixDate(true);
_range.enddate = lastDate;//.fixDate(true);
break;
}
/*
if (typeof(_range.startdate) != typeof(new Date()))
_range.startdate = new Date(_range.startdate);
if (typeof(_range.enddate) != typeof(new Date()))
_range.enddate = new Date(_range.enddate);
if (_range.startdate)
firstDate = _range.startdate;
if (query.startdate < firstDate)
firstDate = query.startdate;
//lastDate = _range.enddate;
if (query.enddate > lastDate)
lastDate = query.enddate;
if (_range.enddate > lastDate)
lastDate = _range.enddate;
*/
//_range.enddate = lastDate;
//_range.startdate = _range.startdate.fixDate(true);
//_range.enddate = new Date(_range.enddate).fixDate(true);
_range.state = 'ready';
_range.key = _range.startdate.clean() + '_' + _range.enddate.clean();//_range.enddate.clean()
_range.debug = '1';
save(key, value, _range);
}
else if (value) {
var rangeToAdd =
{
key: firstDate.clean() + '_' + lastDate.clean(), //query.enddate.clean(),
startdate: firstDate,
enddate: lastDate,//new Date(query.enddate),
state: 'ready',
debug: '2'
};
value.ranges.push(rangeToAdd);
save(key, value, rangeToAdd);
}
else {
var value = {
ranges: [
{
key: firstDate.clean() + '_' + lastDate.clean(),// query.enddate.clean(),
startdate: firstDate,
enddate: lastDate,//new Date(query.enddate),
state: 'ready',
debug: '3'
}
]};
save(key, value, value.ranges[0]);
}
});
}
else {
return callback(datatable, data, error);
}
//});
}
else {
return callback(datatable, data, error);
}
});
};
exports.fetch = function (query, callback) {
joola.logger.debug('Fetch from cache [' + query.datatable.id + '] with range [' + utils.formatDate(query.startdate, 'yyyy-mm-dd hh:nn:ss') + ']-[' + utils.formatDate(query.enddate, 'yyyy-mm-dd hh:nn:ss') + ']');
try {
if (!query.datatable.datasource) query.datatable.datasource = _datasources.get(query.datatable.datasourceid);
var db = createConnection(query.datatable);
db.fetch(query, callback);
}
catch (ex) {
return callback(ex, arguments);
}
};
exports.eagerCache = function (dt, callback) {
var self = this;
joola.logger.debug('Eager cache running for table [' + dt.id + ']...');
var _ds;
//var maxDate = null;
self.maxCacheDate(dt, function (err, maxDate) {
if (maxDate) {
joola.logger.debug('Found max cache date on [' + dt.id + '], setting to: ' + maxDate);
/*_ds = _.find(joola.config.integration.datasources, function (ds) {
return ds.id == dt.datasource.id;
});
if (_ds && _ds.lastEagerCache) {
joola.logger.debug('Missing max cache date on [' + dt.id + '], setting to last used: ' + _ds.lastEagerCache);
maxDate = _ds.lastEagerCache;
}*/
}
else {
joola.logger.debug('Missing max cache date on [' + dt.id + '], setting to: ' + dt.datasource.enddate.value);
maxDate = dt.datasource.enddate.value;
//if (_ds)
// _ds.lastEagerCache = new Date(maxDate);
}
//}
var _fromdate = new Date(maxDate);
var _todate = new Date();
//_todate.setMilliseconds(_todate.getMilliseconds());
//if (_ds)
// _ds.lastEagerCache = new Date(maxDate);
joola.logger.debug('Processing ' + dt.name + '[' + _fromdate + ']:[' + _todate + ']');
var query = connector.createQuery();
query.type = 'eager';
dt.query = _datatables.basequery(dt);
var basequery = dt.query;
query.sql = basequery.sql;
query.datasource = dt.datasource;
var enddate = ce.clone(_todate);
var startdate = ce.clone(_fromdate);
startdate.setMilliseconds(startdate.getMilliseconds() + 1);
query.enddate = enddate;
query.startdate = startdate;
self.cacheTable(dt, ce.clone(query), function (err) {
return callback(err);
});
});
};
module.exports = exports;