hoard
Version:
node.js lib for storing time series data on disk, similar to RRD.
630 lines • 24 kB
JavaScript
var Binary, Buffer, Put, archiveInfoFormat, archiveInfoSize, async, create, fetch, floatFormat, floatSize, fs, info, longFormat, longSize, metadataFormat, metadataSize, pack, path, pointFormat, pointSize, propagate, timestampFormat, timestampSize, underscore, unixTime, update, updateMany, updateManyArchive, valueFormat, valueSize, _;
fs = require('fs');
Buffer = require('buffer').Buffer;
Binary = require('binary');
underscore = _ = require('../lib/underscore');
async = require('../lib/async');
pack = require('../lib/jspack').jspack;
path = require('path');
Put = require('put');
Number.prototype.mod = function(n) {
return ((this % n) + n) % n;
};
longFormat = "!L";
longSize = pack.CalcLength(longFormat);
floatFormat = "!f";
floatSize = pack.CalcLength(floatFormat);
timestampFormat = "!L";
timestampSize = pack.CalcLength(timestampFormat);
valueFormat = "!d";
valueSize = pack.CalcLength(valueFormat);
pointFormat = "!Ld";
pointSize = pack.CalcLength(pointFormat);
metadataFormat = "!2LfL";
metadataSize = pack.CalcLength(metadataFormat);
archiveInfoFormat = "!3L";
archiveInfoSize = pack.CalcLength(archiveInfoFormat);
unixTime = function() {
return parseInt(new Date().getTime() / 1000);
};
create = function(filename, archives, xFilesFactor, cb) {
var a, archive, archiveOffset, buffer, encodeFloat, headerSize, oldest, points, secondsPerPoint, _i, _len;
archives.sort(function(a, b) {
return a[0] - b[0];
});
if (path.existsSync(filename)) {
cb(new Error('File ' + filename + ' already exists'));
}
oldest = ((function() {
var _i, _len, _results;
_results = [];
for (_i = 0, _len = archives.length; _i < _len; _i++) {
a = archives[_i];
_results.push(a[0] * a[1]);
}
return _results;
})()).sort().reverse()[0];
encodeFloat = function(value) {
var buffer;
buffer = new Buffer(4);
require('buffer_ieee754').writeIEEE754(buffer, 0.5, 0, 'big', 23, 4);
return buffer;
};
buffer = Put().word32be(unixTime()).word32be(oldest).put(encodeFloat(xFilesFactor)).word32be(archives.length);
headerSize = metadataSize + (archiveInfoSize * archives.length);
archiveOffset = headerSize;
for (_i = 0, _len = archives.length; _i < _len; _i++) {
archive = archives[_i];
secondsPerPoint = archive[0];
points = archive[1];
buffer.word32be(archiveOffset);
buffer.word32be(secondsPerPoint);
buffer.word32be(points);
archiveOffset += points * pointSize;
}
buffer.pad(archiveOffset - headerSize);
return fs.writeFile(filename, buffer.buffer(), 'binary', cb);
};
propagate = function(fd, timestamp, xff, higher, lower, cb) {
var lowerIntervalEnd, lowerIntervalStart, packedPoint, parseSeries;
lowerIntervalStart = timestamp - timestamp.mod(lower.secondsPerPoint);
lowerIntervalEnd = lowerIntervalStart + lower.secondsPerPoint;
packedPoint = new Buffer(pointSize);
fs.read(fd, packedPoint, 0, pointSize, higher.offset, function(err, written, buffer) {
var byteDistance, firstSeriesSize, higherBaseInterval, higherBaseValue, higherEnd, higherFirstOffset, higherLastOffset, higherPoints, higherSize, pointDistance, relativeFirstOffset, relativeLastOffset, secondSeriesSize, seriesSize, seriesString, timeDistance, _ref;
if (err) {
cb(err);
}
_ref = pack.Unpack(pointFormat, packedPoint), higherBaseInterval = _ref[0], higherBaseValue = _ref[1];
if (higherBaseInterval === 0) {
higherFirstOffset = higher.offset;
} else {
timeDistance = lowerIntervalStart - higherBaseInterval;
pointDistance = timeDistance / higher.secondsPerPoint;
byteDistance = pointDistance * pointSize;
higherFirstOffset = higher.offset + byteDistance.mod(higher.size);
}
higherPoints = lower.secondsPerPoint / higher.secondsPerPoint;
higherSize = higherPoints * pointSize;
relativeFirstOffset = higherFirstOffset - higher.offset;
relativeLastOffset = (relativeFirstOffset + higherSize).mod(higher.size);
higherLastOffset = relativeLastOffset + higher.offset;
if (higherFirstOffset < higherLastOffset) {
seriesSize = higherLastOffset - higherFirstOffset;
seriesString = new Buffer(seriesSize);
return fs.read(fd, seriesString, 0, seriesSize, higherFirstOffset, function(err, written, buffer) {
return parseSeries(seriesString);
});
} else {
higherEnd = higher.offset + higher.size;
firstSeriesSize = higherEnd - higherFirstOffset;
secondSeriesSize = higherLastOffset - higher.offset;
seriesString = new Buffer(firstSeriesSize + secondSeriesSize);
return fs.read(fd, seriesString, 0, firstSeriesSize, higherFirstOffset, function(err, written, buffer) {
var ret;
if (err) {
cb(err);
}
if (secondSeriesSize > 0) {
return fs.read(fd, seriesString, firstSeriesSize, secondSeriesSize, higher.offset, function(err, written, buffer) {
if (err) {
cb(err);
}
return parseSeries(seriesString);
});
} else {
ret = new Buffer(firstSeriesSize);
seriesString.copy(ret, 0, 0, firstSeriesSize);
return parseSeries(ret);
}
});
}
});
return parseSeries = function(seriesString) {
var aggregateValue, byteOrder, currentInterval, f, i, knownPercent, knownValues, myPackedPoint, neighborValues, pointTime, pointTypes, points, seriesFormat, step, sum, unpackedSeries, v, _ref, _ref2, _step;
_ref = [pointFormat[0], pointFormat.slice(1)], byteOrder = _ref[0], pointTypes = _ref[1];
points = seriesString.length / pointSize;
seriesFormat = byteOrder + ((function() {
var _results;
_results = [];
for (f = 0; 0 <= points ? f < points : f > points; 0 <= points ? f++ : f--) {
_results.push(pointTypes);
}
return _results;
})()).join("");
unpackedSeries = pack.Unpack(seriesFormat, seriesString, 0);
neighborValues = (function() {
var _results;
_results = [];
for (f = 0; 0 <= points ? f < points : f > points; 0 <= points ? f++ : f--) {
_results.push(null);
}
return _results;
})();
currentInterval = lowerIntervalStart;
step = higher.secondsPerPoint;
for (i = 0, _ref2 = unpackedSeries.length, _step = 2; 0 <= _ref2 ? i < _ref2 : i > _ref2; i += _step) {
pointTime = unpackedSeries[i];
if (pointTime === currentInterval) {
neighborValues[i / 2] = unpackedSeries[i + 1];
}
currentInterval += step;
}
knownValues = (function() {
var _i, _len, _results;
_results = [];
for (_i = 0, _len = neighborValues.length; _i < _len; _i++) {
v = neighborValues[_i];
if (v !== null) {
_results.push(v);
}
}
return _results;
})();
if (knownValues.length === 0) {
cb(null, false);
return;
}
sum = function(list) {
var s, x, _i, _len;
s = 0;
for (_i = 0, _len = list.length; _i < _len; _i++) {
x = list[_i];
s += x;
}
return s;
};
knownPercent = knownValues.length / neighborValues.length;
if (knownPercent >= xff) {
aggregateValue = sum(knownValues) / knownValues.length;
myPackedPoint = pack.Pack(pointFormat, [lowerIntervalStart, aggregateValue]);
packedPoint = new Buffer(pointSize);
return fs.read(fd, packedPoint, 0, pointSize, lower.offset, function(err) {
var byteDistance, lowerBaseInterval, lowerBaseValue, mypp, offset, pointDistance, timeDistance, _ref3;
_ref3 = pack.Unpack(pointFormat, packedPoint), lowerBaseInterval = _ref3[0], lowerBaseValue = _ref3[1];
if (lowerBaseInterval === 0) {
offset = lower.offset;
} else {
timeDistance = lowerIntervalStart - lowerBaseInterval;
pointDistance = timeDistance / lower.secondsPerPoint;
byteDistance = pointDistance * pointSize;
offset = lower.offset + byteDistance.mod(lower.size);
}
mypp = new Buffer(myPackedPoint);
return fs.write(fd, mypp, 0, pointSize, offset, function(err) {
return cb(null, true);
});
});
} else {
return cb(null, false);
}
};
};
update = function(filename, value, timestamp, cb) {
info(filename, function(err, header) {
var archive, diff, i, lowerArchives, now, _ref;
if (err) {
cb(err);
}
now = unixTime();
diff = now - timestamp;
if (!(diff < header.maxRetention && diff >= 0)) {
cb(new Error('Timestamp not covered by any archives in this database.'));
return;
}
for (i = 0, _ref = header.archives.length; 0 <= _ref ? i < _ref : i > _ref; 0 <= _ref ? i++ : i--) {
archive = header.archives[i];
if (archive.retention < diff) {
continue;
}
lowerArchives = header.archives.slice(i + 1);
break;
}
return fs.open(filename, 'r+', function(err, fd) {
var myInterval, myPackedPoint, packedPoint, propagateLowerArchives;
if (err) {
cb(err);
}
myInterval = timestamp - timestamp.mod(archive.secondsPerPoint);
myPackedPoint = new Buffer(pack.Pack(pointFormat, [myInterval, value]));
packedPoint = new Buffer(pointSize);
fs.read(fd, packedPoint, 0, pointSize, archive.offset, function(err, bytesRead, buffer) {
var baseInterval, baseValue, byteDistance, myOffset, pointDistance, timeDistance, _ref2;
if (err) {
cb(err);
}
_ref2 = pack.Unpack(pointFormat, packedPoint), baseInterval = _ref2[0], baseValue = _ref2[1];
if (baseInterval === 0) {
return fs.write(fd, myPackedPoint, 0, pointSize, archive.offset, function(err, written, buffer) {
var _ref3;
if (err) {
cb(err);
}
_ref3 = [myInterval, value], baseInterval = _ref3[0], baseValue = _ref3[1];
return propagateLowerArchives();
});
} else {
timeDistance = myInterval - baseInterval;
pointDistance = timeDistance / archive.secondsPerPoint;
byteDistance = pointDistance * pointSize;
myOffset = archive.offset + byteDistance.mod(archive.size);
return fs.write(fd, myPackedPoint, 0, pointSize, myOffset, function(err, written, buffer) {
if (err) {
cb(err);
}
return propagateLowerArchives();
});
}
});
return propagateLowerArchives = function() {
return fs.close(fd, cb);
};
});
});
};
updateMany = function(filename, points, cb) {
points.sort(function(a, b) {
return a[0] - b[0];
}).reverse();
return info(filename, function(err, header) {
if (err) {
cb(err);
}
return fs.open(filename, 'r+', function(err, fd) {
var age, archives, currentArchive, currentArchiveIndex, currentPoints, now, point, updateArchiveCalls, _i, _len;
now = unixTime();
archives = header.archives;
currentArchiveIndex = 0;
currentArchive = header.archives[currentArchiveIndex];
currentPoints = [];
updateArchiveCalls = [];
for (_i = 0, _len = points.length; _i < _len; _i++) {
point = points[_i];
age = now - point[0];
while (currentArchive.retention < age) {
if (currentPoints) {
currentPoints.reverse();
(function(header, currentArchive, currentPoints) {
var f;
f = function(cb) {
return updateManyArchive(fd, header, currentArchive, currentPoints, cb);
};
return updateArchiveCalls.push(f);
})(header, currentArchive, currentPoints);
currentPoints = [];
}
if (currentArchiveIndex < (archives.length - 1)) {
currentArchiveIndex++;
currentArchive = archives[currentArchiveIndex];
} else {
currentArchive = null;
break;
}
}
if (!currentArchive) {
break;
}
currentPoints.push(point);
}
return async.series(updateArchiveCalls, function(err, results) {
if (err) {
throw err;
}
if (currentArchive && currentPoints.length > 0) {
currentPoints.reverse();
return updateManyArchive(fd, header, currentArchive, currentPoints, function(err) {
if (err) {
throw err;
}
return fs.close(fd, cb);
});
} else {
return fs.close(fd, cb);
}
});
});
});
};
updateManyArchive = function(fd, header, archive, points, cb) {
var alignedPoints, ap, currentString, interval, numberOfPoints, p, packedBasePoint, packedStrings, previousInterval, startInterval, step, timestamp, value, _i, _j, _len, _len2;
step = archive.secondsPerPoint;
alignedPoints = [];
for (_i = 0, _len = points.length; _i < _len; _i++) {
p = points[_i];
timestamp = p[0], value = p[1];
alignedPoints.push([timestamp - timestamp.mod(step), value]);
}
packedStrings = [];
previousInterval = null;
currentString = [];
for (_j = 0, _len2 = alignedPoints.length; _j < _len2; _j++) {
ap = alignedPoints[_j];
interval = ap[0], value = ap[1];
if (!previousInterval || (interval === previousInterval + step)) {
currentString.concat(pack.Pack(pointFormat, [interval, value]));
previousInterval = interval;
} else {
numberOfPoints = currentString.length / pointSize;
startInterval = previousInterval - (step * (numberOfPoints - 1));
packedStrings.push([startInterval, new Buffer(currentString)]);
currentString = pack.Pack(pointFormat, [interval, value]);
previousInterval = interval;
}
}
if (currentString.length > 0) {
numberOfPoints = currentString.length / pointSize;
startInterval = previousInterval - (step * (numberOfPoints - 1));
packedStrings.push([startInterval, new Buffer(currentString, 'binary')]);
}
packedBasePoint = new Buffer(pointSize);
return fs.read(fd, packedBasePoint, 0, pointSize, archive.offset, function(err) {
var baseInterval, baseValue, propagateLowerArchives, writePackedString, _ref;
if (err) {
cb(err);
}
_ref = pack.Unpack(pointFormat, packedBasePoint), baseInterval = _ref[0], baseValue = _ref[1];
if (baseInterval === 0) {
baseInterval = packedStrings[0][0];
}
writePackedString = function(ps, callback) {
var archiveEnd, byteDistance, bytesBeyond, myOffset, packedString, pointDistance, timeDistance;
interval = ps[0], packedString = ps[1];
timeDistance = interval - baseInterval;
pointDistance = timeDistance / step;
byteDistance = pointDistance * pointSize;
myOffset = archive.offset + byteDistance.mod(archive.size);
archiveEnd = archive.offset + archive.size;
bytesBeyond = (myOffset + packedString.length) - archiveEnd;
if (bytesBeyond > 0) {
return fs.write(fd, packedString, 0, packedString.length - bytesBeyond, myOffset, function(err) {
if (err) {
cb(err);
}
assert.equal(archiveEnd, myOffset + packedString.length - bytesBeyond);
return fs.write(fd, packedString, packedString.length - bytesBeyond, bytesBeyond, archive.offset, function(err) {
if (err) {
cb(err);
}
return callback();
});
});
} else {
return fs.write(fd, packedString, 0, packedString.length, myOffset, function(err) {
return callback();
});
}
};
async.forEachSeries(packedStrings, writePackedString, function(err) {
if (err) {
throw err;
}
return propagateLowerArchives();
});
return propagateLowerArchives = function() {
var arc, callPropagate, fit, higher, interval, lower, lowerArchives, lowerIntervals, p, propagateCalls, uniqueLowerIntervals, _k, _l, _len3, _len4;
higher = archive;
lowerArchives = (function() {
var _k, _len3, _ref2, _results;
_ref2 = header.archives;
_results = [];
for (_k = 0, _len3 = _ref2.length; _k < _len3; _k++) {
arc = _ref2[_k];
if (arc.secondsPerPoint > archive.secondsPerPoint) {
_results.push(arc);
}
}
return _results;
})();
if (lowerArchives.length > 0) {
propagateCalls = [];
for (_k = 0, _len3 = lowerArchives.length; _k < _len3; _k++) {
lower = lowerArchives[_k];
fit = function(i) {
return i - i.mod(lower.secondsPerPoint);
};
lowerIntervals = (function() {
var _l, _len4, _results;
_results = [];
for (_l = 0, _len4 = alignedPoints.length; _l < _len4; _l++) {
p = alignedPoints[_l];
_results.push(fit(p[0]));
}
return _results;
})();
uniqueLowerIntervals = _.uniq(lowerIntervals);
for (_l = 0, _len4 = uniqueLowerIntervals.length; _l < _len4; _l++) {
interval = uniqueLowerIntervals[_l];
propagateCalls.push({
interval: interval,
header: header,
higher: higher,
lower: lower
});
}
higher = lower;
}
callPropagate = function(args, callback) {
return propagate(fd, args.interval, args.header.xFilesFactor, args.higher, args.lower, function(err, result) {
if (err) {
cb(err);
}
return callback(err, result);
});
};
return async.forEachSeries(propagateCalls, callPropagate, function(err, result) {
if (err) {
throw err;
}
return cb(null);
});
} else {
return cb(null);
}
};
});
};
info = function(path, cb) {
fs.readFile(path, function(err, data) {
var archives, metadata;
if (err) {
cb(err);
}
archives = [];
metadata = {};
return Binary.parse(data).word32bu('lastUpdate').word32bu('maxRetention').buffer('xff', 4).word32bu('archiveCount').tap(function(vars) {
var index, _ref, _results;
metadata = vars;
metadata.xff = pack.Unpack('!f', vars.xff, 0)[0];
this.flush();
_results = [];
for (index = 0, _ref = metadata.archiveCount; 0 <= _ref ? index < _ref : index > _ref; 0 <= _ref ? index++ : index--) {
this.word32bu('offset').word32bu('secondsPerPoint').word32bu('points');
_results.push(this.tap(function(archive) {
this.flush();
archive.retention = archive.secondsPerPoint * archive.points;
archive.size = archive.points * pointSize;
return archives.push(archive);
}));
}
return _results;
}).tap(function() {
return cb(null, {
maxRetention: metadata.maxRetention,
xFilesFactor: metadata.xff,
archives: archives
});
});
});
};
fetch = function(path, from, to, cb) {
info(path, function(err, header) {
var archive, diff, fd, file, fromInterval, now, oldestTime, toInterval, unpack, _i, _len, _ref;
now = unixTime();
oldestTime = now - header.maxRetention;
if (from < oldestTime) {
from = oldestTime;
}
if (!(from < to)) {
throw new Error('Invalid time interval');
}
if (to > now || to < from) {
to = now;
}
diff = now - from;
fd = null;
_ref = header.archives;
for (_i = 0, _len = _ref.length; _i < _len; _i++) {
archive = _ref[_i];
if (archive.retention >= diff) {
break;
}
}
fromInterval = parseInt(from - from.mod(archive.secondsPerPoint)) + archive.secondsPerPoint;
toInterval = parseInt(to - to.mod(archive.secondsPerPoint)) + archive.secondsPerPoint;
file = fs.createReadStream(path);
Binary.stream(file).skip(archive.offset).word32bu('baseInterval').word32bu('baseValue').tap(function(vars) {
var fromOffset, getOffset, n, points, step, timeInfo, toOffset, values;
if (vars.baseInterval === 0) {
step = archive.secondsPerPoint;
points = (toInterval - fromInterval) / step;
timeInfo = [fromInterval, toInterval, step];
values = (function() {
var _results;
_results = [];
for (n = 0; 0 <= points ? n < points : n > points; 0 <= points ? n++ : n--) {
_results.push(null);
}
return _results;
})();
return cb(null, timeInfo, values);
} else {
getOffset = function(interval) {
var a, byteDistance, pointDistance, timeDistance;
timeDistance = interval - vars.baseInterval;
pointDistance = timeDistance / archive.secondsPerPoint;
byteDistance = pointDistance * pointSize;
a = archive.offset + byteDistance.mod(archive.size);
return a;
};
fromOffset = getOffset(fromInterval);
toOffset = getOffset(toInterval);
return fs.open(path, 'r', function(err, fd) {
var archiveEnd, seriesBuffer, size, size1, size2;
if (err) {
throw err;
}
if (fromOffset < toOffset) {
size = toOffset - fromOffset;
seriesBuffer = new Buffer(size);
return fs.read(fd, seriesBuffer, 0, size, fromOffset, function(err, num) {
if (err) {
cb(err);
}
return fs.close(fd, function(err) {
if (err) {
cb(err);
}
return unpack(seriesBuffer);
});
});
} else {
archiveEnd = archive.offset + archive.size;
size1 = archiveEnd - fromOffset;
size2 = toOffset - archive.offset;
seriesBuffer = new Buffer(size1 + size2);
return fs.read(fd, seriesBuffer, 0, size1, fromOffset, function(err, num) {
if (err) {
cb(err);
}
return fs.read(fd, seriesBuffer, size1, size2, archive.offset, function(err, num) {
if (err) {
cb(err);
}
unpack(seriesBuffer);
return fs.close(fd);
});
});
}
});
}
});
return unpack = function(seriesData) {
var currentInterval, f, i, numPoints, pointTime, pointValue, seriesFormat, step, timeInfo, unpackedSeries, valueList, _ref2, _step;
numPoints = seriesData.length / pointSize;
seriesFormat = "!" + ((function() {
var _results;
_results = [];
for (f = 0; 0 <= numPoints ? f < numPoints : f > numPoints; 0 <= numPoints ? f++ : f--) {
_results.push('Ld');
}
return _results;
})()).join("");
unpackedSeries = pack.Unpack(seriesFormat, seriesData);
valueList = (function() {
var _results;
_results = [];
for (f = 0; 0 <= numPoints ? f < numPoints : f > numPoints; 0 <= numPoints ? f++ : f--) {
_results.push(null);
}
return _results;
})();
currentInterval = fromInterval;
step = archive.secondsPerPoint;
for (i = 0, _ref2 = unpackedSeries.length, _step = 2; 0 <= _ref2 ? i < _ref2 : i > _ref2; i += _step) {
pointTime = unpackedSeries[i];
if (pointTime === currentInterval) {
pointValue = unpackedSeries[i + 1];
valueList[i / 2] = pointValue;
}
currentInterval += step;
}
timeInfo = [fromInterval, toInterval, step];
return cb(null, timeInfo, valueList);
};
});
};
exports.create = create;
exports.update = update;
exports.updateMany = updateMany;
exports.info = info;
exports.fetch = fetch;