qminer
Version:
A C++ based data analytics platform for processing large-scale real-time streams containing structured and unstructured data
1,129 lines • 115 kB
JavaScript
/**
* Copyright (c) 2015, Jozef Stefan Institute, Quintelligence d.o.o. and contributors
* All rights reserved.
*
* This source code is licensed under the FreeBSD license found in the
* LICENSE file in the root directory of this source tree.
*/
/**
* QMiner module.
* @module qm
* @example
* // import module
* var qm = require('qminer');
*/
/**
* Stream Aggregate
* @classdesc Represents a stream aggregate. The class can construct these {@link module:qm~StreamAggregator} objects. Also turn to these stream aggregators to see
* which methods are implemented.
* @class
* @param {module:qm.Base} base - The base object on which it's created.
* @param {(module:qm~StreamAggregator | function)} arg - Constructor arguments. There are two argument types:
* <br>1. Using the {@link module:qm~StreamAggregator} object,
* <br>2. using a function/JavaScript class. The function has defined The object containing the schema of the stream aggregate or the function object defining the operations of the stream aggregate.
* @param {(string | Array.<string>)} [storeName] - The store names where the aggregate will be registered.
* @example
* // import qm module
* var qm = require('qminer');
* // create a simple base containing one store
* var base = new qm.Base({
* mode: "createClean",
* schema: [{
* name: "People",
* fields: [
* { name: "Name", type: "string" },
* { name: "Gendre", type: "string" },
* ]
* },
* {
* name: "Laser",
* fields: [
* { name: "Time", type: "datetime" },
* { name: "WaveLength", type: "float" }
* ]
* }]
* });
*
* // create a new stream aggregator for 'People' store, get the length of the record name (with the function object)
* var aggr = new qm.StreamAggr(base, new function () {
* var numOfAdds = 0;
* var numOfUpdates = 0;
* var numOfDeletes = 0;
* var time = "";
* this.name = 'nameLength',
* this.onAdd = function (rec) {
* numOfAdds += 1;
* };
* this.onUpdate = function (rec) {
* numOfUpdates += 1;
* };
* this.onDelete = function (rec) {
* numOfDeletes += 1;
* };
* this.onTime = function (ts) {
* time = ts;
* };
* this.saveJson = function (limit) {
* return { adds: numOfAdds, updates: numOfUpdates, deletes: numOfDeletes, time: time };
* };
* }, "People");
*
* // create a new time series window buffer stream aggregator for 'Laser' store (with the JSON object)
* var wavelength = {
* name: "WaveLengthLaser",
* type: "timeSeriesWinBuf",
* store: "Laser",
* timestamp: "Time",
* value: "WaveLength",
* winsize: 10000
* }
* var sa = base.store("Laser").addStreamAggr(wavelength);
* base.close();
*/
exports.StreamAggr = function (base, json, storeName) { return Object.create(require('qminer').StreamAggr.prototype); };
/**
* @typedef {module:qm.StreamAggr} StreamAggregator
* Stream aggregator types.
* @property {module:qm~StreamAggrTimeSeriesWindow} timeSeries - The time series type.
* @property {module:qm~StreamAggrTimeSeriesWindowVector} timeSeriesBufferVector - The time series buffer vector type.
* @property {module:qm~StreamAggrTimeSeriesTick} tick - The time series tick type.
* @property {module:qm~StreamAggrRecordBuffer} record-buffer - The record buffer type.
* @property {module:qm~StreamAggrFeatureSpace} ftr-space - The feature space type.
* @property {module:qm~StreamAggrSum} sum - The sum type. Calculates the sum of values.
* @property {module:qm~StreamAggrMin} min - The minimal type. Saves the minimal value in the buffer.
* @property {module:qm~StreamAggrMax} max - The maximal type. Saves the maximal value in the buffer.
* @property {module:qm~StreamAggrSparseVecSum} sparse-vec-sum - The sparse-vector-sum type.
* @property {module:qm~StreamAggrMovingAverage} ma - The moving average type. Calculates the average within the window.
* @property {module:qm~StreamAggrEMA} ema - The exponental moving average type. Calculates the exponental average of the values.
* @property {module:qm~StreamAggrEMASpVec} ema-sp-vec - The exponental moving average for sparse vectors type.
* @property {module:qm~StreamAggrMovingVariance} var - The moving variance type. Calculates the variance of values within the window.
* @property {module:qm~StreamAggrMovingCovariance} cov - The moving covariance type. Calculates the covariance of values within the window.
* @property {module:qm~StreamAggrMovingCorrelation} cor - The moving correlation type. Calculates the correlation of values within the window.
* @property {module:qm~StreamAggrResampler} res - The resampler type. Resamples the records so that they come in in the same time interval.
* @property {module:qm~StreamAggrAggrResampler} aggr-res - The aggregating (avg/sum) resampler type. Resamplers the records so that it takes the
* records in the time window and returns one sample.
* @property {module:qm~StreamAggrMerger} mer - The merger type. Merges the records from two stream series.
* @property {module:qm~StreamAggrHistogram} hist - The online histogram type.
* @property {module:qm~StreamAggrSlottedHistogram} slotted-hist - The online slotted-histogram type.
* @property {module:qm~StreamAggrVecDiff} vec-diff - The difference of two vectors (e.g. online histograms) type.
* @property {module:qm~StreamAggrSimpleLinearRegression} lin-reg - The linear regressor type.
* @property {module:qm~StreamAggrAnomalyDetectorNN} detector-nn - The anomaly detector type. Detects anomalies using the k nearest neighbour algorithm.
* @property {module:qm~StreamAggrThreshold} treshold - The threshold indicator type.
* @property {module:qm~StreamAggrTDigest} tdigest - The quantile estimator type. It estimates the quantiles of the given data using {@link module:analytics.TDigest TDigest}.
* @property {module:qm~StreamAggrRecordSwitch} record-switch-aggr - The record switch type.
* @property {module:qm~StreamAggrPageHinkley} pagehinkley - The Page-Hinkley test for concept drift detection type.
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrTimeSeriesWindow
* This stream aggregator represents the time series window buffer. It stores the values inside a moving window.
* It implements all the stream aggregate methods <b>except</b> {@link module:qm.StreamAggr#getFloat} and {@link module:qm.StreamAggr#getTimestamp}.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'timeSeriesWinBuf'`.
* @property {string} store - The name of the store from which to takes the data.
* @property {string} timestamp - The field of the store, where it takes the timestamp.
* @property {string} value - The field of the store, where it takes the values.
* @property {number} winsize - The size of the window, in milliseconds.
* @property {number} delay - Delay in milliseconds.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window is 2 seconds (2000ms).
* var aggr = {
* name: 'TimeSeriesAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius',
* winsize: 2000
* };
* base.store("Heat").addStreamAggr(aggr);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrTimeSeriesWindowVector
* This stream aggregator represents the values read from a time series window buffer.
* It implements {@link module:qm.StreamAggr#getFloatVector}, {@link module:qm.StreamAggr#getFloatAt} and {@link module:qm.StreamAggr#getFloatLength}.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'timeSeriesWinBuf'`.
* @property {string} store - The name of the store from which to takes the data.
* @property {string} inAggr - The name of the window buffer aggregate that represents the input.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [{
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* var store = base.store("Heat");
* var tick = store.addStreamAggr({
* type: 'timeSeriesTick',
* timestamp: 'Time',
* value: 'Celsius'
* });
*
* var winbufvec = store.addStreamAggr({
* type: 'timeSeriesWinBufVector',
* inAggr: tick.name,
* winsize: 2000
* });
*
* store.push({ Time: '2015-06-10T14:13:32.0', Celsius: 1 });
* winbufvec.getFloatVector().print(); // prints 1
* store.push({ Time: '2015-06-10T14:33:30.0', Celsius: 2 });
* winbufvec.getFloatVector().print(); // prints 2
* store.push({ Time: '2015-06-10T14:33:31.0', Celsius: 3 });
* winbufvec.getFloatVector().print(); // prints 2,3
* store.push({ Time: '2015-06-10T14:33:32.0', Celsius: 4 });
* winbufvec.getFloatVector().print(); // prints 2,3,4
*
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrRecordBuffer
* This stream aggregator represents record buffer. It stores the values inside a moving window.
* It implements all the stream aggregate methods <b>except</b> {@link module:qm.StreamAggr#getFloat} and {@link module:qm.StreamAggr#getTimestamp}.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'recordBuffer'`.
* @property {number} size - The size of the window.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store. The size of the window is 3 records.
* var aggr = {
* name: 'Delay',
* type: 'recordBuffer',
* size: 3
* };
* base.store("Heat").addStreamAggr(aggr);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrSum
* This stream aggregator represents the sum moving window buffer. It sums all the values, that are in the connected stream aggregator.
* It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the sum of the values of the records in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'winBufSum'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Income",
* fields: [
* { name: "Amount", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Income' store, that takes the values from the 'Amount' field
* // and the timestamp from the 'Time' field. The size of the window should 1 week.
* var timeser = {
* name: 'TimeSeriesAggr',
* type: 'timeSeriesWinBuf',
* store: 'Income',
* timestamp: 'Time',
* value: 'Amount',
* winsize: 604800000 // 7 days in miliseconds
* };
* var timeSeries = base.store("Income").addStreamAggr(timeser);
*
* // add a sum aggregator, that is connected with the 'TimeSeriesAggr' aggregator
* var sum = {
* name: 'SumAggr',
* type: 'winBufSum',
* store: 'Heat',
* inAggr: 'TimeSeriesAggr'
* };
* var sumAggr = base.store("Income").addStreamAggr(sum);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrMin
* This stream aggregator represents the minimum moving window buffer. It monitors the minimal value in the connected stream aggregator.
* It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the minimal value of the records in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'winBufMin'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window is 1 day.
* var timeser = {
* name: 'TimeSeriesAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius',
* winsize: 86400000 // 1 day in miliseconds
* };
* var timeSeries = base.store("Heat").addStreamAggr(timeser);
*
* // add a min aggregator, that is connected with the 'TimeSeriesAggr' aggregator
* var min = {
* name: 'MinAggr',
* type: 'winBufMin',
* store: 'Heat',
* inAggr: 'TimeSeriesAggr'
* };
* var minimal = base.store("Heat").addStreamAggr(min);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrMax
* This stream aggregator represents the maximum moving window buffer. It monitors the maximal value in the connected stream aggregator.
* It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the maximal value of the records in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type for the stream aggregator. <b>Important:</b> It must be equal to `'winBufMax'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window is 1 day.
* var timeser = {
* name: 'TimeSeriesAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius',
* winsize: 86400000 // one day in miliseconds
* };
* var timeSeries = base.store("Heat").addStreamAggr(timeser);
*
* // add a max aggregator, that is connected with the 'TimeSeriesAggr' aggregator
* var max = {
* name: 'MaxAggr',
* type: 'winBufMax',
* store: 'Heat',
* inAggr: 'TimeSeriesAggr'
* };
* var maximal = base.store("Heat").addStreamAggr(max);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrSparseVecSum
* This stream aggregator represents the sparse-vector-sum moving window buffer. It sums all the sparse-vector values, that are in the connected stream aggregator.
* It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getValueVector} returns the sum of the values of the records in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'winBufSpVecSum'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* @example
* var qm = require('qminer');
* var base = new qm.Base({
* mode: 'createClean',
* schema: [{
* name: 'Docs',
* fields: [
* { name: 'Time', type: 'datetime' },
* { name: 'Text', type: 'string' }
* ]
* }]
* });
* var store = base.store('Docs');
*
* var aggr = {
* name: 'featureSpaceWindow',
* type: 'timeSeriesWinBufFeatureSpace',
* store: 'Docs',
* timestamp: 'Time',
* featureSpace: {
* type: "categorical",
* source: "Docs",
* field: "Text"
* },
* winsize: 1000
* };
* var sa = store.addStreamAggr(aggr);
*
* var aggr2 = {
* name: 'sparseVectorSum',
* type: 'winBufSpVecSum',
* store: 'Docs',
* inAggr: 'featureSpaceWindow'
* };
* var sa2 = store.addStreamAggr(aggr2);
*
* store.push({ Time: '2015-06-10T14:13:32.0', Text: 'a' }); // 0
* store.push({ Time: '2015-06-10T14:13:33.0', Text: 'b' }); // 1
* store.push({ Time: '2015-06-10T14:14:34.0', Text: 'c' }); // 2
* store.push({ Time: '2015-06-10T14:15:35.0', Text: 'd' }); // 3
* store.push({ Time: '2015-06-10T14:15:36.0', Text: 'e' }); // 4
* store.push({ Time: '2015-06-10T14:15:37.0', Text: 'f' }); // 5
*
* var valVec2 = sa2.getValueVector(); // [0, 0, 0, 0, 1, 1] - only vectors 4 and 5 remain in window
*
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrTimeSeriesTick
* This stream aggregator represents the time series tick window buffer. It exposes the data to other stream aggregators
* (similar to {@link module:qm~StreamAggrTimeSeriesWindow}). It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the last value added in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'timeSeriesTick'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} value - The name of the store field, from which it takes the values.
* @property {string} timestamp - The name of the store field, from which it takes the timestamp.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Students",
* fields: [
* { name: "Id", type: "float" },
* { name: "TimeOfGraduation", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series tick stream aggregator for the 'Students' store, that takes the values from the 'Id' field
* // and the timestamp from the 'TimeOfGraduation' field.
* var tick = {
* name: 'TimeSeriesTickAggr',
* type: 'timeSeriesTick',
* store: 'Students',
* timestamp: 'TimeOfGraduation',
* value: 'Id',
* };
* var timeSeriesTick = base.store("Students").addStreamAggr(tick);
* base.close();
*/
/**
* @typedef {module:qmStreamAggr} StreamAggrMovingAverage
* This stream aggregator represents the moving average window buffer. It calculates the moving average value of the connected stream aggregator values.
* It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the average of the values in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name of the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'ma'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window should be 1 day.
* var timeser = {
* name: 'TimeSeriesAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius',
* winsize: 86400000
* };
* var timeSeries = base.store("Heat").addStreamAggr(timeser);
*
* // add a moving average aggregator, that is connected with the 'TimeSeriesAggr' aggregator
* var ma = {
* name: 'movingAverageAggr',
* type: 'ma',
* store: 'Heat',
* inAggr: 'TimeSeriesAggr'
* };
* var movingAverage = base.store("Heat").addStreamAggr(ma);
* base.close();
*/
/**
* @typedef {module:qmStreamAggr} StreamAggrEMA
* This stream aggregator represents the exponential moving average window buffer. It calculates the weighted moving average
* of the values in the connected stream aggregator, where the weights are exponentially decreasing. It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the exponentional average of the values in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'ema'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* It <b>cannot</b> be connect to the {@link module:qm~StreamAggrTimeSeriesWindow}.
* @property {string} emaType - The type of interpolation. Possible options are:
* <br>1. `'previous'` - Interpolates with the previous value.
* <br>2. `'next'` - Interpolates with the next value.
* <br>3. `'linear'` - Makes a linear interpolation.
* @property {number} interval - The time interval defining the decay. It must be greater than `initWindow`.
* @property {number} initWindow - The time window of required values for initialization.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series tick stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window should be 1 hour.
* var timeser = {
* name: 'TimeSeriesAggr',
* type: 'timeSeriesTick',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius'
* };
* var timeSeries = base.store("Heat").addStreamAggr(timeser);
*
* // add an exponentional moving average aggregator, that is connected with the 'TimeSeriesAggr' aggregator.
* // It should interpolate with the previous value, the decay should be 3 seconds and the initWindow should be 2 seconds.
* var ema = {
* name: 'emaAggr',
* type: 'ema',
* store: 'Heat',
* inAggr: 'TimeSeriesAggr',
* emaType: 'previous',
* interval: 3000,
* initWindow: 2000
* };
* var expoMovingAverage = base.store("Heat").addStreamAggr(ema);
* base.close();
*/
/**
* @typedef {module:qmStreamAggr} StreamAggrThreshold
* This stream aggregator represents a threshold indicator. It outputs 1 if the current value in the data streams is
* above the threshold and 0 otherwise. It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the exponentional average of the values in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'ema'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* It <b>cannot</b> be connect to the {@link module:qm~StreamAggrTimeSeriesWindow}.
* @property {string} threshold - The threshold mentioned above.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series tick stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window should be 1 hour.
* var timeser = {
* name: 'TimeSeriesTickAggr',
* type: 'timeSeriesTick',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius'
* };
* var timeSeries = base.store("Heat").addStreamAggr(timeser);
*
* // a threshold aggregator, that is connected wit hthe 'TimeSeriesAggr' aggregator.
* // It should output 1 when the temperature is over 3 degrees Celsius and 0 otherwise
* var thresholdAggr = {
* name: 'thresholdAggr1',
* type: 'threshold',
* store: 'Heat',
* inAggr: 'TimeSeriesTickAggr',
* threshold: 3
* };
* var expoMovingAverage = base.store("Heat").addStreamAggr(thresholdAggr);
* base.close();
*/
/**
* @typedef {module:qmStreamAggr} StreamAggrEMASpVec
* This stream aggregator represents the exponential moving average window buffer for sparse vectors. It calculates the weighted moving average
* of the values in the connected stream aggregator, where the weights are exponentially decreasing. It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getValueVector} returns the exponentional average of the sparse vector values in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'ema'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* It <b>cannot</b> be connect to the {@link module:qm~StreamAggrTimeSeriesWindow}.
* @property {string} emaType - The type of interpolation. Possible options:
* <br>1. `'previous'` - Interpolates with the previous value.
* <br>2. `'next'` - Interpolates with the next value.
* <br>3. `'linear'` - Makes a linear interpolation.
* @property {number} interval - The time interval defining the decay. It must be greater than `initWindow`.
* @property {number} [initWindow=0] - The time window of required values for initialization.
* @property {number} [cuttof=0.001] - Minimal value for any dimension. If value of certain dimension falls bellow this value, the dimension is pruned from average.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Data",
* fields: [
* { name: "Text", type: "string" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
* var store = base.store("Data");
* // create a new time series that emits data as sparse vector, based on text
* var aggr = {
* name: 'featureSpaceWindow',
* type: 'timeSeriesWinBufFeatureSpace',
* store: store.name,
* timestamp: 'Time',
* featureSpace: {
* type: "categorical",
* source: store.name,
* field: "Text"
* },
* winsize: 1 // keep only most recent value in window
* };
* // attach sum
* var sa = store.addStreamAggr(aggr);
* var aggr2 = {
* name: 'sparseVectorSum',
* type: 'winBufSpVecSum',
* store: store.name,
* inAggr: aggr.name // this means that sum is equal to the most recent data
* };
* // ok, now attach EMA
* var sa2 = store.addStreamAggr(aggr2);
* var ema_def = {
* name: 'sparseVectorEma',
* type: 'emaSpVec',
* store: store.name,
* inAggr: aggr2.name,
* emaType: "next",
* interval: 2000,
* initWindow: 0
* };
* var ema = store.addStreamAggr(ema_def);
* // add some data
* store.push({ Time: 1000, Text: 'a' });
* store.push({ Time: 2000, Text: 'b' });
* store.push({ Time: 3000, Text: 'c' });
* // display EMA data
* ema.getValueVector().print();
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrMovingVariance
* This stream aggregator represents the moving variance window buffer. It calculates the moving variance of the stream aggregator, that it's connected to.
* It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the variance of the values in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'variance'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggr - The name of the stream aggregator to which it connects and gets data.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window is 1 day
* var timeser = {
* name: 'TimeSeriesAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius',
* winsize: 86400000
* };
* var timeSeries = base.store("Heat").addStreamAggr(timeser);
*
* // add a variance aggregator, that is connected with the 'TimeSeriesAggr' aggregator
* var variance = {
* name: 'varAggr',
* type: 'variance',
* store: 'Heat',
* inAggr: 'TimeSeriesAggr'
* };
* var varianceAggr = base.store("Heat").addStreamAggr(variance);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrMovingCovariance
* This stream aggregator represents the moving covariance window buffer. It calculates the moving covariance of the two stream aggregators, that it's connected to.
* It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the covariance of the values in its buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in its buffer window.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'covariance'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggrX - The name of the first stream aggregator to which it connects and gets data.
* @property {string} inAggrY - The name of the recond stream aggregator to which it connects and gets data.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "WaterConsumption", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window is 1 day.
* var Celsius = {
* name: 'CelsiusAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius',
* winsize: 86400000
* }; base.store("Heat").addStreamAggr(Celsius);
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'WaterConsumption' field
* // and the timestamp from the 'Time' field. The size of the window is 1 day.
* var water = {
* name: 'WaterAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'WaterConsumption',
* winsize: 86400000
* }; base.store("Heat").addStreamAggr(water);
*
* // add a covariance aggregator, that is connected with the 'CelsiusAggr' and 'WaterAggr' stream aggregators
* var covariance = {
* name: 'covAggr',
* type: 'covariance',
* store: 'Heat',
* inAggrX: 'CelsiusAggr',
* inAggrY: 'WaterAggr'
* };
* var covarianceAggr = base.store("Heat").addStreamAggr(covariance);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrMovingCorrelation
* This stream aggregator represents the moving covariance window buffer. It calculates the moving correlation of the three stream aggregators,
* that it's connected to. It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the correlation of the values in it's buffer window.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the newest record in it's buffer window.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'correlation'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} inAggrCov - The name of the covariance stream aggregator.
* @property {string} inAggrVarX - The name of the first variance stream aggregator.
* @property {string} inAggrVarY - The name of the second variance stream aggregator.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "WaterConsumption", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The size of the window is 1 day
* var Celsius = {
* name: 'CelsiusAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'Celsius',
* winsize: 86400000
* }; base.store("Heat").addStreamAggr(Celsius);
*
* // create a new time series stream aggregator for the 'Heat' store, that takes the values from the 'WaterConsumption' field
* // and the timestamp from the 'Time' field. The size of the window is 1 day
* var water = {
* name: 'WaterAggr',
* type: 'timeSeriesWinBuf',
* store: 'Heat',
* timestamp: 'Time',
* value: 'WaterConsumption',
* winsize: 86400000
* }; base.store("Heat").addStreamAggr(water);
*
* // add a covariance aggregator, that is connected with the 'CelsiusAggr' and 'WaterAggr' aggregators
* var covariance = {
* name: 'covarianceAggr',
* type: 'covariance',
* store: 'Heat',
* inAggrX: 'CelsiusAggr',
* inAggrY: 'WaterAggr'
* }; base.store("Heat").addStreamAggr(covariance);
*
* // add the two variance aggregators, that take from the 'Celsius' and 'WaterConsumption' fields, respectively
* var celVar = {
* name: 'CelsiusVarAggr',
* type: 'variance',
* store: 'Heat',
* inAggr: 'CelsiusAggr'
* }; base.store("Heat").addStreamAggr(celVar);
*
* var waterVar = {
* name: 'waterVarAggr',
* type: 'variance',
* store: 'Heat',
* inAggr: 'WaterAggr'
* }; base.store("Heat").addStreamAggr(waterVar);
*
* // add a correlation aggregator, that is connected to 'CovarianceAggr', 'CelsiusVarAggr' and 'WaterValAggr' aggregators
* var corr = {
* name: 'corrAggr',
* type: 'correlation',
* store: 'Heat',
* inAggrCov: 'covarianceAggr',
* inAggrVarX: 'CelsiusVarAggr',
* inAggrVarY: 'waterVarAggr'
* };
* var correlation = base.store("Heat").addStreamAggr(corr);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrResampler
* This stream aggregator represents the resampler window buffer. It creates new values that are interpolated by using the values from an existing store.
* No methods are implemented for this aggregator.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'resampler'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {string} outStore - The store in which the samples are stored.
* @property {string} timestamp - The store field from which it takes the timestamps.
* @property {Array.<Object>} fields - The array off `field` objects from which it takes the values. The `field` object contain the properties:
* <br>`field.name` - The store field from which it takes the values. Type `string`.
* <br>`field.interpolator` - The type of the interpolation. The options are `'previous'`, `'next'` and `'linear'`. Type `string`.
* @property {boolean} createStore - If true, `outStore` must be created.
* @property {number} interval - The interval size. The frequency on which it makes the interpolated values.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Heat",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* },
* {
* name: "interpolatedValues",
* fields: [
* { name: "Value", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
* // create a new resampler stream aggregator for the 'Heat' store, that takes the values from the 'Celsius' field
* // and the timestamp from the 'Time' field. The interpolated values are stored in the 'interpolatedValues' store.
* // The interpolation should be linear and the interval should be 2 seconds
* var res = {
* name: 'resamplerAggr',
* type: 'resampler',
* store: 'Heat',
* outStore: 'interpolatedValues',
* timestamp: 'Time',
* fields: [{
* name: 'Celsius',
* interpolator: 'linear'
* }],
* createStore: false,
* interval: 2000
* };
* var resampler = base.store("Heat").addStreamAggr(res);
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrAggrResampler
* This stream aggregate resamples an input time series to a new time seris
* of equally spaced measurements. Each new measurement corresponds to an
* aggregate (sum,avg,min,max) computed over an interval. The aggregate
* exposes the following methods.
* <br>1. {@link module:qm.StreamAggr#getFloat} returns the last resampled value.
* <br>2. {@link module:qm.StreamAggr#getTimestamp} returns the timestamp of the last resampled value.
* <br>3. {@link module:qm.StreamAggr#onStep} reads from an input aggregate and tries to resample.
* <br>4. {@link module:qm.StreamAggr#onTime} updates the current time (no data has arrived, but time has passed) and tries to resample.
* <br>5. {@link module:qm.StreamAggr#getParams} returns a parameter object.
* <br>6. {@link module:qm.StreamAggr#setParams} used primarily for setting the out-aggregate.
* <br>The stream aggregate exposes its results through `getFloat` and `getTimestamp` methods (itself represents timeseries).
* The resampler has an input time-series aggregate (supports `getFloat` and `getTimestamp`), from where it reads time series values.
* The reading and resampling occourrs wehen resamplers `onStep()` or `onTime()` methods are called.
* When resampling succeeds (all the data needed for the computation becomes available), the resampler
* will trigger the `onStep()` method of an output stream aggregate that will read the resamplers state through `getFloat` and `getTime`.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'aggrResampler'`.
* @property {number} interval - Interval size in milliseconds
* @property {string} aggType - Must be one of the values: `"sum"`, `"avg"`, `"min"` or `"max"` - represents the function executed on the data values in the interval.
* @property {(string | module:qm.StreamAggr)} inAggr - The name of the input stream aggregate which must implement `getFloat()` and `getTimestamp()` methods.
* @property {(string | number)} [start] - Start time (linux timestamp or a web log date string like `1970-01-01T00:00:00.000`)
* @property {string} [roundStart] - Must be one of the values: `'h'`, `'m'` or `'s'` - represents rounding of the start time when it must be determined by the first observed record. `'h'` will clip minutes, seconds and milliseconds, `'m'` will clip seconds and milliseconds and `'s'` will clip only milliseconds.
* @property {number} [defaultValue=0] - default value for empty intervals (no data available).
* @property {boolean} [skipEmpty=false] - If true, the resampler will not call the `onStep` method of the out-aggregate when the interval is empty (for example, average of an empty set is not defined).
* @property {string} [name] - The given name for the stream aggregator.
* @property {(string | module:qm.StreamAggr)} [outAggr] - The name of the output stream aggregate. Only useful when the `outAggr` is a javascript stream aggregate, otherwise the output must be set by calling `setParam({outAggr: outAggregateName})`.
* @example
* var qm = require('qminer');
* // create a base with a simple timeseries store
* var base = new qm.Base({
* mode: 'createClean',
* schema: [{
* name: 'default',
* fields: [
* { name: 'timestamp', type: 'datetime' },
* { name: 'value', type: 'float' }
* ]
* }]
* });
* var store = base.store('default');
* // the tick aggregate reads from the store (provides time series input to other aggregates)
* var raw = store.addStreamAggr({
* type: 'timeSeriesTick',
* timestamp: 'timestamp',
* value: 'value'
* });
*
* // will compute sums over 1 second intervals
* var resampler = store.addStreamAggr({
* type: 'aggrResample',
* inAggr: raw.name,
* start: '1970-01-01T00:00:00.000',
* defaultValue: 0,
* aggType: 'sum',
* interval: 1000
* });
*
* // will print out resampler state on each resample
* var resamplerOutput = new qm.StreamAggr(base, new function () {
* this.onStep = function () {
* console.log('Resampler emitted the sum: ' + resampler.getFloat() +
* ' for the interval [' + new Date(resampler.getTimestamp()).toISOString() +
* ' - ' + new Date(resampler.getTimestamp() + resampler.getParams().interval).toISOString() + ')');
* }
* });
*
* // IMPORTANT. After the output exists, connect it to resampler
* resampler.setParams({ outAggr: resamplerOutput.name });
*
* store.push({ timestamp: 1, value: 1 });
* store.push({ timestamp: 10, value: 10 });
* store.push({ timestamp: 500, value: 100 });
* store.push({ timestamp: 2000, value: 1000 }); // triggers two resampling steps (two intervals complete)
* // // three measurements for the first interval
* // Resampler emitted the sum: 111 for the interval [1970-01-01T00:00:00.000Z - 1970-01-01T00:00:01.000Z)
* // // zero measurements for the second interval (does not skip empty)
* // Resampler emitted the sum: 0 for the interval [1970-01-01T00:00:01.000Z - 1970-01-01T00:00:02.000Z)
* store.push({ timestamp: 2001, value: 10000 });
* store.push({ timestamp: 3000, value: 100000 }); // triggers one resampling step (one interval complete)
* // // one measurement for the third interval
* // Resampler emitted the sum: 11000 for the interval [1970-01-01T00:00:02.000Z - 1970-01-01T00:00:03.000Z)
*
* base.close();
*/
/**
* @typedef {module:qm.StreamAggr} StreamAggrMerger
* This stream aggregator represents the merger aggregator. It merges records from two or more stores into a new store
* depending on the timestamp. No methods are implemented for this aggregator.
* <image src="pictures/merger.gif" alt="Merger Animation">
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'merger'`.
* @property {string} outStore - The name of the store where it saves the merged records.
* @property {boolean} createStore - If the outStore must be created.
* @property {string} timestamp - The store field of outStore, where the timestamp is saved.
* @property {Array.<Object>} fields - An array of `field` objects. The `field` object contain the properties:
* <br>`field.source` - The name of the store, from which it takes the values. Type `string`.
* <br>`field.inField` - The field name of source, from which it takes the values. Type `string`.
* <br>`field.outField` - The field name of outStore, into which it saves the values. Type `string`.
* <br>`field.interpolation` - The type of the interpolation. The options are: `'previous'`, `'next'` and `'linear'`. Type `string`.
* <br>`field.timestamp` - The field name of source, where the timestamp is saved. Type `string`.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store
* var base = new qm.Base({
* mode: "createClean",
* schema: [
* {
* name: "Cars",
* fields: [
* { name: "NumberOfCars", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* },
* {
* name: "Temperature",
* fields: [
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* },
* {
* name: "Merged",
* fields: [
* { name: "NumberOfCars", type: "float" },
* { name: "Celsius", type: "float" },
* { name: "Time", type: "datetime" }
* ]
* }]
* });
* // create a new merger stream aggregator that mergers the records of the 'Cars' and 'Temperature' stores.
* // The records are interpolated linearly and stored in the 'Merged' store.
* var mer = {
* name: 'MergerAggr',
* type: 'merger',
* outStore: 'Merged',
* createStore: false,
* timestamp: 'Time',
* fields: [
* { source: 'Cars', inField: 'NumberOfCars', outField: 'NumberOfCars', interpolation: 'linear', timestamp: 'Time' },
* { source: 'Temperature', inField: 'Celsius', outField: 'Celsius', interpolation: 'linear', timestamp: 'Time' }
* ]
* };
* var merger = new qm.StreamAggr(base, mer);
* base.close();
*/
/**
* @typedef {module:qmStreamAggr} StreamAggrFeatureSpace
* This stream aggregator creates the feature space and stores the specified features of the last input. It implements the following methods:
* <br>1. {@link module:qm.StreamAggr#getFloatVector} returns the dense feature vectors.
* <br>2. {@link module:qm.StreamAggr#getFeatureSpace} returns the feature space.
* @property {string} name - The given name for the stream aggregator.
* @property {string} type - The type of the stream aggregator. <b>Important:</b> It must be equal to `'featureSpace'`.
* @property {string} store - The name of the store from which it takes the data.
* @property {number} initCount - The number of records needed before it initializes.
* @property {boolean} update - If true, updates the feature space.
* @property {boolean} full - If true, saves the full vector of features.
* @property {boolean} sparse - If true, saves the sparse vector of features.
* @property {module:qm~FeatureExtractor[]} FeatureSpace - Array of feature extractors.
* @example
* // import the qm module
* var qm = require('qminer');
* // create a base with a simple store named Cars with 4 fields
* var base = new qm.Base({
* mode: 'createClean',
* schema: [{
* name: 'Cars',
* fields: [
* { name: 'NumberOfCars', type: 'float' },
* { name: 'Temperature', type: 'float' },
* { name: 'Precipitation',