@modelx/jsonm
Version:
Construct AI & ML models with JSON using Typescript & Tensorflow
1,346 lines (1,311 loc) • 1.54 MB
JavaScript
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
var http = require('http');
var https = require('https');
var validURL = require('valid-url');
var csv$1 = require('csvtojson');
var probabilityDistributions = require('probability-distributions');
var mlStat = require('ml-stat');
var range = require('lodash.range');
var rangeRight = require('lodash.rangeright');
var randomJs = require('random-js');
var mlMatrix = require('ml-matrix');
var ConfusionMatrix = require('ml-confusion-matrix');
var jgsl = require('js-grid-search-lite');
var fpg = require('node-fpgrowth');
var tf = require('@tensorflow/tfjs-node');
require('@tensorflow-models/universal-sentence-encoder');
var Promisie = require('promisie');
var Luxon = require('luxon');
var Outlier = require('outlier');
var flatten$1 = require('flat');
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
function _interopNamespace(e) {
if (e && e.__esModule) return e;
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () {
return e[k];
}
});
}
});
}
n['default'] = e;
return Object.freeze(n);
}
var validURL__default = /*#__PURE__*/_interopDefaultLegacy(validURL);
var csv__default = /*#__PURE__*/_interopDefaultLegacy(csv$1);
var probabilityDistributions__default = /*#__PURE__*/_interopDefaultLegacy(probabilityDistributions);
var range__default = /*#__PURE__*/_interopDefaultLegacy(range);
var rangeRight__default = /*#__PURE__*/_interopDefaultLegacy(rangeRight);
var ConfusionMatrix__default = /*#__PURE__*/_interopDefaultLegacy(ConfusionMatrix);
var jgsl__default = /*#__PURE__*/_interopDefaultLegacy(jgsl);
var fpg__namespace = /*#__PURE__*/_interopNamespace(fpg);
var tf__namespace = /*#__PURE__*/_interopNamespace(tf);
var Promisie__default = /*#__PURE__*/_interopDefaultLegacy(Promisie);
var Luxon__namespace = /*#__PURE__*/_interopNamespace(Luxon);
var Outlier__default = /*#__PURE__*/_interopDefaultLegacy(Outlier);
var flatten__default = /*#__PURE__*/_interopDefaultLegacy(flatten$1);
/**
* Asynchronously loads a CSV from a remote URL and returns an array of objects
* @example
* // returns [{header:value,header2:value2}]
* loadCSVURI('https://raw.githubusercontent.com/repetere/modelscript/master/test/mock/data.csv').then(csvData).catch(console.error)
* @param {string} filepath - URL to CSV path
* @param {Object} [options] - options passed to csvtojson
* @returns {Object[]} returns an array of objects from a csv where each column header is the property name
*/
async function loadCSVURI(filepath, options) {
const reqMethod = (filepath.search(/https/gi) > -1) ? https.get : http.get;
return new Promise((resolve, reject) => {
const csvData = [];
const config = Object.assign({ checkType: true, }, options);
const req = reqMethod(filepath, res => {
csv__default['default'](config).fromStream(res)
.subscribe((json) => {
csvData.push(json);
},
//onError
(err) => {
return reject(err);
},
//onComplete
(error) => {
if (error) {
return reject(error);
}
else {
return resolve(csvData);
}
});
// .on('data', jsonObj => {
// csvData.push(JSON.parse(jsonObj.toString()));
// })
// .on('json', (jsonObj:CSVJSONRow) => {
// csvData.push(jsonObj);
// })
// .on('error', )
// .on('done', );
});
req.on('error', reject);
});
}
/**
* Asynchronously loads a CSV from either a filepath or remote URL and returns an array of objects
* @example
* // returns [{header:value,header2:value2}]
* loadCSV('../mock/invalid-file.csv').then(csvData).catch(console.error)
* @param {string} filepath - URL to CSV path
* @param {Object} [options] - options passed to csvtojson
* @returns {Object[]} returns an array of objects from a csv where each column header is the property name
*/
async function loadCSV(filepath, options) {
if (validURL__default['default'].isUri(filepath)) {
return loadCSVURI(filepath, options);
}
else {
return new Promise((resolve, reject) => {
const csvData = [];
const config = Object.assign({ checkType: true, }, options);
csv__default['default'](config).fromFile(filepath)
.subscribe((json, lineNumber) => {
csvData.push(json);
},
//onError
(err) => {
return reject(err);
},
//onComplete
(error) => {
if (error) {
return reject(error);
}
else {
return resolve(csvData);
}
});
});
}
}
/**
* Asynchronously loads a TSV from either a filepath or remote URL and returns an array of objects
* @example
* // returns [{header:value,header2:value2}]
* loadCSV('../mock/invalid-file.tsv').then(csvData).catch(console.error)
* @param {string} filepath - URL to CSV path
* @param {Object} [options] - options passed to csvtojson
* @returns {Object[]} returns an array of objects from a csv where each column header is the property name
*/
async function loadTSV(filepath, options) {
const tsvOptions = Object.assign({
checkType: true,
}, options, {
delimiter: '\t',
});
return loadCSV(filepath, tsvOptions);
}
var csvUtils = /*#__PURE__*/Object.freeze({
__proto__: null,
loadCSVURI: loadCSVURI,
loadCSV: loadCSV,
loadTSV: loadTSV
});
var stemmer_1 = stemmer;
// Standard suffix manipulations.
var step2list = {
ational: 'ate',
tional: 'tion',
enci: 'ence',
anci: 'ance',
izer: 'ize',
bli: 'ble',
alli: 'al',
entli: 'ent',
eli: 'e',
ousli: 'ous',
ization: 'ize',
ation: 'ate',
ator: 'ate',
alism: 'al',
iveness: 'ive',
fulness: 'ful',
ousness: 'ous',
aliti: 'al',
iviti: 'ive',
biliti: 'ble',
logi: 'log'
};
var step3list = {
icate: 'ic',
ative: '',
alize: 'al',
iciti: 'ic',
ical: 'ic',
ful: '',
ness: ''
};
// Consonant-vowel sequences.
var consonant = '[^aeiou]';
var vowel = '[aeiouy]';
var consonants = '(' + consonant + '[^aeiouy]*)';
var vowels = '(' + vowel + '[aeiou]*)';
var gt0 = new RegExp('^' + consonants + '?' + vowels + consonants);
var eq1 = new RegExp(
'^' + consonants + '?' + vowels + consonants + vowels + '?$'
);
var gt1 = new RegExp('^' + consonants + '?(' + vowels + consonants + '){2,}');
var vowelInStem = new RegExp('^' + consonants + '?' + vowel);
var consonantLike = new RegExp('^' + consonants + vowel + '[^aeiouwxy]$');
// Exception expressions.
var sfxLl = /ll$/;
var sfxE = /^(.+?)e$/;
var sfxY = /^(.+?)y$/;
var sfxIon = /^(.+?(s|t))(ion)$/;
var sfxEdOrIng = /^(.+?)(ed|ing)$/;
var sfxAtOrBlOrIz = /(at|bl|iz)$/;
var sfxEED = /^(.+?)eed$/;
var sfxS = /^.+?[^s]s$/;
var sfxSsesOrIes = /^.+?(ss|i)es$/;
var sfxMultiConsonantLike = /([^aeiouylsz])\1$/;
var step2 = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
var step3 = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/;
var step4 = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/;
// Stem `value`.
// eslint-disable-next-line complexity
function stemmer(value) {
var firstCharacterWasLowerCaseY;
var match;
value = String(value).toLowerCase();
// Exit early.
if (value.length < 3) {
return value
}
// Detect initial `y`, make sure it never matches.
if (
value.charCodeAt(0) === 121 // Lowercase Y
) {
firstCharacterWasLowerCaseY = true;
value = 'Y' + value.slice(1);
}
// Step 1a.
if (sfxSsesOrIes.test(value)) {
// Remove last two characters.
value = value.slice(0, value.length - 2);
} else if (sfxS.test(value)) {
// Remove last character.
value = value.slice(0, value.length - 1);
}
// Step 1b.
if ((match = sfxEED.exec(value))) {
if (gt0.test(match[1])) {
// Remove last character.
value = value.slice(0, value.length - 1);
}
} else if ((match = sfxEdOrIng.exec(value)) && vowelInStem.test(match[1])) {
value = match[1];
if (sfxAtOrBlOrIz.test(value)) {
// Append `e`.
value += 'e';
} else if (sfxMultiConsonantLike.test(value)) {
// Remove last character.
value = value.slice(0, value.length - 1);
} else if (consonantLike.test(value)) {
// Append `e`.
value += 'e';
}
}
// Step 1c.
if ((match = sfxY.exec(value)) && vowelInStem.test(match[1])) {
// Remove suffixing `y` and append `i`.
value = match[1] + 'i';
}
// Step 2.
if ((match = step2.exec(value)) && gt0.test(match[1])) {
value = match[1] + step2list[match[2]];
}
// Step 3.
if ((match = step3.exec(value)) && gt0.test(match[1])) {
value = match[1] + step3list[match[2]];
}
// Step 4.
if ((match = step4.exec(value))) {
if (gt1.test(match[1])) {
value = match[1];
}
} else if ((match = sfxIon.exec(value)) && gt1.test(match[1])) {
value = match[1];
}
// Step 5.
if (
(match = sfxE.exec(value)) &&
(gt1.test(match[1]) ||
(eq1.test(match[1]) && !consonantLike.test(match[1])))
) {
value = match[1];
}
if (sfxLl.test(value) && gt1.test(value)) {
value = value.slice(0, value.length - 1);
}
// Turn initial `Y` back to `y`.
if (firstCharacterWasLowerCaseY) {
value = 'y' + value.slice(1);
}
return value
}
const natural = {
PorterStemmer: {
tokenizeAndStem: (input = '') => {
const stems = input
.split(' ')
.map(word => word.trim())
.filter(word => word)
.map(stemmer_1);
// console.log({ stems });
return stems;
}
}
};
const avg = mlStat.array.mean;
const mean = avg;
const sum = mlStat.array.sum;
const scale = (a, d) => a.map(x => (x - avg(a)) / d);
const max = mlStat.array.max; // (a: number[]) => ArrayStat.max(a);//a.concat([]).sort((x:number, y:number):any => x < y)[0];
const min = mlStat.array.min;
const sd = mlStat.array.standardDeviation; //(a, av) => Math.sqrt(avg(a.map(x => (x - av) * x)));
/**
* Returns an array of the squared different of two arrays
* @memberOf util
* @param {Number[]} left
* @param {Number[]} right
* @returns {Number[]} Squared difference of left minus right array
*/
function squaredDifference(left, right) {
return left.reduce((result, val, index, arr) => {
result.push(Math.pow((right[index] - val), 2));
return result;
}, []);
}
/**
* The standard error of the estimate is a measure of the accuracy of predictions made with a regression line. Compares the estimate to the actual value
* @memberOf util
* @see {@link http://onlinestatbook.com/2/regression/accuracy.html}
* @example
const actuals = [ 2, 4, 5, 4, 5, ];
const estimates = [ 2.8, 3.4, 4, 4.6, 5.2, ];
const SE = ms.util.standardError(actuals, estimates);
SE.toFixed(2) // => 0.89
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} Standard Error of the Estimate
*/
function standardError(actuals = [], estimates = []) {
if (actuals.length !== estimates.length)
throw new RangeError('arrays must have the same length');
const squaredDiff = squaredDifference(actuals, estimates);
return Math.sqrt((sum(squaredDiff)) / (actuals.length - 2));
}
/**
* Calculates the z score of each value in the sample, relative to the sample mean and standard deviation.
* @memberOf util
* @see {@link https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.mstats.zscore.html}
* @param {Number[]} observations - An array like object containing the sample data.
* @returns {Number[]} The z-scores, standardized by mean and standard deviation of input array
*/
function standardScore(observations = []) {
const mean = avg(observations);
const stdDev = sd(observations);
return observations.map(x => ((x - mean) / stdDev));
}
/**
* In statistics, the coefficient of determination, denoted R2 or r2 and pronounced "R squared", is the proportion of the variance in the dependent variable that is predictable from the independent variable(s). Compares distance of estimated values to the mean.
* {\bar {y}}={\frac {1}{n}}\sum _{i=1}^{n}y_{i}
* @example
const actuals = [ 2, 4, 5, 4, 5, ];
const estimates = [ 2.8, 3.4, 4, 4.6, 5.2, ];
const r2 = ms.util.coefficientOfDetermination(actuals, estimates);
r2.toFixed(1) // => 0.6
* @memberOf util
* @see {@link https://en.wikipedia.org/wiki/Coefficient_of_determination} {@link http://statisticsbyjim.com/regression/standard-error-regression-vs-r-squared/}
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} r^2
*/
function coefficientOfDetermination(actuals = [], estimates = []) {
if (actuals.length !== estimates.length)
throw new RangeError('arrays must have the same length');
const actualsMean = mean(actuals);
const totalVariation = sum(actuals.reduce((result, val, index) => {
result.push(Math.pow((actuals[index] - actualsMean), 2));
return result;
}, []));
const unexplainedVariation = sum(actuals.reduce((result, val, index) => {
result.push(Math.pow((actuals[index] - estimates[index]), 2));
return result;
}, []));
const rSquared = ((totalVariation - unexplainedVariation) / totalVariation);
return rSquared;
/*
@see {@link https://math.tutorvista.com/statistics/coefficient-of-determination.html}
Some Properties of Coefficient of Determination are as follow:
It helps to provide the proportion of the variance of one variable which is predictable from the other variable.
It is a way of measurement which allows determining how clear it can be in making predictions from a certain data provided.
It can be taken as a ratio of the explained variation to the total variation.
It denotes the strength of the linear association between the variables.
The square of the coefficient of determination will always b e between 0 and1, which is 0 ≤
≤
r2 ≤
≤
1. Here r2 will always be a positive value.
As r2 gets close to 1, the Y data values get close to the regression line.
As r2 gets close to 0, the Y data values get further from the regression line.
It helps to provide the proportion of the variance of one variable which is predictable from the other variable.
It is a way of measurement which allows determining how clear it can be in making predictions from a certain data provided.
It can be taken as a ratio of the explained variation to the total variation.
It denotes the strength of the linear association between the variables.
*/
}
/**
* You can use the adjusted coefficient of determination to determine how well a multiple regression equation “fits” the sample data. The adjusted coefficient of determination is closely related to the coefficient of determination (also known as R2) that you use to test the results of a simple regression equation.
* @example
const adjr2 = ms.util.adjustedCoefficentOfDetermination({
rSquared: 0.944346527,
sampleSize: 8,
independentVariables: 2,
});
r2.toFixed(3) // => 0.922
* @memberOf util
* @see {@link http://www.dummies.com/education/math/business-statistics/how-to-calculate-the-adjusted-coefficient-of-determination/}
* @param {Object} [options={}]
* @param {Number[]} [options.actuals] - numerical samples
* @param {Number[]} [options.estimates] - estimate values
* @param {Number} [options.rSquared] - coefficent of determination
* @param {Number} [options.sampleSize] - the sample size
* @param {Number} options.independentVariables - the number of independent variables in the regression equation
* @returns {Number} adjusted r^2 for multiple linear regression
*/
function adjustedCoefficentOfDetermination(options) {
const { actuals, estimates, rSquared, independentVariables, sampleSize, } = options;
const r2 = rSquared || coefficientOfDetermination(actuals, estimates);
const n = sampleSize || actuals.length;
const k = independentVariables;
return (1 - (1 - r2) * ((n - 1) / (n - (k + 1))));
}
/**
* The coefficent of Correlation is given by R decides how well the given data fits a line or a curve.
* @example
const actuals = [ 39, 42, 67, 76, ];
const estimates = [ 44, 40, 60, 84, ];
const R = ms.util.coefficientOfCorrelation(actuals, estimates);
R.toFixed(4) // => 0.9408
* @memberOf util
* @see {@link https://calculator.tutorvista.com/r-squared-calculator.html}
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} R
*/
function coefficientOfCorrelation(actuals = [], estimates = []) {
if (actuals.length !== estimates.length)
throw new RangeError('arrays must have the same length');
const sumX = sum(actuals);
const sumY = sum(estimates);
const sumProdXY = actuals.reduce((result, val, index) => {
result = result + (actuals[index] * estimates[index]);
return result;
}, 0);
const sumXSquared = actuals.reduce((result, val) => {
result = result + (val * val);
return result;
}, 0);
const sumYSquared = estimates.reduce((result, val) => {
result = result + (val * val);
return result;
}, 0);
const N = actuals.length;
const R = ((N * sumProdXY - sumX * sumY) /
Math.sqrt((N * sumXSquared - Math.pow(sumX, 2)) * (N * sumYSquared - Math.pow(sumY, 2))));
return R;
}
/**
* The coefficent of determination is given by r^2 decides how well the given data fits a line or a curve.
*
* @param {Number[]} [actuals=[]]
* @param {Number[]} [estimates=[]]
* @returns {Number} r^2
*/
function rSquared(actuals = [], estimates = []) {
return Math.pow(coefficientOfCorrelation(actuals, estimates), 2);
}
/**
* returns an array of vectors as an array of arrays
* @example
const vectors = [ [1,2,3], [1,2,3], [3,3,4], [3,3,3] ];
const arrays = pivotVector(vectors); // => [ [1,2,3,3], [2,2,3,3], [3,3,4,3] ];
* @memberOf util
* @param {Array[]} vectors
* @returns {Array[]}
*/
function pivotVector$1(vectors = []) {
return vectors.reduce((result, val, index /*, arr*/) => {
val.forEach((vecVal, i) => {
(index === 0)
? (result.push([vecVal,]))
: (result[i].push(vecVal));
});
return result;
}, []);
}
/**
* returns a matrix of values by combining arrays into a matrix
* @memberOf util
* @example
const arrays = [
[ 1, 1, 3, 3 ],
[ 2, 2, 3, 3 ],
[ 3, 3, 4, 3 ],
];
pivotArrays(arrays); //=>
// [
// [1, 2, 3,],
// [1, 2, 3,],
// [3, 3, 4,],
// [3, 3, 3,],
// ];
* @param {Array} [vectors=[]] - array of arguments for columnArray to merge columns into a matrix
* @returns {Array} a matrix of column values
*/
function pivotArrays(arrays = []) {
return (arrays.length)
? arrays[0].map((vectorItem, index) => {
const returnArray = [];
arrays.forEach((v, i) => {
returnArray.push(arrays[i][index]);
});
return returnArray;
})
: arrays;
}
////Vector, Matrix,
/**
* Standardize features by removing the mean and scaling to unit variance
Centering and scaling happen independently on each feature by computing the relevant statistics on the samples in the training set. Mean and standard deviation are then stored to be used on later data using the transform method.
Standardization of a dataset is a common requirement for many machine learning estimators: they might behave badly if the individual feature do not more or less look like standard normally distributed data (e.g. Gaussian with 0 mean and unit variance)
* @memberOf util
* @param {number[]} z - array of integers or floats
* @returns {number[]}
*/
const StandardScaler = (z) => scale(z, sd(z));
/** This function returns two functions that can standard scale new inputs and reverse scale new outputs
* @param {Number[]} values - array of numbers
* @returns {Object} - {scale[ Function ], descale[ Function ]}
*/
function StandardScalerTransforms(vector = [], nan_value = -1, return_nan = false, inputComponents = {}) {
const average = typeof inputComponents.average !== 'undefined'
? inputComponents.average
: avg(vector);
const standard_dev = typeof inputComponents.standard_dev !== 'undefined'
? inputComponents.standard_dev
: sd(vector);
const maximum = typeof inputComponents.maximum !== 'undefined'
? inputComponents.maximum
: max(vector);
const minimum = typeof inputComponents.minimum !== 'undefined'
? inputComponents.minimum
: min(vector);
const scale = (z) => {
const scaledValue = (z - average) / standard_dev;
if (isNaN(scaledValue) && return_nan)
return scaledValue;
else if (isNaN(scaledValue) && return_nan === false)
return (isNaN(standard_dev)) ? z : standard_dev;
else
return scaledValue;
}; // equivalent to MinMaxScaler(z)
const descale = (scaledZ) => {
const descaledValue = (scaledZ * standard_dev) + average;
if (isNaN(descaledValue) && return_nan)
return descaledValue;
else if (isNaN(descaledValue) && return_nan === false)
return (isNaN(standard_dev)) ? scaledZ : standard_dev;
else
return descaledValue;
};
const values = vector.map(scale)
.map(val => {
if (isNaN(val))
return nan_value;
else
return val;
});
return {
components: {
average,
standard_dev,
maximum,
minimum,
},
scale,
descale,
values,
};
}
/**
* Transforms features by scaling each feature to a given range.
This estimator scales and translates each feature individually such that it is in the given range on the training set, i.e. between zero and one.
* @memberOf util
* @param {number[]} z - array of integers or floats
* @returns {number[]}
*/
const MinMaxScaler = (z) => scale(z, (max(z) - min(z)));
/** This function returns two functions that can mix max scale new inputs and reverse scale new outputs
* @param {Number[]} values - array of numbers
* @returns {Object} - {scale[ Function ], descale[ Function ]}
*/
function MinMaxScalerTransforms(vector = [], nan_value = -1, return_nan = false, inputComponents = {}) {
const average = typeof inputComponents.average !== 'undefined'
? inputComponents.average
: avg(vector);
const standard_dev = typeof inputComponents.standard_dev !== 'undefined'
? inputComponents.standard_dev
: sd(vector);
const maximum = typeof inputComponents.maximum !== 'undefined'
? inputComponents.maximum
: max(vector);
//@ts-ignore
const minimum = typeof inputComponents.minimum !== 'undefined'
//@ts-ignore
? inputComponents.minimum
: min(vector);
const scale = (z) => {
const scaledValue = (z - average) / (maximum - minimum);
if (isNaN(scaledValue) && return_nan)
return scaledValue;
else if (isNaN(scaledValue) && return_nan === false)
return (isNaN(standard_dev)) ? z : standard_dev;
else
return scaledValue;
}; // equivalent to MinMaxScaler(z)
const descale = (scaledZ) => {
const descaledValue = (scaledZ * (maximum - minimum)) + average;
if (isNaN(descaledValue) && return_nan)
return descaledValue;
else if (isNaN(descaledValue) && return_nan === false)
return (isNaN(standard_dev)) ? scaledZ : standard_dev;
else
return descaledValue;
};
const values = vector.map(scale)
.map(val => {
if (isNaN(val))
return nan_value;
else
return val;
});
return {
components: {
average,
standard_dev,
maximum,
minimum,
},
scale,
descale,
values,
};
}
/**
* Converts z-score into the probability
* @memberOf util
* @see {@link https://stackoverflow.com/questions/36575743/how-do-i-convert-probability-into-z-score}
* @param {number} z - Number of standard deviations from the mean.
* @returns {number} p - p-value
*/
function approximateZPercentile(z, alpha = true) {
// If z is greater than 6.5 standard deviations from the mean
// the number of significant digits will be outside of a reasonable
// range.
if (z < -6.5)
return 0.0;
if (z > 6.5)
return 1.0;
let factK = 1;
let sum = 0;
let term = 1;
let k = 0;
let loopStop = Math.exp(-23);
while (Math.abs(term) > loopStop) {
term = 0.3989422804 * Math.pow(-1, k) * Math.pow(z, k) / (2 * k + 1) /
Math.pow(2, k) * Math.pow(z, k + 1) / factK;
sum += term;
k++;
factK *= k;
}
sum += 0.5;
return (alpha) ? 1 - sum : sum;
}
/**
* returns a safe column name / url slug from a string
* @param {String} name
* @returns {String}
*/
function getSafePropertyName(name) {
return name.replace(/[^\w\s]/gi, '_');
}
/**
* The errors (residuals) from acutals and estimates
* @memberOf util
* @example
const actuals = [ 45, 38, 43, 39 ];
const estimates = [ 41, 43, 41, 42 ];
const errors = ms.util.forecastErrors(actuals, estimates); // => [ 4, -5, 2, -3 ]
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number[]} errors (residuals)
*/
function forecastErrors(actuals, estimates) {
if (actuals.length !== estimates.length)
throw new Error(`Actuals length (${actuals.length}) must equal Estimates length (${estimates.length})`);
return actuals.map((act, i) => act - estimates[i]);
}
/**
* The bias of forecast accuracy
* @memberOf util
* @see {@link https://scm.ncsu.edu/scm-articles/article/measuring-forecast-accuracy-approaches-to-forecasting-a-tutorial}
* @example
const actuals = [ 45, 38, 43, 39 ];
const estimates = [ 41, 43, 41, 42 ];
const MFE = ms.util.meanForecastError(actuals, estimates); // => -0.5
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} MFE (bias)
*/
function meanForecastError(actuals, estimates) {
const errors = forecastErrors(actuals, estimates);
return avg(errors);
}
/**
* Mean Absolute Deviation (MAD) indicates the absolute size of the errors
* @memberOf util
* @see {@link https://scm.ncsu.edu/scm-articles/article/measuring-forecast-accuracy-approaches-to-forecasting-a-tutorial}
* @example
const actuals = [ 45, 38, 43, 39 ];
const estimates = [ 41, 43, 41, 42 ];
const MAD = ms.util.meanAbsoluteDeviation(actuals, estimates); // => 3.5
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} MAD
*/
function meanAbsoluteDeviation(actuals, estimates) {
const errors = forecastErrors(actuals, estimates).map(e => Math.abs(e));
return avg(errors);
}
/**
* Tracking Signal - Used to pinpoint forecasting models that need adjustment
* @memberOf util
* @see {@link https://scm.ncsu.edu/scm-articles/article/measuring-forecast-accuracy-approaches-to-forecasting-a-tutorial}
* @example
const actuals = [ 45, 38, 43, 39 ];
const estimates = [ 41, 43, 41, 42 ];
const trackingSignal = ms.util.trackingSignal(actuals, estimates);
trackingSignal.toFixed(2) // => -0.57
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} trackingSignal
*/
function trackingSignal(actuals, estimates) {
const runningSumOfForecastErrors = sum(forecastErrors(actuals, estimates));
const MAD = meanAbsoluteDeviation(actuals, estimates);
return runningSumOfForecastErrors / MAD;
}
/**
* The standard error of the estimate is a measure of the accuracy of predictions made with a regression line. Compares the estimate to the actual value
* @memberOf util
* @see {@link http://onlinestatbook.com/2/regression/accuracy.html}
* @example
const actuals = [ 45, 38, 43, 39 ];
const estimates = [ 41, 43, 41, 42 ];
const MSE = ms.util.meanSquaredError(actuals, estimates); // => 13.5
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} MSE
*/
function meanSquaredError(actuals, estimates) {
const squaredErrors = forecastErrors(actuals, estimates).map(e => e * e);
return avg(squaredErrors);
}
/**
* MAD over Mean Ratio - The MAD/Mean ratio is an alternative to the MAPE that is better suited to intermittent and low-volume data. As stated previously, percentage errors cannot be calculated when the actual equals zero and can take on extreme values when dealing with low-volume data. These issues become magnified when you start to average MAPEs over multiple time series. The MAD/Mean ratio tries to overcome this problem by dividing the MAD by the Mean—essentially rescaling the error to make it comparable across time series of varying scales
* @memberOf util
* @see {@link https://www.forecastpro.com/Trends/forecasting101August2011.html}
* @example
const actuals = [ 45, 38, 43, 39 ];
const estimates = [ 41, 43, 41, 42 ];
const MMR = ms.util.MADMeanRatio(actuals, estimates);
MAPE.toFixed(2) // => 0.08
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} MMR
*/
function MADMeanRatio(actuals, estimates) {
const MAD = meanAbsoluteDeviation(actuals, estimates);
const mean = avg(actuals);
return MAD / mean;
}
/**
* MAPE (Mean Absolute Percent Error) measures the size of the error in percentage terms
* @memberOf util
* @see {@link https://www.forecastpro.com/Trends/forecasting101August2011.html}
* @example
const actuals = [ 45, 38, 43, 39 ];
const estimates = [ 41, 43, 41, 42 ];
const MAPE = ms.util.meanAbsolutePercentageError(actuals, estimates);
MAPE.toFixed(2) // => 0.86
* @param {Number[]} actuals - numerical samples
* @param {Number[]} estimates - estimates values
* @returns {Number} MAPE
*/
function meanAbsolutePercentageError(actuals, estimates) {
const errors = forecastErrors(actuals, estimates).map(e => Math.abs(e));
const absErrorPercent = errors.map((e, i) => e / actuals[i]);
return avg(absErrorPercent);
}
/**
* @namespace
*/
const util$b = {
range: range__default['default'],
rangeRight: rangeRight__default['default'],
scale,
avg,
mean: avg,
sum,
max,
min,
sd,
StandardScaler,
StandardScalerTransforms,
MinMaxScaler,
MinMaxScalerTransforms,
LogScaler: (z) => z.map(Math.log),
ExpScaler: (z) => z.map(Math.exp),
squaredDifference,
standardError,
coefficientOfDetermination,
coefficientOfCorrelation,
r: coefficientOfCorrelation,
rSquared,
adjustedCoefficentOfDetermination,
rBarSquared: adjustedCoefficentOfDetermination,
adjustedRSquared: adjustedCoefficentOfDetermination,
pivotVector: pivotVector$1,
pivotArrays,
standardScore,
zScore: standardScore,
approximateZPercentile,
// approximatePercentileZ,
getSafePropertyName,
forecastErrors,
meanForecastError,
MFE: meanForecastError,
meanAbsoluteDeviation,
MAD: meanAbsoluteDeviation,
trackingSignal,
TS: trackingSignal,
meanSquaredError,
MSE: meanSquaredError,
MADMeanRatio,
MMR: MADMeanRatio,
meanAbsolutePercentageError,
MAPE: meanAbsolutePercentageError,
};
// import { ml, } from './ml';
const transformConfigMap = {
scale: 'scaleOptions',
descale: 'descaleOptions',
label: 'labelOptions',
labelEncoder: 'labelOptions',
labeldecode: 'labelOptions',
labelDecode: 'labelOptions',
labelDecoder: 'labelOptions',
onehot: 'oneHotOptions',
oneHot: 'oneHotOptions',
oneHotEncode: 'oneHotOptions',
oneHotEncoder: 'oneHotOptions',
reducer: 'reducerOptions',
reduce: 'reducerOptions',
merge: 'mergeData',
};
/**
* class for manipulating an array of objects, typically from CSV data
* @class DataSet
* @memberOf preprocessing
*/
class DataSet {
/**
* creates a new raw data instance for preprocessing data for machine learning
* @example
* const dataset = new ms.DataSet(csvData);
* @param {Object[]} dataset
* @returns {this}
*/
constructor(data = [], options = {}) {
this.config = Object.assign({
debug: true,
}, options);
this.data = [...data,];
this.labels = new Map();
this.encoders = new Map();
this.scalers = new Map();
this.selectColumns = DataSet.selectColumns;
this.columnArray = DataSet.columnArray;
this.encodeObject = DataSet.encodeObject;
this.oneHotEncoder = DataSet.oneHotEncoder;
this.oneHotDecoder = DataSet.oneHotDecoder;
this.columnMatrix = DataSet.columnMatrix;
this.reverseColumnMatrix = DataSet.reverseColumnMatrix;
this.reverseColumnVector = DataSet.reverseColumnVector;
this.getTransforms = DataSet.getTransforms;
if (this.config.labels || this.config.encoders || this.config.scalers) {
this.importFeatures({
labels: this.config.labels,
encoders: this.config.encoders,
scalers: this.config.scalers,
});
}
return this;
}
/**
* Allows for fit transform short hand notation
* @example
DataSet.getTransforms({
Age: ['scale',],
Rating: ['label',], }); //=> [
// {
// name: 'Age', options: { strategy: 'scale', }, },
// },
// {
// name: 'Rating', options: { strategy: 'label', },
// },
// ];
* @param {Object} transforms
* @returns {Array<Object>} returns fit columns, columns property
*/
static getTransforms(transforms = {}) {
return Object.keys(transforms).reduce((result, columnName) => {
const transformColumnObject = transforms[columnName];
const transformObject = {
name: columnName,
options: {
strategy: (Array.isArray(transformColumnObject))
? transformColumnObject[0]
: transformColumnObject,
},
};
if (Array.isArray(transformColumnObject) && transformColumnObject.length > 1) {
//@ts-ignore
transformObject.options[transformConfigMap[transformColumnObject[0]]] = transformColumnObject[1];
}
result.push(transformObject);
return result;
}, []);
}
/**
* returns an array of objects by applying labels to matrix of columns
* @example
const data = [{ Age: '44', Salary: '44' },
{ Age: '27', Salary: '27' }]
const AgeDataSet = new MS.DataSet(data);
const dependentVariables = [ [ 'Age', ], [ 'Salary', ], ];
const AgeSalMatrix = AgeDataSet.columnMatrix(dependentVariables); // =>
// [ [ '44', '72000' ],
// [ '27', '48000' ] ];
MS.DataSet.reverseColumnMatrix({vectors:AgeSalMatrix,labels:dependentVariables}); // => [{ Age: '44', Salary: '44' },
{ Age: '27', Salary: '27' }]
*
* @param {*} options
* @param {Array[]} options.vectors - array of vectors
* @param {String[]} options.labels - array of labels
* @returns {Object[]} an array of objects with properties derived from options.labels
*/
static reverseColumnMatrix(options = {}) {
const { vectors = [], labels = [], } = options;
const features = (Array.isArray(labels) && Array.isArray(labels[0]))
? labels
: labels.map(label => [label,]);
return vectors.reduce((result, val) => {
result.push(val.reduce((prop, value, index) => {
prop[features[index][0]] = val[index];
return prop;
}, {}));
return result;
}, []);
}
static reverseColumnVector(options = {}) {
const { vector = [], labels = [], } = options;
const features = (Array.isArray(labels) && Array.isArray(labels[0]))
? labels
: labels.map(label => [label,]);
return vector.reduce((result, val) => {
result.push({ [features[0][0]]: val, });
return result;
}, []);
}
/**
* Returns an object into an one hot encoded object
* @example
const labels = ['apple', 'orange', 'banana',];
const prefix = 'fruit_';
const name = 'fruit';
const options = { labels, prefix, name, };
const data = {
fruit: 'apple',
};
EncodedCSVDataSet.encodeObject(data, options); // => { fruit_apple: 1, fruit_orange: 0, fruit_banana: 0, }
* @param {Object} data - object to encode
* @param {{labels:Array<String>,prefix:String,name:String}} options - encoded object options
* @returns {Object} one hot encoded object
*/
static encodeObject(data, options) {
const { labels, prefix, name, } = options;
const encodedData = labels.reduce((encodedObj, label) => {
const oneHotLabelArrayName = `${prefix}${label}`;
encodedObj[oneHotLabelArrayName] = (label && data[name] && data[name].toString() === label.toString()) ? 1 : 0;
return encodedObj;
}, {});
return encodedData;
}
/**
* returns a new object of one hot encoded values
* @example
* // [ 'Brazil','Mexico','Ghana','Mexico','Ghana','Brazil','Mexico','Brazil','Ghana', 'Brazil' ]
const originalCountry = dataset.columnArray('Country');
// { originalCountry:
// { Country_Brazil: [ 1, 0, 0, 0, 0, 1, 0, 1, 0, 1 ],
// Country_Mexico: [ 0, 1, 0, 1, 0, 0, 1, 0, 0, 0 ],
// Country_Ghana: [ 0, 0, 1, 0, 1, 0, 0, 0, 1, 0 ] },
// }
const oneHotCountryColumn = dataset.oneHotEncoder('Country');
* @param {string} name - csv column header, or JSON object property name
* @param options
* @see {@link http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html}
* @return {Object}
*/
static oneHotEncoder(name, options) {
const config = Object.assign({
merge: true,
}, options);
const labelData = config.data || this.columnArray(name, config.columnArrayOptions);
const labels = Array.from(new Set(labelData).values());
const prefix = config.prefix || `${name}_`;
const encodedData = labelData.reduce((result, val, index, arr) => {
labels.forEach(encodedLabel => {
const oneHotLabelArrayName = `${prefix}${encodedLabel}`;
const oneHotVal = (val === encodedLabel) ? 1 : 0;
if (Array.isArray(result[oneHotLabelArrayName])) {
result[oneHotLabelArrayName].push(oneHotVal);
}
else {
result[oneHotLabelArrayName] = [oneHotVal,];
}
});
return result;
}, {});
if (this.encoders.has(name) && config.merge) {
this.encoders.get(name).labels = Array.from(new Set(labels.concat(this.encoders.get(name).labels)));
// this.encoders.get(name);
}
else {
this.encoders.set(name, {
name,
labels,
prefix,
});
}
return encodedData;
}
/**
* Return one hot encoded data
* @example
const csvData = [{
'Country': 'Brazil',
'Age': '44',
'Salary': '72000',
'Purchased': 'N',
},
{
'Country': 'Mexico',
'Age': '27',
'Salary': '48000',
'Purchased': 'Yes',
},
...
];
const EncodedCSVDataSet = new ms.preprocessing.DataSet(csvData);
EncodedCSVDataSet.fitColumns({
columns: [
{
name: 'Country',
options: { strategy: 'onehot', },
},
],
});
EncodedCSVDataSet.oneHotDecoder('Country);// =>
// [ { Country: 'Brazil' },
// { Country: 'Mexico' },
// { Country: 'Ghana' },
// { Country: 'Mexico' },
// ...]
* @param {string} name - column name
* @param options
* @returns {Array<Object>} returns an array of objects from an one hot encoded column
*/
static oneHotDecoder(name, options) {
const config = Object.assign({
// handle_unknown: 'error'
}, options);
const encoderMap = config.encoders || this.encoders;
const prefix = config.prefix || encoderMap.get(name).prefix;
const labels = config.labels || encoderMap.get(name).labels;
const encodedData = config.data || this.oneHotColumnArray(name, config.oneHotColumnArrayOptions);
// console.log({ encodedData, encoderMap, prefix });
return encodedData.reduce((result, val) => {
const columnNames = Object.keys(val).filter(prop => val[prop] === 1 && (labels.indexOf(prop.replace(prefix, '')) !== -1 || labels.map((label) => String(label)).indexOf(prop.replace(prefix, '')) !== -1));
const columnName = columnNames[0] || '';
// console.log({ columnName, columnNames, labels, val},Object.keys(val));
const datum = {
[name]: columnName.replace(prefix, ''),
};
result.push(datum);
return result;
}, []);
}
static oneHotColumnArray(name, oneHotColumnArrayOptions) {
throw new Error("Method not implemented.");
}
/**
* returns a list of objects with only selected columns as properties
* @example
const data = [{ Age: '44', Salary: '44' , Height: '34' },
{ Age: '27', Salary: '44' , Height: '50' }]
const AgeDataSet = new MS.DataSet(data);
const cols = [ 'Age', 'Salary' ];
const selectedCols = CSVDataSet.selectColumns(cols); // => [{ Age: '44', Salary: '44' },
{ Age: '27', Salary: '27' }]
*
* @param {String[]} names - array of selected columns
* @param {*} options
* @returns {Object[]} an array of objects with properties derived from names
*/
static selectColumns(names, options = {}) {
const config = Object.assign({}, options);
const data = config.data || this.data;
return data.reduce((result, val) => {
const selectedData = {};
names.forEach((name) => {
selectedData[name] = val[name];
});
result.push(selectedData);
return result;
}, []);
}
/**
* returns a new array of a selected column from an array of objects, can filter, scale and replace values
* @example
* //column Array returns column of data by name
// [ '44','27','30','38','40','35','','48','50', '37' ]
const OringalAgeColumn = dataset.columnArray('Age');
* @param {string} name - csv column header, or JSON object property name
* @param options
* @param {function} [options.prefilter=(arr[val])=>true] - prefilter values to return
* @param {function} [options.filter=(arr[val])=>true] - filter values to return
* @param {function} [options.replace.test=undefined] - test function for replacing values (arr[val])
* @param {(string|number|function)} [options.replace.value=undefined] - value to replace (arr[val]) if replace test is true, if a function (result,val,index,arr,name)=>your custom value
* @param {number} [options.parseIntBase=10] - radix value for parseInt
* @param {boolean} [options.parseFloat=false] - convert values to floats
* @param {boolean} [options.parseInt=false] - converts values to ints
* @param {boolean} [options.scale=false] - standard or minmax feature scale values
* @returns {array}
*/
static columnArray(name, options = {}) {
const config = Object.assign({
prefilter: () => true,
filter: () => true,
replace: {
test: undefined,
value: undefined,
},
parseInt: false,
parseIntBase: 10,
parseFloat: (options.scale) ? true : false,
scale: false,
}, options);
const data = config.data || this.data;
const modifiedColumn = data
.filter(config.prefilter)
.reduce((result, val, index, arr) => {
let objVal = val[name];
let returnVal = (typeof config.replace.test === 'function') ?
config.replace.test(objVal) ?
typeof config.replace.value === 'function' ?
config.replace.value(result, val, index, arr, name) :
config.replace.value :
objVal :
objVal;
if (config.filter(returnVal)) {
if (config.parseInt)
result.push(parseInt(returnVal, config.parseIntBase));
else if (config.parseFloat)
result.push(parseFloat(returnVal));
else
result.push(returnVal);
}
return result;
}, []);
if (typeof config.scale === 'function') {
return modifiedColumn.map(config.scale);
}
else if (config.scale) {
switch (config.scale) {
case 'standard':
return util$b.StandardScaler(modifiedColumn);
case 'log':
return util$b.LogScaler(modifiedColumn);
case 'exp':
return util$b.ExpScaler(modifiedColumn);
case 'normalize':
default:
return util$b.MinMaxScaler(modifiedColumn);
}
}
else {
return modifiedColumn;
}
}
/**
* returns a matrix of values by combining column arrays into a matrix
* @example const csvObj = new DataSet([{col1:1,col2:5},{col1:2,col2:6}]);
csvObj.columnMatrix([['col1',{parseInt:true}],['col2']]); // =>
//[
// [1,5],
// [2,6],
//]
* @param {Array} [vectors=[]] - array of arguments for columnArray to merge columns into a matrix
* @param {Array} [data=[]] - array of data to convert to matrix
* @returns {Array} a matrix of column values
*/
static columnMatrix(vectors = [], data = []) {
const options = (data.length) ? { data, } : {};
const columnVectors = (Array.isArray(vectors) && Array.isArray(vectors[0]))
? vectors
: vectors.map(vector => [vector, options,]);
const vectorArrays = columnVectors
//@ts-ignore
.map((vec) => DataSet.columnArray.call(this, ...vec));
return util$b.pivotArrays(vectorArrays);
}
/**
* returns a JavaScript Object from a Map (supports nested Map Objects)
* @example const csvObj = new DataSet([{col1:1,col2:5},{col1:2,col2:6}]);
csvObj.columnMatrix([['col1',{parseInt:true}],['col2']]); // =>
//[
// [1,5],
// [2,6],
//]
* @param {Map} mapObj - Map to convert into JavaScript Object
* @returns {Object} JavaScript Object converted from a Map
*/
static mapToObject(mapObj = new Map()) {
return Array.from(mapObj.keys())
.reduce((result, val) => {
const mapVal = mapObj.get(val);
if (mapVal instanceof Map) {
result[val] = DataSet.mapToObject(mapVal);
}
else if (typeof mapVal === 'function') {
result[val] = `[Function ${mapVal.name}]`;
}
else {
result[val] = JSON.parse(JSON.stringify(mapVal));
}
return result;
}, {});
}
/**
* returns 0 or 1 depending on the input value
* @example DataSet.getBinaryValue('true') // => 1
DataSet.getBinaryValue('false') // => 0
DataSet.getBinaryValue('No') // => 0
DataSet.getBinaryValue(false) // => 0
* @param {String|Number} [value=''] - value to convert to a 1 or a 0
* @returns {Number} 0 or 1 depending on truthiness of value
*/
static getBinaryValue(value = '') {
if (!value)
return 0;
switch (value) {
case false:
case 'N':
case 'n':
case 'NO':
case 'No':
case 'no':
case 'False':
case 'F':
case 'f':
return 0;
default:
return 1;
}
}
/**
* returns Object of all encoders and scalers
* @example const csvObj = new DataSet([{col1:1,col2:5},{col1:false,col2:6}]);
DataSet.fitColumns({col1:['label',{binary:true}]});
Dataset.data // => [{col1:true,col2:5},{col1:false,col2:6}]
Dataset.exportFeatures() //=> { labels: { col1: { "0": false, "1": true, "N": 0, "Yes": 1, "No": 0, "f": 0, "false": 1, } } }
* @param {Funct