brain.js
Version:
Neural networks in JavaScript
1,512 lines (1,478 loc) • 349 kB
JavaScript
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
var gpu_js = require('gpu.js');
/**
* Relu Activation, aka Rectified Linear Unit Activation
* @description https://en.wikipedia.org/wiki/Rectifier_(neural_networks)
*/
function activate$3(weight) {
return Math.max(0, weight);
}
/**
* Relu derivative
*/
function measure$3(weight, delta) {
if (weight <= 0) {
return 0;
}
return delta;
}
var relu$2 = /*#__PURE__*/Object.freeze({
__proto__: null,
activate: activate$3,
measure: measure$3
});
/**
* sigmoid activation
*/
function activate$2(value) {
return 1 / (1 + Math.exp(-value));
}
/**
* sigmoid derivative
*/
function measure$2(weight, error) {
return weight * (1 - weight) * error;
}
var sigmoid$2 = /*#__PURE__*/Object.freeze({
__proto__: null,
activate: activate$2,
measure: measure$2
});
/**
* Hyperbolic tan
*/
function activate$1(weight) {
return Math.tanh(weight);
}
/**
* @description grad for z = tanh(x) is (1 - z^2)
*/
function measure$1(weight, error) {
return (1 - weight * weight) * error;
}
var tanh$2 = /*#__PURE__*/Object.freeze({
__proto__: null,
activate: activate$1,
measure: measure$1
});
/**
* Leaky Relu Activation, aka Leaky Rectified Linear Unit Activation
* @description https://en.wikipedia.org/wiki/Rectifier_(neural_networks)
*/
function activate(weight) {
return weight > 0 ? weight : 0.01 * weight;
}
/**
* Leaky Relu derivative
*/
function measure(weight, error) {
return weight > 0 ? error : 0.01 * error;
}
var leakyRelu$1 = /*#__PURE__*/Object.freeze({
__proto__: null,
activate: activate,
measure: measure
});
var index$1 = /*#__PURE__*/Object.freeze({
__proto__: null,
relu: relu$2,
sigmoid: sigmoid$2,
tanh: tanh$2,
leakyRelu: leakyRelu$1
});
/* Functions for turning sparse hashes into arrays and vice versa */
const lookup = {
/**
* Performs `[{a: 1}, {b: 6, c: 7}] -> {a: 0, b: 1, c: 2}`
* @param {Object} hashes
* @returns {Object}
*/
toTable(hashes) {
const hash = hashes.reduce((memo, hash) => {
return Object.assign(memo, hash);
}, {});
return lookup.toHash(hash);
},
/**
* Performs `[{a: 1}, {b: 6, c: 7}] -> {a: 0, b: 1, c: 2}`
*/
toTable2D(objects2D) {
const table = {};
let valueIndex = 0;
for (let i = 0; i < objects2D.length; i++) {
const objects = objects2D[i];
for (let j = 0; j < objects.length; j++) {
const object = objects[j];
for (const p in object) {
if (object.hasOwnProperty(p) && !table.hasOwnProperty(p)) {
table[p] = valueIndex++;
}
}
}
}
return table;
},
toInputTable2D(data) {
const table = {};
let tableIndex = 0;
for (let dataIndex = 0; dataIndex < data.length; dataIndex++) {
const input = data[dataIndex].input;
for (let i = 0; i < input.length; i++) {
const object = input[i];
for (const p in object) {
if (!object.hasOwnProperty(p))
continue;
if (!table.hasOwnProperty(p)) {
table[p] = tableIndex++;
}
}
}
}
return table;
},
toOutputTable2D(data) {
const table = {};
let tableIndex = 0;
for (let dataIndex = 0; dataIndex < data.length; dataIndex++) {
const output = data[dataIndex].output;
for (let i = 0; i < output.length; i++) {
const object = output[i];
for (const p in object) {
if (!object.hasOwnProperty(p))
continue;
if (!table.hasOwnProperty(p)) {
table[p] = tableIndex++;
}
}
}
}
return table;
},
/**
* performs `{a: 6, b: 7} -> {a: 0, b: 1}`
*/
toHash(hash) {
const lookup = {};
let index = 0;
const keys = Object.keys(hash);
for (let i = 0; i < keys.length; i++) {
lookup[keys[i]] = index++;
}
return lookup;
},
/**
* performs `{a: 0, b: 1}, {a: 6} -> [6, 0]`
*/
toArray(lookup, object, arrayLength) {
const result = new Float32Array(arrayLength);
for (const p in lookup) {
if (!lookup.hasOwnProperty(p))
continue;
result[lookup[p]] = object.hasOwnProperty(p) ? object[p] : 0;
}
return result;
},
toArrayShort(lookup, object) {
const result = [];
for (const p in lookup) {
if (!lookup.hasOwnProperty(p))
continue;
if (!object.hasOwnProperty(p))
break;
result[lookup[p]] = object[p];
}
return Float32Array.from(result);
},
toArrays(lookup, objects, arrayLength) {
const result = [];
for (let i = 0; i < objects.length; i++) {
result.push(this.toArray(lookup, objects[i], arrayLength));
}
return result;
},
/**
* performs `{a: 0, b: 1}, [6, 7] -> {a: 6, b: 7}`
* @param {Object} lookup
* @param {Array} array
* @returns {Object}
*/
toObject(lookup, array) {
const object = {};
for (const p in lookup) {
if (!lookup.hasOwnProperty(p))
continue;
object[p] = array[lookup[p]];
}
return object;
},
toObjectPartial(lookup, array, offset = 0, limit = 0) {
const object = {};
let i = 0;
for (const p in lookup) {
if (!lookup.hasOwnProperty(p))
continue;
if (offset > 0) {
if (i++ < offset)
continue;
}
if (limit > 0) {
if (i++ >= limit)
continue;
}
object[p] = array[lookup[p] - offset];
}
return object;
},
dataShape(data) {
const shape = [];
let lastData;
if (data.hasOwnProperty('input')) {
shape.push('datum');
lastData = data.input;
}
else if (Array.isArray(data)) {
if (data[0] &&
data[0].input) {
shape.push('array', 'datum');
lastData = data[0].input;
}
else if (Array.isArray(data[0])) {
shape.push('array');
lastData = data[0];
}
else {
lastData = data;
}
}
else {
lastData = data;
}
let p;
while (lastData) {
p = Object.keys(lastData)[0];
if (Array.isArray(lastData) ||
typeof lastData.buffer === 'object') {
shape.push('array');
const possibleNumber = lastData[parseInt(p)];
if (typeof possibleNumber === 'number') {
shape.push('number');
break;
}
else {
lastData = possibleNumber;
}
}
else if (typeof lastData === 'object' &&
typeof lastData.buffer !== 'object') {
shape.push('object');
const possibleNumber = lastData[p];
if (typeof possibleNumber === 'number') {
shape.push('number');
break;
}
else {
lastData = possibleNumber;
}
}
else {
throw new Error('unhandled signature');
}
}
return shape;
},
addKeys(value, table) {
if (Array.isArray(value))
return table;
let i = Object.keys(table).length;
for (const p in value) {
if (!value.hasOwnProperty(p))
continue;
if (table.hasOwnProperty(p))
continue;
table[p] = i++;
}
return table;
},
};
var commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {};
function createCommonjsModule(fn, basedir, module) {
return module = {
path: basedir,
exports: {},
require: function (path, base) {
return commonjsRequire(path, (base === undefined || base === null) ? module.path : base);
}
}, fn(module, module.exports), module.exports;
}
function commonjsRequire () {
throw new Error('Dynamic requires are not currently supported by @rollup/plugin-commonjs');
}
var thaw_1 = createCommonjsModule(function (module, exports) {
var __assign = (commonjsGlobal && commonjsGlobal.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.thaw = exports.Thaw = void 0;
/**
* thaw an array of items
*/
var Thaw = /** @class */ (function () {
function Thaw(items, options) {
var _this = this;
if (options === void 0) { options = {}; }
var _a = __assign(__assign({}, Thaw.defaultSettings), options), each = _a.each, done = _a.done;
this.i = 0;
this.isStopped = false;
this.items = items;
this.options = options;
this.tick = function () {
if (_this.isStopped)
return;
_this.timeout = setTimeout(_this.tick, 0);
if (Thaw.thawing)
return;
var item = _this.items[_this.i];
if (_this.i >= _this.items.length) {
if (done !== null) {
Thaw.thawing = true;
done();
Thaw.thawing = false;
}
_this.isStopped = true;
clearTimeout(_this.timeout);
return;
}
if (each !== null) {
Thaw.thawing = true;
each(item, _this.i);
Thaw.thawing = false;
}
else if (item !== undefined) {
item();
}
_this.i++;
};
Thaw.thaws.push(this);
if (!options.delay) {
this.tick();
}
}
Object.defineProperty(Thaw, "isThawing", {
/**
* returns if Thaw.js is thawing
*/
get: function () {
return Thaw.thawing;
},
enumerable: false,
configurable: true
});
/**
* Stops all Thaw instances
*/
Thaw.stopAll = function () {
for (var i = 0; i < Thaw.thaws.length; i++) {
Thaw.thaws[i].stop();
}
};
/**
* readies thaw to continue
*/
Thaw.prototype.makeReady = function () {
if (this.isStopped) {
this.isStopped = false;
return true;
}
return false;
};
/**
* Adds an item to the end of this instance of Thaw and readies Thaw to process it
*/
Thaw.prototype.add = function (item) {
this.items.push(item);
if (this.makeReady()) {
this.tick();
}
return this;
};
/**
* Inserts an item just after the current item being processed in Thaw and readies Thaw to process it
*/
Thaw.prototype.insert = function (item) {
this.items.splice(this.i, 0, item);
if (this.makeReady()) {
this.tick();
}
return this;
};
/**
* Adds an Array to the end of this instance of Thaw and readies Thaw to process it
*/
Thaw.prototype.addArray = function (items) {
this.items = this.items.concat(items);
if (this.makeReady()) {
this.tick();
}
return this;
};
/**
* Inserts an Array just after the current item being processed in Thaw and readies Thaw to process them
*/
Thaw.prototype.insertArray = function (items) {
var before = this.items.splice(0, this.i);
var after = this.items;
this.items = before.concat(items, after);
if (this.makeReady()) {
this.tick();
}
return this;
};
/**
* Stops this instance of Thaw
*/
Thaw.prototype.stop = function () {
this.isStopped = true;
clearTimeout(this.timeout);
if (this.options.done) {
this.options.done();
}
return this;
};
Thaw.thawing = false;
Thaw.thaws = [];
Thaw.defaultSettings = {
each: null,
done: null
};
return Thaw;
}());
exports.Thaw = Thaw;
/**
* simple thaw
*/
function thaw(items, options) {
return new Thaw(items, options);
}
exports.thaw = thaw;
});
var block = createCommonjsModule(function (module, exports) {
Object.defineProperty(exports, "__esModule", { value: true });
exports.Block = void 0;
var Block = /** @class */ (function () {
function Block(options, count) {
if (count === void 0) { count = 200; }
this.index = 0;
this.thaws = [];
this.count = count;
this.options = options;
}
/**
* add an item to the end of items
*/
Block.prototype.add = function (item) {
var next = this.next();
next.add(item);
return this;
};
/**
* add an Array to the end of items
*/
Block.prototype.addArray = function (items) {
var next = this.next();
next.addArray(items);
return this;
};
/**
* insert an item into items @ current position
*/
Block.prototype.insert = function (item) {
var next = this.next();
next.insert(item);
return this;
};
/**
* insert and array into items @ current position
*/
Block.prototype.insertArray = function (items) {
var next = this.next();
next.insertArray(items);
return this;
};
/**
* Stops all thaws in this block
*/
Block.prototype.stop = function () {
for (var i = 0; i < this.thaws.length; i++) {
this.thaws[i].stop();
}
return this;
};
/**
* Get next available in block
*/
Block.prototype.next = function () {
var thaw;
var thaws = this.thaws;
if (thaws.length < this.count) {
thaw = new thaw_1.Thaw([], this.options);
thaws.push(thaw);
}
else {
thaw = thaws[this.index] || null;
}
this.index++;
if (this.index >= this.count) {
this.index = 0;
}
return thaw;
};
return Block;
}());
exports.Block = Block;
});
var dist = createCommonjsModule(function (module, exports) {
Object.defineProperty(exports, "__esModule", { value: true });
exports.Block = exports.thaw = exports.Thaw = void 0;
Object.defineProperty(exports, "Thaw", { enumerable: true, get: function () { return thaw_1.Thaw; } });
Object.defineProperty(exports, "thaw", { enumerable: true, get: function () { return thaw_1.thaw; } });
Object.defineProperty(exports, "Block", { enumerable: true, get: function () { return block.Block; } });
if (typeof window !== 'undefined') {
// @ts-ignore
window.Thaw = thaw_1.Thaw;
// @ts-ignore
window.thaw = thaw_1.thaw;
// @ts-ignore
window.Thaw.Block = block.Block;
}
});
function arraysToFloat32Arrays(arrays) {
const result = [];
for (let i = 0; i < arrays.length; i++) {
result.push(Float32Array.from(arrays[i]));
}
return result;
}
function inputOutputArraysToFloat32Arrays(input, output) {
const result = [];
for (let i = 0; i < input.length; i++) {
result.push(Float32Array.from(input[i]));
}
for (let i = 0; i < output.length; i++) {
result.push(Float32Array.from(output[i]));
}
return result;
}
function arrayToFloat32Arrays(array) {
const result = [];
for (let i = 0; i < array.length; i++) {
result.push(Float32Array.from([array[i]]));
}
return result;
}
function inputOutputArrayToFloat32Arrays(input, output) {
const result = [];
for (let i = 0; i < input.length; i++) {
result.push(Float32Array.from([input[i]]));
}
for (let i = 0; i < output.length; i++) {
result.push(Float32Array.from([output[i]]));
}
return result;
}
function arrayToFloat32Array(array) {
return Float32Array.from(array);
}
function inputOutputObjectsToFloat32Arrays(input, output, inputTable, outputTable, inputLength, outputLength) {
const results = [];
for (let i = 0; i < input.length; i++) {
const object = input[i];
const result = new Float32Array(inputLength);
for (const p in object) {
if (object.hasOwnProperty(p)) {
result[inputTable[p]] = object[p];
}
}
results.push(result);
}
for (let i = 0; i < output.length; i++) {
const object = output[i];
const result = new Float32Array(outputLength);
for (const p in object) {
if (object.hasOwnProperty(p)) {
result[outputTable[p]] = object[p];
}
}
results.push(result);
}
return results;
}
function objectToFloat32Arrays(object) {
const result = [];
for (const p in object) {
if (!object.hasOwnProperty(p))
continue;
result.push(Float32Array.from([object[p]]));
}
return result;
}
function inputOutputObjectToFloat32Arrays(input, output) {
const result = [];
for (const p in input) {
if (!input.hasOwnProperty(p))
continue;
result.push(Float32Array.from([input[p]]));
}
for (const p in output) {
if (!output.hasOwnProperty(p))
continue;
result.push(Float32Array.from([output[p]]));
}
return result;
}
function objectToFloat32Array(object, table, length) {
const result = new Float32Array(length);
for (const p in object) {
if (object.hasOwnProperty(p)) {
result[table[p]] = object[p];
}
}
return result;
}
class LookupTable {
constructor(data, prop) {
this.prop = null;
this.table = {};
this.length = 0;
const table = this.table;
if (prop) {
this.prop = prop;
for (let i = 0; i < data.length; i++) {
const datum = data[i];
const object = datum[prop];
for (const p in object) {
if (!object.hasOwnProperty(p))
continue;
if (table.hasOwnProperty(p))
continue;
table[p] = this.length++;
}
}
}
else if (Array.isArray(data) && Array.isArray(data[0])) {
for (let i = 0; i < data.length; i++) {
const array = data[i];
for (let j = 0; j < array.length; j++) {
const object = array[j];
for (const p in object) {
if (!object.hasOwnProperty(p))
continue;
if (table.hasOwnProperty(p))
continue;
table[p] = this.length++;
}
}
}
}
else {
for (let i = 0; i < data.length; i++) {
const object = data[i];
for (const p in object) {
if (!object.hasOwnProperty(p))
continue;
if (table.hasOwnProperty(p))
continue;
table[p] = this.length++;
}
}
}
}
}
function max(values) {
if (Array.isArray(values) || values instanceof Float32Array) {
return Math.max(...values);
}
else {
return Math.max(...Object.values(values));
}
}
function mse$1(errors) {
// mean squared error
let sum = 0;
for (let i = 0; i < errors.length; i++) {
sum += errors[i] ** 2;
}
return sum / errors.length;
}
function randomWeight() {
return Math.random() * 0.4 - 0.2;
}
/**
* Returns a random float between given min and max bounds (inclusive)
* @param min Minimum value of the ranfom float
* @param max Maximum value of the random float
*/
function randomFloat(min, max) {
return Math.random() * (max - min) + min;
}
/**
* Complicated math. All you need to know is that it returns a random number.
* More info: https://en.wikipedia.org/wiki/Normal_distribution
*/
function gaussRandom() {
if (gaussRandom.returnV) {
gaussRandom.returnV = false;
return gaussRandom.vVal;
}
const u = 2 * Math.random() - 1;
const v = 2 * Math.random() - 1;
const r = u * u + v * v;
if (r === 0 || r > 1) {
return gaussRandom();
}
const c = Math.sqrt((-2 * Math.log(r)) / r);
gaussRandom.vVal = v * c; // cache this
gaussRandom.returnV = true;
return u * c;
}
/**
* Returns a random integer between given min and max bounds
* @param min Minimum value of the random integer
* @param max Maximum value of the random integer
*/
function randomInteger(min, max) {
return Math.floor(Math.random() * (max - min) + min);
}
/**
* If you know what this is: https://en.wikipedia.org/wiki/Normal_distribution
* @param mu
* @param std
*/
function randomN(mu, std) {
return mu + gaussRandom() * std;
}
gaussRandom.returnV = false;
gaussRandom.vVal = 0;
var random$1 = /*#__PURE__*/Object.freeze({
__proto__: null,
randomFloat: randomFloat,
gaussRandom: gaussRandom,
randomInteger: randomInteger,
randomN: randomN
});
/**
* Returns an array of given size, full of randomness
*/
function randos(size, std = null) {
const array = new Float32Array(size);
if (std === null) {
for (let i = 0; i < size; i++) {
array[i] = randomWeight();
}
}
else {
for (let i = 0; i < size; i++) {
array[i] = randomFloat(-std, std);
}
}
return array;
}
/**
* Returns a 2D matrix of given size, full of randomness
*/
function randos2D(width, height, std) {
const result = new Array(height);
for (let y = 0; y < height; y++) {
result[y] = randos(width, std);
}
return result;
}
/**
* Returns a 3D tensor of given size, full of randomness
*/
function randos3D(width, height, depth, std) {
const result = new Array(depth);
for (let z = 0; z < depth; z++) {
result[z] = randos2D(width, height, std);
}
return result;
}
/**
* Returns an array of zeros
*/
function zeros$1(size) {
return new Float32Array(size);
}
function getTypedArrayFn(value, table) {
if (value.buffer instanceof ArrayBuffer) {
return null;
}
if (Array.isArray(value)) {
return arrayToFloat32Array;
}
if (!table)
throw new Error('table is not Object');
const { length } = Object.keys(table);
return (v) => {
const array = new Float32Array(length);
for (const p in table) {
if (!table.hasOwnProperty(p))
continue;
if (typeof v[p] !== 'number')
continue;
array[table[p]] = v[p] || 0;
}
return array;
};
}
function defaults$8() {
return {
inputSize: 0,
outputSize: 0,
binaryThresh: 0.5,
};
}
function trainDefaults$3() {
return {
activation: 'sigmoid',
iterations: 20000,
errorThresh: 0.005,
log: false,
logPeriod: 10,
leakyReluAlpha: 0.01,
learningRate: 0.3,
momentum: 0.1,
callbackPeriod: 10,
timeout: Infinity,
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
};
}
class NeuralNetwork {
constructor(options = {}) {
this.options = defaults$8();
this.trainOpts = trainDefaults$3();
this.sizes = [];
this.outputLayer = -1;
this.biases = [];
this.weights = []; // weights for bias nodes
this.outputs = [];
// state for training
this.deltas = [];
this.changes = []; // for momentum
this.errors = [];
this.errorCheckInterval = 1;
this.inputLookup = null;
this.inputLookupLength = 0;
this.outputLookup = null;
this.outputLookupLength = 0;
this._formatInput = null;
this._formatOutput = null;
this.runInput = (input) => {
this.setActivation();
return this.runInput(input);
};
this.calculateDeltas = (output) => {
this.setActivation();
return this.calculateDeltas(output);
};
// adam
this.biasChangesLow = [];
this.biasChangesHigh = [];
this.changesLow = [];
this.changesHigh = [];
this.iterations = 0;
this.options = { ...this.options, ...options };
this.updateTrainingOptions(options);
const { inputSize, hiddenLayers, outputSize } = this.options;
if (inputSize && outputSize) {
this.sizes = [inputSize].concat(hiddenLayers !== null && hiddenLayers !== void 0 ? hiddenLayers : []).concat([outputSize]);
}
}
/**
*
* Expects this.sizes to have been set
*/
initialize() {
if (!this.sizes.length) {
throw new Error('Sizes must be set before initializing');
}
this.outputLayer = this.sizes.length - 1;
this.biases = new Array(this.outputLayer); // weights for bias nodes
this.weights = new Array(this.outputLayer);
this.outputs = new Array(this.outputLayer);
// state for training
this.deltas = new Array(this.outputLayer);
this.changes = new Array(this.outputLayer); // for momentum
this.errors = new Array(this.outputLayer);
for (let layerIndex = 0; layerIndex <= this.outputLayer; layerIndex++) {
const size = this.sizes[layerIndex];
this.deltas[layerIndex] = zeros$1(size);
this.errors[layerIndex] = zeros$1(size);
this.outputs[layerIndex] = zeros$1(size);
if (layerIndex > 0) {
this.biases[layerIndex] = randos(size);
this.weights[layerIndex] = new Array(size);
this.changes[layerIndex] = new Array(size);
for (let nodeIndex = 0; nodeIndex < size; nodeIndex++) {
const prevSize = this.sizes[layerIndex - 1];
this.weights[layerIndex][nodeIndex] = randos(prevSize);
this.changes[layerIndex][nodeIndex] = zeros$1(prevSize);
}
}
}
this.setActivation();
if (this.trainOpts.praxis === 'adam') {
this._setupAdam();
}
}
setActivation(activation) {
const value = activation !== null && activation !== void 0 ? activation : this.trainOpts.activation;
switch (value) {
case 'sigmoid':
this.runInput = this._runInputSigmoid;
this.calculateDeltas = this._calculateDeltasSigmoid;
break;
case 'relu':
this.runInput = this._runInputRelu;
this.calculateDeltas = this._calculateDeltasRelu;
break;
case 'leaky-relu':
this.runInput = this._runInputLeakyRelu;
this.calculateDeltas = this._calculateDeltasLeakyRelu;
break;
case 'tanh':
this.runInput = this._runInputTanh;
this.calculateDeltas = this._calculateDeltasTanh;
break;
default:
throw new Error(`Unknown activation ${value}. Available activations are: 'sigmoid', 'relu', 'leaky-relu', 'tanh'`);
}
}
get isRunnable() {
return this.sizes.length > 0;
}
run(input) {
if (!this.isRunnable) {
throw new Error('network not runnable');
}
let formattedInput;
if (this.inputLookup) {
formattedInput = lookup.toArray(this.inputLookup, input, this.inputLookupLength);
}
else {
formattedInput = input;
}
this.validateInput(formattedInput);
const output = this.runInput(formattedInput).slice(0);
if (this.outputLookup) {
return lookup.toObject(this.outputLookup, output);
}
return output;
}
_runInputSigmoid(input) {
this.outputs[0] = input; // set output state of input layer
let output = null;
for (let layer = 1; layer <= this.outputLayer; layer++) {
const activeLayer = this.sizes[layer];
const activeWeights = this.weights[layer];
const activeBiases = this.biases[layer];
const activeOutputs = this.outputs[layer];
for (let node = 0; node < activeLayer; node++) {
const weights = activeWeights[node];
let sum = activeBiases[node];
for (let k = 0; k < weights.length; k++) {
sum += weights[k] * input[k];
}
// sigmoid
activeOutputs[node] = 1 / (1 + Math.exp(-sum));
}
output = input = activeOutputs;
}
if (!output) {
throw new Error('output was empty');
}
return output;
}
_runInputRelu(input) {
this.outputs[0] = input; // set output state of input layer
let output = null;
for (let layer = 1; layer <= this.outputLayer; layer++) {
const activeSize = this.sizes[layer];
const activeWeights = this.weights[layer];
const activeBiases = this.biases[layer];
const activeOutputs = this.outputs[layer];
for (let node = 0; node < activeSize; node++) {
const weights = activeWeights[node];
let sum = activeBiases[node];
for (let k = 0; k < weights.length; k++) {
sum += weights[k] * input[k];
}
// relu
activeOutputs[node] = sum < 0 ? 0 : sum;
}
output = input = activeOutputs;
}
if (!output) {
throw new Error('output was empty');
}
return output;
}
_runInputLeakyRelu(input) {
this.outputs[0] = input; // set output state of input layer
const { leakyReluAlpha } = this.trainOpts;
let output = null;
for (let layer = 1; layer <= this.outputLayer; layer++) {
const activeSize = this.sizes[layer];
const activeWeights = this.weights[layer];
const activeBiases = this.biases[layer];
const activeOutputs = this.outputs[layer];
for (let node = 0; node < activeSize; node++) {
const weights = activeWeights[node];
let sum = activeBiases[node];
for (let k = 0; k < weights.length; k++) {
sum += weights[k] * input[k];
}
// leaky relu
activeOutputs[node] = Math.max(sum, leakyReluAlpha * sum);
}
output = input = activeOutputs;
}
if (!output) {
throw new Error('output was empty');
}
return output;
}
_runInputTanh(input) {
this.outputs[0] = input; // set output state of input layer
let output = null;
for (let layer = 1; layer <= this.outputLayer; layer++) {
const activeSize = this.sizes[layer];
const activeWeights = this.weights[layer];
const activeBiases = this.biases[layer];
const activeOutputs = this.outputs[layer];
for (let node = 0; node < activeSize; node++) {
const weights = activeWeights[node];
let sum = activeBiases[node];
for (let k = 0; k < weights.length; k++) {
sum += weights[k] * input[k];
}
// tanh
activeOutputs[node] = Math.tanh(sum);
}
output = input = activeOutputs;
}
if (!output) {
throw new Error('output was empty');
}
return output;
}
/**
*
* Verifies network sizes are initialized
* If they are not it will initialize them based off the data set.
*/
verifyIsInitialized(preparedData) {
if (this.sizes.length && this.outputLayer > 0)
return;
this.sizes = [];
this.sizes.push(preparedData[0].input.length);
if (!this.options.hiddenLayers) {
this.sizes.push(Math.max(3, Math.floor(preparedData[0].input.length / 2)));
}
else {
this.options.hiddenLayers.forEach((size) => {
this.sizes.push(size);
});
}
this.sizes.push(preparedData[0].output.length);
this.initialize();
}
updateTrainingOptions(trainOpts) {
const merged = { ...this.trainOpts, ...trainOpts };
this.validateTrainingOptions(merged);
this.trainOpts = merged;
this.setLogMethod(this.trainOpts.log);
}
validateTrainingOptions(options) {
const validations = {
activation: () => {
return ['sigmoid', 'relu', 'leaky-relu', 'tanh'].includes(options.activation);
},
iterations: () => {
const val = options.iterations;
return typeof val === 'number' && val > 0;
},
errorThresh: () => {
const val = options.errorThresh;
return typeof val === 'number' && val > 0 && val < 1;
},
log: () => {
const val = options.log;
return typeof val === 'function' || typeof val === 'boolean';
},
logPeriod: () => {
const val = options.logPeriod;
return typeof val === 'number' && val > 0;
},
leakyReluAlpha: () => {
const val = options.leakyReluAlpha;
return typeof val === 'number' && val > 0 && val < 1;
},
learningRate: () => {
const val = options.learningRate;
return typeof val === 'number' && val > 0 && val < 1;
},
momentum: () => {
const val = options.momentum;
return typeof val === 'number' && val > 0 && val < 1;
},
callback: () => {
const val = options.callback;
return typeof val === 'function' || val === undefined;
},
callbackPeriod: () => {
const val = options.callbackPeriod;
return typeof val === 'number' && val > 0;
},
timeout: () => {
const val = options.timeout;
return typeof val === 'number' && val > 0;
},
praxis: () => {
const val = options.praxis;
return !val || val === 'adam';
},
beta1: () => {
const val = options.beta1;
return val > 0 && val < 1;
},
beta2: () => {
const val = options.beta2;
return val > 0 && val < 1;
},
epsilon: () => {
const val = options.epsilon;
return val > 0 && val < 1;
},
};
for (const p in validations) {
const v = options;
if (!validations[p]()) {
throw new Error(`[${p}, ${v[p]}] is out of normal training range, your network will probably not train.`);
}
}
}
/**
*
* Gets JSON of trainOpts object
* NOTE: Activation is stored directly on JSON object and not in the training options
*/
getTrainOptsJSON() {
const { activation, iterations, errorThresh, log, logPeriod, leakyReluAlpha, learningRate, momentum, callbackPeriod, timeout, praxis, beta1, beta2, epsilon, } = this.trainOpts;
return {
activation,
iterations,
errorThresh,
log: typeof log === 'function'
? true
: typeof log === 'boolean'
? log
: false,
logPeriod,
leakyReluAlpha,
learningRate,
momentum,
callbackPeriod,
timeout: timeout === Infinity ? 'Infinity' : timeout,
praxis,
beta1,
beta2,
epsilon,
};
}
setLogMethod(log) {
if (typeof log === 'function') {
this.trainOpts.log = log;
}
else if (log) {
this.trainOpts.log = this.logTrainingStatus;
}
else {
this.trainOpts.log = false;
}
}
logTrainingStatus(status) {
console.log(`iterations: ${status.iterations}, training error: ${status.error}`);
}
calculateTrainingError(data) {
let sum = 0;
for (let i = 0; i < data.length; ++i) {
sum += this.trainPattern(data[i], true);
}
return sum / data.length;
}
trainPatterns(data) {
for (let i = 0; i < data.length; ++i) {
this.trainPattern(data[i]);
}
}
trainingTick(data, status, endTime) {
const { callback, callbackPeriod, errorThresh, iterations, log, logPeriod, } = this.trainOpts;
if (status.iterations >= iterations ||
status.error <= errorThresh ||
Date.now() >= endTime) {
return false;
}
status.iterations++;
if (log && status.iterations % logPeriod === 0) {
status.error = this.calculateTrainingError(data);
log(status);
}
else if (status.iterations % this.errorCheckInterval === 0) {
status.error = this.calculateTrainingError(data);
}
else {
this.trainPatterns(data);
}
if (callback && status.iterations % callbackPeriod === 0) {
callback({
iterations: status.iterations,
error: status.error,
});
}
return true;
}
prepTraining(data, options = {}) {
this.updateTrainingOptions(options);
const preparedData = this.formatData(data);
const endTime = Date.now() + this.trainOpts.timeout;
const status = {
error: 1,
iterations: 0,
};
this.verifyIsInitialized(preparedData);
this.validateData(preparedData);
return {
preparedData,
status,
endTime,
};
}
train(data, options = {}) {
const { preparedData, status, endTime } = this.prepTraining(data, options);
while (true) {
if (!this.trainingTick(preparedData, status, endTime)) {
break;
}
}
return status;
}
async trainAsync(data, options = {}) {
const { preparedData, status, endTime } = this.prepTraining(data, options);
return await new Promise((resolve, reject) => {
try {
const thawedTrain = new dist.Thaw(new Array(this.trainOpts.iterations), {
delay: true,
each: () => this.trainingTick(preparedData, status, endTime) ||
thawedTrain.stop(),
done: () => resolve(status),
});
thawedTrain.tick();
}
catch (trainError) {
reject(trainError);
}
});
}
trainPattern(value, logErrorRate) {
// forward propagate
this.runInput(value.input);
// back propagate
this.calculateDeltas(value.output);
this.adjustWeights();
if (logErrorRate) {
return mse$1(this.errors[this.outputLayer]);
}
return null;
}
_calculateDeltasSigmoid(target) {
for (let layer = this.outputLayer; layer >= 0; layer--) {
const activeSize = this.sizes[layer];
const activeOutput = this.outputs[layer];
const activeError = this.errors[layer];
const activeDeltas = this.deltas[layer];
const nextLayer = this.weights[layer + 1];
for (let node = 0; node < activeSize; node++) {
const output = activeOutput[node];
let error = 0;
if (layer === this.outputLayer) {
error = target[node] - output;
}
else {
const deltas = this.deltas[layer + 1];
for (let k = 0; k < deltas.length; k++) {
error += deltas[k] * nextLayer[k][node];
}
}
activeError[node] = error;
activeDeltas[node] = error * output * (1 - output);
}
}
}
_calculateDeltasRelu(target) {
for (let layer = this.outputLayer; layer >= 0; layer--) {
const currentSize = this.sizes[layer];
const currentOutputs = this.outputs[layer];
const nextWeights = this.weights[layer + 1];
const nextDeltas = this.deltas[layer + 1];
const currentErrors = this.errors[layer];
const currentDeltas = this.deltas[layer];
for (let node = 0; node < currentSize; node++) {
const output = currentOutputs[node];
let error = 0;
if (layer === this.outputLayer) {
error = target[node] - output;
}
else {
for (let k = 0; k < nextDeltas.length; k++) {
error += nextDeltas[k] * nextWeights[k][node];
}
}
currentErrors[node] = error;
currentDeltas[node] = output > 0 ? error : 0;
}
}
}
_calculateDeltasLeakyRelu(target) {
const alpha = this.trainOpts.leakyReluAlpha;
for (let layer = this.outputLayer; layer >= 0; layer--) {
const currentSize = this.sizes[layer];
const currentOutputs = this.outputs[layer];
const nextDeltas = this.deltas[layer + 1];
const nextWeights = this.weights[layer + 1];
const currentErrors = this.errors[layer];
const currentDeltas = this.deltas[layer];
for (let node = 0; node < currentSize; node++) {
const output = currentOutputs[node];
let error = 0;
if (layer === this.outputLayer) {
error = target[node] - output;
}
else {
for (let k = 0; k < nextDeltas.length; k++) {
error += nextDeltas[k] * nextWeights[k][node];
}
}
currentErrors[node] = error;
currentDeltas[node] = output > 0 ? error : alpha * error;
}
}
}
_calculateDeltasTanh(target) {
for (let layer = this.outputLayer; layer >= 0; layer--) {
const currentSize = this.sizes[layer];
const currentOutputs = this.outputs[layer];
const nextDeltas = this.deltas[layer + 1];
const nextWeights = this.weights[layer + 1];
const currentErrors = this.errors[layer];
const currentDeltas = this.deltas[layer];
for (let node = 0; node < currentSize; node++) {
const output = currentOutputs[node];
let error = 0;
if (layer === this.outputLayer) {
error = target[node] - output;
}
else {
for (let k = 0; k < nextDeltas.length; k++) {
error += nextDeltas[k] * nextWeights[k][node];
}
}
currentErrors[node] = error;
currentDeltas[node] = (1 - output * output) * error;
}
}
}
/**
*
* Changes weights of networks
*/
adjustWeights() {
const { learningRate, momentum } = this.trainOpts;
for (let layer = 1; layer <= this.outputLayer; layer++) {
const incoming = this.outputs[layer - 1];
const activeSize = this.sizes[layer];
const activeDelta = this.deltas[layer];
const activeChanges = this.changes[layer];
const activeWeights = this.weights[layer];
const activeBiases = this.biases[layer];
for (let node = 0; node < activeSize; node++) {
const delta = activeDelta[node];
for (let k = 0; k < incoming.length; k++) {
let change = activeChanges[node][k];
change = learningRate * delta * incoming[k] + momentum * change;
activeChanges[node][k] = change;
activeWeights[node][k] += change;
}
activeBiases[node] += learningRate * delta;
}
}
}
_setupAdam() {
this.biasChangesLow = [];
this.biasChangesHigh = [];
this.changesLow = [];
this.changesHigh = [];
this.iterations = 0;
for (let layer = 0; layer <= this.outputLayer; layer++) {
const size = this.sizes[layer];
if (layer > 0) {
this.biasChangesLow[layer] = zeros$1(size);
this.biasChangesHigh[layer] = zeros$1(size);
this.changesLow[layer] = new Array(size);
this.changesHigh[layer] = new Array(size);
for (let node = 0; node < size; node++) {
const prevSize = this.sizes[layer - 1];
this.changesLow[layer][node] = zeros$1(prevSize);
this.changesHigh[layer][node] = zeros$1(prevSize);
}
}
}
this.adjustWeights = this._adjustWeightsAdam;
}
_adjustWeightsAdam() {
this.iterations++;
const { iterations } = this;
const { beta1, beta2, epsilon, learningRate } = this.trainOpts;
for (let layer = 1; layer <= this.outputLayer; layer++) {
const incoming = this.outputs[layer - 1];
const currentSize = this.sizes[layer];
const currentDeltas = this.deltas[layer];
const currentChangesLow = this.changesLow[layer];
const currentChangesHigh = this.changesHigh[layer];
const currentWeights = this.weights[layer];
const currentBiases = this.biases[layer];
const currentBiasChangesLow = this.biasChangesLow[layer];
const currentBiasChangesHigh = this.biasChangesHigh[layer];
for (let node = 0; node < currentSize; node++) {
const delta = currentDeltas[node];
for (let k = 0; k < incoming.length; k++) {
const gradient = delta * incoming[k];
const changeLow = currentChangesLow[node][k] * beta1 + (1 - beta1) * gradient;
const changeHigh = currentChangesHigh[node][k] * beta2 +
(1 - beta2) * gradient * gradient;
const momentumCorrection = changeLow / (1 - Math.pow(beta1, iterations));
const gradientCorrection = changeHigh / (1 - Math.pow(beta2, iterations));
currentChangesLow[node][k] = changeLow;
currentChangesHigh[node][k] = changeHigh;
currentWeights[node][k] +=
(learningRate * momentum