neataptic
Version:
Architecture-free neural network library with genetic algorithm implementations
1,642 lines (1,380 loc) • 117 kB
JavaScript
/*!
* The MIT License (MIT)
*
* Copyright 2017 Thomas Wagenaar <wagenaartje@protonmail.com>. Copyright for
* portions of Neataptic are held by Copyright 2017 Juan Cazala - cazala.com, as a
* part of project Synaptic.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE
*
*/
(function webpackUniversalModuleDefinition(root, factory) {
if(typeof exports === 'object' && typeof module === 'object')
module.exports = factory();
else if(typeof define === 'function' && define.amd)
define([], factory);
else if(typeof exports === 'object')
exports["neataptic"] = factory();
else
root["neataptic"] = factory();
})(this, function() {
return /******/ (function(modules) { // webpackBootstrap
/******/ // The module cache
/******/ var installedModules = {};
/******/
/******/ // The require function
/******/ function __webpack_require__(moduleId) {
/******/
/******/ // Check if module is in cache
/******/ if(installedModules[moduleId])
/******/ return installedModules[moduleId].exports;
/******/
/******/ // Create a new module (and put it into the cache)
/******/ var module = installedModules[moduleId] = {
/******/ i: moduleId,
/******/ l: false,
/******/ exports: {}
/******/ };
/******/
/******/ // Execute the module function
/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
/******/
/******/ // Flag the module as loaded
/******/ module.l = true;
/******/
/******/ // Return the exports of the module
/******/ return module.exports;
/******/ }
/******/
/******/
/******/ // expose the modules object (__webpack_modules__)
/******/ __webpack_require__.m = modules;
/******/
/******/ // expose the module cache
/******/ __webpack_require__.c = installedModules;
/******/
/******/ // identity function for calling harmony imports with the correct context
/******/ __webpack_require__.i = function(value) { return value; };
/******/
/******/ // define getter function for harmony exports
/******/ __webpack_require__.d = function(exports, name, getter) {
/******/ if(!__webpack_require__.o(exports, name)) {
/******/ Object.defineProperty(exports, name, {
/******/ configurable: false,
/******/ enumerable: true,
/******/ get: getter
/******/ });
/******/ }
/******/ };
/******/
/******/ // getDefaultExport function for compatibility with non-harmony modules
/******/ __webpack_require__.n = function(module) {
/******/ var getter = module && module.__esModule ?
/******/ function getDefault() { return module['default']; } :
/******/ function getModuleExports() { return module; };
/******/ __webpack_require__.d(getter, 'a', getter);
/******/ return getter;
/******/ };
/******/
/******/ // Object.prototype.hasOwnProperty.call
/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); };
/******/
/******/ // __webpack_public_path__
/******/ __webpack_require__.p = "";
/******/
/******/ // Load entry module and return exports
/******/ return __webpack_require__(__webpack_require__.s = 21);
/******/ })
/************************************************************************/
/******/ ([
/* 0 */
/***/ (function(module, exports, __webpack_require__) {
/*******************************************************************************
METHODS
*******************************************************************************/
var methods = {
activation: __webpack_require__(9),
mutation: __webpack_require__(16),
selection: __webpack_require__(18),
crossover: __webpack_require__(14),
cost: __webpack_require__(13),
gating: __webpack_require__(15),
connection: __webpack_require__(12),
rate: __webpack_require__(17)
};
/** Export */
module.exports = methods;
/***/ }),
/* 1 */
/***/ (function(module, exports, __webpack_require__) {
/* Export */
module.exports = Node;
/* Import */
var methods = __webpack_require__(0);
var Connection = __webpack_require__(3);
var config = __webpack_require__(2);
/*******************************************************************************
NODE
*******************************************************************************/
function Node (type) {
this.bias = (type === 'input') ? 0 : Math.random() * 0.2 - 0.1;
this.squash = methods.activation.LOGISTIC;
this.type = type || 'hidden';
this.activation = 0;
this.state = 0;
this.old = 0;
// For dropout
this.mask = 1;
// For tracking momentum
this.previousDeltaBias = 0;
// Batch training
this.totalDeltaBias = 0;
this.connections = {
in: [],
out: [],
gated: [],
self: new Connection(this, this, 0)
};
// Data for backpropagation
this.error = {
responsibility: 0,
projected: 0,
gated: 0
};
}
Node.prototype = {
/**
* Activates the node
*/
activate: function (input) {
// Check if an input is given
if (typeof input !== 'undefined') {
this.activation = input;
return this.activation;
}
this.old = this.state;
// All activation sources coming from the node itself
this.state = this.connections.self.gain * this.connections.self.weight * this.state + this.bias;
// Activation sources coming from connections
var i;
for (i = 0; i < this.connections.in.length; i++) {
var connection = this.connections.in[i];
this.state += connection.from.activation * connection.weight * connection.gain;
}
// Squash the values received
this.activation = this.squash(this.state) * this.mask;
this.derivative = this.squash(this.state, true);
// Update traces
var nodes = [];
var influences = [];
for (i = 0; i < this.connections.gated.length; i++) {
let conn = this.connections.gated[i];
let node = conn.to;
let index = nodes.indexOf(node);
if (index > -1) {
influences[index] += conn.weight * conn.from.activation;
} else {
nodes.push(node);
influences.push(conn.weight * conn.from.activation +
(node.connections.self.gater === this ? node.old : 0));
}
// Adjust the gain to this nodes' activation
conn.gain = this.activation;
}
for (i = 0; i < this.connections.in.length; i++) {
let connection = this.connections.in[i];
// Elegibility trace
connection.elegibility = this.connections.self.gain * this.connections.self.weight *
connection.elegibility + connection.from.activation * connection.gain;
// Extended trace
for (var j = 0; j < nodes.length; j++) {
let node = nodes[j];
let influence = influences[j];
let index = connection.xtrace.nodes.indexOf(node);
if (index > -1) {
connection.xtrace.values[index] = node.connections.self.gain * node.connections.self.weight *
connection.xtrace.values[index] + this.derivative * connection.elegibility * influence;
} else {
// Does not exist there yet, might be through mutation
connection.xtrace.nodes.push(node);
connection.xtrace.values.push(this.derivative * connection.elegibility * influence);
}
}
}
return this.activation;
},
/**
* Back-propagate the error, aka learn
*/
propagate: function (rate, momentum, update, target) {
momentum = momentum || 0;
rate = rate || 0.3;
// Error accumulator
var error = 0;
// Output nodes get their error from the enviroment
if (this.type === 'output') {
this.error.responsibility = this.error.projected = target - this.activation;
} else { // the rest of the nodes compute their error responsibilities by backpropagation
// error responsibilities from all the connections projected from this node
var i;
for (i = 0; i < this.connections.out.length; i++) {
let connection = this.connections.out[i];
let node = connection.to;
// Eq. 21
error += node.error.responsibility * connection.weight * connection.gain;
}
// Projected error responsibility
this.error.projected = this.derivative * error;
// Error responsibilities from all connections gated by this neuron
error = 0;
for (i = 0; i < this.connections.gated.length; i++) {
let conn = this.connections.gated[i];
let node = conn.to;
let influence = node.connections.self.gater === this ? node.old : 0;
influence += conn.weight * conn.from.activation;
error += node.error.responsibility * influence;
}
// Gated error responsibility
this.error.gated = this.derivative * error;
// Error responsibility
this.error.responsibility = this.error.projected + this.error.gated;
}
if (this.type === 'constant') return;
// Adjust all the node's incoming connections
for (i = 0; i < this.connections.in.length; i++) {
let connection = this.connections.in[i];
let gradient = this.error.projected * connection.elegibility;
for (var j = 0; j < connection.xtrace.nodes.length; j++) {
let node = connection.xtrace.nodes[j];
let value = connection.xtrace.values[j];
gradient += node.error.responsibility * value;
}
// Adjust weight
let deltaWeight = rate * gradient * this.mask;
connection.totalDeltaWeight += deltaWeight;
if (update) {
connection.totalDeltaWeight += momentum * connection.previousDeltaWeight;
connection.weight += connection.totalDeltaWeight;
connection.previousDeltaWeight = connection.totalDeltaWeight;
connection.totalDeltaWeight = 0;
}
}
// Adjust bias
var deltaBias = rate * this.error.responsibility;
this.totalDeltaBias += deltaBias;
if (update) {
this.totalDeltaBias += momentum * this.previousDeltaBias;
this.bias += this.totalDeltaBias;
this.previousDeltaBias = this.totalDeltaBias;
this.totalDeltaBias = 0;
}
},
/**
* Creates a connection from this node to the given node
*/
connect: function (target, weight) {
var connections = [];
if (typeof target.bias !== 'undefined') { // must be a node!
if (target === this) {
// Turn on the self connection by setting the weight
if (this.connections.self.weight !== 0) {
if (config.warnings) console.warn('This connection already exists!');
} else {
this.connections.self.weight = weight || 1;
}
connections.push(this.connections.self);
} else if (this.isProjectingTo(target)) {
throw new Error('Already projecting a connection to this node!');
} else {
let connection = new Connection(this, target, weight);
target.connections.in.push(connection);
this.connections.out.push(connection);
connections.push(connection);
}
} else { // should be a group
for (var i = 0; i < target.nodes.length; i++) {
let connection = new Connection(this, target.nodes[i], weight);
target.nodes[i].connections.in.push(connection);
this.connections.out.push(connection);
target.connections.in.push(connection);
connections.push(connection);
}
}
return connections;
},
/**
* Disconnects this node from the other node
*/
disconnect: function (node, twosided) {
if (this === node) {
this.connections.self.weight = 0;
return;
}
for (var i = 0; i < this.connections.out.length; i++) {
let conn = this.connections.out[i];
if (conn.to === node) {
this.connections.out.splice(i, 1);
let j = conn.to.connections.in.indexOf(conn);
conn.to.connections.in.splice(j, 1);
if (conn.gater !== null) this.ungate(conn);
break;
}
}
if (twosided) {
node.disconnect(this);
}
},
/**
* Make this node gate a connection
*/
gate: function (connections) {
if (!Array.isArray(connections)) {
connections = [connections];
}
for (var i = 0; i < connections.length; i++) {
var connection = connections[i];
this.connections.gated.push(connection);
connection.gater = this;
}
},
/**
* Removes the gates from this node from the given connection(s)
*/
ungate: function (connections) {
if (!Array.isArray(connections)) {
connections = [connections];
}
for (var i = connections.length - 1; i >= 0; i--) {
var connection = connections[i];
var index = this.connections.gated.indexOf(connection);
this.connections.gated.splice(index, 1);
connection.gater = null;
}
},
/**
* Clear the context of the node
*/
clear: function () {
for (var i = 0; i < this.connections.in.length; i++) {
var connection = this.connections.in[i];
connection.elegibility = 0;
connection.xtrace = {
nodes: [],
values: []
};
}
for (i = 0; i < this.connections.gated.length; i++) {
let conn = this.connections.gated[i];
conn.gain = 0;
}
this.error.responsibility = this.error.projected = this.error.gated = 0;
this.old = this.state = this.activation = 0;
},
/**
* Mutates the node with the given method
*/
mutate: function (method) {
if (typeof method === 'undefined') {
throw new Error('No mutate method given!');
} else if (!(method.name in methods.mutation)) {
throw new Error('This method does not exist!');
}
switch (method) {
case methods.mutation.MOD_ACTIVATION:
// Can't be the same squash
var squash = method.allowed[(method.allowed.indexOf(this.squash) + Math.floor(Math.random() * (method.allowed.length - 1)) + 1) % method.allowed.length];
this.squash = squash;
break;
case methods.mutation.MOD_BIAS:
var modification = Math.random() * (method.max - method.min) + method.min;
this.bias += modification;
break;
}
},
/**
* Checks if this node is projecting to the given node
*/
isProjectingTo: function (node) {
for (var i = 0; i < this.connections.out.length; i++) {
var conn = this.connections.out[i];
if (conn.to === node) {
return true;
}
}
if (node === this && this.connections.self.weight !== 0) return true;
return false;
},
/**
* Checks if the given node is projecting to this node
*/
isProjectedBy: function (node) {
for (var i = 0; i < this.connections.in.length; i++) {
var conn = this.connections.in[i];
if (conn.from === node) {
return true;
}
}
if (node === this && this.connections.self.weight !== 0) return true;
return false;
},
/**
* Converts the node to a json object
*/
toJSON: function () {
var json = {
bias: this.bias,
type: this.type,
squash: this.squash.name,
mask: this.mask
};
return json;
}
};
/**
* Convert a json object to a node
*/
Node.fromJSON = function (json) {
var node = new Node();
node.bias = json.bias;
node.type = json.type;
node.mask = json.mask;
for (var squash in methods.activation) {
if (methods.activation[squash].name === json.squash) {
node.squash = methods.activation[squash];
break;
}
}
return node;
};
/***/ }),
/* 2 */
/***/ (function(module, exports) {
/*******************************************************************************
CONFIG
*******************************************************************************/
// Config
var config = {
warnings: false
};
/* Export */
module.exports = config;
/***/ }),
/* 3 */
/***/ (function(module, exports) {
/* Export */
module.exports = Connection;
/*******************************************************************************
CONNECTION
*******************************************************************************/
function Connection (from, to, weight) {
this.from = from;
this.to = to;
this.gain = 1;
this.weight = (typeof weight === 'undefined') ? Math.random() * 0.2 - 0.1 : weight;
this.gater = null;
this.elegibility = 0;
// For tracking momentum
this.previousDeltaWeight = 0;
// Batch training
this.totalDeltaWeight = 0;
this.xtrace = {
nodes: [],
values: []
};
}
Connection.prototype = {
/**
* Converts the connection to a json object
*/
toJSON: function () {
var json = {
weight: this.weight
};
return json;
}
};
/**
* Returns an innovation ID
* https://en.wikipedia.org/wiki/Pairing_function (Cantor pairing function)
*/
Connection.innovationID = function (a, b) {
return 1 / 2 * (a + b) * (a + b + 1) + b;
};
/***/ }),
/* 4 */
/***/ (function(module, exports, __webpack_require__) {
/* Export */
module.exports = Group;
/* Import */
var methods = __webpack_require__(0);
var config = __webpack_require__(2);
var Layer = __webpack_require__(5);
var Node = __webpack_require__(1);
/*******************************************************************************
Group
*******************************************************************************/
function Group (size) {
this.nodes = [];
this.connections = {
in: [],
out: [],
self: []
};
for (var i = 0; i < size; i++) {
this.nodes.push(new Node());
}
}
Group.prototype = {
/**
* Activates all the nodes in the group
*/
activate: function (value) {
var values = [];
if (typeof value !== 'undefined' && value.length !== this.nodes.length) {
throw new Error('Array with values should be same as the amount of nodes!');
}
for (var i = 0; i < this.nodes.length; i++) {
var activation;
if (typeof value === 'undefined') {
activation = this.nodes[i].activate();
} else {
activation = this.nodes[i].activate(value[i]);
}
values.push(activation);
}
return values;
},
/**
* Propagates all the node in the group
*/
propagate: function (rate, momentum, target) {
if (typeof target !== 'undefined' && target.length !== this.nodes.length) {
throw new Error('Array with values should be same as the amount of nodes!');
}
for (var i = this.nodes.length - 1; i >= 0; i--) {
if (typeof target === 'undefined') {
this.nodes[i].propagate(rate, momentum);
} else {
this.nodes[i].propagate(rate, momentum, target[i]);
}
}
},
/**
* Connects the nodes in this group to nodes in another group or just a node
*/
connect: function (target, method, weight) {
var connections = [];
var i, j;
if (target instanceof Group) {
if (typeof method === 'undefined') {
if (this !== target) {
if (config.warnings) console.warn('No group connection specified, using ALL_TO_ALL');
method = methods.connection.ALL_TO_ALL;
} else {
if (config.warnings) console.warn('No group connection specified, using ONE_TO_ONE');
method = methods.connection.ONE_TO_ONE;
}
}
if (method === methods.connection.ALL_TO_ALL || method === methods.connection.ALL_TO_ELSE) {
for (i = 0; i < this.nodes.length; i++) {
for (j = 0; j < target.nodes.length; j++) {
if (method === methods.connection.ALL_TO_ELSE && this.nodes[i] === target.nodes[j]) continue;
let connection = this.nodes[i].connect(target.nodes[j], weight);
this.connections.out.push(connection[0]);
target.connections.in.push(connection[0]);
connections.push(connection[0]);
}
}
} else if (method === methods.connection.ONE_TO_ONE) {
if (this.nodes.length !== target.nodes.length) {
throw new Error('From and To group must be the same size!');
}
for (i = 0; i < this.nodes.length; i++) {
let connection = this.nodes[i].connect(target.nodes[i], weight);
this.connections.self.push(connection[0]);
connections.push(connection[0]);
}
}
} else if (target instanceof Layer) {
connections = target.input(this, method, weight);
} else if (target instanceof Node) {
for (i = 0; i < this.nodes.length; i++) {
let connection = this.nodes[i].connect(target, weight);
this.connections.out.push(connection[0]);
connections.push(connection[0]);
}
}
return connections;
},
/**
* Make nodes from this group gate the given connection(s)
*/
gate: function (connections, method) {
if (typeof method === 'undefined') {
throw new Error('Please specify Gating.INPUT, Gating.OUTPUT');
}
if (!Array.isArray(connections)) {
connections = [connections];
}
var nodes1 = [];
var nodes2 = [];
var i, j;
for (i = 0; i < connections.length; i++) {
var connection = connections[i];
if (!nodes1.includes(connection.from)) nodes1.push(connection.from);
if (!nodes2.includes(connection.to)) nodes2.push(connection.to);
}
switch (method) {
case methods.gating.INPUT:
for (i = 0; i < nodes2.length; i++) {
let node = nodes2[i];
let gater = this.nodes[i % this.nodes.length];
for (j = 0; j < node.connections.in.length; j++) {
let conn = node.connections.in[j];
if (connections.includes(conn)) {
gater.gate(conn);
}
}
}
break;
case methods.gating.OUTPUT:
for (i = 0; i < nodes1.length; i++) {
let node = nodes1[i];
let gater = this.nodes[i % this.nodes.length];
for (j = 0; j < node.connections.out.length; j++) {
let conn = node.connections.out[j];
if (connections.includes(conn)) {
gater.gate(conn);
}
}
}
break;
case methods.gating.SELF:
for (i = 0; i < nodes1.length; i++) {
let node = nodes1[i];
let gater = this.nodes[i % this.nodes.length];
if (connections.includes(node.connections.self)) {
gater.gate(node.connections.self);
}
}
}
},
/**
* Sets the value of a property for every node
*/
set: function (values) {
for (var i = 0; i < this.nodes.length; i++) {
if (typeof values.bias !== 'undefined') {
this.nodes[i].bias = values.bias;
}
this.nodes[i].squash = values.squash || this.nodes[i].squash;
this.nodes[i].type = values.type || this.nodes[i].type;
}
},
/**
* Disconnects all nodes from this group from another given group/node
*/
disconnect: function (target, twosided) {
twosided = twosided || false;
// In the future, disconnect will return a connection so indexOf can be used
var i, j, k;
if (target instanceof Group) {
for (i = 0; i < this.nodes.length; i++) {
for (j = 0; j < target.nodes.length; j++) {
this.nodes[i].disconnect(target.nodes[j], twosided);
for (k = this.connections.out.length - 1; k >= 0; k--) {
let conn = this.connections.out[k];
if (conn.from === this.nodes[i] && conn.to === target.nodes[j]) {
this.connections.out.splice(k, 1);
break;
}
}
if (twosided) {
for (k = this.connections.in.length - 1; k >= 0; k--) {
let conn = this.connections.in[k];
if (conn.from === target.nodes[j] && conn.to === this.nodes[i]) {
this.connections.in.splice(k, 1);
break;
}
}
}
}
}
} else if (target instanceof Node) {
for (i = 0; i < this.nodes.length; i++) {
this.nodes[i].disconnect(target, twosided);
for (j = this.connections.out.length - 1; j >= 0; j--) {
let conn = this.connections.out[j];
if (conn.from === this.nodes[i] && conn.to === target) {
this.connections.out.splice(j, 1);
break;
}
}
if (twosided) {
for (j = this.connections.in.length - 1; j >= 0; j--) {
var conn = this.connections.in[j];
if (conn.from === target && conn.to === this.nodes[i]) {
this.connections.in.splice(j, 1);
break;
}
}
}
}
}
},
/**
* Clear the context of this group
*/
clear: function () {
for (var i = 0; i < this.nodes.length; i++) {
this.nodes[i].clear();
}
}
};
/***/ }),
/* 5 */
/***/ (function(module, exports, __webpack_require__) {
/* Export */
module.exports = Layer;
/* Import */
var methods = __webpack_require__(0);
var Group = __webpack_require__(4);
var Node = __webpack_require__(1);
/*******************************************************************************
Group
*******************************************************************************/
function Layer () {
this.output = null;
this.nodes = [];
this.connections = { in: [],
out: [],
self: []
};
}
Layer.prototype = {
/**
* Activates all the nodes in the group
*/
activate: function (value) {
var values = [];
if (typeof value !== 'undefined' && value.length !== this.nodes.length) {
throw new Error('Array with values should be same as the amount of nodes!');
}
for (var i = 0; i < this.nodes.length; i++) {
var activation;
if (typeof value === 'undefined') {
activation = this.nodes[i].activate();
} else {
activation = this.nodes[i].activate(value[i]);
}
values.push(activation);
}
return values;
},
/**
* Propagates all the node in the group
*/
propagate: function (rate, momentum, target) {
if (typeof target !== 'undefined' && target.length !== this.nodes.length) {
throw new Error('Array with values should be same as the amount of nodes!');
}
for (var i = this.nodes.length - 1; i >= 0; i--) {
if (typeof target === 'undefined') {
this.nodes[i].propagate(rate, momentum);
} else {
this.nodes[i].propagate(rate, momentum, target[i]);
}
}
},
/**
* Connects the nodes in this group to nodes in another group or just a node
*/
connect: function (target, method, weight) {
var connections;
if (target instanceof Group || target instanceof Node) {
connections = this.output.connect(target, method, weight);
} else if (target instanceof Layer) {
connections = target.input(this, method, weight);
}
return connections;
},
/**
* Make nodes from this group gate the given connection(s)
*/
gate: function (connections, method) {
this.output.gate(connections, method);
},
/**
* Sets the value of a property for every node
*/
set: function (values) {
for (var i = 0; i < this.nodes.length; i++) {
var node = this.nodes[i];
if (node instanceof Node) {
if (typeof values.bias !== 'undefined') {
node.bias = values.bias;
}
node.squash = values.squash || node.squash;
node.type = values.type || node.type;
} else if (node instanceof Group) {
node.set(values);
}
}
},
/**
* Disconnects all nodes from this group from another given group/node
*/
disconnect: function (target, twosided) {
twosided = twosided || false;
// In the future, disconnect will return a connection so indexOf can be used
var i, j, k;
if (target instanceof Group) {
for (i = 0; i < this.nodes.length; i++) {
for (j = 0; j < target.nodes.length; j++) {
this.nodes[i].disconnect(target.nodes[j], twosided);
for (k = this.connections.out.length - 1; k >= 0; k--) {
let conn = this.connections.out[k];
if (conn.from === this.nodes[i] && conn.to === target.nodes[j]) {
this.connections.out.splice(k, 1);
break;
}
}
if (twosided) {
for (k = this.connections.in.length - 1; k >= 0; k--) {
let conn = this.connections.in[k];
if (conn.from === target.nodes[j] && conn.to === this.nodes[i]) {
this.connections.in.splice(k, 1);
break;
}
}
}
}
}
} else if (target instanceof Node) {
for (i = 0; i < this.nodes.length; i++) {
this.nodes[i].disconnect(target, twosided);
for (j = this.connections.out.length - 1; j >= 0; j--) {
let conn = this.connections.out[j];
if (conn.from === this.nodes[i] && conn.to === target) {
this.connections.out.splice(j, 1);
break;
}
}
if (twosided) {
for (k = this.connections.in.length - 1; k >= 0; k--) {
let conn = this.connections.in[k];
if (conn.from === target && conn.to === this.nodes[i]) {
this.connections.in.splice(k, 1);
break;
}
}
}
}
}
},
/**
* Clear the context of this group
*/
clear: function () {
for (var i = 0; i < this.nodes.length; i++) {
this.nodes[i].clear();
}
}
};
Layer.Dense = function (size) {
// Create the layer
var layer = new Layer();
// Init required nodes (in activation order)
var block = new Group(size);
layer.nodes.push(block);
layer.output = block;
layer.input = function (from, method, weight) {
if (from instanceof Layer) from = from.output;
method = method || methods.connection.ALL_TO_ALL;
return from.connect(block, method, weight);
};
return layer;
};
Layer.LSTM = function (size) {
// Create the layer
var layer = new Layer();
// Init required nodes (in activation order)
var inputGate = new Group(size);
var forgetGate = new Group(size);
var memoryCell = new Group(size);
var outputGate = new Group(size);
var outputBlock = new Group(size);
inputGate.set({
bias: 1
});
forgetGate.set({
bias: 1
});
outputGate.set({
bias: 1
});
// Set up internal connections
memoryCell.connect(inputGate, methods.connection.ALL_TO_ALL);
memoryCell.connect(forgetGate, methods.connection.ALL_TO_ALL);
memoryCell.connect(outputGate, methods.connection.ALL_TO_ALL);
var forget = memoryCell.connect(memoryCell, methods.connection.ONE_TO_ONE);
var output = memoryCell.connect(outputBlock, methods.connection.ALL_TO_ALL);
// Set up gates
forgetGate.gate(forget, methods.gating.SELF);
outputGate.gate(output, methods.gating.OUTPUT);
// Add to nodes array
layer.nodes = [inputGate, forgetGate, memoryCell, outputGate, outputBlock];
// Define output
layer.output = outputBlock;
layer.input = function (from, method, weight) {
if (from instanceof Layer) from = from.output;
method = method || methods.connection.ALL_TO_ALL;
var connections = [];
var input = from.connect(memoryCell, method, weight);
connections = connections.concat(input);
connections = connections.concat(from.connect(inputGate, method, weight));
connections = connections.concat(from.connect(outputGate, method, weight));
connections = connections.concat(from.connect(forgetGate, method, weight));
inputGate.gate(input, methods.gating.INPUT);
return connections;
};
return layer;
};
Layer.GRU = function (size) {
// Create the layer
var layer = new Layer();
var updateGate = new Group(size);
var inverseUpdateGate = new Group(size);
var resetGate = new Group(size);
var memoryCell = new Group(size);
var output = new Group(size);
var previousOutput = new Group(size);
previousOutput.set({
bias: 0,
squash: methods.activation.IDENTITY,
type: 'constant'
});
memoryCell.set({
squash: methods.activation.TANH
});
inverseUpdateGate.set({
bias: 0,
squash: methods.activation.INVERSE,
type: 'constant'
});
updateGate.set({
bias: 1
});
resetGate.set({
bias: 0
});
// Update gate calculation
previousOutput.connect(updateGate, methods.connection.ALL_TO_ALL);
// Inverse update gate calculation
updateGate.connect(inverseUpdateGate, methods.connection.ONE_TO_ONE, 1);
// Reset gate calculation
previousOutput.connect(resetGate, methods.connection.ALL_TO_ALL);
// Memory calculation
var reset = previousOutput.connect(memoryCell, methods.connection.ALL_TO_ALL);
resetGate.gate(reset, methods.gating.OUTPUT); // gate
// Output calculation
var update1 = previousOutput.connect(output, methods.connection.ALL_TO_ALL);
var update2 = memoryCell.connect(output, methods.connection.ALL_TO_ALL);
updateGate.gate(update1, methods.gating.OUTPUT);
inverseUpdateGate.gate(update2, methods.gating.OUTPUT);
// Previous output calculation
output.connect(previousOutput, methods.connection.ONE_TO_ONE, 1);
// Add to nodes array
layer.nodes = [updateGate, inverseUpdateGate, resetGate, memoryCell, output, previousOutput];
layer.output = output;
layer.input = function (from, method, weight) {
if (from instanceof Layer) from = from.output;
method = method || methods.connection.ALL_TO_ALL;
var connections = [];
connections = connections.concat(from.connect(updateGate, method, weight));
connections = connections.concat(from.connect(resetGate, method, weight));
connections = connections.concat(from.connect(memoryCell, method, weight));
return connections;
};
return layer;
};
Layer.Memory = function (size, memory) {
// Create the layer
var layer = new Layer();
// Because the output can only be one group, we have to put the nodes all in óne group
var previous = null;
var i;
for (i = 0; i < memory; i++) {
var block = new Group(size);
block.set({
squash: methods.activation.IDENTITY,
bias: 0,
type: 'constant'
});
if (previous != null) {
previous.connect(block, methods.connection.ONE_TO_ONE, 1);
}
layer.nodes.push(block);
previous = block;
}
layer.nodes.reverse();
for (i = 0; i < layer.nodes.length; i++) {
layer.nodes[i].nodes.reverse();
}
// Because output can only be óne group, fit all memory nodes in óne group
var outputGroup = new Group(0);
for (var group in layer.nodes) {
outputGroup.nodes = outputGroup.nodes.concat(layer.nodes[group].nodes);
}
layer.output = outputGroup;
layer.input = function (from, method, weight) {
if (from instanceof Layer) from = from.output;
method = method || methods.connection.ALL_TO_ALL;
if (from.nodes.length !== layer.nodes[layer.nodes.length - 1].nodes.length) {
throw new Error('Previous layer size must be same as memory size');
}
return from.connect(layer.nodes[layer.nodes.length - 1], methods.connection.ONE_TO_ONE, 1);
};
return layer;
};
/***/ }),
/* 6 */
/***/ (function(module, exports, __webpack_require__) {
/* Export */
module.exports = Network;
/* Import */
var multi = __webpack_require__(7);
var methods = __webpack_require__(0);
var Connection = __webpack_require__(3);
var config = __webpack_require__(2);
var Neat = __webpack_require__(8);
var Node = __webpack_require__(1);
/* Easier variable naming */
var mutation = methods.mutation;
/*******************************************************************************
NETWORK
*******************************************************************************/
function Network (input, output) {
if (typeof input === 'undefined' || typeof output === 'undefined') {
throw new Error('No input or output size given');
}
this.input = input;
this.output = output;
// Store all the node and connection genes
this.nodes = []; // Stored in activation order
this.connections = [];
this.gates = [];
this.selfconns = [];
// Regularization
this.dropout = 0;
// Create input and output nodes
var i;
for (i = 0; i < this.input + this.output; i++) {
var type = (i < this.input) ? 'input' : 'output';
this.nodes.push(new Node(type, this.nodes.length));
}
// Connect input nodes with output nodes directly
for (i = 0; i < this.input; i++) {
for (var j = this.input; j < this.output + this.input; j++) {
// https://stats.stackexchange.com/a/248040/147931
var weight = Math.random() * this.input * Math.sqrt(2 / this.input);
this.connect(this.nodes[i], this.nodes[j], weight);
}
}
}
Network.prototype = {
/**
* Activates the network
*/
activate: function (input, training) {
var output = [];
// Activate nodes chronologically
for (var i = 0; i < this.nodes.length; i++) {
if (this.nodes[i].type === 'input') {
this.nodes[i].activate(input[i]);
} else if (this.nodes[i].type === 'output') {
var activation = this.nodes[i].activate();
output.push(activation);
} else {
if (training) this.nodes[i].mask = Math.random() < this.dropout ? 0 : 1;
this.nodes[i].activate();
}
}
return output;
},
/**
* Backpropagate the network
*/
propagate: function (rate, momentum, update, target) {
if (typeof target !== 'undefined' && target.length !== this.output) {
throw new Error('Output target length should match network output length');
}
var targetIndex = target.length;
// Propagate output nodes
var i;
for (i = this.nodes.length - 1; i >= this.nodes.length - this.output; i--) {
this.nodes[i].propagate(rate, momentum, update, target[--targetIndex]);
}
// Propagate hidden and input nodes
for (i = this.nodes.length - this.output - 1; i >= this.input; i--) {
this.nodes[i].propagate(rate, momentum, update);
}
},
/**
* Clear the context of the network
*/
clear: function () {
for (var i = 0; i < this.nodes.length; i++) {
this.nodes[i].clear();
}
},
/**
* Connects the from node to the to node
*/
connect: function (from, to, weight) {
var connections = from.connect(to, weight);
for (var i = 0; i < connections.length; i++) {
var connection = connections[i];
if (from !== to) {
this.connections.push(connection);
} else {
this.selfconns.push(connection);
}
}
return connections;
},
/**
* Disconnects the from node from the to node
*/
disconnect: function (from, to) {
// Delete the connection in the network's connection array
var connections = from === to ? this.selfconns : this.connections;
for (var i = 0; i < connections.length; i++) {
var connection = connections[i];
if (connection.from === from && connection.to === to) {
if (connection.gater !== null) this.ungate(connection);
connections.splice(i, 1);
break;
}
}
// Delete the connection at the sending and receiving neuron
from.disconnect(to);
},
/**
* Gate a connection with a node
*/
gate: function (node, connection) {
if (this.nodes.indexOf(node) === -1) {
throw new Error('This node is not part of the network!');
} else if (connection.gater != null) {
if (config.warnings) console.warn('This connection is already gated!');
return;
}
node.gate(connection);
this.gates.push(connection);
},
/**
* Remove the gate of a connection
*/
ungate: function (connection) {
var index = this.gates.indexOf(connection);
if (index === -1) {
throw new Error('This connection is not gated!');
}
this.gates.splice(index, 1);
connection.gater.ungate(connection);
},
/**
* Removes a node from the network
*/
remove: function (node) {
var index = this.nodes.indexOf(node);
if (index === -1) {
throw new Error('This node does not exist in the network!');
}
// Keep track of gaters
var gaters = [];
// Remove selfconnections from this.selfconns
this.disconnect(node, node);
// Get all its inputting nodes
var inputs = [];
for (var i = node.connections.in.length - 1; i >= 0; i--) {
let connection = node.connections.in[i];
if (mutation.SUB_NODE.keep_gates && connection.gater !== null && connection.gater !== node) {
gaters.push(connection.gater);
}
inputs.push(connection.from);
this.disconnect(connection.from, node);
}
// Get all its outputing nodes
var outputs = [];
for (i = node.connections.out.length - 1; i >= 0; i--) {
let connection = node.connections.out[i];
if (mutation.SUB_NODE.keep_gates && connection.gater !== null && connection.gater !== node) {
gaters.push(connection.gater);
}
outputs.push(connection.to);
this.disconnect(node, connection.to);
}
// Connect the input nodes to the output nodes (if not already connected)
var connections = [];
for (i = 0; i < inputs.length; i++) {
let input = inputs[i];
for (var j = 0; j < outputs.length; j++) {
let output = outputs[j];
if (!input.isProjectingTo(output)) {
var conn = this.connect(input, output);
connections.push(conn[0]);
}
}
}
// Gate random connections with gaters
for (i = 0; i < gaters.length; i++) {
if (connections.length === 0) break;
let gater = gaters[i];
let connIndex = Math.floor(Math.random() * connections.length);
this.gate(gater, connections[connIndex]);
connections.splice(connIndex, 1);
}
// Remove gated connections gated by this node
for (i = node.connections.gated.length - 1; i >= 0; i--) {
let conn = node.connections.gated[i];
this.ungate(conn);
}
// Remove selfconnection
this.disconnect(node, node);
// Remove the node from this.nodes
this.nodes.splice(index, 1);
},
/**
* Mutates the network with the given method
*/
mutate: function (method) {
if (typeof method === 'undefined') {
throw new Error('No (correct) mutate method given!');
}
var i, j;
switch (method) {
case mutation.ADD_NODE:
// Look for an existing connection and place a node in between
var connection = this.connections[Math.floor(Math.random() * this.connections.length)];
var gater = connection.gater;
this.disconnect(connection.from, connection.to);
// Insert the new node right before the old connection.to
var toIndex = this.nodes.indexOf(connection.to);
var node = new Node('hidden', this.nodes.length);
// Random squash function
node.mutate(mutation.MOD_ACTIVATION);
// Place it in this.nodes
var minBound = Math.min(toIndex, this.nodes.length - this.output);
this.nodes.splice(minBound, 0, node);
// Now create two new connections
var newConn1 = this.connect(connection.from, node)[0];
var newConn2 = this.connect(node, connection.to)[0];
// Check if the original connection was gated
if (gater != null) {
this.gate(gater, Math.random() >= 0.5 ? newConn1 : newConn2);
}
break;
case mutation.SUB_NODE:
// Check if there are nodes left to remove
if (this.nodes.length === this.input + this.output) {
if (config.warnings) console.warn('No more nodes left to remove!');
break;
}
// Select a node which isn't an input or output node
var index = Math.floor(Math.random() * (this.nodes.length - this.output - this.input) + this.input);
this.remove(this.nodes[index]);
break;
case mutation.ADD_CONN:
// Create an array of all uncreated (feedforward) connections
var available = [];
for (i = 0; i < this.nodes.length - this.output; i++) {
let node1 = this.nodes[i];
for (j = Math.max(i + 1, this.input); j < this.nodes.length; j++) {
let node2 = this.nodes[j];
if (!node1.isProjectingTo(node2)) available.push([node1, node2]);
}
}
if (available.length === 0) {
if (config.warnings) console.warn('No more connections to be made!');
break;
}
var pair = available[Math.floor(Math.random() * available.length)];
this.connect(pair[0], pair[1]);
break;
case mutation.SUB_CONN:
// List of possible connections that can be removed
var possible = [];
for (i = 0; i < this.connections.length; i++) {
let conn = this.connections[i];
// Check if it is not disabling a node
if (conn.from.connections.out.length > 1 && conn.to.connections.in.length > 1 && this.nodes.indexOf(conn.to) > this.nodes.indexOf(conn.from)) {
possible.push(conn);
}
}
if (possible.length === 0) {
if (config.warnings) console.warn('No connections to remove!');
break;
}
var randomConn = possible[Math.floor(Math.random() * possible.length)];
this.disconnect(randomConn.from, randomConn.to);
break;
case mutation.MOD_WEIGHT:
var connection = this.connections[Math.floor(Math.random() * this.connections.length)];
var modification = Math.random() * (method.max - method.min) + method.min;
connection.weight += modification;
break;
case mutation.MOD_BIAS:
// Has no effect on input node, so they are excluded
var index = Math.floor(Math.random() * (this.nodes.length - this.input) + this.input);
var node = this.nodes[index];
node.mutate(method);
break;
case mutation.MOD_ACTIVATION:
// Has no effect on input node, so they are excluded
if (!method.mutateOutput && this.input + this.output === this.nodes.length) {
if (config.warnings) console.warn('No nodes that allow mutation of activation function');
break;
}
var index = Math.floor(Math.random() * (this.nodes.length - (method.mutateOutput ? 0 : this.output) - this.input) + this.input);
var node = this.nodes[index];
node.mutate(method);
break;
case mutation.ADD_SELF_CONN:
// Check which nodes aren't selfconnected yet
var possible = [];
for (i = this.input; i < this.nodes.length; i++) {
let node = this.nodes[i];
if (this.selfconns.indexOf(node.connections.self) === -1) {
possible.push(node);
}
}
if (possible.length === 0) {
if (config.warnings) console.warn('No more self-connections to add!');
break;
}
// Select a random node
var node = possible[Math.floor(Math.random() * possible.length)];
// Connect it to himself
this.connect(node, node);
break;
case mutation.SUB_SELF_CONN:
if (this.selfconns.length === 0) {
if (config.warnings) console.warn('No more self-connections to