greataptic/src/index.js

A simplistic neural network library.

/* @flow */

var util = require('util');
var { Vector } = require('./vector.js');
var shuffle = require('shuffle-array');

let words = ['above-mentioned', 'above-listed', 'before-mentioned', 'aforementioned', 'abundance', 'accelerate', 'accentuate', 'accommodation', 'accompany', 'accomplish', 'accorded', 'accordingly', 'accrue', 'accurate', 'acquiesce', 'acquire', 'additional', 'address', 'addressees', 'adjustment', 'admissible', 'advantageous', 'advise', 'aggregate', 'aircraft', 'alleviate', 'allocate', 'alternatively', 'ameliorate', 'and/or', 'anticipate', 'applicant', 'application', 'apparent', 'apprehend', 'appreciable', 'appropriate', 'approximate', 'ascertain', 'attain', 'attempt', 'authorize', 'beg', 'belated', 'beneficial', 'bestow', 'beverage', 'capability', 'caveat', 'cease', 'chauffeur', 'clearly', 'obviously', 'combined', 'commence', 'complete', 'component', 'comprise', 'conceal', 'concerning', 'consequently', 'consolidate', 'constitutes', 'contains', 'convene', 'corridor', 'currently', 'deem', 'delete', 'demonstrate', 'depart', 'designate', 'desire', 'determine', 'disclose', 'different', 'discontinue', 'disseminate', 'duly', 'authorized', 'signed', 'each...apiece', 'economical', 'elect', 'eliminate', 'elucidate', 'emphasize', 'employ', 'encounter', 'endeavor', 'end', 'result', 'product', 'enquiry', 'ensure', 'entitlement', 'enumerate', 'equipments', 'equitable', 'equivalent', 'establish', 'evaluate', 'evidenced', 'evident', 'evince', 'excluding', 'exclusively', 'exhibit', 'expedite', 'expeditious', 'expend', 'expertise', 'expiration', 'facilitate', 'fauna', 'feasible', 'females', 'finalize', 'flora', 'following', 'forfeit', 'formulate', 'forward', 'frequently', 'function', 'furnish', 'grant', 'herein', 'heretofore', 'herewith', 'thereof', 'wherefore', 'wherein', 'however', 'identical', 'identify', 'immediately', 'impacted', 'implement', 'inasmuch', 'inception', 'indicate', 'indication', 'initial', 'initiate', 'interface', 'irregardless', 'liaison', '-ly', 'doubtless', 'fast', 'ill', 'much', 'seldom', 'thus', 'magnitude', 'maintain', 'majority', 'maximum', 'merge', 'methodology', 'minimize', 'minimum', 'modify', 'monitor', 'moreover', 'multiple', 'necessitate', 'nevertheless', 'notify', 'not...unless', 'not...except', 'not...until', 'notwithstanding', 'numerous', 'objective', 'obligate', 'observe', 'obtain', 'operate', 'optimum', 'option', 'orientate', '...out', 'calculate', 'cancel,', 'distribute', 'segregate', 'separate', 'overall', 'parameters', 'participate', 'particulars', 'perchance', 'perform', 'permit', 'perspire', 'peruse', 'place', 'portion', 'possess', 'potentiality', 'practicable', 'preclude', 'preowned', 'previously', 'prioritize', 'proceed', 'procure', 'proficiency', 'promulgate', 'provide', 'purchase', 'reflect', 'regarding', 'relocate', 'remain', 'remainder', 'remuneration', 'render', 'represents', 'request', 'require', 'requirement', 'reside', 'residence', 'respectively', 'retain', 'retire', 'rigorous', 'selection', 'separate', 'shall', 'solicit', 'state-of-the-art', 'strategize', 'subject', 'submit', 'subsequent', 'subsequently', 'substantial', 'sufficient', 'terminate', 'therefore', 'therein', 'timely', 'transpire', 'transmit', 'type', 'validate', 'variation', 'very', 'viable', 'warrant', 'whereas', 'whosoever', 'whomsoever', 'witnessed'];

type NetworkBasicEvolveOptions = {
    population: ?number;
    maxGens: ?number;
    maxMutation: ?number;
    mutationDecay: ?number;
    stepFactor: ?number;
    quota: ?number;
    random: ?boolean;
    debug: ?boolean;
}

type NetworkDynamicEvolveOptions = {
    population: ?number;
    maxGens: ?number;
    maxMutation: ?number;
    mutationDecay: ?number;
    stepFactor: ?number;
    quota: ?number;
    random: ?boolean;
    debug: ?boolean;

    step: (currentStep: NeuralNetwork, generation: ?number) => number;
    postStep: ?(bestSoFar: NeuralNetwork) => void;
}

type NetworkDynamicBatchEvolveOptions = {
    population: ?number;
    maxGens: ?number;
    maxMutation: ?number;
    mutationDecay: ?number;
    stepFactor: ?number;
    quota: ?number;
    random: ?boolean;
    debug: ?boolean;
    batchPop: number;

    stepAll: (steps: NeuralNetwork[], generation: number | null, batchIndex: number) => void;
    postStep: ?(bestSoFar: NeuralNetwork) => void;
}

type NetworkEvolveOptions = NetworkBasicEvolveOptions | NetworkDynamicEvolveOptions | NetworkDynamicBatchEvolveOptions;

function _logit(x) {
    return Math.log(x / (1 - x));
}

function gaussRand() {
    // adapted from https://stackoverflow.com/a/49434653/5129091
    var u = Math.random(), v = Math.random();

    if (u === 0) u = 0.5;
    if (v === 0) v = 0.5;

    let res = Math.sqrt( -2.0 * Math.log(u) ) * Math.cos( 2.0 * Math.PI * v ) / 10 + .5;
    if (res > 1 || res < 0) return gaussRand();

    return res;
}

let greataptic = {};
greataptic.Vector = Vector;

type VectorLike = number[] | Vector;

greataptic.$vec = function $vec(b: VectorLike) {
    return new Vector(b);
}

function mutateNum(x: number, amount: number) {
    return x + 2 * amount * (Math.random() - 0.5);
}

function mutateList(l: number[], amount: number) {
    return l.map<number>((x) => mutateNum(x, amount));
}

function stepInterp(a: number, b: number, alpha: number) {
    return (1 - alpha) * a + alpha * b;
}

function stepInterpList(la: number[], lb: number[], alpha: number) {
    let res = [];

    for (let i = 0; i < Math.min(la.length, lb.length); i++)
        res[i] = stepInterp(la[i], lb[i], alpha);

    return res;
}

class Layer {
    size: number;
    type: string;
    next: ?string;
    name: ?string;
    data: any;
    net: ?any;

    constructor(next: ?string, name: ?string = null, data: ?any = null) {
        this.size = 0;
        this.next = next;
        this.type = this.typename();
        this.data = data;
        this.name = null;
    }

    process(vec: Vector) : Vector {
        return vec;
    }

    remake(data: ?any) {
        return new (this.constructor)(this.next, this.name, data);
    }

    _load(data: ?any) {
        this.data = data;
    }

    _save() {
        return this.data;
    }
    
    static load(data: any) {
        let res = new (this)(data.next, data.name, null);
        res._load(data.data);

        return res;
    }

    save() {
        return {
            type: this.type,
            next: this.next,
            name: this.name,
            data: this._save()
        };
    }

    typename() {
        throw new Error("Don't use Layer directly! (tried to call method 'name')");
    }

    mutate(amount: ?number = null) {
        return this;
    }

    breed(layer: Layer) {
        return Math.random() > 0.5 ? layer : this;
    }

    applyStep(layer: Layer, fitness: number) {
        return this;
    }
};

class ActivationLayer extends Layer {
    mutate() {}
    breed(layer) { return this; }
    applyStep(layer, fitness) { return this; }

    process(vec: Vector) : Vector {
        return vec.map(this.activate);
    }
    
    activate(n: number) {
        return n;
    }
}

// List of layer types.
let LSTMLayer;
let types = {
    // Sequential layer group.
    sequence: class GroupLayer extends Layer {
        data: Layer[];

        typename() {
            return 'sequence';
        }

        constructor(next: ?string, name: ?string = null, parts: Layer[]) {
            super(next, name);

            this.data = parts;
            this.size = parts.slice(-1)[0].size;
        }

        process(vec: Vector) : Vector {
            let res = vec;

            this.data.forEach((l) => res = l.process(res));
            return res;
        }

        mutate(amount: ?number = null) {
            this.data.forEach((l) => {
                if (l.mutate)
                    l.mutate(amount);
            });
        }

        breed(layer: Layer) {
            let p = this.data.map((l, i) => types[l.type].breed(l, layer.data[i]));
            return this.remake(p);
        }

        applyStep(layer: Layer, fitness: number) {
            let p = this.data.map((l, i) => types[l.type].applyStep(l, layer.data[i], fitness));
            return this.remake(p);
        }
    },

    combo: class ConcatLayer extends Layer {
        data: Layer[];

        typename() {
            return 'combo';
        }

        process(vec: Vector) : Vector {
            let res = this.data.reduce((a, ln) => a.concat(types[ln.type].process(vec, ln).data), []);
            return new Vector(res);
        }

        constructor(next: ?string = null, name: ?string = null, parts: Layer[]) {
            super(next, name = null);

            this.data = parts;
            this.size = parts.map((p: Layer) => p.size).reduce((a, b) => a + b, 0);
        }

        mutate(amount: ?number = null) {
            this.data.forEach((l) => {
                if (l.mutate)
                    l.mutate(amount);
            });
        }

        breed(layer2: Layer) {
            let p = this.data.map((l, i) => types[l.type].breed(l, layer2.data[i]));
            return this.remake(p);
        }

        applyStep(layer2: Layer, fitness: number) {
            let p = this.data.map((l, i) => types[l.type].applyStep(l, layer2.data[i], fitness));
            return this.remake(p);
        }
    },

    sigmoid: class ASigmoidLayer extends Layer {
        typename() {
            return 'sigmoid';
        }

        activate(n: number) {
            return 1 / (1 + Math.exp(-n));
        }
    },

    tanh: class ATanhLayer extends Layer {
        typename() {
            return 'tanh';
        }

        activate(n: number) {
            return Math.tanh(n);
        }
    },

    logit: class ALogitLayer extends Layer {
        typename() {
            return 'logit';
        }

        activate(n: number) {
            return Math.log(n / (1 - n));
        }
    },

    spiking: class SpikingLayer extends Layer {
        typename() {
            return 'spiking';
        }

        constructor(next: ?string, name: ?string = null, nodes: any[]) {
            super(next, name);

            this.data = nodes;
        }

        process(vec: Vector) : Vector {
            let res = new Array(this.data.length).fill(0);

            this.data.forEach((node, ni) => {
                if ((node.power += Math.max(new Vector(node.weights).multiplyVec(vec).sum(), 0)) > node.limit) {
                    node.power = 0;
                    res[ni] = node.output;
                }
            });

            return new Vector(res);
        }

        mutate(amount: ?number = null) {
            function _mut(x) {
                if (amount != null)
                    return x + 2 * amount * (Math.random() - 0.5);

                else
                    return x + _logit(0.25 + 0.5 * Math.random());
            }

            this.data = this.data.map((n) => ({
                power: n.power,
                output: n.output,
                limit: _mut(n.limit),
                weights: n.weights.map(_mut)
            }));
        }

        breed(layer2: Layer) {
            return this.remake(
                this.data.map((node, ni) => ({
                    power: choice([node.power, layer2.data[ni].power]),
                    output: choice([node.output, layer2.data[ni].output]),
                    limit: choice([node.limit, layer2.data[ni].limit]),
                    weights: node.weights.map((x, xi) => Math.random() > 0.5 ? x : layer2.data[ni].weights[xi])
                }))
            );
        }

        applyStep(layer2: Layer, fitness: number) {
            return this.remake(
                this.data.map((node, ni) => ({
                    power: stepInterp(node.power, layer2.data[ni].power, fitness),
                    output: stepInterp(node.output, layer2.data[ni].output, fitness),
                    limit: stepInterp(node.limit, layer2.data[ni].limit, fitness),
                    weights: node.weights.map((x, xi) => stepInterp(x, layer2.data[ni].weights[xi], fitness))
                }))
            );
        }
    },

    square: class SquareLayer extends Layer {
        typename() {
            return 'square';
        }

        constructor(next: ?string, name: ?string = null, nodes: any[]) {
            super(next, name);
            this.data = nodes;
        }

        process(vec: Vector) : Vector {
            return new Vector(this.data.map((n) => {
                return new Vector(n.weights.linear).multiplyVec(vec).add(new Vector(n.weights.square).multiplyVec(vec.pow(2))).sum() + n.offset;
            }));
        }

        mutate(amount: ?number = null) {
            function _mut(x) {
                if (amount != null)
                    return x + 2 * amount * (Math.random() - 0.5);

                else
                    return x + _logit(0.25 + 0.5 * Math.random());
            }

            this.data = this.data.map((l) => ({
                weights: {
                    linear: l.weights.linear.map(_mut),
                    square: l.weights.square.map(_mut),
                },
                offset: _mut(l.offset)
            }));
        }

        breed(layer2: Layer) {
            return this.remake(
                this.data.map((n, ni) => ({
                    offset: choice([n.offset, layer2.data[ni].offset]),
                    weights: {
                        linear: n.weights.linear.map((x, xi) => (Math.random() > 0.5) ? x : layer2.data[ni].weights.linear[xi]),
                        square: n.weights.square.map((x, xi) => (Math.random() > 0.5) ? x : layer2.data[ni].weights.square[xi])
                    }
                })),
            );
        }

        applyStep(layer2: Layer, fitness: number) {
            return this.remake(
                this.data.map((n, ni) => ({
                    offset: choice([n.offset, layer2.data[ni].offset]),
                    weights: {
                        linear: n.weights.linear.map((x, xi) => (Math.random() > 0.5) ? x : layer2.data[ni].weights.linear[xi]),
                        square: n.weights.square.map((x, xi) => (Math.random() > 0.5) ? x : layer2.data[ni].weights.square[xi])
                    }
                }))
            );
        }
    },

    linear: class LinearLayer extends Layer {
        typename() {
            return 'linear';
        }

        constructor(next: ?string, name: ?string = null, nodes: any[]) {
            super(next, name);
            this.data = nodes;
        }

        process(vec: Vector) : Vector {
            return new Vector(this.data.map((n) => {
                return new Vector(n.weights).multiplyVec(vec).sum() + n.offset;
            }));
        }

        mutate(amount: ?number = null) {
            function _mut(x) {
                if (amount != null)
                    return x + 2 * amount * (Math.random() - 0.5);

                else
                    return x + _logit(0.25 + 0.5 * Math.random());
            }

            this.data = this.data.map((l) => ({
                weights: l.weights.map(_mut),
                offset: _mut(l.offset)
            }));
        }

        breed(layer2: Layer) {
            return this.remake(this.data.map((n, ni) => ({
                offset: choice([n.offset, layer2.data[ni].offset]),
                weights: n.weights.map((x, xi) => (Math.random() > 0.5) ? x : layer2.data[ni].weights[xi])
            })))
        }

        applyStep(layer2: Layer, fitness: number) {
            return this.remake(
                this.data.map((n, ni) => ({
                    offset: stepInterp(n.offset, layer2.data[ni].offset, fitness),
                    weights: n.weights.map((x, xi) => stepInterp(x , layer2.data[ni].weights[xi], fitness))
                }))
            )
        }
    },

    lstm: LSTMLayer = class LSTMLayer extends Layer {
        typename() {
            return 'lstm';
        }

        static randomGates(lastSize: number, size: number) {
            return {
                input: {
                    weights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    hiddenWeights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    offset: Vector.random(size).data,
                },
                
                output: {
                    weights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    hiddenWeights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    offset: Vector.random(size).data,
                },
                
                forget: {
                    weights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    hiddenWeights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    offset: Vector.random(size).data,
                },

                cell: {
                    weights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    hiddenWeights: new Array(size).fill(0).map<number[]>(() => Vector.random(lastSize).data),
                    offset: Vector.random(size).data,
                },
            };
        }

        static randomStates(size: number) {
            return {
                cell: Array.apply(null, Array(size)).map<number>(Number.prototype.valueOf, 0),
                hidden: Array.apply(null, Array(size)).map<number>(Number.prototype.valueOf, 0),
            };
        }

        constructor(next: ?string, name: ?string, gates: any, states: any, activation: string = 'sigmoid', cellActivation: string = 'tanh') {
            super(next, name, {
                gates: gates,
                states: states,

                activation: activation,
                cellActivation: cellActivation,
            });
        }

        process(vec: Vector) : Vector {
            vec = new Vector(vec);
            
            let { forget, input, output } = this.data.gates;
            let gCell = this.data.gates.cell;
            let act = this.data.activation;
            let cAct = this.data.cellActivation;

            let sHidden = new Vector(this.data.states.hidden);
            let sCell = new Vector(this.data.states.cell);

            let gat = {
                forget: {
                    weights: forget.weights.map<Vector>((w) => new Vector(w)),
                    hiddenWeights: forget.hiddenWeights.map<Vector>((w) => new Vector(w)),
                },

                input: {
                    weights: input.weights.map<Vector>((w) => new Vector(w)),
                    hiddenWeights: input.hiddenWeights.map<Vector>((w) => new Vector(w))
                },

                output: {
                    weights: output.weights.map<Vector>((w) => new Vector(w)),
                    hiddenWeights: output.hiddenWeights.map<Vector>((w) => new Vector(w))
                },

                cell: {
                    weights: gCell.weights.map<Vector>((w) => new Vector(w)),
                    hiddenWeights: gCell.hiddenWeights.map<Vector>((w) => new Vector(w))
                },
            };

            let forgVec = new Vector(gat.forget.weights.map((n, i) => (
                greataptic.activate(act, n.multiplyVec(vec).sum() + gat.forget.hiddenWeights[i].multiplyVec(sHidden).sum() + forget.offset[i])
            )));

            let inpVec = new Vector(gat.input.weights.map((n, i) => (
                greataptic.activate(act, n.multiplyVec(vec).sum() + gat.input.hiddenWeights[i].multiplyVec(sHidden).sum() + input.offset[i])
            )));

            let outVec = new Vector(gat.output.weights.map((n, i) => (
                greataptic.activate(act, n.multiplyVec(vec).sum() + gat.output.hiddenWeights[i].multiplyVec(sHidden).sum() + output.offset[i])
            )));

            this.data.states.cell = forgVec.multiplyVec(sCell).add(inpVec.multiplyVec(
                gat.cell.weights.map((n, i) => (
                    greataptic.activate(cAct, n.multiplyVec(vec).sum() + gat.cell.hiddenWeights[i].multiplyVec(sHidden).sum() + gCell.offset[i])
                ))
            )).data;

            this.data.states.hidden = outVec.multiplyVec(this.data.states.cell.map((x) => greataptic.activate(cAct, x))).data;

            return new Vector(this.data.states.hidden);
        }

        breedGate(g1: any, g2: any, choices: any[]) {
            return {
                weights: g1.weights.map((w, i) => choices[i] ? w : g2.weights[i]),
                hiddenWeights: g1.hiddenWeights.map((w, i) => choices[i] ? w : g2.hiddenWeights[i]),
                offset: g1.offset.map((o, i) => choices[i] ? o : g2.offset[i])
            };
        }

        breedGates(g1: any, g2: any, choices: any[]) {
            if (!choices) {
                for (let i = 0; i < g1.hidden.weights.length; i++)
                    choices.push(+(Math.random() > 0.5));
            }

            return {
                hidden: this.breedGate(g1.hidden, g2.hidden, choices),
                input: this.breedGate(g1.input, g2.input, choices),
                output: this.breedGate(g1.output, g2.output, choices),
                cell: this.breedGate(g1.cell, g2.cell, choices),
            };
        }

        breedState(s1: any, s2: any, choices: any[]) {
            return s1.map((x, i) => choices[i] ? x : s2[i]);
        }

        breedStates(s1: any, s2: any, choices: any[]) {
            return {
                cell: this.breedState(s1.cell, s2.cell, choices),
                hidden: this.breedState(s1.hidden, s2.hidden, choices),
            };
        }

        breed(layer2: Layer) {
            let choices = [];
            let bred = new (this.constructor)(
                this.next,
                this.name,
                this.breedGates(this.data.gates, layer2.data.gates, choices),
                this.breedStates(this.data.states, layer2.data.states, choices),
                this.data.activation,
                this.data.cellActivation
            );
            
            return bred;
        }

        stepGate(g1: any, g2: any, fit: number) {
            return {
                weights: g1.weights.map((w, i) => stepInterpList(w, g2.weights[i], fit)),
                hiddenWeights: g1.hiddenWeights.map((w, i) => stepInterpList(w, g2.hiddenWeights[i], fit)),
                offset: stepInterpList(g1.offset, g2.offset, fit)
            };
        }

        stepGates(g1: any, g2: any, fit: number) {
            return {
                forget: this.stepGate(g1.forget, g2.forget, fit),
                input: this.stepGate(g1.input, g2.input, fit),
                output: this.stepGate(g1.output, g2.output, fit),
                cell: this.stepGate(g1.cell, g2.cell, fit),
            };
        }

        stepState(s1: any, s2: any, fit: number) {
            return s1.map((x, i) => stepInterp(x, s2[i], fit));
        }

        stepStates(s1: any, s2: any, fit: number) {
            return {
                cell: this.stepState(s1.cell, s2.cell, fit),
                hidden: this.stepState(s1.hidden, s2.hidden, fit),
            };
        }

        applyStep(layer2: Layer, fit: number) {
            let stepped = new (this.constructor)(
                this.next,
                this.name,
                this.stepGates(this.data.gates, layer2.data.gates, fit),
                this.stepStates(this.data.states, layer2.data.states, fit),
                this.data.activation,
                this.data.cellActivation
            );

            stepped.name = this.name;
            
            return stepped;
        }

        mutateGate(g1: any, amount: number) {
            return {
                weights: g1.weights.map((w) => mutateList(w, amount)),
                hiddenWeights: g1.hiddenWeights.map((w) => mutateList(w, amount)),
                offset: mutateList(g1.offset, amount)
            };
        }

        mutateGates(g1: any, amount: number) {
            g1.forget = this.mutateGate(g1.forget, amount);
            g1.input = this.mutateGate(g1.input, amount);
            g1.output = this.mutateGate(g1.output, amount);
            g1.cell = this.mutateGate(g1.cell, amount);
        }

        mutateState(s1: any, amount: number) {
            return mutateList(s1, amount);
        }

        mutateStates(s1: any, amount: number) {
            s1.cell = this.mutateState(s1.cell, amount);
            s1.hidden = this.mutateState(s1.hidden, amount);
        }

        mutate(amount: ?number = null) {
            if (amount == null) amount = _logit(0.25 + 0.5 * Math.random());

            this.mutateGates(this.data.gates, (amount: any));
            this.mutateStates(this.data.states, (amount: any));
        }
    },

    lstm_peephole: class LSTMPeepholLayer extends LSTMLayer {
        typename() {
            return 'lstm_peephole';
        }

        process(vec: Vector) : Vector {
            let { forget, input, output } = this.data.gates;
            let gCell = this.data.gates.cell;
            let act = this.data.activation;
            let cAct = this.data.cellActivation;

            let sCell = new Vector(this.data.states.cell);

            let gat = {
                forget: {
                    weights: forget.weights.map((w) => new Vector(w)),
                    hiddenWeights: forget.hiddenWeights.map((w) => new Vector(w)),
                },

                input: {
                    weights: input.weights.map((w) => new Vector(w)),
                    hiddenWeights: input.hiddenWeights.map((w) => new Vector(w))
                },

                output: {
                    weights: output.weights.map((w) => new Vector(w)),
                    hiddenWeights: output.hiddenWeights.map((w) => new Vector(w))
                },

                cell: {
                    weights: gCell.weights.map((w) => new Vector(w)),
                    hiddenWeights: gCell.hiddenWeights.map((w) => new Vector(w))
                },
            };

            let forgVec = new Vector(gat.forget.weights.map((n, i) => (
                greataptic.activate(act, n.multiplyVec(vec).sum() + gat.forget.hiddenWeights[i].multiplyVec(sCell).sum() + forget.offset[i])
            )));

            let inpVec = new Vector(gat.input.weights.map((n, i) => (
                greataptic.activate(act, n.multiplyVec(vec).sum() + gat.input.hiddenWeights[i].multiplyVec(sCell).sum() + input.offset[i])
            )));

            let outVec = new Vector(gat.output.weights.map((n, i) => (
                greataptic.activate(act, n.multiplyVec(vec).sum() + gat.output.hiddenWeights[i].multiplyVec(sCell).sum() + output.offset[i])
            )));

            this.data.states.cell = forgVec.multiplyVec(sCell).add(inpVec.multiplyVec(
                gat.cell.weights.map((n, i) => (
                    greataptic.activate(cAct, n.multiplyVec(vec).sum() + gCell.offset[i])
                ))
            )).data;

            this.data.states.hidden = outVec.multiplyVec(this.data.states.cell.map((x) => greataptic.activate(cAct, x))).data;

            return new Vector(this.data.states.hidden);
        }
    }
};

greataptic.layerTypes = types;

greataptic.isNetworkData = function(n: any): bool {
    return !!(n && n.layers && (n.first != null));
};

function choice(l: any[]): any {
    return l[Math.floor(l.length * Math.random())];
}

greataptic.breed = function breed(nets: NeuralNetwork[]) {
    let res = nets[0].clone();
    let newLayers = {};

    Object.keys(res.data.layers).forEach((i) => {
        let l = res.data.layers[i];

        let n2 = choice(nets.slice(1));
        let l2 = n2.data.layers[i];

        if (n2 !== nets[0] && l2.type === l.type && types[l.type].breed)
            l = types[l.type].breed(l, l2);

        newLayers[i] = l;
    });

    res.data.layers = newLayers;
    return res;
};

let $net = greataptic.$net = function(data: any) {
    return new NeuralNetwork(data);
}

let NeuralNetwork = greataptic.NeuralNetwork = class NeuralNetwork {
    data: {
        layers: { [string]: Layer },
        first: string,
    };
    id: string;
    fitness: ?number;

    constructor(net: {
        layers: { [string]: Layer },
        first: string,
    }) {
        if (!greataptic.isNetworkData(net))
            throw new Error(`The net parameter passed to NeuralNetwork's constructor (${util.inspect(net)}) is invalid!`);

        this.data = net;
        this.id = new Array(5).fill(0).map(() => choice(words)).join('-') + '-' + Math.ceil(Math.random() * 10000);
    }

    json() {
        let lays = {};

        Object.entries(this.data.layers).forEach((l) => {
            let [ id: string, lay: Layer ] = l;
            lays[id] = (lay: any).save();
        });

        return JSON.stringify({
            layers: lays,
            first: this.data.first
        });
    }

    compute(vec: VectorLike): Vector {
        if (!Vector.is(vec))
            vec = new Vector(vec);

        let li;
        let layer = this.data.layers[li = this.data.first];
        let res: Vector = (vec: any);
        let i = 0;

        // eslint-disable-next-line no-constant-condition
        while (true) {
            if (layer == null)
                throw new Error(`Layer '${li}' in network pool (#${i + 1} in sequence) does not exist!`);

            //let old = res;
            layer.net = this;
            res = layer.process(res);
            delete layer.net;

            if (layer.next == null)
                break;

            else {
                layer = this.data.layers[li = layer.next];
                i++;
            }
        }

        return res;
    }
    
    computeAsync(vec: Vector) : Promise<Vector> {
        if (!Vector.is(vec))
            vec = new Vector(vec);

        let li;
        let layer = this.data.layers[li = this.data.first];
        let res = vec;
        let i = 0;

        return new Promise(((resolve) => {
        // eslint-disable-next-line no-constant-condition
            function pass() {
                if (layer == null)
                    throw new Error(`Layer '${li}' in network pool (#${i + 1} in sequence) does not exist!`);

                let t = types[layer.type];

                if (t == null)
                    throw new Error(`No such layer type ${util.inspect(layer.type)}!`);

                //let old = res;
                layer.net = this;
                res = t.process(res, layer);
                delete layer.net;

                if (layer.next == null)
                    resolve(res);

                else {
                    layer = ((this: any): NeuralNetwork).data.layers[li = layer.next];
                    i++;

                    setTimeout(pass, 0);
                }
            }

            pass();
        }).bind(this));
    }

    clone() : NeuralNetwork {
        return greataptic.fromJSON(this.json());
    }

    mutate(amount: ?number = null) : NeuralNetwork {
        let c = this.clone();

        Array.from(Object.entries(c.data.layers)).forEach((e) => {
            let l: Layer = (e[1]: any);

            if (l.type && types[l.type].mutate) {
                types[l.type].mutate(l, amount);
            }
        });

        return c;
    }

    applyStep(otherNet: NeuralNetwork, fitness: number) {
        if (fitness === 0) return;

        Array.from(Object.entries(this.data.layers)).forEach((e) => {
            let id = e[0];
            let l = ((e[1]: any): Layer);

            if (l.type && types[l.type].applyStep && otherNet.data.layers[id] && otherNet.data.layers[id].type === l.type)
                this.data.layers[id] = types[l.type].applyStep(l, otherNet.data.layers[id], fitness);
        });
    }

    error(inputSet: number[][], expectedSet: number[][]) {
        let outputSet: Vector[] = inputSet.map<Vector>((i: number[]) => this.compute(i));
        let res = outputSet.map((os, si) => os.data.map((o, i) => Math.pow((o - expectedSet[si][i]), 2)).reduce((a, b) => a + b, 0)).reduce((a, b) => a + b, 0);

        return res / inputSet.length;
    }

    staticFitness(inputSet: number[][], expectedSet: number[][]) {
        let err = this.error(inputSet, expectedSet);
        return 1 / (1 + err);
    }

    _evolveStep(ctx: any) : Promise<NeuralNetwork> {
        let populateSteps = () => {
            let res = ctx.steps = new Array(ctx.population).fill(0).map(() => ctx.makeStep(this));
            return res;
        }

        let tryStep = (step) => {
            return Promise.resolve(ctx.getFitness(step)).then((fit) => {
                step.fitness = fit;
            });
        }

        let concludeSteps = () => {
            let res = this.clone();
            let newFit = 0;
            let denom = 1;
            
            ctx.steps.forEach((s) => {
                res.applyStep(s, s.fitness * ctx.stepFactor);

                newFit = (newFit * (denom - 1) + s.fitness) / denom++;
            });
            
            res.fitness = newFit;
            (res: any).trainCtx = ctx;

            if (ctx.options.postStep)
                ctx.options.postStep(res);
            
            return res;
        }

        populateSteps();

        let prom = Promise.resolve();

        if (ctx.options.stepAll) {
            let batchPop = ctx.options.batchPop;
            let i = 0;
            let batchIndex = 1;

            // eslint-disable-next-line no-constant-condition
            while (true) {
                if (i >= ctx.steps.length)
                    break;

                batchPop = Math.min(ctx.steps.length - i, batchPop);
                
                let ls = ctx.steps.slice(i, i + batchPop);
                let gen = ctx.options.generation || null;
                let bi = batchIndex;
                
                prom = prom.then(() => Promise.resolve(ctx.options.stepAll(ls, gen, bi)));
                
                i += batchPop;
                batchIndex++;
            }
        }
        
        else
            ctx.steps.forEach((step) => {
                prom = prom.then(() => tryStep(step));
            });

        return new Promise((resolve) => {
            prom.then(() => {
                resolve(concludeSteps());
            });
        });
    }

    evolveStep(options: any = {}) {
        let getFitness = (net) => {
            return new Promise(() => {
                if (options.step)
                    return Promise.resolve(options.step(net, options.generation || null));

                else
                    return 0;
                    
            }).then((res) => {
                if (options.inputSet && options.expectedSet && !options.callbackOnly)
                    res += this.staticFitness(options.inputSet, options.expectedSet);

                return res;
            });
        }

        let makeStep = () => {
            if (options.random)
                return this.mutate(100);

            else
                return this.mutate(gaussRand() * ctx.maxMutation * Math.pow(options.mutationDecay, (options.generation || 0) - 1));
        }

        let ctx = {
            options: options,
            
            population: options.population || 50,
            maxGens: options.maxGens || 500,
            maxMutation: options.maxMutation || 0.5,
            mutationDecay: options.mutationDecay || 0.85,
            stepFactor: options.stepFactor || 0.75,
            
            getFitness: getFitness,
            makeStep: makeStep,

            steps: null
        };

        return this._evolveStep(ctx);
    }

    evolve(options: any = {}) : Promise<NeuralNetwork> {
        let fitQuota = options.quota != null ? options.quota : Infinity;
        let cur = this;
        let gen = 1;

        options.random = options.random || false;
        
        return new Promise((resolve) => {
            function iter() {
                options.generation = gen;
                cur.evolveStep(options).then((evolved) => {
                    options.random = false;
                    let ctx = (evolved: any).trainCtx;

                    if (options.debug) // print debug status
                        console.log(`[DEBUG] Generation ${gen}/${ctx.maxGens}: fitness is now ${(evolved.fitness: any)}.`);

                    if ((evolved.fitness: any) >= fitQuota || ++gen > (evolved: any).trainCtx.maxGens)
                        resolve(evolved);

                    else {
                        if (!cur.fitness || (evolved.fitness: any) > cur.fitness) cur = evolved;
                        setTimeout(iter, 0);
                    }
                });
            }
            
            setTimeout(iter, 0);
        });
    }
}

greataptic.sequential = function sequential(inputSize: number, layers: any[]) {
    if ( layers.length === 0 ) {
        layers = [{ size: inputSize, post: 'sigmoid' }];
    }

    let res = {
        layers: {},
        first: '1'
    };

    let lastLayer: ?Layer = null;
    let lastSize: ?number = null;
    
    function buildLayer(l: any, index: any, after: any) {
        let subres = [];
        let postAfter = after;
        let preAfter = index + 'm';
        let midAfter = (l.post == null ? postAfter : index + 'p');
        let preId = index;
        let midId = (l.pre == null ? index : index + 'm');
        let postId = index + 'p';
        let subs = 0;

        function addLayer(l: any, id: any, size: number) {
            lastSize = size;
            lastLayer = l;
            l.name = id;
            subres.push(l);
            res.layers[id] = l;
        }

        let layerInput = lastSize == null ? inputSize : lastSize;
        
        if (l.pre != null) {
            addLayer(new (types[l.pre])(preAfter, null, l.data != null ? l.data : null), preId, layerInput);
            layerInput = lastSize == null ? inputSize : lastSize;
        }

        if (l.type === 'combo') {
            addLayer(new (types.combo)(midAfter, null, (l: any).parts.map<any>((ls, i, a) => new (types.sequence)(
                null,
                index + '_presub' + subs,
                buildLayer(ls, index + '_sub' + (++subs), (
                    i + 1 === a.length ? null : index + '_sub' + (subs + 1)
                )
            )), null)), midId, l.parts.reduce((a, ls) => a + ls.size, 0));
        }

        else if (l.type === 'sequence')
            addLayer(new (types.sequence)(midAfter, null, (l: any).parts.map<any>((ls, i, a) => buildLayer(ls, index + '_sub' + (++subs), (
                i + 1 === a.length ? null : index + '_sub' + (subs + 1)
            ))).reduce((a, b) => a.concat(b), [])), midId, l.parts.reduce((a, b) => b.size || a, null));

        else if (l.size != null) {
            if (l.type === 'spiking')
                addLayer(new (types.spiking)(
                    midAfter,
                    null,
                    new Array(l.size).fill(0).map(() => ({
                        weights: Vector.random(layerInput).data,
                        power: 0,
                        output: Math.pow(Math.random() * 3, 2),
                        limit: _logit(Math.random() * 0.45 + 0.5)
                    }))
                ), midId, l.size);

            else if (l.type === 'linear' || l.type == null) {
                addLayer(new (types.linear)(
                    midAfter,
                    null,
                    new Array(l.size).fill(0).map(() => ({
                        offset: _logit(0.225 + 0.55 * Math.random()),
                        weights: Vector.random(layerInput).data
                    })),
                ), midId, l.size);
            }

            else if (l.type === 'square')
                addLayer(new (types.square)(
                    midAfter,
                    null,
                    new Array(l.size).fill(0).map(() => ({
                        offset: _logit(0.25 + 0.5 * Math.random()),
                        weights: {
                            linear: Vector.random(layerInput).data,
                            square: Vector.random(layerInput).data
                        }
                    })),
                ), midId, l.size);

            else if (l.type.startsWith('lstm') && types[l.type])
                addLayer(new (types[l.type])(
                    midAfter,
                    null,
                    types[l.type].randomGates(lastSize, l.size),
                    types[l.type].randomStates(l.size),
                    l.activation || 'sigmoid',
                    l.stateActivation || 'tanh'
                ), midId, l.size);
        }

        else
            addLayer(new(types[l.type])(midAfter), midId, layerInput);

        if (l.post != null)
            addLayer(new(types[l.post])(postAfter), postId, lastSize != null ? lastSize : layerInput);

        return subres;
    }

    layers.forEach((l, i) => buildLayer(l, '' + (i + 1), '' + (i + 2)));

    if (lastLayer) (lastLayer: any).next = null;
    return greataptic.$net(res);
};

greataptic.fromJSON = function fromJSON(j: string) {
    let res = JSON.parse(j);
    let lays = res.layers;

    Object.entries(lays).forEach((l: any) => {
        let [ id: string, lay: Layer ] = l;
        lays[id] = types[lay.type].load(lay);
    });

    return new greataptic.NeuralNetwork(res);
};

function inputNoise(size: number, options: any = {}) {
    let range = options.range || 1;
    let negative = (options.negative == null || options.negative);

    return new Array(size).fill(0).map<number>(() => Math.random() * (negative ? range * 2 : range) - (negative ? range : 0));
}

greataptic.createVectorifier = function createVectorifier(args: any) {
    let vectorifier = {
        args: new Map<string, any>(),

        getSize: function () {
            let res = 0;

            vectorifier.args.forEach((a) => {
                res += a.getSize();
            });

            return res;
        },

        encode: function (obj: any) {
            let enc = [];

            vectorifier.args.forEach((a) => {
                if (obj[a.name] === undefined)
                    throw new Error(`Required argument ${a.name} not passed to vectorizer!`);

                else
                    enc = enc.concat(a.encode(obj[a.name]));
            });

            return enc;
        },

        decode: function (vect: number[]) {
            let cursor = 0;
            let res = {};

            vectorifier.args.forEach((a) => {
                res[a.name] = a.decode(vect.slice(cursor, cursor + a.getSize()));
                cursor += a.getSize();
            });

            return res;
        }
    };

    args.forEach((arg) => {
        let aname = arg.name;
        const strspace = ' 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';

        if (arg.type.toLowerCase() === 'simplestring')
            vectorifier.args.set(aname, {
                name: aname,

                getSize: function getSize() {
                    return arg.size;
                },

                encode: function encode(s) {
                    let sl = s.slice(0, arg.size).split('').map((x) => strspace.indexOf(x) === -1 ? strspace[0] : x).join('');

                    while (sl.length < arg.size)
                        sl += arg.default || ' ';

                    return Array.from(sl).map((c) => {
                        return strspace.indexOf(c) / strspace.length;
                    });
                },

                decode: function decode(v) {
                    let x = v.map((i) => strspace[Math.floor(i * strspace.length)]).join('');
                    return x;
                }
            });

        else if (arg.type.toLowerCase() === 'number')
            vectorifier.args.set(aname, {
                name: aname,

                getSize: function getSize() {
                    return 1;
                },

                encode: function encode(n) {
                    return [(n - arg.min) / (arg.max - arg.min)];
                },
                
                decode: function decode(v) {
                    let res = v[0] * (arg.max - arg.min) + arg.min;
                    if (arg.rounded) res = Math.round(res);
                    
                    return res;
                }
            });

        else if (arg.type.toLowerCase() === 'numbers') {
            vectorifier.args.set(aname, {
                name: aname,

                getSize: function getSize() {
                    return arg.size;
                },

                encode: function encode(a) {
                    a = a.slice(0, arg.size);

                    while (a.length < arg.size)
                        a.push(0);

                    return a.map((n) => (n - arg.min) / (arg.max - arg.min));
                },

                decode: function decode(v) {
                    return v.map((n) => {
                        let res = n * (arg.max - arg.min) + arg.min;
                        if (arg.rounded) res = Math.round(res);
                        
                        return res;
                    });
                }
            });
        } else if (arg.type.toLowerCase() === 'string')
            vectorifier.args.set(aname, {
                name: aname,

                getSize: function getSize() {
                    return arg.size;
                },

                encode: function encode(s) {
                    let sl = s.slice(0, arg.size);

                    while (sl.length < arg.size)
                        sl += arg.default || ' ';

                    return Array.from(sl).map((c) => {
                        return c.charCodeAt(0) / 256;
                    });
                },

                decode: function decode(v) {
                    return v.map((i) => String.fromCharCode(Math.floor(i * 256))).join('');
                }
            });
    });

    return vectorifier;
};

greataptic.GAN = class GAN {
    size: { noise: number, output: number };
    generator: NeuralNetwork;
    discriminator: NeuralNetwork;
    noiseOptions: any;

    constructor(properties: any) {
        properties.size = properties.size || {
            output: 40
        };

        this.size = {
            noise: +properties.size.noise || 15,
            output: +properties.size.output
        };

        this.generator = greataptic.sequential(this.size.noise, (properties.generatorLayers || this.generatorDefaultLayers()).concat([{
            size: this.size.output,
            type: properties.outputType || 'linear',
            post: 'sigmoid'
        }]));
        this.discriminator = greataptic.sequential(this.size.output, (properties.discriminatorLayers || this.discriminatorDefaultLayers()).concat([{
            size: 1,
            post: 'sigmoid'
        }]));
        this.noiseOptions = properties.noise || {};
    }

    generatorDefaultLayers() {
        return [
            {
                size: this.size.noise * 2 + 10,
                post: 'sigmoid'
            },
            {
                size: Math.ceil(this.size.noise * 1.5 + this.size.output) + 1,
                post: 'sigmoid'
            }
        ];
    }

    discriminatorDefaultLayers() {
        return [
            {
                size: Math.ceil(this.size.noise * 2 + this.size.output) + 5,
                post: 'sigmoid'
            },
            {
                size: Math.ceil(this.size.output / 1.5) + 2,
                post: 'sigmoid'
            },
        ];
    }

    rate(data: VectorLike) {
        let input = Array.from((data: any).data || (data: any)).concat(new Array(Math.max(0, this.size.output - ((data: any).data || (data: any)).length)).fill(0)).slice(0, this.size.output);
        return this.discriminator.compute(new Vector(input)).data[0];
    }

    evolve(realData: VectorLike[], options: any) : Promise<NeuralNetwork> {
        realData = realData.map((rd) => new Vector(rd));

        let fake = [];
        let _step = options.step;
        let _postStep = options.postStep;

        Object.assign(options, {
            step: (net) => {
                let f = this.generate(net);
                fake.push(f);
                
                if (_step != null) _step(net);

                return this.rate(f);
            },

            postStep: (net) => {
                this._evolveCycleDiscGrade(realData, fake, (options.discriminatorTrainOptions || {}).comparisonSize || 4, options.discriminatorTrainOptions || null);
                net.fake = fake;
                if (_postStep != null) _postStep(net);
            }
        });

        return this.generator.evolve(options).then((newGenerator) => {
            return this.generator = newGenerator;
        });
    }

    _evolveCycleDiscGrade(realData: VectorLike[], fakeData: VectorLike[], maxComparisonSize: number = 4, discriminatorOptions: ?any = null) {
        maxComparisonSize = Math.min(maxComparisonSize, realData.length, fakeData.length);

        realData = shuffle(realData, {
            copy: true
        }).slice(0, maxComparisonSize);
        fakeData = shuffle(fakeData, {
            copy: true
        }).slice(0, maxComparisonSize);

        let expec = new Array(realData.length).fill([1]);
        let notExpec = new Array(realData.length).fill([0]);

        let _opts = {
            inputSet: realData.concat(fakeData),
            expectedSet: expec.concat(notExpec),
            maxGens: 15,
            maxMutation: 0.3,
            population: 30
        };

        Object.assign(_opts, discriminatorOptions);

        return this.discriminator.evolve(_opts).then((disc) => {
            this.discriminator = disc;
        });
    }

    makeNoise() {
        return inputNoise(this.size.noise, this.noiseOptions);
    }

    generate(net: ?Neural