node_neural_net/index.js

A utility to create and train a feed-forward neural network in Node.js

const fs = require('fs');

// sigmoid(z) computes the value of the sigmoid function at z
// requires: z is a number
function sigmoid(z) {
    return 1 / (1 + Math.exp(-z));
}

// dot_prod(v1, v2) determines the dot product between two vectors
// requires: v1 and v2 are vectors
function dot_prod(v1, v2) {
    let sum = 0;
    for (let i = 0; i < v1.length; ++i) {
        sum += v1[i] * v2[i];
    }
    return sum;
}


// matrix_vector_mult(m, v) determines the matrix-vector product of m and v
// requires: m is a matrix, v is a vector
// time: O(c * r) where c and r are the dimensions of m
function matrix_vector_mult(m, v) {
    let out = new Array(m.length);
    for (let i = 0; i < m.length; ++i) {
        out[i] = dot_prod(m[i], v);
    }
    return out;
}

// vector_add(v1, v2) computes the sum of v1 and v2, represented as a new vector
// requires: v1 and v2 are vectors
function vector_add(v1, v2) {
    let out = new Array(v1.length);
    for (let i = 0; i < v1.length; ++i) {
        out[i] = v1[i] + v2[i];
    }
    return out;
}

// vector_sigmoid(v) computes the sigmoid function on each component of v,
//   creating a new vector representing these values
// requires: v is a vector
function vector_sigmoid(v) {
    let out = new Array(v.length);
    for (let i = 0; i < v.length; ++i) {
        out[i] = sigmoid(v[i]);
    }
    return out;
}

// copy_tensor_dim(t) creates a new tensor of the same rank and dimensions of t,
//   where each entry is filled with a 0
// requires: t is a tensor (number or n-dimensional array)
// time: O(r) where r is the rank of t
function copy_tensor_dim(t) {
    if (typeof t === Number) {
        return 0;
    }
    let out = new Array(t.length);
    for (let i = 0; i < t.length; ++i) {
        out[i] = copy_tensor_dim(t[i]);
    }
    return out;
}


// tensor_add_mutate(t1, t2) adds two tensors together by mutating t1
// requires: t1 and t2 are tensors of same rank and dimensions,
//           t1 and t2 are at least rank 1 (otherwise just use +=)
function tensor_add_mutate(t1, t2) {
    if (t1[0].constructor === Array) {
        for (let i = 0; i < t1.length; ++i) {
            tensor_add_mutate(t1[i], t2[i]);
        }
    } else {
        for (let i = 0; i < t1.length; ++i) {
            t1[i] += t2[i];
        }
    }
}

class NeuralNet {
    constructor(layer_sizes, weights, biases) {
        this.layers_len = layer_sizes.length;
        if (weights === undefined || biases === undefined) {
            this.weights = new Array(layer_sizes.length - 1);
            this.biases = new Array(layer_sizes.length - 1);
            for (let l = 0; l < this.weights.length; ++l) {
                this.weights[l] = new Array(layer_sizes[l + 1]);
                this.biases[l] = new Array(layer_sizes[l + 1]);
                for (let j = 0; j < this.weights[l].length; ++j) {
                    this.weights[l][j] = new Array(layer_sizes[l]);
                    this.biases[l][j] = Math.random() * 2 - 1;
                    for (let k = 0; k < this.weights[l][j].length; ++k) {
                        this.weights[l][j][k] = Math.random() * 2 - 1;
                    }
                }
            }
        } else {
            this.weights = weights;
            this.biases = biases;
        }
    }

    // save_weights(dir) creates a new js file at dir with the weights and biases
    //   of this model encoded as a module
    // requires: dir is a valid path ending in a .js file
    save_weights(dir) {
        fs.writeFileSync(dir, "let model_weights = "
            + JSON.stringify(this.weights) + "; \n let model_biases = "
            + JSON.stringify(this.biases) + "; \n exports.weights = model_weights; exports.biases = model_biases;");
    }

    // eval(input_nodes) computes the forward propogation of this network
    // requires: input_nodes is a vector of length layer_sizes[0]
    eval(input_nodes) {
        let net_state = new Array(this.layers_len);
        net_state[0] = input_nodes;
        for (let i = 1; i < this.layers_len; ++i) {
            net_state[i] = vector_sigmoid(vector_add(matrix_vector_mult(this.weights[i - 1], net_state[i - 1]), this.biases[i - 1]));
        }
        return net_state;
    }

    // train(input, output, batch_size, step_size) mutates the weights and biases of this network
    //   to lower the error of evaluating input compared to the ground-truth output.
    //   This is done by computing the gradient of our parameters dw, db, (backpropogation!) and
    //   moving in the direction of steepest descent (gradient descent!).
    train(input, output, batch_size, step_size) {
        for (let batch_number = 0; batch_number + batch_size < input.length; batch_number += batch_size) {
            let dw = copy_tensor_dim(this.weights);
            let db = copy_tensor_dim(this.biases);
            for (let sample_in_batch = 0; sample_in_batch < batch_size; ++sample_in_batch) {

                let current_index = batch_number + sample_in_batch;

                // calculate the network on given input in batch
                let net_state = this.eval(input[current_index]);
                let delta = copy_tensor_dim(net_state);

                // last layer da
                for (let j = 0; j < delta[delta.length - 1].length; ++j) {
                    delta[delta.length - 1][j] = (net_state[delta.length - 1][j] - output[current_index][j])
                }

                // other layers da
                for (let l = delta.length - 2; l >= 0; --l) {
                    for (let k = 0; k < delta[l].length; ++k) {
                        let effects_j = 0;
                        for (let j = 0; j < delta[l + 1].length; ++j) {
                            effects_j += this.weights[l][j][k] * delta[l + 1][j];
                        }
                        delta[l][k] = (net_state[l][k]) * (1 - net_state[l][k]) * effects_j;
                    }
                }

                // dw and db
                for (let l = 0; l < net_state.length - 1; ++l) {
                    for (let j = 0; j < net_state[l + 1].length; ++j) {
                        for (let k = 0; k < net_state[l].length; ++k) {
                            dw[l][j][k] -= (step_size * delta[l + 1][j] * net_state[l][k]) / batch_size;
                        }
                        db[l][j] -= (step_size * delta[l + 1][j]) / batch_size;
                    }
                }
            }
            tensor_add_mutate(this.weights, dw);
            tensor_add_mutate(this.biases, db);
        }
    }
}

module.exports = NeuralNet;