rlab/dnn/lib/MLP.js

Javascript scientific library like R

/**
 * Created by joonkukang on 2014. 1. 14..
 */
var math = require('./utils').math;
var M = R.M;
MLP = module.exports = function (settings) {
    var self = this;
    self.x = settings['input'];
    self.y = settings['label'];
    self.sigmoidLayers = [];
    self.nLayers = settings['hidden_layer_sizes'].length;
    self.settings = {
        'log level' : 1 // 0 : nothing, 1 : info, 2: warn
    };
    var i;
    for(i=0 ; i<self.nLayers+1 ; i++) {
        var inputSize, layerInput;
        if(i == 0)
            inputSize = settings['n_ins'];
        else
            inputSize = settings['hidden_layer_sizes'][i-1];

        if(i == 0)
            layerInput = self.x;
        else
            layerInput = self.sigmoidLayers[self.sigmoidLayers.length-1].sampleHgivenV();

        var sigmoidLayer;
        if(i == self.nLayers) {
            sigmoidLayer = new HiddenLayer({
                'input' : layerInput,
                'n_in' : inputSize,
                'n_out' : settings['n_outs'],
                'activation' : R.NN.sigmoid, // math.sigmoid,
                'W' : (typeof settings['w_array'] === 'undefined')? undefined : settings['w_array'][i],
                'b' : (typeof settings['b_array'] === 'undefined')? undefined : settings['b_array'][i]
            });
        } else {
            sigmoidLayer = new HiddenLayer({
                'input' : layerInput,
                'n_in' : inputSize,
                'n_out' : settings['hidden_layer_sizes'][i],
                'activation' : R.NN.sigmoid, // math.sigmoid,
                'W' : (typeof settings['w_array'] === 'undefined')? undefined : settings['w_array'][i],
                'b' : (typeof settings['b_array'] === 'undefined')? undefined : settings['b_array'][i]
            });
        }
        self.sigmoidLayers.push(sigmoidLayer);
    }
};

MLP.prototype.train = function(settings) {
    var self = this;
    var lr = 0.6, epochs = 1000;
    if(typeof settings['lr'] !== 'undefined')
        lr = settings['lr'];
    if(typeof settings['epochs'] !== 'undefined')
        epochs = settings['epochs'];


    var epoch;
    var currentProgress = 1;
    for(epoch=0 ; epoch < epochs ; epoch++) {

        // Feed Forward
        var i;
        var layerInput = [];
        layerInput.push(self.x);
        for(i=0; i<self.nLayers+1 ; i++) {
            layerInput.push(self.sigmoidLayers[i].output(layerInput[i]));
        }
        var output = layerInput[self.nLayers+1];
        // Back Propagation
        var delta = new Array(self.nLayers + 1);
				
        delta[self.nLayers] = m.mulMatElementWise(m.minusMat(self.y, output),
            m.activateMat(self.sigmoidLayers[self.nLayers].linearOutput(layerInput[self.nLayers]), m.dSigmoid));

//				var linearOutput=this.sigmoidLayers[this.nLayers].linearOutput(layerInput[this.nLayers]).map1(R.NN.dSigmoid);
//				delta[self.nLayers] = this.y.sub(output).mul(linearOutput);
        /*
         self.nLayers = 3 (3 hidden layers)
         delta[3] : ouput layer
         delta[2] : 3rd hidden layer, delta[0] : 1st hidden layer
         */
        for(i = self.nLayers - 1; i>=0 ; i--) {
//            delta[i] = m.mulMatElementWise(self.sigmoidLayers[i+1].backPropagate(delta[i+1]),                m.activateMat(self.sigmoidLayers[i].linearOutput(layerInput[i]), m.dSigmoid));
					  var o = this.sigmoidLayers[i].linearOutput(layerInput[i]).map1(R.NN.dSigmoid);
					  delta[i] = this.sigmoidLayers[i+1].backPropagate(delta[i+1]).mul(o);
        }
        // Update Weight, Bias
        for(var i=0; i<self.nLayers+1 ; i++) {
						
//            var deltaW = m.activateMat(m.mulMat(m.transpose(layerInput[i]),delta[i]),function(x){return 1. * x / self.x.length;})
//            var deltaB = m.meanMatAxis(delta[i],0);
//            self.sigmoidLayers[i].W = m.addMat(self.sigmoidLayers[i].W,deltaW);
//            self.sigmoidLayers[i].b = m.addVec(self.sigmoidLayers[i].b,deltaB);
						
						var deltaW = layerInput[i].tr().dot(delta[i]).map1((x)=>x/this.x.length);
						var deltaB = delta[i].colMean();
						self.sigmoidLayers[i].W = this.sigmoidLayers[i].W.add(deltaW);
						self.sigmoidLayers[i].b = this.sigmoidLayers[i].b.add(deltaB);
        }

        if(self.settings['log level'] > 0) {
            var progress = (1.*epoch/epochs)*100;
            if(progress > currentProgress) {
                console.log("MLP",progress.toFixed(0),"% Completed.");
                currentProgress+=8;
            }
        }
    }
    if(self.settings['log level'] > 0)
        console.log("MLP Final Cross Entropy : ",self.getReconstructionCrossEntropy());
};

MLP.prototype.getReconstructionCrossEntropy = function() {
    var self = this;
    var reconstructedOutput = self.predict(self.x);
		var a = R.map2(self.y, reconstructedOutput, function(x,y) {
			return x*Math.log(y);
		})
//    var a = math.activateTwoMat(self.y,reconstructedOutput,function(x,y){
//        return x*Math.log(y);
//    });

		var b = R.map2(self.y, reconstructedOutput, function(x,y) {
        return (1-x)*Math.log(1-y);			
		});
//    var b = math.activateTwoMat(self.y,reconstructedOutput,function(x,y){
//        return (1-x)*Math.log(1-y);
//    });

//    var crossEntropy = -math.meanVec(math.sumMatAxis(math.addMat(a,b),1));
    var crossEntropy = a.add(b).colMean().neg(); // -(a+b).colMean()
    return crossEntropy
}

MLP.prototype.predict = function(x) {
    var self = this;
    var output = x;
    for(i=0; i<self.nLayers+1 ; i++) {
        output = self.sigmoidLayers[i].output(output);
    }
    return output;
};

MLP.prototype.set = function(property,value) {
    var self = this;
    self.settings[property] = value;
}