recurrent-js-gpu
Version:
GPU-accelerated Deep Recurrent Neural Networks and LSTMs in Typescript. Ported, object-oriented and refactored version of Andrej Karpathy's recurrent-js (https://github.com/karpathy/recurrentjs)
75 lines • 4.18 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const Mat_1 = require("./Mat");
const RandMat_1 = require("./RandMat");
const NNModel_1 = require("./NNModel");
class LSTM extends NNModel_1.NNModel {
constructor(inputSize, hiddenSizes, outputSize, needsBackProp = true) {
super(needsBackProp);
this.inputSize = inputSize;
this.hiddenSizes = hiddenSizes;
this.outputSize = outputSize;
for (let i = 0; i < hiddenSizes.length; i++) {
const prevSize = i === 0 ? inputSize : hiddenSizes[i - 1];
const hiddenSize = hiddenSizes[i];
this.model.inputWx[i] = new RandMat_1.RandMat(hiddenSize, prevSize, 0, 0.08);
this.model.inputWh[i] = new RandMat_1.RandMat(hiddenSize, hiddenSize, 0, 0.08);
this.model.inputb[i] = new Mat_1.Mat(hiddenSize, 1);
this.model.forgetWx[i] = new RandMat_1.RandMat(hiddenSize, prevSize, 0, 0.08);
this.model.forgetWh[i] = new RandMat_1.RandMat(hiddenSize, hiddenSize, 0, 0.08);
this.model.forgetb[i] = new Mat_1.Mat(hiddenSize, 1);
this.model.outputWx[i] = new RandMat_1.RandMat(hiddenSize, prevSize, 0, 0.08);
this.model.outputWh[i] = new RandMat_1.RandMat(hiddenSize, hiddenSize, 0, 0.08);
this.model.outputb[i] = new Mat_1.Mat(hiddenSize, 1);
this.model.cellWx[i] = new RandMat_1.RandMat(hiddenSize, prevSize, 0, 0.08);
this.model.cellWh[i] = new RandMat_1.RandMat(hiddenSize, hiddenSize, 0, 0.08);
this.model.cellb[i] = new Mat_1.Mat(hiddenSize, 1);
}
this.model.decoderWh = new RandMat_1.RandMat(outputSize, (hiddenSizes.length - 1), 0, 0.08);
this.model.decoderb = new Mat_1.Mat(outputSize, 1);
}
forward(observations, previousOutput, graph = this.graph) {
let hiddenPrevs, cellPrevs;
if (previousOutput == null || typeof previousOutput.h === 'undefined') {
hiddenPrevs = new Array();
cellPrevs = new Array();
for (let d = 0; d < this.hiddenSizes.length; d++) {
hiddenPrevs.push(new Mat_1.Mat(this.hiddenSizes[d], 1));
cellPrevs.push(new Mat_1.Mat(this.hiddenSizes[d], 1));
}
}
else {
hiddenPrevs = previousOutput.h;
cellPrevs = previousOutput.c;
}
const hidden = [];
const cell = [];
for (let d = 0; d < this.hiddenSizes.length; d++) {
const inputVector = (d === 0) ? observations : hidden[d - 1];
const hiddenPrev = hiddenPrevs[d];
const cellPrev = cellPrevs[d];
const h0 = graph.mul(this.model.inputWx[d], inputVector);
const h1 = graph.mul(this.model.inputWh[d], hiddenPrev);
const inputGate = graph.sigmoid(graph.add(graph.add(h0, h1), this.model.inputb[d]));
const h2 = graph.mul(this.model.forgetWx[d], inputVector);
const h3 = graph.mul(this.model.forgetWh[d], hiddenPrev);
const forgetGate = graph.sigmoid(graph.add(graph.add(h2, h3), this.model.forgetb[d]));
const h4 = graph.mul(this.model.outputWx[d], inputVector);
const h5 = graph.mul(this.model.outputWh[d], hiddenPrev);
const outputGate = graph.sigmoid(graph.add(graph.add(h4, h5), this.model.outputb[d]));
const h6 = graph.mul(this.model.cellWx[d], inputVector);
const h7 = graph.mul(this.model.cellWh[d], hiddenPrev);
const cellWrite = graph.tanh(graph.add(graph.add(h6, h7), this.model.cellb[d]));
const retainCell = graph.eltmul(forgetGate, cellPrev);
const writeCell = graph.eltmul(inputGate, cellWrite);
const cellD = graph.add(retainCell, writeCell);
const hiddenD = graph.eltmul(outputGate, graph.tanh(cellD));
hidden.push(hiddenD);
cell.push(cellD);
}
const output = graph.add(graph.mul(this.model.decoderWh, hidden[hidden.length - 1]), this.model.decoderb);
return { 'h': hidden, 'c': cell, 'o': output };
}
}
exports.LSTM = LSTM;
//# sourceMappingURL=LSTM.js.map