recurrent-js-gpu/dist/Solver.js

GPU-accelerated Deep Recurrent Neural Networks and LSTMs in Typescript. Ported, object-oriented and refactored version of Andrej Karpathy's recurrent-js (https://github.com/karpathy/recurrentjs)

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const Mat_1 = require("./Mat");
class Solver {
    constructor(decayRate = 0.999, smoothEps = 1e-8) {
        this.decayRate = decayRate;
        this.smoothEps = smoothEps;
        this.stepCache = {};
    }
    reset() {
        this.stepNumberOfClippings = 0;
        this.stepTotalNumber = 0;
    }
    step(model, stepSize, l2Regularization, clippingValue) {
        this.reset();
        const solverStats = { ratioClipped: 0 };
        for (const key in model) {
            if (model.hasOwnProperty(key)) {
                this.iterateModelLayer(model, key, clippingValue, l2Regularization, stepSize);
            }
        }
        solverStats.ratioClipped = this.stepNumberOfClippings * 1.0 / this.stepTotalNumber;
        return solverStats;
    }
    iterateModelLayer(model, key, clipval, regc, stepSize) {
        const currentModelLayer = model[key];
        if (!(this.stepCache.hasOwnProperty(key))) {
            this.stepCache[key] = new Mat_1.Mat(currentModelLayer.n, currentModelLayer.d);
        }
        const currentStepCache = this.stepCache[key];
        for (let i = 0; i < currentModelLayer.w.length; i++) {
            let mdwi = this.RMSprop(currentModelLayer, i, currentStepCache);
            mdwi = this.gradientClipping(mdwi, clipval);
            this.update(currentModelLayer, i, stepSize, mdwi, currentStepCache, regc);
            this.resetGradients(currentModelLayer, i);
        }
    }
    RMSprop(modelLayer, i, stepCache) {
        const mdwi = modelLayer.dw[i];
        stepCache.w[i] = stepCache.w[i] * this.decayRate + (1.0 - this.decayRate) * mdwi * mdwi;
        return mdwi;
    }
    gradientClipping(mdwi, clipval) {
        if (mdwi > clipval) {
            mdwi = clipval;
            this.stepNumberOfClippings++;
        }
        else if (mdwi < -clipval) {
            mdwi = -clipval;
            this.stepNumberOfClippings++;
        }
        this.stepTotalNumber++;
        return mdwi;
    }
    update(m, i, stepSize, mdwi, stepCache, regc) {
        m.w[i] += -stepSize * mdwi / Math.sqrt(stepCache.w[i] + this.smoothEps) - regc * m.w[i];
    }
    resetGradients(currentModelLayer, i) {
        currentModelLayer.dw[i] = 0;
    }
}
exports.Solver = Solver;
//# sourceMappingURL=Solver.js.map