lysergic/dist/ast/activations.js

Synaptic's neural network compiler

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const operations_1 = require("./operations");
exports.WHOLE_LAYER_ACTIVATION_KIND = 128;
var ActivationTypes;
(function (ActivationTypes) {
    ActivationTypes[ActivationTypes["IDENTITY"] = 0] = "IDENTITY";
    ActivationTypes[ActivationTypes["LOGISTIC_SIGMOID"] = 1] = "LOGISTIC_SIGMOID";
    ActivationTypes[ActivationTypes["TANH"] = 2] = "TANH";
    ActivationTypes[ActivationTypes["INVERSE_IDENTITY"] = 5] = "INVERSE_IDENTITY";
    ActivationTypes[ActivationTypes["EXP"] = 6] = "EXP";
    ActivationTypes[ActivationTypes["SOFTPLUS"] = 7] = "SOFTPLUS";
    ActivationTypes[ActivationTypes["SOFTSIGN"] = 8] = "SOFTSIGN";
    ActivationTypes[ActivationTypes["GAUSSIAN"] = 9] = "GAUSSIAN";
    ActivationTypes[ActivationTypes["STEP"] = 11] = "STEP";
    ActivationTypes[ActivationTypes["RELU"] = 12] = "RELU";
    ActivationTypes[ActivationTypes["PRELU"] = 13] = "PRELU";
    ActivationTypes[ActivationTypes["RELU_PLUSONE"] = 14] = "RELU_PLUSONE";
    ActivationTypes[ActivationTypes["ELU"] = 15] = "ELU";
    ActivationTypes[ActivationTypes["PELU"] = 16] = "PELU";
    ActivationTypes[ActivationTypes["POW"] = 17] = "POW";
    ActivationTypes[ActivationTypes["POW_MINUS1"] = 18] = "POW_MINUS1";
    ActivationTypes[ActivationTypes["AVG_POOLING"] = exports.WHOLE_LAYER_ACTIVATION_KIND | 1] = "AVG_POOLING";
    ActivationTypes[ActivationTypes["MAX_POOLING"] = exports.WHOLE_LAYER_ACTIVATION_KIND | 2] = "MAX_POOLING";
    ActivationTypes[ActivationTypes["MAXOUT"] = exports.WHOLE_LAYER_ACTIVATION_KIND | 3] = "MAXOUT";
    ActivationTypes[ActivationTypes["SOFTMAX"] = exports.WHOLE_LAYER_ACTIVATION_KIND | 4] = "SOFTMAX";
    ActivationTypes[ActivationTypes["SHARPEN"] = exports.WHOLE_LAYER_ACTIVATION_KIND | 5] = "SHARPEN";
    ActivationTypes[ActivationTypes["BATCH_NORM"] = exports.WHOLE_LAYER_ACTIVATION_KIND | 6] = "BATCH_NORM";
})(ActivationTypes = exports.ActivationTypes || (exports.ActivationTypes = {}));
function buildActivationFunction(state, type) {
    if (type & exports.WHOLE_LAYER_ACTIVATION_KIND)
        return null;
    switch (type) {
        case ActivationTypes.LOGISTIC_SIGMOID:
            return operations_1.div(operations_1.number(1), operations_1.sum(operations_1.number(1), operations_1.exp(operations_1.neg(state))));
        case ActivationTypes.TANH:
            return operations_1.div(operations_1.sub(operations_1.exp(state), operations_1.div(operations_1.number(1), operations_1.exp(state))), operations_1.sum(operations_1.exp(state), operations_1.div(operations_1.number(1), operations_1.exp(state))));
        case ActivationTypes.STEP:
            return operations_1.conditional(operations_1.gt(state, operations_1.number(0)), operations_1.number(1), operations_1.number(0));
        case ActivationTypes.RELU_PLUSONE:
            return operations_1.sum(operations_1.number(1), operations_1.max(state, operations_1.number(0)));
        case ActivationTypes.RELU:
            return operations_1.max(state, operations_1.number(0));
        case ActivationTypes.SOFTPLUS:
            return operations_1.ln(operations_1.sum(operations_1.number(1), operations_1.exp(state)));
        case ActivationTypes.SOFTSIGN:
            return operations_1.div(state, operations_1.sum(operations_1.number(1), operations_1.abs(state)));
        case ActivationTypes.EXP:
            return operations_1.exp(state);
        case ActivationTypes.POW:
            return operations_1.mul(state, state);
        case ActivationTypes.POW_MINUS1:
            return operations_1.div(operations_1.number(1), state);
        case ActivationTypes.GAUSSIAN:
            return operations_1.exp(operations_1.neg(operations_1.mul(state, state)));
        case ActivationTypes.INVERSE_IDENTITY:
            return operations_1.sub(operations_1.number(1), state);
        case ActivationTypes.IDENTITY:
            return state;
    }
    return state;
}
exports.buildActivationFunction = buildActivationFunction;
function buildDerivativeFunction(state, activation, type) {
    if ((type & exports.WHOLE_LAYER_ACTIVATION_KIND) != 0)
        return null;
    switch (type) {
        case ActivationTypes.IDENTITY:
            return operations_1.number(1);
        case ActivationTypes.LOGISTIC_SIGMOID:
            return operations_1.mul(activation, operations_1.sub(operations_1.number(1), activation));
        case ActivationTypes.TANH:
            return operations_1.sub(operations_1.number(1), operations_1.mul(activation, activation));
        case ActivationTypes.STEP:
            return operations_1.number(0);
        case ActivationTypes.RELU_PLUSONE:
            return operations_1.conditional(operations_1.gte(state, operations_1.number(0)), operations_1.number(1), operations_1.number(0));
        case ActivationTypes.RELU:
            return operations_1.conditional(operations_1.gte(state, operations_1.number(0)), operations_1.number(1), operations_1.number(0));
        case ActivationTypes.SOFTPLUS:
            return operations_1.div(operations_1.number(1), operations_1.sum(operations_1.number(1), operations_1.exp(operations_1.neg(state))));
        case ActivationTypes.SOFTSIGN:
            return operations_1.div(operations_1.number(1), operations_1.pow(operations_1.sum(operations_1.number(1), operations_1.abs(state)), operations_1.number(2)));
        case ActivationTypes.EXP:
            return activation;
        case ActivationTypes.POW:
            return operations_1.mul(operations_1.number(2), state);
        case ActivationTypes.POW_MINUS1:
            return operations_1.div(operations_1.number(1), operations_1.mul(state, state));
        case ActivationTypes.GAUSSIAN:
            return operations_1.mul(operations_1.mul(operations_1.number(-2), state), activation);
        case ActivationTypes.INVERSE_IDENTITY:
            return operations_1.number(-1);
    }
    return operations_1.number(1);
}
exports.buildDerivativeFunction = buildDerivativeFunction;
//# sourceMappingURL=activations.js.map