lysergic
Version:
Synaptic's neural network compiler
519 lines • 29.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const nodes = require("./ast/nodes");
exports.nodes = nodes;
const operations_1 = require("./ast/operations");
const activations_1 = require("./ast/activations");
class AST {
constructor(options) {
this.inputs = [];
this.outputs = [];
this.targets = [];
this.document = operations_1.document();
const { topology } = options;
this.topology = topology;
}
reset() {
this.inputs = [];
this.outputs = [];
this.targets = [];
this.document = operations_1.document();
}
isArrayOrdered(items) {
let ordered = true;
let prev = null;
items.forEach(i => {
if (prev !== null) {
ordered = ordered && (prev == i - 1);
}
prev = i;
});
return ordered;
}
isOrdered(from, to, accessor) {
let ordered = true;
let prev = null;
for (let i = from; i < to; i++) {
let args = [...accessor, i];
const actual = this.topology.heap.getVariable(...args);
if (prev) {
ordered = ordered && (prev.position == (actual.position - 1));
}
prev = actual;
}
return ordered;
}
isSameActivationTypeOrdered(from, to, accessor) {
}
build() {
this.reset();
const layers = this.topology.layers;
let outputLayer = layers.length - 1;
const activationFunction = operations_1.func('activate');
this.document.addNode(activationFunction);
const propagationFunction = operations_1.func('propagate');
this.document.addNode(propagationFunction);
for (let layer = 0; layer < layers.length; layer++) {
if (layer != 0) {
for (let unit = 0; unit < layers[layer].length; unit++) {
if (this.topology.activationFunction[layers[layer][unit]] !== activations_1.ActivationTypes.MAX_POOLING) {
this.buildComputeState(layers[layer][unit], layer);
}
}
}
let sameActivationFunction = true;
let prevActivationFunction = null;
for (let unit = 0; unit < layers[layer].length; unit++) {
const activationFunction = this.topology.activationFunction[layers[layer][unit]];
if (unit > 0) {
sameActivationFunction = sameActivationFunction && (activationFunction == prevActivationFunction);
}
prevActivationFunction = activationFunction;
}
for (let unit = 0; unit < layers[layer].length; unit++) {
let activationJ;
switch (layer) {
case 0:
activationJ = this.topology.heap.getVariable('activation', layers[layer][unit]);
this.inputs.push(activationJ);
break;
case outputLayer:
activationJ = this.buildActivation(layers[layer][unit], layer);
this.outputs.push(activationJ);
break;
default:
this.buildActivation(layers[layer][unit], layer);
}
}
let buckets = [];
for (let unitIndex = 0; unitIndex < layers[layer].length; unitIndex++) {
const unit = layers[layer][unitIndex];
const type = this.topology.activationFunction[unit];
if (type & activations_1.WHOLE_LAYER_ACTIVATION_KIND) {
let bucket = buckets.find(bucket => bucket.type === type);
if (bucket == null) {
bucket = {
units: [],
type,
layer
};
buckets.push(bucket);
}
bucket.units.push(unit);
}
}
buckets.forEach(bucket => this.buildWholeLayerActivation(bucket));
for (let unit = 0; unit < layers[layer].length; unit++) {
switch (layer) {
case 0:
break;
default:
this.buildActivationDerivative(layers[layer][unit], layer);
}
}
for (let unit = 0; unit < layers[layer].length; unit++) {
switch (layer) {
case 0:
break;
default:
if (this.topology.activationFunction[layers[layer][unit]] !== activations_1.ActivationTypes.MAX_POOLING) {
this.buildActivationTraces(layers[layer][unit], layer);
}
}
}
}
for (let unit = layers[outputLayer].length - 1; unit >= 0; unit--) {
let targetJ = this.topology.heap.setVariable(`target`, unit, 0);
this.targets.push(targetJ);
this.buildPropagation(layers[outputLayer][unit], outputLayer, targetJ);
}
for (let layer = layers.length - 2; layer > 0; layer--) {
for (let unit = layers[layer].length - 1; unit >= 0; unit--) {
this.buildPropagation(layers[layer][unit], layer);
}
}
this.targets.reverse();
}
getFunctionBodyNode(functionName) {
let activationFunction = this.document.children.find(node => node instanceof nodes.FunctionNode && node.name === functionName);
return activationFunction.body;
}
buildActivationDerivative(j, layerJ, targetFunction = 'activate') {
const layerNode = this.getFunctionBodyNode(targetFunction);
const blockNode = new nodes.BlockNode();
blockNode.name = `Activation derivative ${layerJ}:${j}`;
layerNode.addNode(blockNode);
const statement = (node) => blockNode.addNode(node);
const stateJ = this.topology.heap.getVariable(`state`, j);
const activationJ = this.topology.heap.getVariable(`activation`, j);
const derivativeJ = this.topology.heap.getVariable(`derivative`, j);
const derivativeFunction = activations_1.buildDerivativeFunction(stateJ, activationJ, this.topology.activationFunction[j]);
if (derivativeFunction) {
statement(operations_1.assign(derivativeJ, derivativeFunction));
}
return derivativeJ;
}
buildActivationTraces(j, layerJ) {
const topology = this.topology;
const layerNode = this.getFunctionBodyNode('activate');
const blockNode = new nodes.BlockNode();
blockNode.name = `Traces of ${layerJ}:${j}`;
layerNode.addNode(blockNode);
const statement = (node) => blockNode.addNode(node);
const activationJ = this.topology.heap.getVariable(`activation`, j);
const derivativeJ = this.topology.heap.getVariable(`derivative`, j);
const isSelfConnected = topology.connections.some(connection => connection.to === j && connection.from === j);
const isSelfConnectionGated = topology.gates.some(gate => gate.to === j && gate.from === j);
let i, k, h, g, l, a, to, from;
for (h = 0; h < topology.inputSet[j].length; h++) {
i = topology.inputSet[j][h];
const elegibilityTraceJI = this.topology.heap.getVariable(`elegibilityTrace`, j, i);
const activationI = this.topology.heap.getVariable(`activation`, i);
const gainJI = this.topology.heap.getVariable(`gain`, j, i);
if (isSelfConnected && isSelfConnectionGated) {
const gainJJ = this.topology.heap.getVariable(`gain`, j, j);
const weightJJ = this.topology.heap.getVariable(`weight`, j, j);
statement(operations_1.assign(elegibilityTraceJI, operations_1.sum(operations_1.mul(operations_1.mul(gainJJ, weightJJ), elegibilityTraceJI), operations_1.mul(gainJI, activationI))));
}
else if (isSelfConnected) {
const weightJJ = this.topology.heap.getVariable(`weight`, j, j);
statement(operations_1.assign(elegibilityTraceJI, operations_1.sum(operations_1.mul(weightJJ, elegibilityTraceJI), operations_1.mul(gainJI, activationI))));
}
else {
statement(operations_1.assign(elegibilityTraceJI, operations_1.mul(gainJI, activationI)));
}
for (g = 0; g < topology.gatedBy[j].length; g++) {
k = topology.gatedBy[j][g];
const isSelfConnectedK = topology.connections.some(connection => connection.to === k && connection.from === k);
const isSelfConnectionGatedK = topology.gates.some(gate => gate.to === k && gate.from === k);
const bigParenthesisTermResult = this.topology.heap.setVariable('bigParenthesisTermResult', 0);
let keepBigParenthesisTerm = false;
let initializeBigParenthesisTerm = false;
if (isSelfConnectedK && this.topology.heap.hasVariable('derivativeTerm', k, j) && this.topology.heap.getVariable('derivativeTerm', k, j).initialValue) {
const stateK = this.topology.heap.getVariable(`state`, k);
statement(operations_1.assign(bigParenthesisTermResult, stateK));
keepBigParenthesisTerm = true;
}
else {
initializeBigParenthesisTerm = true;
}
for (l = 0; l < topology.inputsOfGatedBy[k][j].length; l++) {
a = topology.inputsOfGatedBy[k][j][l];
if (a !== k) {
if (initializeBigParenthesisTerm) {
statement(operations_1.assign(bigParenthesisTermResult, operations_1.number(0)));
initializeBigParenthesisTerm = false;
}
const weightKA = this.topology.heap.getVariable(`weight`, k, a);
const activationA = this.topology.heap.getVariable(`activation`, a);
statement(operations_1.assignSum(bigParenthesisTermResult, operations_1.mul(weightKA, activationA)));
keepBigParenthesisTerm = true;
}
}
const extendedElegibilityTraceJIK = this.topology.heap.getVariable(`extendedElegibilityTrace`, j, i, k);
if (isSelfConnectedK && isSelfConnectionGatedK) {
const gainKK = this.topology.heap.getVariable(`gain`, k, k);
const weightKK = this.topology.heap.getVariable(`weight`, k, k);
if (keepBigParenthesisTerm) {
statement(operations_1.assign(extendedElegibilityTraceJIK, operations_1.sum(operations_1.mul(operations_1.mul(gainKK, weightKK), extendedElegibilityTraceJIK), operations_1.mul(operations_1.mul(derivativeJ, elegibilityTraceJI), bigParenthesisTermResult))));
}
else {
statement(operations_1.assign(extendedElegibilityTraceJIK, operations_1.mul(operations_1.mul(gainKK, weightKK), extendedElegibilityTraceJIK)));
}
}
else if (isSelfConnectedK) {
const weightKK = this.topology.heap.getVariable(`weight`, k, k);
if (keepBigParenthesisTerm) {
statement(operations_1.assign(extendedElegibilityTraceJIK, operations_1.sum(operations_1.mul(weightKK, extendedElegibilityTraceJIK), operations_1.mul(operations_1.mul(derivativeJ, elegibilityTraceJI), bigParenthesisTermResult))));
}
else {
statement(operations_1.assign(extendedElegibilityTraceJIK, operations_1.mul(weightKK, extendedElegibilityTraceJIK)));
}
}
else {
if (keepBigParenthesisTerm) {
statement(operations_1.assign(extendedElegibilityTraceJIK, operations_1.mul(operations_1.mul(derivativeJ, elegibilityTraceJI), bigParenthesisTermResult)));
}
}
}
}
for (h = 0; h < topology.gatedBy[j].length; h++) {
to = topology.gatedBy[j][h];
for (g = 0; g < topology.inputsOfGatedBy[to][j].length; g++) {
from = topology.inputsOfGatedBy[to][j][g];
const gainToFrom = this.topology.heap.getVariable(`gain`, to, from);
statement(operations_1.assign(gainToFrom, activationJ));
}
}
}
buildWholeLayerActivation(bucket) {
const units = bucket.units;
const type = bucket.type;
const layerJ = bucket.layer;
const layerNode = this.getFunctionBodyNode('activate');
const blockNode = new nodes.BlockNode();
blockNode.name = `Whole Layer Activation (${type}) ${layerJ}`;
layerNode.addNode(blockNode);
const statement = (node) => blockNode.addNode(node);
switch (type) {
case activations_1.ActivationTypes.MAX_POOLING: {
units.forEach(unit => {
const inputs = this.topology.inputsOf[unit];
const maximum = this.topology.heap.getVariable('activation', unit);
const activations = inputs.map(input => this.topology.heap.getVariable(`activation`, input));
activations.forEach((activation, index) => {
statement(operations_1.assign(maximum, index === 0 ? activation : operations_1.max(maximum, activation)));
});
inputs.forEach(input => {
const activation = this.topology.heap.getVariable(`activation`, input);
const weight = this.topology.heap.getVariable(`weight`, unit, input);
const derivative = this.topology.heap.getVariable(`derivative`, input);
statement(operations_1.assign(weight, operations_1.krnonecker(activation, maximum)));
statement(operations_1.assign(derivative, operations_1.krnonecker(activation, maximum)));
});
});
break;
}
case activations_1.ActivationTypes.SOFTMAX: {
const activations = units.map(unit => this.topology.heap.getVariable(`activation`, unit));
const derivatives = units.map(unit => this.topology.heap.getVariable(`derivative`, unit));
const states = units.map(unit => this.topology.heap.getVariable(`state`, unit));
const maximum = this.topology.heap.setVariable(`softmaxMaximum`, layerJ, 0);
const denominator = this.topology.heap.setVariable(`softmaxDenominator`, layerJ, 0);
const nominators = [];
units.forEach((unit, i) => {
const nominator = this.topology.heap.setVariable(`softmaxNominators`, layerJ, unit, 0);
nominators.push(nominator);
});
statement(operations_1.assign(maximum, operations_1.number(0)));
statement(operations_1.assign(denominator, operations_1.number(0)));
states.forEach(state => {
statement(operations_1.assign(maximum, operations_1.max(maximum, state)));
});
states.forEach((state, i) => {
statement(operations_1.assign(activations[i], operations_1.exp(operations_1.sub(state, maximum))));
});
activations.forEach(activation => statement(operations_1.assignSum(denominator, activation)));
activations.forEach(activation => {
statement(operations_1.assign(activation, operations_1.div(activation, denominator)));
});
states.forEach((state, i) => {
statement(operations_1.assign(derivatives[i], operations_1.mul(state, operations_1.sub(operations_1.number(1), state))));
states.forEach((state, j) => {
if (i !== j) {
statement(operations_1.assignSub(derivatives[i], operations_1.mul(state, state)));
}
});
});
break;
}
default:
}
}
buildComputeState(j, layerJ, targetFunction = 'activate') {
const topology = this.topology;
const layerNode = this.getFunctionBodyNode(targetFunction);
const blockNode = new nodes.BlockNode();
blockNode.name = `State ${layerJ}:${j}`;
layerNode.addNode(blockNode);
const statement = (node) => blockNode.addNode(node);
let i, h;
const stateJ = this.topology.heap.getVariable(`state`, j);
const isSelfConnected = topology.connections.some(connection => connection.to === j && connection.from === j);
const isSelfConnectionGated = topology.gates.some(gate => gate.to === j && gate.from === j);
if (isSelfConnected && isSelfConnectionGated) {
const gainJJ = this.topology.heap.getVariable(`gain`, j, j);
const weightJJ = this.topology.heap.getVariable(`weight`, j, j);
statement(operations_1.assignMul(stateJ, operations_1.mul(gainJJ, weightJJ)));
}
else if (isSelfConnected) {
const weightJJ = this.topology.heap.getVariable(`weight`, j, j);
statement(operations_1.assignMul(stateJ, weightJJ));
}
else {
statement(operations_1.assign(stateJ, operations_1.number(0)));
}
for (h = 0; h < topology.inputSet[j].length; h++) {
i = topology.inputSet[j][h];
const isGated = topology.gates.some(gate => gate.from === i && gate.to === j);
if (isGated) {
const stateJ = this.topology.heap.getVariable(`state`, j);
const gainJI = this.topology.heap.getVariable(`gain`, j, i);
const weightJI = this.topology.heap.getVariable(`weight`, j, i);
const activationI = this.topology.heap.getVariable(`activation`, i);
statement(operations_1.assignSum(stateJ, operations_1.mul(operations_1.mul(gainJI, weightJI), activationI)));
}
else {
const stateJ = this.topology.heap.getVariable(`state`, j);
const weightJI = this.topology.heap.getVariable(`weight`, j, i);
const activationI = this.topology.heap.getVariable(`activation`, i);
statement(operations_1.assignSum(stateJ, operations_1.mul(weightJI, activationI)));
}
}
return stateJ;
}
buildActivation(j, layerJ, targetFunction = 'activate') {
const layerNode = this.getFunctionBodyNode(targetFunction);
const blockNode = new nodes.BlockNode();
blockNode.name = `Activation ${layerJ}:${j}`;
layerNode.addNode(blockNode);
const statement = (node) => blockNode.addNode(node);
const stateJ = this.topology.heap.getVariable(`state`, j);
const activationJ = this.topology.heap.getVariable(`activation`, j);
const activationFunction = activations_1.buildActivationFunction(stateJ, this.topology.activationFunction[j]);
if (activationFunction) {
statement(operations_1.assign(activationJ, activationFunction));
}
return activationJ;
}
buildPropagation(j, layerJ, targetJ) {
const layerNode = this.getFunctionBodyNode('propagate');
const blockNode = new nodes.BlockNode();
blockNode.name = `Propagation ${layerJ}:${j}`;
layerNode.addNode(blockNode);
const statement = (node) => blockNode.addNode(node);
let hasProjectedError = this.topology.projectionSet[j].length > 0;
const hasGatedError = this.topology.gateSet[j].length > 0;
if (typeof targetJ !== 'undefined') {
hasProjectedError = true;
const errorResponsibilityJ = this.topology.heap.getVariable(`errorResponsibility`, j);
const projectedErrorResponsibilityJ = this.topology.heap.getVariable(`projectedErrorResponsibility`, j);
const activationJ = this.topology.heap.getVariable(`activation`, j);
statement(operations_1.assign(errorResponsibilityJ, operations_1.sub(targetJ, activationJ)));
statement(operations_1.assign(projectedErrorResponsibilityJ, errorResponsibilityJ));
}
else {
const projectedErrorResponsibilityJ = this.topology.heap.getVariable(`projectedErrorResponsibility`, j);
if (hasProjectedError) {
statement(operations_1.assign(projectedErrorResponsibilityJ, operations_1.number(0)));
}
for (let h = 0; h < this.topology.projectionSet[j].length; h++) {
const k = this.topology.projectionSet[j][h];
const errorResponsibilityK = this.topology.heap.getVariable(`errorResponsibility`, k);
const isGated = this.topology.gates.some(gate => gate.to === k && gate.from === j);
if (isGated) {
const weightKJ = this.topology.heap.getVariable(`weight`, k, j);
const gainKJ = this.topology.heap.getVariable(`gain`, k, j);
statement(operations_1.assignSum(projectedErrorResponsibilityJ, operations_1.mul(operations_1.mul(gainKJ, weightKJ), errorResponsibilityK)));
}
else {
const weightKJ = this.topology.heap.getVariable(`weight`, k, j);
statement(operations_1.assignSum(projectedErrorResponsibilityJ, operations_1.mul(weightKJ, errorResponsibilityK)));
}
}
const derivativeJ = this.topology.heap.getVariable(`derivative`, j);
if (hasProjectedError) {
statement(operations_1.assignMul(projectedErrorResponsibilityJ, derivativeJ));
}
const gatedErrorResponsibilityJ = this.topology.heap.getVariable(`gatedErrorResponsibility`, j);
if (hasGatedError) {
statement(operations_1.assignMul(gatedErrorResponsibilityJ, operations_1.number(0)));
}
for (let h = 0; h < this.topology.gateSet[j].length; h++) {
const k = this.topology.gateSet[j][h];
const isSelfConnectedK = this.topology.connections.some(connection => connection.to === k && connection.from === k);
const bigParenthesisTermResult = this.topology.heap.setVariable('bigParenthesisTermResult', null);
let keepBigParenthesisTerm = false;
let initializeBigParenthesisTerm = false;
if (isSelfConnectedK && this.topology.heap.hasVariable('derivativeTerm', k, j)) {
const stateK = this.topology.heap.getVariable(`state`, k);
statement(operations_1.assign(bigParenthesisTermResult, stateK));
keepBigParenthesisTerm = true;
}
else {
initializeBigParenthesisTerm = true;
}
for (let l = 0; l < this.topology.inputsOfGatedBy[k][j].length; l++) {
const a = this.topology.inputsOfGatedBy[k][j][l];
if (a !== k) {
if (initializeBigParenthesisTerm) {
statement(operations_1.assign(bigParenthesisTermResult, operations_1.number(0)));
initializeBigParenthesisTerm = false;
}
const weightKA = this.topology.heap.getVariable(`weight`, k, a);
const activationA = this.topology.heap.getVariable(`activation`, a);
statement(operations_1.assignSum(bigParenthesisTermResult, operations_1.mul(weightKA, activationA)));
keepBigParenthesisTerm = true;
}
}
if (keepBigParenthesisTerm) {
const errorResponsibilityK = this.topology.heap.getVariable(`errorResponsibility`, k);
statement(operations_1.assignSum(gatedErrorResponsibilityJ, operations_1.mul(errorResponsibilityK, bigParenthesisTermResult)));
}
}
if (hasGatedError) {
statement(operations_1.assignMul(gatedErrorResponsibilityJ, derivativeJ));
}
const errorResponsibilityJ = this.topology.heap.getVariable(`errorResponsibility`, j);
if (hasProjectedError && hasGatedError) {
statement(operations_1.assign(errorResponsibilityJ, operations_1.sum(projectedErrorResponsibilityJ, gatedErrorResponsibilityJ)));
}
else if (hasProjectedError) {
statement(operations_1.assign(errorResponsibilityJ, projectedErrorResponsibilityJ));
}
else if (hasGatedError) {
statement(operations_1.assign(errorResponsibilityJ, gatedErrorResponsibilityJ));
}
}
if (this.topology.activationFunction[j] === activations_1.ActivationTypes.MAX_POOLING) {
return;
}
const learningRate = this.topology.heap.getVariable('learningRate');
const momentum = this.topology.heap.getVariable('momentum');
for (let h = 0; h < this.topology.inputSet[j].length; h++) {
const i = this.topology.inputSet[j][h];
const weightJI = this.topology.heap.getVariable(`weight`, j, i);
statement(operations_1.assignSum(weightJI, operations_1.mul(weightJI, momentum)));
}
for (let h = 0; h < this.topology.inputSet[j].length; h++) {
const i = this.topology.inputSet[j][h];
const Δw = this.topology.heap.getVariable(`gradient`, j, i);
if (hasProjectedError && hasGatedError) {
const projectedErrorResponsibilityJ = this.topology.heap.getVariable(`projectedErrorResponsibility`, j);
const elegibilityTraceJI = this.topology.heap.getVariable(`elegibilityTrace`, j, i);
statement(operations_1.assign(Δw, operations_1.mul(projectedErrorResponsibilityJ, elegibilityTraceJI)));
for (let g = 0; g < this.topology.gateSet[j].length; g++) {
const k = this.topology.gateSet[j][g];
const errorResponsibilityK = this.topology.heap.getVariable(`errorResponsibility`, k);
const extendedElegibilityTraceJIK = this.topology.heap.getVariable(`extendedElegibilityTrace`, j, i, k);
statement(operations_1.assignSum(Δw, operations_1.mul(errorResponsibilityK, extendedElegibilityTraceJIK)));
}
statement(operations_1.assignMul(Δw, learningRate));
}
else if (hasProjectedError) {
const projectedErrorResponsibilityJ = this.topology.heap.getVariable(`projectedErrorResponsibility`, j);
const elegibilityTraceJI = this.topology.heap.getVariable(`elegibilityTrace`, j, i);
statement(operations_1.assign(Δw, operations_1.mul(operations_1.mul(projectedErrorResponsibilityJ, elegibilityTraceJI), learningRate)));
}
else if (hasGatedError) {
statement(operations_1.assign(Δw, operations_1.number(0)));
for (let g = 0; g < this.topology.gateSet[j].length; g++) {
const k = this.topology.gateSet[j][g];
const errorResponsibilityK = this.topology.heap.getVariable(`errorResponsibility`, k);
const extendedElegibilityTraceJIK = this.topology.heap.getVariable(`extendedElegibilityTrace`, j, i, k);
statement(operations_1.assignSum(Δw, operations_1.mul(errorResponsibilityK, extendedElegibilityTraceJIK)));
}
statement(operations_1.assignMul(Δw, learningRate));
}
}
for (let h = 0; h < this.topology.inputSet[j].length; h++) {
const i = this.topology.inputSet[j][h];
const gradient = this.topology.heap.getVariable(`gradient`, j, i);
const weightJI = this.topology.heap.getVariable(`weight`, j, i);
if (this.topology.unitParameters[j].l1) {
statement(operations_1.assignSum(weightJI, operations_1.mul(operations_1.sign(weightJI), operations_1.number(this.topology.unitParameters[j].l1))));
}
if (this.topology.unitParameters[j].l2) {
statement(operations_1.assignSub(weightJI, operations_1.mul(weightJI, operations_1.number(this.topology.unitParameters[j].l2))));
}
statement(operations_1.assignSum(weightJI, gradient));
}
}
getDocument() {
return this.document;
}
}
AST.nodes = nodes;
exports.AST = AST;
exports.default = AST;
//# sourceMappingURL=AST.js.map