@ai-on-browser/data-analysis-models/lib/model/nns/layer/base.js

Data analysis model package without any dependencies

import Matrix from '../../../util/matrix.js'
import Tensor from '../../../util/tensor.js'

const layerClasses = {}

/**
 * Exception for neuralnetwork layer class
 */
export class NeuralnetworkLayerException extends Error {
	/**
	 * @param {string} message Error message
	 * @param {*} value Some value
	 */
	constructor(message, value) {
		super(message)
		this.value = value
		this.name = 'NeuralnetworkLayerException'
	}
}

/**
 * Neuralnetwork layer
 */
export default class Layer {
	/**
	 * Returns layer from JSON.
	 * @param {import("./index").PlainLayerObject} obj Object represented a layer
	 * @returns {Layer} Layer
	 */
	static fromObject(obj) {
		const cls = layerClasses[obj.type]
		if (!cls) {
			throw new NeuralnetworkLayerException(`Invalid layer type: ${obj.type}`)
		}
		return new cls(obj)
	}

	/**
	 * Regist layer class.
	 * @param {string} [name] Name of the layer
	 * @param {Layer} [cls] Layer class
	 */
	static registLayer(name, cls) {
		cls ||= this
		if (!name && cls !== Layer) {
			name = this.name
				.replace(/Layer$/, '')
				.replace(/[A-Z]/g, s => `_${s.toLowerCase()}`)
				.slice(1)
			cls = this
		}
		if (layerClasses[name]) {
			throw new NeuralnetworkLayerException(`Layer name '${name}' already exists.`)
		}
		layerClasses[name] = cls
	}

	/**
	 * List of names of other layers dependent on this layer.
	 * @type {string[]}
	 */
	get dependentLayers() {
		return []
	}

	/**
	 * Bind pre-condition values.
	 * @param {object} values Binding object
	 * @param {Matrix | Tensor | {[key: string]: Matrix | Tensor}} values.input Input data for neuralnetwork
	 * @param {Matrix} [values.supervisor] Supervisor data
	 * @param {number} values.n Data count
	 * @param {*} values.rest Some other values
	 */
	bind(values) {}

	/**
	 * Returns calculated values.
	 * @param {...(Matrix | Tensor)} x Input values
	 * @returns {Matrix | Tensor | Array<Matrix | Tensor>} Output values
	 */
	calc(...x) {
		throw new NeuralnetworkLayerException('Not impleneted', this)
	}

	/**
	 * Returns gradient values.
	 * @param {...(Matrix | Tensor)} bo Input value of backpropagation
	 * @returns {Matrix | Tensor | Array<Matrix | Tensor>} Output value of backpropagation
	 */
	grad(...bo) {
		throw new NeuralnetworkLayerException('Not impleneted', this)
	}

	/**
	 * Update parameters.
	 * @param {{lr: number, delta: (function (string, Matrix): Matrix)}} optimizer Optimizer for this layer
	 */
	update(optimizer) {}

	/**
	 * Returns object of this layer.
	 * @returns {import("./index").PlainLayerObject} Object represented this layer
	 */
	toObject() {
		for (const name of Object.keys(layerClasses)) {
			if (this.constructor === layerClasses[name]) {
				return { type: name }
			}
		}
		return {}
	}
}

/**
 * Base class for Flow-based generative model
 */
export class FlowLayer extends Layer {
	/**
	 * Returns inverse values.
	 * @param {...(Matrix | Tensor)} y Input value of inverse calculation
	 * @returns {Matrix | Tensor | Array<Matrix | Tensor>} Output value of inverse calculation
	 */
	inverse(...y) {
		throw new NeuralnetworkLayerException('Not impleneted', this)
	}

	/**
	 * Returns determinant of the Jacobian.
	 * @returns {number} Determinant of the Jacobian
	 */
	jacobianDeterminant() {
		throw new NeuralnetworkLayerException('Not impleneted', this)
	}
}

const buildUnaryLayer = (name, calcFunc, gradFunc) => {
	class TempLayer extends Layer {
		calc(x) {
			this._i = x
			if (!calcFunc) {
				this._o = x
				return this._o
			}
			this._o = x.copy()
			this._o.map(calcFunc)
			return this._o
		}

		grad(bo) {
			if (!gradFunc) {
				return bo
			}
			const bi = bo.copy()
			for (let i = 0; i < bi.length; i++) {
				bi.value[i] *= gradFunc(this._i.value[i], this._o.value[i])
			}
			return bi
		}
	}
	Object.defineProperty(TempLayer, 'name', {
		value: `${name.split('_').reduce((s, nm) => s + nm[0].toUpperCase() + nm.substring(1).toLowerCase(), '')}Layer`,
	})
	TempLayer.registLayer(name)
}

const unaryLayers = {
	abs: {
		calc: Math.abs,
		grad: i => (i < 0 ? -1 : 1),
	},
	acos: {
		calc: Math.acos,
		grad: i => -1 / (Math.sqrt(1 - i ** 2) + 1.0e-4),
	},
	acosh: {
		calc: Math.acosh,
		grad: i => 1 / (Math.sqrt(i ** 2 - 1) + 1.0e-4),
	},
	asin: {
		calc: Math.asin,
		grad: i => 1 / (Math.sqrt(1 - i ** 2) + 1.0e-4),
	},
	asinh: {
		calc: Math.asinh,
		grad: i => 1 / Math.sqrt(1 + i ** 2),
	},
	atan: {
		calc: Math.atan,
		grad: i => 1 / (1 + i ** 2),
	},
	atanh: {
		calc: Math.atanh,
		grad: i => 1 / (1 - i ** 2),
	},
	bent_identity: {
		calc: v => (Math.sqrt(v ** 2 + 1) - 1) / 2 + v,
		grad: i => i / (2 * Math.sqrt(i ** 2 + 1)) + 1,
	},
	bitwise_not: {
		calc: v => ~v,
		grad: () => 0,
	},
	ceil: { calc: Math.ceil },
	cloglog: {
		calc: v => 1 - Math.exp(-Math.exp(v)),
		grad: (i, o) => Math.exp(i) * (1 - o),
	},
	cloglogm: {
		calc: v => 1 - 2 * Math.exp(-0.7 * Math.exp(v)),
		grad: i => 1.4 * Math.exp(i - 0.7 * Math.exp(i)),
	},
	cos: {
		calc: Math.cos,
		grad: i => -Math.sin(i),
	},
	cosh: {
		calc: Math.cosh,
		grad: i => Math.sinh(i),
	},
	detach: { grad: () => 0 },
	elish: {
		calc: v => (v >= 0 ? v : Math.exp(v) - 1) / (1 + Math.exp(-v)),
		grad: (i, o) => ((i >= 0 ? 1 : Math.exp(i)) + o * Math.exp(-i)) / (1 + Math.exp(-i)),
	},
	elliott: {
		calc: v => v / (2 * (1 + Math.abs(v))) + 0.5,
		grad: i => 1 / (2 * (1 + Math.abs(i)) ** 2),
	},
	erf: {
		calc: v => {
			const p = 0.3275911
			const a1 = 0.254829592
			const a2 = -0.284496736
			const a3 = 1.421413741
			const a4 = -1.453152027
			const a5 = 1.061405429

			const sign = v < 0 ? -1 : 1
			const t = 1 / (1 + p * Math.abs(v))
			const erf = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-v * v)
			return sign * erf
		},
		grad: i => (2 * Math.exp(-(i ** 2))) / Math.sqrt(Math.PI),
	},
	exp: {
		calc: Math.exp,
		grad: (_, o) => o,
	},
	floor: { calc: Math.floor },
	gelu: {
		calc: v => {
			const erf = v => {
				const p = 0.3275911
				const a1 = 0.254829592
				const a2 = -0.284496736
				const a3 = 1.421413741
				const a4 = -1.453152027
				const a5 = 1.061405429

				const sign = v < 0 ? -1 : 1
				const t = 1 / (1 + p * Math.abs(v))
				const erf = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-v * v)
				return sign * erf
			}
			return (v * (1 + erf(v / Math.sqrt(2)))) / 2
		},
		grad: (i, o) => (i === 0 ? 0.5 : o / i) + (i * Math.exp(-((i / Math.sqrt(2)) ** 2))) / Math.sqrt(2 * Math.PI),
	},
	hard_elish: {
		calc: v => Math.max(0, Math.min(1, (v + 1) / 2)) * (v >= 0 ? v : Math.exp(v) - 1),
		grad: i => (i <= -1 ? 0 : i >= 1 ? (i >= 0 ? 1 : Math.exp(i)) : i >= 0 ? i : ((1 + i) * Math.exp(i)) / 2),
	},
	hard_swish: {
		calc: v => (v <= -3 ? 0 : 3 <= v ? v : (v * (v + 3)) / 6),
		grad: i => (i <= -3 ? 0 : 3 <= i ? 1 : (2 * i + 3) / 6),
	},
	identity: {},
	is_inf: {
		calc: v => v === Infinity || v === -Infinity,
		grad: () => 0,
	},
	is_nan: {
		calc: v => Number.isNaN(v),
		grad: () => 0,
	},
	lisht: {
		calc: v => v * Math.tanh(v),
		grad: i => Math.tanh(i) + i * (1 - Math.tanh(i) ** 2),
	},
	log: {
		calc: Math.log,
		grad: i => 1 / i,
	},
	loglog: {
		calc: v => Math.exp(-Math.exp(-v)),
		grad: (i, o) => Math.exp(-i) * o,
	},
	logsigmoid: {
		calc: v => Math.log(1 / (1 + Math.exp(-v))),
		grad: i => 1 / (1 + Math.exp(i)),
	},
	mish: {
		calc: v => v * Math.tanh(Math.log(1 + Math.exp(v))),
		grad: i =>
			(Math.exp(i) * (4 * (i + 1) + 4 * Math.exp(2 * i) + Math.exp(3 * i) + Math.exp(i) * (4 * i + 6))) /
			(2 * Math.exp(i) + Math.exp(2 * i) + 2) ** 2,
	},
	negative: {
		calc: v => -v,
		grad: () => -1,
	},
	not: {
		calc: v => !v,
		grad: () => 0,
	},
	reciprocal: {
		calc: v => 1 / v,
		grad: i => -1 / i ** 2,
	},
	relu: {
		calc: v => (v > 0 ? v : 0),
		grad: i => (i > 0 ? 1 : 0),
	},
	resech: {
		calc: v => v / Math.cosh(v),
		grad: (i, o) => 1 / Math.cosh(i) - o * Math.tanh(i),
	},
	reu: {
		calc: v => (v > 0 ? v : v * Math.exp(v)),
		grad: (i, o) => (i > 0 ? 1 : Math.exp(i) + o),
	},
	rootsig: {
		calc: v => v / (1 + Math.sqrt(1 + v ** 2)),
		grad: i => 1 / (Math.sqrt(i ** 2 + 1) + i ** 2 + 1),
	},
	round: { calc: Math.round },
	sign: { calc: Math.sign },
	silu: {
		calc: v => v / (1 + Math.exp(-v)),
		grad: (i, o) => 1 / (1 + Math.exp(-i)) + o * (1 - o / i),
	},
	sin: {
		calc: Math.sin,
		grad: i => Math.cos(i),
	},
	sinh: {
		calc: Math.sinh,
		grad: i => Math.cosh(i),
	},
	softsign: {
		calc: v => v / (1 + Math.abs(v)),
		grad: i => 1 / (1 + Math.abs(i)) ** 2,
	},
	sqrt: {
		calc: Math.sqrt,
		grad: (_, o) => 1 / (2 * o),
	},
	square: {
		calc: v => v ** 2,
		grad: i => i * 2,
	},
	ssigmoid: {
		calc: v => 4 / (1 + Math.exp(-v)) - 2,
		grad: i => (4 * Math.exp(-i)) / (1 + Math.exp(-i)) ** 2,
	},
	tan: {
		calc: Math.tan,
		grad: i => 1 / Math.cos(i) ** 2,
	},
	tanh: {
		calc: Math.tanh,
		grad: (_, o) => 1 - o ** 2,
	},
	tanhexp: {
		calc: v => v * Math.tanh(Math.exp(v)),
		grad: i => Math.tanh(Math.exp(i)) - i * Math.exp(i) * (Math.tanh(Math.exp(i)) ** 2 - 1),
	},
	tanhshrink: {
		calc: v => v - Math.tanh(v),
		grad: i => Math.tanh(i) ** 2,
	},
}

for (const name of Object.keys(unaryLayers)) {
	buildUnaryLayer(name, unaryLayers[name].calc, unaryLayers[name].grad)
}

const buildBinaryLayer = (name, calcFunc, gradFunc) => {
	class TempLayer extends Layer {
		calc(...x) {
			this._i = x
			this._o = x[0].copy()
			for (let i = 1; i < x.length; i++) {
				this._o.broadcastOperate(x[i], calcFunc)
			}
			return this._o
		}

		grad(bo) {
			const idx = Array(bo.dimension).fill(0)
			const bi = this._i.map(x => {
				const bi = x.copy()
				bi.fill(0)
				return bi
			})
			const dimdiff = this._i.map(x => bo.dimension - x.dimension)
			do {
				const p = this._i.map((x, k) => idx.slice(dimdiff[k]).map((v, i) => v % x.sizes[i]))
				const val = this._i.map((x, k) => x.at(p[k]))
				const bov = bo.at(idx)
				const ov = this._o.at(idx)
				bi.forEach((xb, k) => {
					xb.operateAt(p[k], v => v + gradFunc(k, val, ov) * bov)
				})
				for (let i = 0; i < idx.length; i++) {
					idx[i]++
					if (idx[i] < bo.sizes[i]) {
						break
					}
					idx[i] = 0
				}
			} while (idx.some(v => v > 0))
			return bi
		}
	}
	Object.defineProperty(TempLayer, 'name', {
		value: `${name.split('_').reduce((s, nm) => s + nm[0].toUpperCase() + nm.substring(1).toLowerCase(), '')}Layer`,
	})
	TempLayer.registLayer(name)
}

const binaryLayers = {
	add: {
		calc: (x1, x2) => x1 + x2,
		gradFunc: () => 1,
	},
	and: {
		calc: (x1, x2) => x1 && x2,
		gradFunc: () => 0,
	},
	bitwise_and: {
		calc: (x1, x2) => x1 & x2,
		gradFunc: () => 0,
	},
	bitwise_or: {
		calc: (x1, x2) => x1 | x2,
		gradFunc: () => 0,
	},
	bitwise_xor: {
		calc: (x1, x2) => x1 ^ x2,
		gradFunc: () => 0,
	},
	div: {
		calc: (x1, x2) => x1 / x2,
		gradFunc: (k, x) => {
			const den = x.slice(1).reduce((s, v) => s * v, 1)
			if (k === 0) {
				return 1 / den
			}
			let v = -x[0] / den ** 2
			for (let i = 1; i < x.length; i++) {
				if (i === k) continue
				v *= x[i]
			}
			return v
		},
	},
	left_bitshift: {
		calc: (x1, x2) => x1 << x2,
		gradFunc: () => 0,
	},
	max: {
		calc: Math.max,
		gradFunc: (k, x) => {
			let max_v = -Infinity
			let max_k = -1
			for (let i = 0; i < x.length; i++) {
				if (max_v < x[i]) {
					max_v = x[i]
					max_k = i
				}
			}
			return max_k === k ? 1 : 0
		},
	},
	min: {
		calc: Math.min,
		gradFunc: (k, x) => {
			let min_v = Infinity
			let min_k = -1
			for (let i = 0; i < x.length; i++) {
				if (min_v > x[i]) {
					min_v = x[i]
					min_k = i
				}
			}
			return min_k === k ? 1 : 0
		},
	},
	mod: {
		calc: (x1, x2) => x1 % x2,
		gradFunc: () => 0,
	},
	mult: {
		calc: (x1, x2) => x1 * x2,
		gradFunc: (k, x) => {
			let v = 1
			for (let i = 0; i < x.length; i++) {
				if (i === k) continue
				v *= x[i]
			}
			return v
		},
	},
	or: {
		calc: (x1, x2) => x1 || x2,
		gradFunc: () => 0,
	},
	power: {
		calc: (x1, x2) => x1 ** x2,
		gradFunc: (k, [x1, x2]) => {
			return k === 0 ? x2 * x1 ** (x2 - 1) : x1 ** x2 * Math.log(x1)
		},
	},
	right_bitshift: {
		calc: (x1, x2) => x1 >> x2,
		gradFunc: () => 0,
	},
	sub: {
		calc: (x1, x2) => x1 - x2,
		gradFunc: k => (k === 0 ? 1 : -1),
	},
	xor: {
		calc: (x1, x2) => (x1 || x2) && !(x1 && x2),
		gradFunc: () => 0,
	},
}

for (const name of Object.keys(binaryLayers)) {
	buildBinaryLayer(name, binaryLayers[name].calc, binaryLayers[name].gradFunc)
}

const buildCompareLayer = (name, calcFunc) => {
	class TempLayer extends Layer {
		calc(...x) {
			this._i = x
			this._o = x[0].copy()
			this._o.map(() => true)
			for (let i = 1; i < x.length; i++) {
				const xi = x[i - 1].copy()
				xi.broadcastOperate(x[i], calcFunc)
				this._o.broadcastOperate(xi, (a, b) => a && b)
			}
			return this._o
		}

		grad() {
			const bi = this._i.map(x => {
				const bi = x.copy()
				bi.fill(0)
				return bi
			})
			return bi
		}
	}
	Object.defineProperty(TempLayer, 'name', {
		value: `${name.split('_').reduce((s, nm) => s + nm[0].toUpperCase() + nm.substring(1).toLowerCase(), '')}Layer`,
	})
	TempLayer.registLayer(name)
}

const compareLayers = {
	equal: { calc: (x1, x2) => x1 === x2 },
	greater: { calc: (x1, x2) => x1 > x2 },
	greater_or_equal: { calc: (x1, x2) => x1 >= x2 },
	less: { calc: (x1, x2) => x1 < x2 },
	less_or_equal: { calc: (x1, x2) => x1 <= x2 },
}

for (const name of Object.keys(compareLayers)) {
	buildCompareLayer(name, compareLayers[name].calc)
}