handwritten-mathematics-recogniser
Version:
Easy and abstracted way to recognise handwritten mathematics in a browser or in a web view.
174 lines (172 loc) • 6.82 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const model_1 = require("./model");
const transformer_1 = require("../../utility/transformer");
const segmenter_1 = require("../../utility/segmenter");
const converter_1 = require("../../utility/converter");
const classes = [
'(',
')',
'+',
'-',
'.',
'0',
'1',
'2',
'3',
'4',
'5',
'6',
'7',
'8',
'9',
'\\sqrt',
'\\times',
];
class HandwrittenExpressionRecogniserDNN {
static recognise(canvas) {
const timeStart = +new Date();
const image = converter_1.Converter.convertCanvasToImage(canvas);
const layers = segmenter_1.Segmenter.getLayers(image)
.map((layer) => {
const recognitions = this.feedForward(transformer_1.Transformer.flattenImage(layer.image));
const best = recognitions.indexOf(Math.max(...recognitions));
const recognition = classes[best];
return Object.assign({}, layer, { recognition });
})
.map((layer) => {
if (layer.width < 25 && layer.height < 25) {
layer.recognition = '.';
}
return layer;
});
const latex = this.format(layers);
console.debug([
new Date().toString(),
`took ${+new Date() - timeStart}ms to recognise`,
latex
].join('\n'));
return latex;
}
static feedForward(image) {
const start = +new Date();
const activation2 = [];
for (let i = 0; i < model_1.w1[0].length; i++) {
activation2[i] = model_1.b1[i];
for (let j = 0; j < model_1.w1.length; j++) {
activation2[i] += image[j] * model_1.w1[j][i];
}
activation2[i] = Math.max(activation2[i], 0);
}
const activation3 = [];
for (let i = 0; i < model_1.w2[0].length; i++) {
activation3[i] = model_1.b2[i];
for (let j = 0; j < model_1.w2.length; j++) {
activation3[i] += activation2[j] * model_1.w2[j][i];
}
}
const maxProbability = Math.max(...activation3);
const normalisedProbabilities = activation3.map((probability) => Math.exp(probability - maxProbability));
const totalProbability = normalisedProbabilities.reduce((partialProbability, probability) => partialProbability + probability);
const recognitions = normalisedProbabilities.map((probability) => probability / totalProbability);
return recognitions;
}
static format(layers, level = 0) {
if (layers.length === 0) {
return '';
}
if (layers.length === 1) {
return layers[0].recognition + '}'.repeat(Math.abs(level));
}
const [layer1, layer2, layer3] = layers;
console.log(layer1, layer2, layer3);
if (layers.length >= 3 &&
layer1.recognition === '.' &&
layer2.recognition === '-' &&
layer3.recognition === '.') {
return `\\div ${this.format(layers.slice(3), level)}`;
}
if (layer1.recognition === '-' &&
layer2.recognition === '-') {
return `= ${this.format(layers.slice(2), level)}`;
}
if (layers.some((layer) => layer.recognition === '-')) {
const indexOfBar = layers.map((layer) => layer.recognition).indexOf('-');
const bar = layers[indexOfBar];
let index = 0;
for (let i = 0; i < layers.length; i++) {
if (i !== indexOfBar &&
layers[i].boundingBox.max.y > bar.center.y &&
layers[i].boundingBox.min.y < bar.center.y) {
break;
}
else {
index = i;
}
}
if (index >= 2) {
const numeratorLayers = layers.slice(0, index + 1)
.filter((layer) => layer.recognition !== '-')
.filter((layer) => layer.boundingBox.max.y < bar.center.y);
const denominatorLayers = layers.slice(0, index + 1)
.filter((layer) => layer.recognition !== '-')
.filter((layer) => layer.boundingBox.min.y > bar.center.y);
const remainingLayers = layers.slice(index + 1);
if (numeratorLayers.length > 0 && denominatorLayers.length > 0) {
return `\\frac{${this.format(numeratorLayers, level)}}{${this.format(denominatorLayers, level)}} ${this.format(remainingLayers, level)}`;
}
}
}
if (this.isSuperscript(layer1, layer2, level)) {
return this.formatSuperscript(layer1, layers, level);
}
if (this.isSubscript(layer1, layer2, level)) {
return this.formatSubscript(layer1, layers, level);
}
return this.formatSameLevel(layer1, layers, level);
}
static isSuperscript(layer1, layer2, level) {
if (layer1.recognition.toString() === '.' ||
layer2.recognition.toString() === '.' ||
this.isOperation(layer1.recognition.toString()) ||
level >= 0 && this.isOperation(layer2.recognition.toString())) {
return false;
}
return layer1.center.y > layer2.boundingBox.max.y;
}
static formatSuperscript(layer, layers, level) {
level++;
if (level > 0) {
return `${layer.recognition}^{${this.format(layers.slice(1), level)}`;
}
else {
return `${layer.recognition}}${this.format(layers.slice(1), level)}`;
}
}
static isSubscript(layer1, layer2, level) {
if (layer1.recognition.toString() === '.' ||
layer2.recognition.toString() === '.' ||
this.isOperation(layer1.recognition.toString()) ||
level <= 0 && this.isOperation(layer2.recognition.toString())) {
return false;
}
return layer1.center.y < layer2.boundingBox.min.y;
}
static formatSubscript(layer, layers, level) {
level--;
if (level < 0) {
return `${layer.recognition}_{${this.format(layers.slice(1), level)}`;
}
else {
return `${layer.recognition}}${this.format(layers.slice(1), level)}`;
}
}
static formatSameLevel(layer, layers, level) {
return `${layer.recognition}${this.format(layers.slice(1), level)}`;
}
static isOperation(symbol) {
return ['+', '-', '\\times', '\\div', '\\sqrt', '=', '.'].indexOf(symbol) > -1;
}
}
exports.HandwrittenExpressionRecogniserDNN = HandwrittenExpressionRecogniserDNN;
//# sourceMappingURL=handwritten-expression-recogniser-dnn.js.map