@yozora/tokenizer-math/lib/esm/index.mjs

Tokenizer for processing fenced math block (formulas)

import { calcTrimBoundaryOfCodePoints, calcStringFromNodePoints, AsciiCodePoint } from '@yozora/character';
import FencedBlockTokenizer, { fencedBlockMatch } from '@yozora/tokenizer-fenced-block';
import { MathType } from '@yozora/ast';
import { mergeContentLinesFaithfully, TokenizerPriority } from '@yozora/core-tokenizer';

const match = function (api) {
    const { markers } = this;
    const hook = fencedBlockMatch.call(this, api);
    return {
        ...hook,
        isContainingBlock: false,
        eatOpener,
        eatAndInterruptPreviousSibling,
    };
    function eatOpener(line, parentToken) {
        const result = hook.eatOpener(line, parentToken);
        if (result == null)
            return null;
        const { token } = result;
        const [lft, rht] = calcTrimBoundaryOfCodePoints(token.infoString);
        if (lft >= rht)
            return result;
        let i = rht - 1;
        for (; i >= lft; --i) {
            const c = token.infoString[i].codePoint;
            if (!markers.includes(c))
                break;
        }
        const countOfTailingMarker = rht - i - 1;
        if (countOfTailingMarker !== token.markerCount)
            return null;
        const mathToken = {
            ...token,
            infoString: [],
            lines: [
                {
                    nodePoints: token.infoString,
                    startIndex: 0,
                    endIndex: rht - countOfTailingMarker,
                    firstNonWhitespaceIndex: lft,
                    countOfPrecedeSpaces: 0,
                },
            ],
        };
        return { token: mathToken, nextIndex: line.endIndex, saturated: true };
    }
    function eatAndInterruptPreviousSibling(line, prevSiblingToken, parentToken) {
        const result = eatOpener(line, parentToken);
        if (result == null)
            return null;
        return {
            token: result.token,
            nextIndex: result.nextIndex,
            remainingSibling: prevSiblingToken,
            saturated: result.saturated,
        };
    }
};

const parse = function (api) {
    return {
        parse: tokens => tokens.map(token => {
            const contents = mergeContentLinesFaithfully(token.lines);
            let value = calcStringFromNodePoints(contents);
            if (!/\n$/.test(value))
                value += '\n';
            const node = api.shouldReservePosition
                ? { type: MathType, position: token.position, value }
                : { type: MathType, value };
            return node;
        }),
    };
};

const uniqueName = '@yozora/tokenizer-math';

class MathTokenizer extends FencedBlockTokenizer {
    constructor(props = {}) {
        super({
            name: props.name ?? uniqueName,
            priority: props.priority ?? TokenizerPriority.FENCED_BLOCK,
            nodeType: MathType,
            markers: [AsciiCodePoint.DOLLAR_SIGN],
            markersRequired: 2,
        });
    }
    match = match;
    parse = parse;
}

export { MathTokenizer, uniqueName as MathTokenizerName, MathTokenizer as default, match as mathMatch, parse as mathParse };