functionalscript/djs/tokenizer/module.f.js

FunctionalScript is a purely functional subset of JavaScript

import * as list from "../../types/list/module.f.js";
const { empty, flat, stateScan } = list;
import * as bf from "../../types/bigfloat/module.f.js";
const { multiply } = bf;
import * as jsTokenizer from "../../js/tokenizer/module.f.js";
const mapToken = input => {
    switch (input.kind) {
        case 'id':
        case 'bigint':
        case '{':
        case '}':
        case ':':
        case ',':
        case '[':
        case ']':
        case '.':
        case '=':
        case 'true':
        case 'false':
        case 'null':
        case 'string':
        case 'number':
        case 'ws':
        case 'nl':
        case 'undefined':
        case '//':
        case '/*':
        case 'eof':
        case 'error': return [input];
        default: return jsTokenizer.isKeywordToken(input) ? [{ kind: 'id', value: input.kind }] : [{ kind: 'error', message: 'invalid token' }];
    }
};
const parseDefaultState = input => {
    switch (input.kind) {
        case 'eof': return [[{ kind: 'eof' }], { kind: 'def' }];
        case '-': return [empty, { kind: '-' }];
        default: return [mapToken(input), { kind: 'def' }];
    }
};
const parseMinusState = input => {
    switch (input.kind) {
        case 'eof': return [[{ kind: 'error', message: 'invalid token' }, { kind: 'eof' }], { kind: 'def' }];
        case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }];
        case 'bigint': return [[{ kind: 'bigint', value: -1n * input.value }], { kind: 'def' }];
        case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def' }];
        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapToken(input) }, { kind: 'def' }];
    }
};
const scanToken = state => input => {
    switch (state.kind) {
        case '-': return parseMinusState(input);
        default: return parseDefaultState(input);
    }
};
const mapTokenWithMetadata = metadata => token => { return { token, metadata }; };
const scanTokenWithMetadata = state => (input) => {
    const [djsTokens, newState] = scanToken(state)(input.token);
    const djsTokensWithMetadata = list.map(mapTokenWithMetadata(input.metadata))(djsTokens);
    return [djsTokensWithMetadata, newState];
};
export const tokenize = input => path => {
    const jsTokens = jsTokenizer.tokenize(input)(path);
    return flat(stateScan(scanTokenWithMetadata)({ kind: 'def' })(jsTokens));
};