humanity-deno/esm/deps/deno.land/std@0.146.0/datetime/tokenizer.js

Humanity is a library for humanizing data in a human-readable form.

// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
// This module is browser compatible.
export class Tokenizer {
    constructor(rules = []) {
        Object.defineProperty(this, "rules", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.rules = rules;
    }
    addRule(test, fn) {
        this.rules.push({ test, fn });
        return this;
    }
    tokenize(string, receiver = (token) => token) {
        function* generator(rules) {
            let index = 0;
            for (const rule of rules) {
                const result = rule.test(string);
                if (result) {
                    const { value, length } = result;
                    index += length;
                    string = string.slice(length);
                    const token = { ...rule.fn(value), index };
                    yield receiver(token);
                    yield* generator(rules);
                }
            }
        }
        const tokenGenerator = generator(this.rules);
        const tokens = [];
        for (const token of tokenGenerator) {
            tokens.push(token);
        }
        if (string.length) {
            throw new Error(`parser error: string not fully parsed! ${string.slice(0, 25)}`);
        }
        return tokens;
    }
}