hx-tokenizer/tokenize.test.js

Split text into tokens.

import test from "ava";
import tokenizer from "./tokenizer";

function macro(t, input, expected) {
  t.deepEqual(tokenizer.tokenize(input), expected);
}
macro.title = (title, input) => `"${input}"`;

test(macro, "one two", ["one", "two"]);
test(macro, "one  two", ["one", "two"]);
test(macro, "one two   ", ["one", "two"]);
test(macro, "   one     two     ", ["one", "two"]);
test(macro, "One tWo", ["One", "tWo"]);
test(macro, "don't you", ["don't", "you"]);
test(macro, "one-two", ["one-two"]);
test(macro, "see https://t.co/Ao47NBwbOB", ["see", "https://t.co/Ao47NBwbOB"]);
test(macro, "hey :)", ["hey", ":)"]);
test(macro, "hey 😅", ["hey", "😅"]);
test(macro, "@men_tion @some2", ["@men_tion", "@some2"]);
test(macro, "¿cuántos años?", ["¿", "cuántos", "años", "?"]);
test(macro, "v8 compiler", ["v8", "compiler"]);
test(macro, "$12 billion", ["$12", "billion"]);
test(macro, "u.s.a.", ["u.s.a."]);
test(macro, "soon...", ["soon", ".", ".", "."]);
test(macro, "one. two", ["one", ".", "two"]);
test(macro, "one two.", ["one", "two", "."]);
test(macro, "one (two)", ["one", "(", "two", ")"]);
test(macro, '"one two" three', ['"', "one", "two", '"', "three"]);
test(macro, "one & two", ["one", "&", "two"]);
test(macro, "one: two", ["one", ":", "two"]);
test(macro, "one.two", ["one", ".", "two"]);
test(macro, "one… two", ["one", "…", "two"]);
test(macro, "barça", ["barça"]);
test(macro, "Verão", ["Verão"]);
test(macro, "déjà vu", ["déjà", "vu"]);
test(macro, "We’re going", ["We’re", "going"]);