@thi.ng/text-analysis/vocab.js

Text tokenization, transformation & analysis transducers, utilities, stop words, porter stemming, vector encodings, similarities

import { bidirIndexFromJSON, defBidirIndex } from "@thi.ng/bidir-index";
import { isIterable } from "@thi.ng/checks/is-iterable";
import { isString } from "@thi.ng/checks/is-string";
import { mapcat } from "@thi.ng/transducers/mapcat";
function defVocab(src, start) {
  return isIterable(src) ? defBidirIndex(
    mapcat((x) => isString(x) ? [x] : x, src),
    { start }
  ) : bidirIndexFromJSON(src);
}
export {
  defVocab
};