lexed
Version:
English word and sentence tokenizer, for natural language processing.
31 lines (30 loc) • 789 B
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const abbreviations_1 = require("./abbreviations");
exports.abbreviations = abbreviations_1.default;
const sentence_level_1 = require("./sentence_level");
const token_level_1 = require("./token_level");
class Lexed {
constructor(input) {
this.input = input;
}
;
sentenceLevel() {
this.sentences = sentence_level_1.default(this.input);
return this.sentences;
}
;
tokenLevel() {
this.tokens = this.sentences.map((sentence) => token_level_1.default(sentence));
return this.tokens;
}
;
lexer() {
this.sentenceLevel();
this.tokenLevel();
return this;
}
;
}
exports.Lexed = Lexed;
exports.default = Lexed;