synonym-optimizer
Version:
Finds the text which has the least number of repetitions
46 lines • 1.56 kB
JavaScript
;
/**
* @license
* Copyright 2019 Ludan Stoecklé
* SPDX-License-Identifier: Apache-2.0
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.LanguageSyn = void 0;
// import * as tokenizer from 'wink-tokenizer';
const wink_tokenizer_1 = __importDefault(require("wink-tokenizer"));
const rosaenlg_filter_1 = require("rosaenlg-filter");
class LanguageSyn {
constructor(iso2, stemmer) {
this.iso2 = iso2;
this.stemmer = stemmer;
}
extractWords(input) {
// console.log(`tokenizing: ${input}`);
const myTokenizer = new wink_tokenizer_1.default();
myTokenizer.defineConfig({
currency: false,
number: false,
punctuation: false,
symbol: false,
time: false,
});
const tokenized = myTokenizer.tokenize(input);
// console.log(`tokenized: ${tokenized}`);
const res = [];
tokenized.forEach((elt) => {
// no alien tags and no html elements
if (elt.tag != 'alien' &&
rosaenlg_filter_1.blockLevelHtmlElts.indexOf(elt.value) == -1 &&
rosaenlg_filter_1.inlineHtmlElts.indexOf(elt.value) == -1) {
res.push(elt.value);
}
});
// console.log(`res: ${res}`);
return res;
}
}
exports.LanguageSyn = LanguageSyn;
//# sourceMappingURL=LanguageSyn.js.map