yoastseo
Version:
Yoast client-side content analysis
27 lines (26 loc) • 841 B
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
var _lodash = require("lodash");
var _tinySegmenter = _interopRequireDefault(require("tiny-segmenter"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
/**
* Split sentence into tokens.
*
* @param {string} sentenceText The sentence text to split.
*
* @returns {Token[]} The tokens.
*/
function splitIntoTokensCustom(sentenceText) {
// Return empty string if sentence is empty.
if (sentenceText === "") {
return [];
}
// Split sentences into words that are also tokens.
const words = new _tinySegmenter.default().segment(sentenceText);
return (0, _lodash.map)(words);
}
var _default = exports.default = splitIntoTokensCustom;
//# sourceMappingURL=splitIntoTokensCustom.js.map