yoastseo/build/languageProcessing/helpers/sentence/getSentences.js

Yoast client-side content analysis

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = _default;
var _lodash = require("lodash");
var _html = require("../html/html.js");
var _imageInText = require("../image/imageInText");
var _stripHTMLTags = require("../sanitize/stripHTMLTags");
var _unifyWhitespace = require("../sanitize/unifyWhitespace");
var _memoizedSentenceTokenizer = _interopRequireDefault(require("./memoizedSentenceTokenizer"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
// Lodash imports.

// Internal dependencies.

// Character classes.
const newLines = "\n\r|\n|\r";

// Regular expressions.
const newLineRegex = new RegExp(newLines);
const paragraphTagsRegex = new RegExp("^(<p>|</p>)$");

/**
 * Returns sentences in a string.
 *
 * @param {String}      text                The string to count sentences in.
 * @param {function}    memoizedTokenizer   The memoized sentence tokenizer.
 *
 * @returns {Array} Sentences found in the text.
 */
function _default(text, memoizedTokenizer = _memoizedSentenceTokenizer.default) {
  // We don't remove the other HTML tags here since removing them might lead to incorrect results when running the sentence tokenizer.
  // Unify only non-breaking spaces and not the other whitespaces since a whitespace could signify a sentence break or a new line.
  text = (0, _unifyWhitespace.unifyNonBreakingSpace)(text);
  /*
   * Remove images from text before tokenizing it into sentences.
   * This is necessary since the highlighting feature doesn't work if the yoastmark tags are enclosing a sentence starting with an image.
   * This step is done here so that applying highlight in captions is possible for all assessments that use this helper.
   */
  text = text.replace(_imageInText.imageRegex, "");
  let blocks = (0, _html.getBlocks)(text);

  // Split each block on newlines.
  blocks = (0, _lodash.flatMap)(blocks, function (block) {
    return block.split(newLineRegex);
  });

  /*
   * Filter blocks that contain only paragraph tags. This step is necessary
   * since switching between editors might add extra paragraph tags with a new line tag in the end
   * that are incorrectly converted into separate blocks.
   */
  blocks = blocks.filter(block => !paragraphTagsRegex.test(block));

  /*
   * We use the `map` method followed by `flat` instead of `flatMap` because `flatMap` would override the second
   * argument of the memoizedTokenizer with the index of the iteratee.
   */
  let sentences = blocks.map(block => memoizedTokenizer(block)).flat();

  /*
   * Strip block tags from the start and/or the end of each sentence and whitespaces if present.
   * After tokenized, sometimes there are still block tags present in the beginning/end of a sentence.
   * Unstripped, these tags could potentially break the highlighting functionality.
   */
  sentences = sentences.map(sentence => (0, _stripHTMLTags.stripBlockTagsAtStartEnd)(sentence).trim());
  return (0, _lodash.filter)(sentences, (0, _lodash.negate)(_lodash.isEmpty));
}
//# sourceMappingURL=getSentences.js.map