yoastseo/build/languageProcessing/helpers/prominentWords/determineProminentWords.js

Yoast client-side content analysis

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.collapseProminentWordsOnStem = collapseProminentWordsOnStem;
exports.default = void 0;
exports.filterProminentWords = filterProminentWords;
exports.getProminentWords = getProminentWords;
exports.getProminentWordsFromPaperAttributes = getProminentWordsFromPaperAttributes;
exports.retrieveAbbreviations = retrieveAbbreviations;
exports.sortProminentWords = sortProminentWords;
var _lodash = require("lodash");
var _getWords = _interopRequireDefault(require("../word/getWords"));
var _quotes = require("../sanitize/quotes");
var _ProminentWord = _interopRequireDefault(require("../../values/ProminentWord"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
const specialCharacters = /[1234567890‘’“”"'.…?!:;,¿¡«»&*@#±^%$|~=+§`[\](){}⟨⟩<>/\\–\-\u2014\u00d7\s]/g;

/**
 * Returns only those prominent words that occur more than a certain number of times and do not consist of special characters.
 *
 * @param {ProminentWord[]} prominentWords A list of prominent words.
 * @param {int} [minimalNumberOfOccurrences] A minimal number of occurrences that is needed for a relevant prominentWord, default 2.
 *
 * @returns {ProminentWord[]} Only relevant words.
 */
function filterProminentWords(prominentWords, minimalNumberOfOccurrences = 2) {
  prominentWords = prominentWords.filter(function (word) {
    return word.getOccurrences() >= minimalNumberOfOccurrences && word.getWord().replace(specialCharacters, "") !== "";
  });
  return prominentWords;
}

/**
 * Sorts prominent words based on their number of occurrences and length.
 *
 * @param {ProminentWord[]} prominentWords The prominent words to sort.
 *
 * @returns {void}
 */
function sortProminentWords(prominentWords) {
  prominentWords.sort(function (wordA, wordB) {
    const difference = wordB.getOccurrences() - wordA.getOccurrences();
    // The word with the highest number of occurrences comes first.
    if (difference !== 0) {
      return difference;
    }

    // In case of a tie on occurrence number, the alphabetically first word comes first.
    return wordA.getStem().localeCompare(wordB.getStem());
  });
}

/**
 * Collapses prominent words that have the same stem.
 *
 * @param {ProminentWord[]} prominentWords All prominentWords.
 *
 * @returns {ProminentWord[]} The original array with collapsed duplicates.
 */
function collapseProminentWordsOnStem(prominentWords) {
  if (prominentWords.length === 0) {
    return [];
  }

  // Sort the input array by stem
  prominentWords.sort(function (wordA, wordB) {
    return wordA.getStem().localeCompare(wordB.getStem());
  });
  const collapsedProminentWords = [];
  let previousWord = new _ProminentWord.default(prominentWords[0].getWord(), prominentWords[0].getStem(), prominentWords[0].getOccurrences());
  for (let i = 1; i < prominentWords.length; i++) {
    const currentWord = new _ProminentWord.default(prominentWords[i].getWord(), prominentWords[i].getStem(), prominentWords[i].getOccurrences());

    /*
     * Compare the stem of the current word in the loop with the previously available stem.
     * If they are equal, the word should be collapsed.
     * When collapsing, the numbers of occurrences get summed.
     * If the stem happens to equal the real word that occurred in the text, we can be sure it's ok to display it
     * to the customer. So, the stem reassigns the word.
     */
    if (currentWord.getStem() === previousWord.getStem()) {
      previousWord.setOccurrences(previousWord.getOccurrences() + currentWord.getOccurrences());
      if (currentWord.getWord() === previousWord.getStem() || currentWord.getWord().toLocaleLowerCase() === previousWord.getStem()) {
        previousWord.setWord(currentWord.getWord());
      }
    } else {
      collapsedProminentWords.push(previousWord);
      previousWord = currentWord;
    }
  }
  collapsedProminentWords.push(previousWord);
  return collapsedProminentWords;
}

/**
 * Retrieves a list of all abbreviations from the text. Returns an empty array if the input text is empty.
 *
 * @param {string} text A text.
 *
 * @returns {string[]} A list of abbreviations from the list.
 */
function retrieveAbbreviations(text) {
  const words = (0, _getWords.default)((0, _quotes.normalizeSingle)(text));
  const abbreviations = [];
  words.forEach(function (word) {
    if (word.length > 1 && word.length < 5 && word === word.toLocaleUpperCase()) {
      abbreviations.push(word.toLocaleLowerCase());
    }
  });
  return (0, _lodash.uniq)(abbreviations);
}

/**
 * Computes prominent words from an array of words. In order to do so, checks whether the word is included in the list of
 * function words and determines the number of occurrences for every word. Then checks if any two words have the same stem
 * and if so collapses over them.
 *
 * @param {string[]} words          The words to determine relevance for.
 * @param {string[]} abbreviations  Abbreviations that should not be stemmed.
 * @param {Function} stemmer        The available stemmer.
 * @param {Array} functionWords     The available function words list.
 *
 * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.
 */
function computeProminentWords(words, abbreviations, stemmer, functionWords) {
  if (words.length === 0) {
    return [];
  }
  const uniqueContentWords = (0, _lodash.uniq)(words.filter(word => !functionWords.includes(word.trim())));
  const prominentWords = [];
  uniqueContentWords.forEach(function (word) {
    if (abbreviations.includes(word)) {
      prominentWords.push(new _ProminentWord.default(word.toLocaleUpperCase(), word, words.filter(element => element === word).length));
    } else {
      prominentWords.push(new _ProminentWord.default(word, stemmer(word), words.filter(element => element === word).length));
    }
  });
  return collapseProminentWordsOnStem(prominentWords);
}

/**
 * Compute hash for an array of words.
 *
 * @param {string[]} arr An array of words.
 *
 * @returns {number} A calculated hash.
 */
const computeHash = arr => {
  let hash = 0;
  if (!arr.length) {
    return hash;
  }
  for (let i = 0; i < arr.length; i++) {
    const item = arr[i];
    for (let j = 0; j < item.length; j++) {
      // eslint-disable-next-line no-bitwise
      hash = hash * 31 + item.charCodeAt(j) | 0;
    }
  }
  return hash;
};

/**
 * Caches prominent words depending on text words.
 * Only the words and abbreviations are used as the cache key as the stemmer and function words
 * are config and shouldn't change in the scope of one request.
 *
 * @param {string[]} words          The words to determine relevance for.
 * @param {string[]} abbreviations  Abbreviations that should not be stemmed.
 * @param {Function} stemmer        The available stemmer.
 * @param {Array} functionWords     The available function words list.
 *
 * @returns {function} The function that collects prominent words for a given set of text words, language and morphologyData.
 */
const computeProminentWordsMemoized = (0, _lodash.memoize)((words, abbreviations, stemmer, functionWords) => {
  return computeProminentWords(words, abbreviations, stemmer, functionWords);
}, (words, abbreviations, stemmer, functionWords) => {
  return computeHash(words) + "|" + computeHash(abbreviations) + "|" + computeHash(functionWords);
});

/**
 * Gets prominent words from the paper text.
 *
 * @param {string} text             The text to retrieve the prominent words from.
 * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.
 * @param {Function} stemmer        The available stemmer.
 * @param {Array} functionWords     The available function words list.
 * @param {function} getWordsCustomHelper   The custom helper to get words.
 *
 * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.
 */
function getProminentWords(text, abbreviations, stemmer, functionWords, getWordsCustomHelper) {
  if (text === "") {
    return [];
  }
  const words = getWordsCustomHelper ? getWordsCustomHelper((0, _quotes.normalizeSingle)(text).toLocaleLowerCase()) : (0, _getWords.default)((0, _quotes.normalizeSingle)(text).toLocaleLowerCase());
  return computeProminentWordsMemoized(words, abbreviations, stemmer, functionWords);
}

/**
 * Gets prominent words from keyphrase and synonyms, metadescription, title, and subheadings.
 *
 * @param {string[]} attributes     The array with attributes to process.
 * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.
 * @param {Function} stemmer        The available stemmer.
 * @param {Array} functionWords     The available function words list.
 * @param {function} getWordsCustomHelper   The custom helper to get words.
 *
 * @returns {ProminentWord[]} Prominent words from the paper attributes.
 */
function getProminentWordsFromPaperAttributes(attributes, abbreviations, stemmer, functionWords, getWordsCustomHelper) {
  const wordsFromAttributes = getWordsCustomHelper ? getWordsCustomHelper(attributes.join(" ").toLocaleLowerCase()) : (0, _getWords.default)(attributes.join(" ").toLocaleLowerCase());
  return computeProminentWords(wordsFromAttributes, abbreviations, stemmer, functionWords);
}
var _default = exports.default = {
  getProminentWords,
  getProminentWordsFromPaperAttributes,
  filterProminentWords,
  sortProminentWords,
  collapseProminentWordsOnStem,
  retrieveAbbreviations
};
//# sourceMappingURL=determineProminentWords.js.map