yoastseo
Version:
Yoast client-side content analysis
107 lines (99 loc) • 5.66 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
var _lodash = require("lodash");
var _countWords = _interopRequireDefault(require("../helpers/word/countWords"));
var _determineProminentWords = require("../helpers/prominentWords/determineProminentWords");
var _getSubheadings = require("../helpers/html/getSubheadings");
var _baseStemmer = _interopRequireDefault(require("../helpers/morphology/baseStemmer"));
var _removeURLs = _interopRequireDefault(require("../helpers/sanitize/removeURLs.js"));
var _removeEmailAddresses = _interopRequireDefault(require("../helpers/sanitize/removeEmailAddresses"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
/**
* Removes URLs and email addresses from the text.
*
* @param {string} text The text to sanitize.
*
* @returns {string} The text without URLs and email addresses.
*/
const sanitizeText = function (text) {
text = (0, _removeURLs.default)(text);
return (0, _removeEmailAddresses.default)(text);
};
/**
* @typedef ProminentWordsForInternalLinking
* @property {ProminentWord[]} prominentWords Prominent words for this paper, filtered and sorted.
* @property {boolean} hasMetaDescription Whether the metadescription is available in the input paper.
* @property {boolean} hasTitle Whether the title is available in the input paper.
*/
/**
* Retrieves the prominent words from the given paper.
*
* @param {Paper} paper The paper to determine the prominent words of.
* @param {Researcher} researcher The researcher to use for analysis.
*
* @returns {ProminentWordsForInternalLinking} result A compound result object.
*/
function getProminentWordsForInternalLinking(paper, researcher) {
const functionWords = researcher.getConfig("functionWords");
// An optional custom helper to return custom function to return the stem of a word.
const customStemmer = researcher.getHelper("customGetStemmer");
const stemmer = customStemmer ? customStemmer(researcher) : researcher.getHelper("getStemmer")(researcher);
// An optional custom helper to get words from the text.
const getWordsCustomHelper = researcher.getHelper("getWordsCustomHelper");
// An optional custom helper to count length to use instead of countWords.
const customCountLength = researcher.getHelper("customCountLength");
const text = sanitizeText(paper.getText());
const metadescription = sanitizeText(paper.getDescription());
const title = sanitizeText(paper.getTitle());
const result = {};
result.hasMetaDescription = metadescription !== "";
result.hasTitle = title !== "";
result.prominentWords = [];
/**
* We only want to return suggestions (and spend time calculating prominent words) if the text is at least 100 words.
* And when a customCountLength is available, we only want to return the suggestions if the text has at least 200 characters.
*/
if (customCountLength) {
if (customCountLength(text) < 200) {
return result;
}
} else if ((0, _countWords.default)(text) < 100) {
return result;
}
const subheadings = (0, _getSubheadings.getSubheadingsTopLevel)(text).map(subheading => subheading[2]);
const attributes = [paper.getKeyword(), paper.getSynonyms(), title, metadescription, subheadings.join(" ")];
// If the language has a custom helper to get words from the text, we don't retrieve the abbreviation.
const abbreviations = getWordsCustomHelper ? [] : (0, _determineProminentWords.retrieveAbbreviations)(text.concat(attributes.join(" ")));
const removedSubheadingText = (0, _getSubheadings.removeSubheadingsTopLevel)(text);
const prominentWordsFromText = (0, _determineProminentWords.getProminentWords)(removedSubheadingText, abbreviations, stemmer, functionWords, getWordsCustomHelper);
const prominentWordsFromPaperAttributes = (0, _determineProminentWords.getProminentWordsFromPaperAttributes)(attributes, abbreviations, stemmer, functionWords, getWordsCustomHelper);
/*
* If a word is used in any of the attributes, its weight is automatically high.
* To make sure the word survives weight filters and gets saved in the database, make the number of occurrences times-3.
*/
prominentWordsFromPaperAttributes.forEach(relevantWord => relevantWord.setOccurrences(relevantWord.getOccurrences() * 3));
const collapsedWords = (0, _determineProminentWords.collapseProminentWordsOnStem)(prominentWordsFromPaperAttributes.concat(prominentWordsFromText));
(0, _determineProminentWords.sortProminentWords)(collapsedWords);
/*
* If morphology data are available for a language, the minimum number of occurrences to consider a word to be prominent is 4.
* This minimum number was chosen in order to avoid premature suggestions of words from the paper attributes.
* These get a times-3 boost and would therefore be prominent with just 1 occurrence.
*
* If morphology data are not available, and therefore word forms are not recognized, the minimum threshold is lowered to 2.
*/
let minimumNumberOfOccurrences = 4;
if (stemmer === _baseStemmer.default) {
minimumNumberOfOccurrences = 2;
}
/*
* Return the 100 top items from the collapsed and sorted list. The number is picked deliberately to prevent larger
* articles from getting too long of lists.
*/
result.prominentWords = (0, _lodash.take)((0, _determineProminentWords.filterProminentWords)(collapsedWords, minimumNumberOfOccurrences), 100);
return result;
}
var _default = exports.default = getProminentWordsForInternalLinking;
//# sourceMappingURL=getProminentWordsForInternalLinking.js.map