gmx-word-counter
Version:
Fast, GMX-V compliant word and character counter. Can count both logographic and non-logographic languages correctly. Also supports generic counting for cases when language is unknown.
83 lines • 4.69 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.countWords = countWords;
const characterCounter_1 = require("./characterCounter");
const logographicCounter_1 = require("./logographicCounter");
const NON_LOGOGRAPHIC_LANGUAGE_REGEX = /[\p{L}\p{M}]+(?:[-'’](?=[\p{L}\p{M}])[\p{L}\p{M}]+)*|(?<=\s|^)\d+[a-zA-Z]?(?=\s|$)|\d+(?:[.,:]\d+)*|\d+/gu;
// A decimal or thousands-separated number counted as a single token.
const NUMBER = String.raw `\d+(?:[.,:]\d+)*`;
// Accented letter sets, defined once and reused for both the base word and its
// apostrophe/hyphen continuation so the two halves cannot drift apart.
const ES_LETTERS = 'A-Za-z0-9áéíóúüñÁÉÍÓÚÜÑ';
const FR_LETTERS = '\\wàâäéèêëîïôöùûüçœæÀÂÄÉÈÊËÎÏÔÖÙÛÜÇŒÆ';
const DE_LETTERS = '\\wäöüÄÖÜß';
// Arabic-script languages join letter runs across the zero-width non-joiner
// (U+200C), zero-width joiner (U+200D) and tatweel (U+0640), so a single word
// split by any of those must stay one token instead of being counted as
// several. Matching on \p{Script=Arabic} keeps this correct for every
// Arabic-script language, not just Persian: Urdu, Pashto, Sindhi, Central
// Kurdish and Uyghur all use the ZWNJ inside words the same way.
// eslint-disable-next-line no-misleading-character-class
const ARABIC_SCRIPT_REGEX = /[\p{Script=Arabic}\p{M}]+(?:[ـ]+(?=[\p{Script=Arabic}\p{M}])[\p{Script=Arabic}\p{M}]+)*|(?<=\s|^)\d+[a-zA-Z]?(?=\s|$)|\d+(?:[.,:]\d+)*|\d+/gu;
const localeRegexMap = {
// persian and the other Arabic-script languages share one regex (see above)
fa: ARABIC_SCRIPT_REGEX,
ar: ARABIC_SCRIPT_REGEX,
ur: ARABIC_SCRIPT_REGEX, // urdu
ps: ARABIC_SCRIPT_REGEX, // pashto
sd: ARABIC_SCRIPT_REGEX, // sindhi
ckb: ARABIC_SCRIPT_REGEX, // central kurdish (sorani)
ug: ARABIC_SCRIPT_REGEX, // uyghur
ta: /[\u0B80-\u0BFF]+|\d+(?:[.,:]\d+)*/g,
// dash is sometimes used in modern Telugu, but shouldn't be counted as a separate word,
// while two words combined by a hyphen should be considered one word
te: /(?:[\u0C00-\u0C7F]|(?<=[\u0C00-\u0C7F])-(?=[\u0C00-\u0C7F]))+|\d+(?:[.,:]\d+)*/g,
kn: /[\u0C80-\u0CFF]+|\d+(?:[.,:]\d+)*/g,
ml: /[\u0D00-\u0D7F]+(?:[\u200C\u200D][\u0D00-\u0D7F]+)*|\d+(?:[.,:]\d+)*/g,
// no \b anchors on es/fr/de: JS \b is ASCII-based and drops words that both
// start and end with a non-ASCII letter (e.g. standalone "à"). Hyphens are
// connectors between letter runs so a lone "-" is not counted as a word, and
// es matches numbers first so a decimal like "3,14" stays a single token.
es: new RegExp(`${NUMBER}|[${ES_LETTERS}]+(?:-[${ES_LETTERS}]+)*`, 'g'),
pt: /[\wàáéíóúâêôãõçüÀÁÉÍÓÚÂÊÔÃÕÇÜ]+/g,
fr: new RegExp(`[${FR_LETTERS}]+(?:['’][${FR_LETTERS}]+)?`, 'g'),
it: /[\w'àèéìòóù]+(?:(?:’[\w'àèéìòóù]+)?)/gi,
de: new RegExp(`[${DE_LETTERS}]+(?:-[${DE_LETTERS}]+)*`, 'g'),
en: /\b[a-zA-Z0-9]+(?:['’-][a-zA-Z0-9]+)*\b/g,
};
function countWordsLogographic(text, languageSubTag) {
const factor = (0, logographicCounter_1.getCharacterCountFactor)(languageSubTag);
const characters = (0, characterCounter_1.countCharacters)(text);
return Math.round(characters.characters / factor);
}
/**
*
* @param {string} text - Text to count words in
* @param {string} languageSubTag - BCP47 language subtag (a full tag like 'zh-CN' is reduced to its primary subtag)
*/
function countWords(text, languageSubTag) {
if (!text) {
return 0;
}
// Reduce full BCP47 tags like 'zh-CN' or 'EN' to the primary language subtag,
// so that logographic detection and locale-specific regexes still apply
const subTags = languageSubTag ? languageSubTag.toLowerCase().split('-') : [''];
const primarySubTag = subTags[0];
// A Latin script subtag (e.g. 'zh-Latn' pinyin, 'ja-Latn' romaji) marks
// romanized text, which must be word-split rather than divided as a
// logographic script or discarded as an unsupported one.
const isRomanized = subTags[1] === 'latn';
if (!isRomanized) {
if ((0, logographicCounter_1.isLogographicScript)(primarySubTag)) {
return countWordsLogographic(text, primarySubTag);
}
if ((0, logographicCounter_1.isUnsupportedLogographicScript)(primarySubTag)) {
return 0;
}
}
// Let's see if we have locale-specific regex
const regex = localeRegexMap[primarySubTag];
const matches = regex ? text.match(regex) : text.match(NON_LOGOGRAPHIC_LANGUAGE_REGEX);
return matches?.length ?? 0;
}
//# sourceMappingURL=wordCounter.js.map