tokenize-words/dist/bundle.js

Break down text into array of words.

class CustomError extends Error {
  /**
   * Create a `CustomError` (same as `Error` but with an extra property called `library`).
   * @param message Argument to be passed to base constructor (`Error()`).
   * @param library Name of the library in which error will be thrown.
   */
  constructor(message, library) {
    super(message);
    this.library = library;
  }
  toString() {
    return `Error in \`${this.library}\` library:\n\t${this.message}`;
  }
}

// messages aren't inside an object because
// tree-shaking isn't possible when exporting an object
// https://medium.com/@rauschma/note-that-default-exporting-objects-is-usually-an-anti-pattern-if-you-want-to-export-the-cf674423ac38#.nibatprx3
// used at `fullfiller/src/validate`
const notEnoughWordsInWordsArray = (minimum, received) => `Given \`text\` doesn't have enough keywords to construct \`wordsArray\` containing the minimum quantity of words required. Minimum number of words required: ${minimum}. Number of words received: ${received}.`;

function isNumeric(word) {
  return /^[\d.,:%$]+$/.test(word) && /\d/.test(word);
}
function escapeRegExp(regexpString) {
  return regexpString.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
/**
 * Apply `functions` to `input` left to right.
 * @param input
 * @param functions Array of functions.
 *  Function only argument and return value must be of the same type as `input`.
 * @returns Value of the same type as `input`.
 */
function reduce(input, functions) {
  return functions.reduce((acc, fn) => fn(acc), input);
}

const escapeAndMakeDotOptional = str => escapeRegExp(str).replace('\\.', '\\.?');
/* eslint-disable import/prefer-default-export */
/**
 * Convert to lowercase if both are true:
 * - text contain word in lowercase
 * - all capitalized occurrences are preceded by dot or string beginning
 */
function getCorrectWordCase(wordCapitalized, text) {
  // check if the first argument really starts with uppercase letter; anything else
  // (e.g.: lowercase letter, number, dot...) won't need to go through this function
  if (!/\p{Lu}/u.test(wordCapitalized[0])) return wordCapitalized;
  const capitalizedRE = new RegExp(`[^.\\s]\\s+${escapeAndMakeDotOptional(wordCapitalized)}(\\s|\\.|$)`);
  const textHasCapitalizedWordNotPrecededByDotOrStringBeginning = capitalizedRE.test(text);
  const wordLowercase = wordCapitalized.toLowerCase();
  const lowercaseRE = new RegExp(`(^|\\s|\\.)${escapeAndMakeDotOptional(wordLowercase)}(\\s|\\.|$)`);
  const textContainLowercaseWord = lowercaseRE.test(text);
  const correctCase = textContainLowercaseWord && !textHasCapitalizedWordNotPrecededByDotOrStringBeginning ? wordLowercase : wordCapitalized;
  return correctCase;
}
/* eslint-enable import/prefer-default-export */

/**
 * Preserve capitalization in words preceded by dot or convert to lowercase.
 */
function handleCapitalizedLetterPrecededByDotOrStringBeginning(text) {
  return text.replace(/(?:^|\.\s+)(\p{Lu}\S*)/gu,
  // capitalizedLetterPrecededByDotOrStringBeginning
  (match, wordAfterDot, _, whole) => /^[\p{Lu}]+$/u.test(wordAfterDot) ? match // if word is acronym (all uppercase), leave as it is
  : match.replace(wordAfterDot, getCorrectWordCase(wordAfterDot, whole)));
}

/**
 * Preserve dot if word containing leading dot occurs more than once.
 */
function handleLeadingDot(wordContainingLeadingDot, text) {
  const wordWithoutDot = wordContainingLeadingDot.substring(1);
  const doesTextContainsMultipleOccurrences = (text.match(new RegExp(`(^|[^\\p{L}\\p{Nd}])${escapeRegExp(wordContainingLeadingDot)}(?=([^\\p{L}\\p{Nd}]|$))`, 'gu')) || []).length > 1;
  const correctWordForm = doesTextContainsMultipleOccurrences ? wordContainingLeadingDot : getCorrectWordCase(wordWithoutDot, text);
  return correctWordForm;
}

function shouldPreserveTrailingDot(wordContainingTrailingDot, text) {
  const wordWithoutDot = wordContainingTrailingDot.replace(/\.$/, '');
  const wordWithDotNumberOfOccurrences = (text.match(new RegExp(`(^|\\s)${escapeRegExp(wordContainingTrailingDot)}(?=\\s|$)`, 'g')) || []).length;
  if (wordWithDotNumberOfOccurrences === 1) return false;
  const wordWithoutDotNumberOfOccurrences = (text.match(new RegExp(`(^|\\s)${escapeRegExp(wordWithoutDot)}(?=\\s|$)`, 'g')) || []).length;
  return wordWithDotNumberOfOccurrences > wordWithoutDotNumberOfOccurrences;
}
/**
 * Preserve dot if word containing trailing dot happens more
 * than once and more often than word without trailing dot.
 */
function handleTrailingDot(wordContainingTrailingDot, text) {
  const wordWithoutDot = wordContainingTrailingDot.slice(0, -1);
  const preserveTrailingDot = shouldPreserveTrailingDot(wordContainingTrailingDot, text);
  const handled = preserveTrailingDot ? wordContainingTrailingDot : wordWithoutDot;
  return handled;
}

/**
 * Handles something like 'word.Word', which is very common in Wikipedia API's response.
 * This occurs when a paragraph ends with a citation (i.e. superscript number in brackets).
 *
 * @summary Replace dot with space and fix the case of the word after dot.
 */
function replaceMiddleDotWithSpace(match, text) {
  const [, wordBeforeDot, wordAfterDot] = /^(.+)\.(.+)$/.exec(match);
  return `${wordBeforeDot} ${getCorrectWordCase(wordAfterDot, text)}`;
}

function replacer(wordContainingDot, offset, whole) {
  if (/^[.]+$/.test(wordContainingDot)) return '';
  // string containing only numeric values
  if (isNumeric(wordContainingDot)) {
    return wordContainingDot.replace(/\.$/, ''); // preserve dot(s), except trailing
  }
  // preserve or remove leading/trailing dot
  if (/^\.|\.$/.test(wordContainingDot) && wordContainingDot.match(/\./g)?.length === 1) {
    return wordContainingDot.startsWith('.') ? handleLeadingDot(wordContainingDot, whole) : handleTrailingDot(wordContainingDot, whole);
  }
  // fix something like `word.Word`
  if (/[\p{Ll}\p{Nd}]\.[\p{Lu}\p{Nd}]/u.test(wordContainingDot) && wordContainingDot.match(/\./g)?.length === 1) {
    return replaceMiddleDotWithSpace(wordContainingDot, whole);
  }
  // else, preserve dot in:
  // - words with multiple dots
  //   - including abbreviations like X.X.X and x.x.
  // - words with dot in the middle that aren't matched previously
  //   - letter before dot is uppercase &/or letter after dot is lowercase
  return wordContainingDot;
}
function preserveRemoveOrReplaceDot(text) {
  return text.replace(/\S*\.\S*/g,
  // word containing dot(s) at any position
  replacer);
}

function preserveCommaOrColonIfSurroundedByNumbers(_, before, commaOrColon, after) {
  const isCommaOrColonSurroundedByNumbers = [before, after].every(char => /^\d$/.test(char));
  const preserved = before + commaOrColon;
  const removed = `${before} `;
  return isCommaOrColonSurroundedByNumbers ? preserved : removed;
}
function removeUselessStuff(string) {
  const removed = string
  // remove useless punctuation
  // `.?!,:;-–—<>[]{}()'"…` = 15 punctuations signs in english
  // em dash/en dash and opening/closing are counted as the same
  // only hyphen & apostrophe will be always preserved
  // single dot will be handled at `preserveRemoveOrReplaceDot` function
  .replace(/["()[\]{}<>–—;?!]+/g, ' ').replace(/(^|.)(,|:)(?=(.|$))/g, preserveCommaOrColonIfSurroundedByNumbers).replace(/\.{2,}|…/g, ' ')
  // remove line breaks
  .replace(/\n+/g, ' ')
  // remove space between initials
  .replace(/(^|\s)(\p{Lu}\.(\s|$)){2,}/gu, initials => ` ${initials.replace(/\s/g, '')} `);
  return removed;
}

/** Remove words that doesn't contain at least one alphanumeric character. */
function removeWordsNotContainingAlphanumericChar(text) {
  return text.replace(/(^|\s)[^\s\p{L}\p{Nd}]+(?=(\s|$))/gu, '');
}

function normalizeText(text) {
  const normalized = reduce(text, [removeUselessStuff, handleCapitalizedLetterPrecededByDotOrStringBeginning, preserveRemoveOrReplaceDot, removeWordsNotContainingAlphanumericChar]);
  return normalized;
}

const optionsDefault = {
  lengthMin: 0 // don't error even if return array is empty
};
/**
 * Break down text string into array of words.
 * @param text
 * @param optionsArg Miscellaneous options.
 * @throws Error if `wordsArray` length is less than `options.lengthMin`.
 * @returns Array of words.
 */
function tokenizeWords(text, optionsArg = {}) {
  const options = {
    ...optionsDefault,
    ...optionsArg
  };
  const wordsArray = normalizeText(text).match(/\S+/g) || [];
  const wordsArrayLength = wordsArray.length;
  if (wordsArrayLength < options.lengthMin) {
    throw new CustomError(notEnoughWordsInWordsArray(options.lengthMin, wordsArrayLength), 'tokenize-words');
  }
  return wordsArray;
}

export { tokenizeWords as default };