@atlaskit/editor-plugin-show-diff/dist/esm/pm-plugins/calculateDiff/smart/segmentText.js

ShowDiff plugin for @atlaskit/editor-core

function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t.return || t.return(); } finally { if (u) throw o; } } }; }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
/**
 * A half-open offset span `[from, to)` measured in *content offsets* relative to the
 * start of a textblock (i.e. the same coordinate space as `buildCharsByOffset`).
 */

/**
 * Minimal local typing for `Intl.Segmenter`. The TS lib target configured for this
 * package does not include the `Intl.Segmenter` declarations, so we declare just the
 * surface we use and access it via a runtime-checked cast. Remove when the lib target is
 * bumped to include `es2022.intl`/`esnext.intl`.
 */

// A character is treated as part of a word unless it is whitespace or common punctuation.
// We deliberately avoid Unicode property escapes (`\p{...}`) and the `u` flag here: the `u`
// flag is only valid when targeting es6+, and the declaration-generation build config for
// this package targets an older lib, which rejects it with TS1501. This negated approach
// keeps word segmentation reasonable across scripts (any non-punctuation, non-space glyph
// counts as a word char) without needing Unicode-aware regex.
var NON_WORD_CHARS = ' \t\n\r\f\v.,;:!?…。！？、"\'`“”‘’()[]{}<>/\\|@#$%^&*-+=~';
var NON_WORD_CHAR_SET = new Set(NON_WORD_CHARS.split(''));
var isWordChar = function isWordChar(ch) {
  return !NON_WORD_CHAR_SET.has(ch);
};
// Whitespace is checked via a Set rather than a regex to avoid both the `require-unicode-regexp`
// lint rule and the TS1501 error the `u` flag triggers under this package's build lib target.
var WHITESPACE_CHAR_SET = new Set([' ', '\t', '\n', '\r', '\f', '\v', "\xA0"]);
var isWhitespaceChar = function isWhitespaceChar(ch) {
  return WHITESPACE_CHAR_SET.has(ch);
};
// Sentence terminators used by the regex fallback. We intentionally keep this small and
// conservative — the Intl.Segmenter path is the source of truth when available.
var SENTENCE_TERMINATORS = new Set(['.', '!', '?', '…', '。', '！', '？']);

/**
 * Build a per-content-offset view of a textblock's characters.
 *
 * `chars[i]` is the character at content offset `i`, or `null` when that offset lies
 * inside a non-text inline node (mention, date, emoji, hardBreak, …). A `null` acts as an
 * *opaque single token*: it counts as one word and never terminates a sentence.
 */
export var buildCharsByOffset = function buildCharsByOffset(parent) {
  var chars = Array.from({
    length: parent.content.size
  }, function () {
    return null;
  });
  parent.content.forEach(function (child, offset) {
    var _child$text;
    if (!child.isText) {
      return;
    }
    var text = (_child$text = child.text) !== null && _child$text !== void 0 ? _child$text : '';
    for (var i = 0; i < text.length; i++) {
      chars[offset + i] = text[i];
    }
  });
  return chars;
};
var getSegmenterCtor = function getSegmenterCtor() {
  if (typeof Intl === 'undefined') {
    return undefined;
  }
  var ctor = Intl.Segmenter;
  return typeof ctor === 'function' ? ctor : undefined;
};
var cachedLocale;
var cachedSentenceSegmenter;
var cachedWordSegmenter;
var getSegmenters = function getSegmenters(locale) {
  var Ctor = getSegmenterCtor();
  if (!Ctor) {
    return undefined;
  }
  if (cachedLocale !== locale || !cachedSentenceSegmenter || !cachedWordSegmenter) {
    cachedLocale = locale;
    cachedSentenceSegmenter = new Ctor(locale, {
      granularity: 'sentence'
    });
    cachedWordSegmenter = new Ctor(locale, {
      granularity: 'word'
    });
  }
  return {
    sentence: cachedSentenceSegmenter,
    word: cachedWordSegmenter
  };
};

/**
 * Convert a char-view into a plain string plus an index map back to content offsets.
 *
 * Opaque inline tokens (`null`) are represented in the string by a single sentinel
 * character (Unicode Object Replacement Character, U+FFFC) so that:
 * - the segmenter treats them as a word-like glyph rather than whitespace/punctuation;
 * - they never look like a sentence terminator.
 * `indexMap[j]` gives the content offset of string index `j`.
 */
var OBJECT_REPLACEMENT = "\uFFFC";
var flattenChars = function flattenChars(chars) {
  var text = '';
  var indexMap = [];
  for (var offset = 0; offset < chars.length; offset++) {
    var ch = chars[offset];
    text += ch === null ? OBJECT_REPLACEMENT : ch;
    indexMap.push(offset);
  }
  return {
    text: text,
    indexMap: indexMap
  };
};
var stringIndexToOffset = function stringIndexToOffset(indexMap, strIdx, fallback) {
  if (strIdx < 0) {
    return fallback;
  }
  if (strIdx >= indexMap.length) {
    // One-past-the-end maps to the offset after the last char.
    return indexMap.length > 0 ? indexMap[indexMap.length - 1] + 1 : fallback;
  }
  return indexMap[strIdx];
};

/**
 * Segment a textblock char-view into sentence spans (content-offset coordinates).
 * Uses Intl.Segmenter when available, otherwise a conservative regex fallback.
 */
export var segmentSentences = function segmentSentences(chars, locale) {
  if (chars.length === 0) {
    return [];
  }
  var _flattenChars = flattenChars(chars),
    text = _flattenChars.text,
    indexMap = _flattenChars.indexMap;
  var segmenters = getSegmenters(locale);
  if (segmenters) {
    var _spans = [];
    var _iterator = _createForOfIteratorHelper(segmenters.sentence.segment(text)),
      _step;
    try {
      for (_iterator.s(); !(_step = _iterator.n()).done;) {
        var seg = _step.value;
        var startOffset = stringIndexToOffset(indexMap, seg.index, 0);
        var endOffset = stringIndexToOffset(indexMap, seg.index + seg.segment.length, chars.length);
        if (endOffset > startOffset) {
          _spans.push({
            from: startOffset,
            to: endOffset
          });
        }
      }
    } catch (err) {
      _iterator.e(err);
    } finally {
      _iterator.f();
    }
    return _spans.length > 0 ? _spans : [{
      from: 0,
      to: chars.length
    }];
  }

  // Regex fallback: split after a sentence terminator followed by whitespace/EOL.
  // A `null` (OBJECT_REPLACEMENT) is never a terminator.
  var spans = [];
  var start = 0;
  for (var i = 0; i < chars.length; i++) {
    var ch = chars[i];
    if (ch !== null && SENTENCE_TERMINATORS.has(ch)) {
      // Consume trailing spaces as part of this sentence.
      var end = i + 1;
      while (end < chars.length && chars[end] !== null && isWhitespaceChar(chars[end])) {
        end++;
      }
      spans.push({
        from: start,
        to: end
      });
      start = end;
    }
  }
  if (start < chars.length) {
    spans.push({
      from: start,
      to: chars.length
    });
  }
  return spans.length > 0 ? spans : [{
    from: 0,
    to: chars.length
  }];
};

/**
 * Count word-like tokens within a content-offset range of the char-view.
 * Each opaque inline token (`null`) counts as exactly one word.
 */
export var countWords = function countWords(chars, span, locale) {
  var slice = chars.slice(span.from, span.to);
  if (slice.length === 0) {
    return 0;
  }
  var segmenters = getSegmenters(locale);
  var _flattenChars2 = flattenChars(slice),
    text = _flattenChars2.text;
  if (segmenters) {
    var _count = 0;
    var _iterator2 = _createForOfIteratorHelper(segmenters.word.segment(text)),
      _step2;
    try {
      for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
        var seg = _step2.value;
        // Object-replacement sentinel is word-like; count it.
        if (seg.isWordLike || seg.segment.includes(OBJECT_REPLACEMENT)) {
          _count++;
        }
      }
    } catch (err) {
      _iterator2.e(err);
    } finally {
      _iterator2.f();
    }
    return _count;
  }

  // Regex fallback: a run of word chars OR a single opaque token is one word.
  var count = 0;
  var inWord = false;
  var _iterator3 = _createForOfIteratorHelper(slice),
    _step3;
  try {
    for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
      var ch = _step3.value;
      if (ch === null) {
        count++;
        inWord = false;
        continue;
      }
      if (isWordChar(ch)) {
        if (!inWord) {
          count++;
          inWord = true;
        }
      } else {
        inWord = false;
      }
    }
  } catch (err) {
    _iterator3.e(err);
  } finally {
    _iterator3.f();
  }
  return count;
};

/**
 * Return the word-token spans (content-offset coordinates) within a range. Used to decide
 * which words a change overlaps. Each opaque inline token is its own span.
 */
export var segmentWordSpans = function segmentWordSpans(chars, span, locale) {
  var segmenters = getSegmenters(locale);
  var slice = chars.slice(span.from, span.to);
  var _flattenChars3 = flattenChars(slice),
    text = _flattenChars3.text,
    indexMap = _flattenChars3.indexMap;
  var spans = [];
  if (segmenters) {
    var _iterator4 = _createForOfIteratorHelper(segmenters.word.segment(text)),
      _step4;
    try {
      for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
        var seg = _step4.value;
        if (seg.isWordLike || seg.segment.includes(OBJECT_REPLACEMENT)) {
          var from = span.from + stringIndexToOffset(indexMap, seg.index, 0);
          var to = span.from + stringIndexToOffset(indexMap, seg.index + seg.segment.length, slice.length);
          spans.push({
            from: from,
            to: to
          });
        }
      }
    } catch (err) {
      _iterator4.e(err);
    } finally {
      _iterator4.f();
    }
    return spans;
  }
  var wordStart = -1;
  for (var i = 0; i < slice.length; i++) {
    var ch = slice[i];
    if (ch === null) {
      if (wordStart !== -1) {
        spans.push({
          from: span.from + wordStart,
          to: span.from + i
        });
        wordStart = -1;
      }
      spans.push({
        from: span.from + i,
        to: span.from + i + 1
      });
    } else if (isWordChar(ch)) {
      if (wordStart === -1) {
        wordStart = i;
      }
    } else if (wordStart !== -1) {
      spans.push({
        from: span.from + wordStart,
        to: span.from + i
      });
      wordStart = -1;
    }
  }
  if (wordStart !== -1) {
    spans.push({
      from: span.from + wordStart,
      to: span.from + slice.length
    });
  }
  return spans;
};