everyutil/string/tokenizeSmart.js

A comprehensive library of lightweight, reusable utility functions for JavaScript and TypeScript, designed to streamline common programming tasks such as string manipulation, array processing, date handling, and more.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.tokenizeSmart = void 0;
/**
 * Splits string into tokens, respecting quotes, punctuation, emojis, etc.
 *
 * Example: tokenizeSmart('Hello, "Mr. Smith" 👋') → ['Hello', '"Mr. Smith"', '👋']
 *
 * @author @dailker
 * @param {string} str - The input string.
 * @returns {string[]} Array of tokens.
 */
function tokenizeSmart(str) {
    const regex = /"([^"]+)"|'([^']+)'|[\p{Emoji}\p{P}]+|\w+/gu;
    const tokens = [];
    let match;
    while ((match = regex.exec(str))) {
        tokens.push(match[1] || match[2] || match[0]);
    }
    return tokens;
}
exports.tokenizeSmart = tokenizeSmart;