@sconedev/ai_toolkit/dist/utils/processText.js

Simplify AI integration in web apps with local and offline model support

/**
 * Processes text for analysis by cleaning, normalizing, and applying optional transformations
 * @param text The input text to process
 * @param options Processing options
 * @returns The processed text
 */
export function processText(text, options) {
    if (!text)
        return '';
    let processed = text;
    // Apply lowercase transformation if requested
    if (options?.lowercase !== false) {
        processed = processed.toLowerCase();
    }
    // Remove special characters if requested
    if (options?.removeSpecialChars) {
        processed = processed.replace(/[^\w\s.,!?-]/g, ' ');
    }
    // Remove punctuation if requested
    if (options?.removePunctuation) {
        processed = processed.replace(/[.,\/#!$%\^&\*;:{}=\-_`~()]/g, ' ');
    }
    // Remove extra whitespace
    if (options?.removeExtraSpaces !== false) {
        processed = processed.replace(/\s+/g, ' ').trim();
    }
    // Truncate if necessary
    if (options?.maxLength && processed.length > options.maxLength) {
        processed = processed.substring(0, options.maxLength);
    }
    return processed;
}