typo-tolerance-synonyms/index.js

This package provides an advanced multilingual search engine with phonetic matching and extensive language support, designed to filter input for typographical errors and return refined search results.

// index.js
const natural = require('natural');
const WordNet = require('node-wordnet');
const franc = require('franc');
const Metaphone = require('metaphone');
const doubleMetaphone = require('double-metaphone');
const stopwords = require('stopwords-iso');
const { transliterate } = require('transliteration');

class TypoTolerantSearch {
    constructor(options = {}) {
        this.wordnet = new WordNet();
        this.typoTolerance = options.typoTolerance || 2;
        this.caseSensitive = options.caseSensitive || false;
        this.synonymCache = new Map();
        this.phoneticCache = new Map();
        this.languages = options.languages || ['eng'];
        this.usePhonetic = options.usePhonetic !== false;
        this.metaphone = new Metaphone();
    }

    // Language detection
    detectLanguage(text) {
        return franc(text, { minLength: 3 });
    }

    // Get stopwords for a language
    getStopwords(langCode) {
        return stopwords[langCode] || [];
    }

    // Phonetic matching using multiple algorithms
    getPhoneticCodes(word) {
        if (this.phoneticCache.has(word)) {
            return this.phoneticCache.get(word);
        }

        const codes = {
            metaphone: this.metaphone.process(word),
            doubleMetaphone: doubleMetaphone(word),
            soundex: natural.SoundEx.process(word)
        };

        this.phoneticCache.set(word, codes);
        return codes;
    }

    // Enhanced Levenshtein with support for diacritics
    levenshteinDistance(str1, str2) {
        // Normalize strings by removing diacritics if needed
        str1 = this.normalizeString(str1);
        str2 = this.normalizeString(str2);

        const m = str1.length;
        const n = str2.length;
        const dp = Array(m + 1).fill().map(() => Array(n + 1).fill(0));

        for (let i = 0; i <= m; i++) dp[i][0] = i;
        for (let j = 0; j <= n; j++) dp[0][j] = j;

        for (let i = 1; i <= m; i++) {
            for (let j = 1; j <= n; j++) {
                if (str1[i - 1] === str2[j - 1]) {
                    dp[i][j] = dp[i - 1][j - 1];
                } else {
                    dp[i][j] = 1 + Math.min(
                        dp[i - 1][j],
                        dp[i][j - 1],
                        dp[i - 1][j - 1]
                    );
                }
            }
        }

        return dp[m][n];
    }

    // Normalize string (remove diacritics, handle special characters)
    normalizeString(text) {
        return transliterate(text)
            .toLowerCase()
            .replace(/[^\w\s]/g, '');
    }

    // Get language-specific synonyms
    async getSynonyms(word, lang = 'eng') {
        const cacheKey = `${word}_${lang}`;
        if (this.synonymCache.has(cacheKey)) {
            return this.synonymCache.get(cacheKey);
        }

        // Handle different languages
        switch (lang) {
            case 'eng':
                return this.getEnglishSynonyms(word);
            case 'spa':
                return this.getSpanishSynonyms(word);
            // Add more languages as needed
            default:
                return this.getEnglishSynonyms(word);
        }
    }

    // English-specific synonym lookup
    async getEnglishSynonyms(word) {
        return new Promise((resolve, reject) => {
            this.wordnet.lookupAsync(word).then(results => {
                const synonyms = new Set();
                results.forEach(result => {
                    result.synonyms.forEach(syn => {
                        if (syn !== word) synonyms.add(syn);
                    });
                });
                const synonymArray = Array.from(synonyms);
                this.synonymCache.set(word, synonymArray);
                resolve(synonymArray);
            }).catch(() => resolve([]));
        });
    }

    // Spanish-specific synonym lookup (example)
    async getSpanishSynonyms(word) {
        // Implement Spanish thesaurus lookup
        // This is a placeholder - you'd need to integrate a Spanish language resource
        return [];
    }

    // Calculate phonetic similarity score
    getPhoneticSimilarity(word1, word2) {
        const codes1 = this.getPhoneticCodes(word1);
        const codes2 = this.getPhoneticCodes(word2);

        let score = 0;
        
        // Compare Metaphone
        if (codes1.metaphone === codes2.metaphone) score += 0.4;
        
        // Compare Double Metaphone
        if (codes1.doubleMetaphone[0] === codes2.doubleMetaphone[0]) score += 0.3;
        if (codes1.doubleMetaphone[1] === codes2.doubleMetaphone[1]) score += 0.2;
        
        // Compare Soundex
        if (codes1.soundex === codes2.soundex) score += 0.1;

        return score;
    }

    // Enhanced search with multi-language and phonetic support
    async search(searchText, targetArray, options = {}) {
        const lang = options.language || this.detectLanguage(searchText);
        const searchWords = searchText.split(' ')
            .filter(word => !this.getStopwords(lang).includes(word));

        const results = [];
        const synonymPromises = searchWords.map(word => this.getSynonyms(word, lang));
        const synonymsArrays = await Promise.all(synonymPromises);

        for (const target of targetArray) {
            const targetWords = target.split(' ')
                .filter(word => !this.getStopwords(lang).includes(word));
            
            let score = 0;
            let matches = 0;

            for (let i = 0; i < searchWords.length; i++) {
                const searchWord = searchWords[i];
                const synonyms = synonymsArrays[i];
                let bestWordScore = 0;

                for (const targetWord of targetWords) {
                    // Exact match
                    if (this.normalizeString(targetWord) === this.normalizeString(searchWord)) {
                        bestWordScore = 1;
                        break;
                    }

                    // Synonym match
                    if (synonyms.some(syn => 
                        this.normalizeString(targetWord) === this.normalizeString(syn))) {
                        bestWordScore = 0.9;
                        break;
                    }

                    // Phonetic match
                    if (this.usePhonetic) {
                        const phoneticScore = this.getPhoneticSimilarity(targetWord, searchWord);
                        bestWordScore = Math.max(bestWordScore, phoneticScore);
                    }

                    // Levenshtein distance match
                    const levenScore = 1 - (this.levenshteinDistance(targetWord, searchWord) / 
                        Math.max(targetWord.length, searchWord.length));
                    bestWordScore = Math.max(bestWordScore, levenScore);
                }

                if (bestWordScore > 0.3) {
                    matches++;
                    score += bestWordScore;
                }
            }

            if (matches > 0) {
                score = score / searchWords.length;
                if (score >= 0.3) {
                    results.push({
                        text: target,
                        score,
                        language: lang,
                        matches
                    });
                }
            }
        }

        return results.sort((a, b) => b.score - a.score);
    }
}

module.exports = MultilingualSearch;