smart-thinking-mcp/build/similarity-engine.js

Un serveur MCP avancé pour le raisonnement multi-dimensionnel, adaptatif et collaboratif

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SimilarityEngine = void 0;
class SimilarityEngine {
    stopWords;
    tokenCache = new Map();
    cacheExpiration = 60 * 60 * 1000; // 1 hour
    constructor() {
        this.stopWords = new Set([
            'the', 'and', 'for', 'avec', 'dans', 'that', 'this', 'from', 'pour', 'avec', 'les', 'des', 'une', 'un',
            'qui', 'que', 'quoi', 'dont', 'mais', 'car', 'donc', 'or', 'nor', 'not', 'pas', 'sur', 'sous', 'par',
            'est', 'sont', 'été', 'être', 'will', 'shall', 'would', 'should', 'could', 'can', 'cannot', 'ne', 'la',
            'le', 'de', 'du', 'au', 'aux', 'their', 'there', 'here', 'very', 'have', 'has', 'had', 'avoir', 'avais',
            'avait', 'été', 'tout', 'tous', 'vous', 'nous', 'ils', 'elles', 'she', 'him', 'her', 'his', 'hers', 'its'
        ]);
    }
    normalize(text) {
        const decomposed = text.normalize('NFKD');
        return decomposed.replace(/[\u0300-\u036f]/g, '').toLowerCase();
    }
    tokenize(text) {
        const cached = this.tokenCache.get(text);
        if (cached && (Date.now() - cached.timestamp) < this.cacheExpiration) {
            return cached.tokens;
        }
        const normalized = this.normalize(text);
        const rawTokens = normalized.split(/[\W_]+/u).filter(Boolean);
        const tokens = rawTokens
            .filter(token => token.length > 2 && !this.stopWords.has(token))
            .map(token => token.trim());
        this.tokenCache.set(text, { tokens, timestamp: Date.now() });
        return tokens;
    }
    computeTermFrequency(tokens) {
        const tf = {};
        if (tokens.length === 0) {
            return tf;
        }
        for (const token of tokens) {
            tf[token] = (tf[token] || 0) + 1;
        }
        const invLength = 1 / tokens.length;
        for (const token of Object.keys(tf)) {
            tf[token] = tf[token] * invLength;
        }
        return tf;
    }
    computeInverseDocumentFrequency(tokensList) {
        const idf = {};
        const documentFrequency = {};
        const totalDocs = tokensList.length || 1;
        for (const tokens of tokensList) {
            const uniqueTokens = new Set(tokens);
            for (const token of uniqueTokens) {
                documentFrequency[token] = (documentFrequency[token] || 0) + 1;
            }
        }
        for (const token of Object.keys(documentFrequency)) {
            const df = documentFrequency[token];
            // Smooth the IDF to avoid division by zero and negative values
            idf[token] = Math.log((totalDocs + 1) / (df + 1)) + 1;
        }
        return idf;
    }
    buildVector(tf, idf) {
        const vector = {};
        for (const term of Object.keys(tf)) {
            const weight = tf[term] * (idf[term] ?? 0);
            if (weight > 0) {
                vector[term] = weight;
            }
        }
        return vector;
    }
    buildVectorsFromTokens(tokensList) {
        if (tokensList.length === 0) {
            return [];
        }
        const idf = this.computeInverseDocumentFrequency(tokensList);
        return tokensList.map(tokens => this.buildVector(this.computeTermFrequency(tokens), idf));
    }
    calculateCosineSimilarity(vectorA, vectorB) {
        const keysA = Object.keys(vectorA);
        const keysB = Object.keys(vectorB);
        if (keysA.length === 0 || keysB.length === 0) {
            return 0;
        }
        let dotProduct = 0;
        let normA = 0;
        let normB = 0;
        const shorterKeys = keysA.length < keysB.length ? keysA : keysB;
        for (const key of shorterKeys) {
            const valueA = vectorA[key];
            const valueB = vectorB[key];
            if (valueA !== undefined && valueB !== undefined) {
                dotProduct += valueA * valueB;
            }
        }
        for (const key of Object.keys(vectorA)) {
            const value = vectorA[key];
            if (value !== undefined) {
                normA += value * value;
            }
        }
        for (const key of Object.keys(vectorB)) {
            const value = vectorB[key];
            if (value !== undefined) {
                normB += value * value;
            }
        }
        if (normA === 0 || normB === 0) {
            return 0;
        }
        const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
        return Math.max(0, Math.min(1, similarity));
    }
    async generateVectors(texts) {
        const tokensList = texts.map(text => this.tokenize(text));
        return this.buildVectorsFromTokens(tokensList);
    }
    async getVector(text, corpus = []) {
        const tokensList = [text, ...corpus].map(item => this.tokenize(item));
        const vectors = this.buildVectorsFromTokens(tokensList);
        return vectors[0] ?? {};
    }
    async findSimilarTexts(referenceText, candidateTexts, limit = 5, threshold = 0.3) {
        if (candidateTexts.length === 0) {
            return [];
        }
        const allTexts = [referenceText, ...candidateTexts];
        const vectors = await this.generateVectors(allTexts);
        if (vectors.length === 0) {
            return [];
        }
        const referenceVector = vectors[0];
        const results = [];
        for (let i = 0; i < candidateTexts.length; i++) {
            const candidateVector = vectors[i + 1] ?? {};
            const similarity = this.calculateCosineSimilarity(referenceVector, candidateVector);
            results.push({ text: candidateTexts[i], score: similarity });
        }
        return results
            .filter(result => result.score >= threshold)
            .sort((a, b) => b.score - a.score)
            .slice(0, limit);
    }
    async calculateTextSimilarity(textA, textB) {
        const results = await this.findSimilarTexts(textA, [textB], 1, 0);
        return results.length > 0 ? results[0].score : 0;
    }
}
exports.SimilarityEngine = SimilarityEngine;
//# sourceMappingURL=similarity-engine.js.map