algoritms.ai/frequency.js

The Algoritms.AI build tools to create better and lightweight AI models with Acuuracy, Context, Frequency, Memory, Maths in Natural Language, Natural Language Processing, Probablities and Vectors.

function tokenize(text) {
    return text.toLowerCase().match(/\b[a-zA-Z]+(?:-[a-zA-Z]+)?\b/g) || [];
}

function sentenceSplit(text) {
    return text.match(/[^.!?]+[.!?]/g) || [text];
}

function computeTF(words) {
    let tf = {};
    words.forEach(word => tf[word] = (tf[word] || 0) + 1);
    let totalWords = words.length;
    Object.keys(tf).forEach(word => tf[word] /= totalWords);
    return tf;
}

function computeIDF(sentences) {
    let idf = {};
    let totalDocs = sentences.length;
    sentences.forEach(sentence => {
        let words = new Set(tokenize(sentence));
        words.forEach(word => idf[word] = (idf[word] || 0) + 1);
    });
    Object.keys(idf).forEach(word => idf[word] = Math.log(totalDocs / (idf[word] + 1)));
    return idf;
}

function computeTFIDF(sentences, tf, idf, keywordBoost = {}) {
    let sentenceScores = {};
    sentences.forEach(sentence => {
        let words = tokenize(sentence);
        let score = words.reduce((sum, word) => sum + ((tf[word] || 0) * (idf[word] || 0) * (keywordBoost[word] || 1)), 0);
        sentenceScores[sentence] = score;
    });
    return sentenceScores;
}

function cosineSimilarity(sentence1, sentence2) {
    let words1 = tokenize(sentence1), words2 = tokenize(sentence2);
    let allWords = new Set([...words1, ...words2]);
    let vec1 = Array.from(allWords).map(word => words1.includes(word) ? 1 : 0);
    let vec2 = Array.from(allWords).map(word => words2.includes(word) ? 1 : 0);
    let dotProduct = vec1.reduce((sum, val, i) => sum + val * vec2[i], 0);
    let magnitude1 = Math.sqrt(vec1.reduce((sum, val) => sum + val ** 2, 0));
    let magnitude2 = Math.sqrt(vec2.reduce((sum, val) => sum + val ** 2, 0));
    return dotProduct / (magnitude1 * magnitude2 || 1);
}

function removeSimilarSentences(sentences, threshold = 0.6) {
    let uniqueSentences = [];
    sentences.forEach(sentence => {
        if (!uniqueSentences.some(existing => cosineSimilarity(sentence, existing) > threshold)) {
            uniqueSentences.push(sentence);
        }
    });
    return uniqueSentences;
}

function compressSentence(sentence) {
    return sentence.replace(/\b(the|is|a|that|which|can|and|of|with|to|on)\b/gi, "").trim();
}

function summarizeText(text, compressionRatio = 0.4) {
    let sentences = sentenceSplit(text);
    let words = tokenize(text);

    let keywordBoost = { "ai": 2, "machines": 1.5, "intelligence": 2, "human": 1.5, "learn": 1.3, "decisions": 1.3 };

    let tf = computeTF(words);
    let idf = computeIDF(sentences);
    let sentenceScores = computeTFIDF(sentences, tf, idf, keywordBoost);

    let sortedSentences = Object.entries(sentenceScores).sort((a, b) => b[1] - a[1]);

    let summarySize = Math.max(2, Math.ceil(sentences.length * compressionRatio));
    let selectedSentences = sortedSentences.slice(0, summarySize).map(entry => entry[0]);

    let uniqueSentences = removeSimilarSentences(selectedSentences);

    let finalSummary = uniqueSentences.map(compressSentence);

    return finalSummary.join(". ");
}
// USAGE: console.log("\nSummarized Text:\n", summarizeText(test_text));