algoritms.ai/nlp.js

The Algoritms.AI build tools to create better and lightweight AI models with Acuuracy, Context, Frequency, Memory, Maths in Natural Language, Natural Language Processing, Probablities and Vectors.

const fs = require("fs");
const path = require("path");

// Probability function for weighted random selection
function weightedRandomSelection(weightedOutcomes) {
    let totalWeight = weightedOutcomes.reduce((sum, item) => sum + item.weight, 0);
    let randomNum = Math.random() * totalWeight;
    let currentSum = 0;

    for (let item of weightedOutcomes) {
        currentSum += item.weight;
        if (randomNum <= currentSum) {
            return item.word;
        }
    }
}

// Load NLP dataset safely
function loadDataset(filePath) {
    if (!fs.existsSync(filePath)) {
        console.error(`Dataset not found: ${filePath}`);
        return {};
    }

    const data = fs.readFileSync(filePath, "utf8").split("\n");
    let mappings = {};

    // Parse dataset format: [hoq, how] -> how
    const regex = /\[([^\]]+)\]\s*->\s*(\w+)/;

    for (let line of data) {
        let match = line.match(regex);
        if (match) {
            let words = match[1].split(",").map(w => w.trim().toLowerCase());
            let correctWord = match[2].trim().toLowerCase();
            words.forEach(word => (mappings[word] = correctWord));
        }
    }

    return mappings;
}

// Core NLP correction function
function correctTextBackend(text, datasetPath) {
    const wordMappings = loadDataset(datasetPath);
    if (Object.keys(wordMappings).length === 0) return text; // Return original if no dataset

    // Tokenize words while preserving spaces & punctuation
    let tokens = text.match(/\b\w+\b|\s+|[^\w\s]/g) || [];

    return tokens.map(token => {
        let lowerToken = token.toLowerCase();

        if (wordMappings[lowerToken]) {
            let correction = weightedRandomSelection([
                { word: wordMappings[lowerToken], weight: 99 },
                { word: token, weight: 1 }
            ]);

            // Preserve case
            return token[0] === token[0].toUpperCase()
                ? correction.charAt(0).toUpperCase() + correction.slice(1)
                : correction;
        }
        return token; // Keep original if no match
    }).join("");
}

// Frontend CLI for user input
function correctTextFrontend() {
    const readline = require("readline");
    const rl = readline.createInterface({
        input: process.stdin,
        output: process.stdout
    });

    const datasetPath = path.join(__dirname, "../data/nlp.txt");

    rl.question("AI: Enter text for correction 😋 ", (userInput) => {
        console.log("\nCorrected Text: ", correctTextBackend(userInput, datasetPath));
        rl.close();
    });
}

// Run test with sample text
const datasetPath = path.join(__dirname, "../data/nlp.txt");
console.log(correctTextBackend("pie", datasetPath));

// Export functions
module.exports = { correctTextBackend, correctTextFrontend };