wink-embeddings-small-en-50d/dist/scripts/convert-glove.js

Small English 50-dimensional word-embedding dataset compatible with wink-nlp.

#!/usr/bin/env ts-node
"use strict";
/**
 * Convert GloVe txt format (word followed by 50 floats per line) into
 * a JSON object mapping word -> number[50]. Optionally limit vocabulary size.
 *
 * Usage:
 *   npm run convert:glove -- <path-to-glove.txt> [output.json] [vocabSize]
 *
 * Example converting 6B.50d:
 *   npm run convert:glove -- ./glove.6B.50d.txt src/embeddings.json 10000
 */
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const node_fs_1 = __importDefault(require("node:fs"));
const node_path_1 = __importDefault(require("node:path"));
const node_readline_1 = __importDefault(require("node:readline"));
const [, , inputFile, outputFile = node_path_1.default.resolve(__dirname, '../src/embeddings.json'), vocabSizeArg] = process.argv;
if (!inputFile) {
    console.error('Error: Path to input .txt file is required.');
    process.exit(1);
}
const vocabSize = vocabSizeArg ? parseInt(vocabSizeArg, 10) : undefined;
if (vocabSize !== undefined && (isNaN(vocabSize) || vocabSize <= 0)) {
    console.error('Error: vocabSize must be a positive integer.');
    process.exit(1);
}
const embeddings = {};
(async () => {
    const rl = node_readline_1.default.createInterface({
        input: node_fs_1.default.createReadStream(inputFile),
        crlfDelay: Infinity,
    });
    let count = 0;
    for await (const line of rl) {
        const parts = line.trim().split(/\s+/);
        const word = parts.shift();
        if (!word)
            continue;
        const vector = parts.map(Number).slice(0, 50);
        if (vector.length !== 50 || vector.some((n) => Number.isNaN(n)))
            continue;
        embeddings[word] = vector;
        count += 1;
        if (vocabSize && count >= vocabSize) {
            break;
        }
    }
    // Ensure directory exists
    node_fs_1.default.mkdirSync(node_path_1.default.dirname(outputFile), { recursive: true });
    node_fs_1.default.writeFileSync(outputFile, JSON.stringify(embeddings, null, 2));
    console.log(`Wrote ${count} embeddings to ${outputFile}`);
})();