bpe-tokenizer-encoding/cli.js

A simple JavaScript implementation of Byte Pair Encoding (BPE)

#!/usr/bin/env node
const { bytePairEncoding } = require("./index");

const input = process.argv[2] || "low_low_lower";
const merges = parseInt(process.argv[3] || "5", 10);

const result = bytePairEncoding(input, merges, true);
console.log("\nFinal Tokens:", result);