kitoken/package.json

Fast tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization

{
    "name": "kitoken",
    "displayName": "Kitoken",
    "description": "Fast tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization",
    "version": "0.11.0",
    "author": {
        "name": "Christian Sdunek",
        "email": "me@systemcluster.me"
    },
    "license": "BSD-2-Clause",
    "homepage": "https://kitoken.dev",
    "repository": {
        "type": "git",
        "url": "github:Systemcluster/kitoken"
    },
    "readme": "README.md",
    "keywords": [
        "tokenizer",
        "nlp",
        "bpe",
        "unigram",
        "wordpiece"
    ],
    "engines": {
        "node": ">=18"
    },
    "type": "module",
    "main": "./dist/index.js",
    "types": "./dist/index.d.ts",
    "exports": {
        ".": {
            "types": "./dist/index.d.ts",
            "default": "./dist/index.js"
        },
        "./node": {
            "types": "./dist/node.d.ts",
            "default": "./dist/node.cjs"
        },
        "./web": {
            "types": "./dist/web.d.ts",
            "default": "./dist/web.js"
        },
        "./minimal": {
            "types": "./dist/minimal.d.ts",
            "default": "./dist/minimal.js"
        },
        "./full": {
            "types": "./dist/full.d.ts",
            "default": "./dist/full.js"
        }
    },
    "files": [
        "./dist/*"
    ],
    "scripts": {
        "build:node": "cross-env RUSTFLAGS=\"-Ctarget-feature=+crt-static -Clto -Cembed-bitcode=yes\" wasm-pack build --release --no-pack --target nodejs --out-dir dist --out-name node --features full -Zbuild-std=panic_abort,core,alloc,std && move-file dist/node.js dist/node.cjs",
        "build:web-all": "cross-env RUSTFLAGS=\"-Ctarget-feature=+crt-static,+sign-ext,+bulk-memory,+nontrapping-fptoint,+multivalue,+reference-types -Clto -Cembed-bitcode=yes -Zwasm-c-abi=spec\" concurrently -m 1 'npm:build:web' 'npm:build:bundler' 'npm:build:minimal' 'npm:build:full'",
        "build:web": "wasm-pack build --release --no-pack --target web --out-dir dist --out-name web --features default,convert,inspect -Zbuild-std=panic_abort,core,alloc,std",
        "build:bundler": "wasm-pack build --release --no-pack --target bundler --out-dir dist --out-name index --features default,convert,inspect -Zbuild-std=panic_abort,core,alloc,std",
        "build:minimal": "wasm-pack build --release --no-pack --target bundler --out-dir dist --out-name minimal --features default -Zbuild-std=panic_abort,core,alloc,std",
        "build:full": "wasm-pack build --release --no-pack --target bundler --out-dir dist --out-name full --features full -Zbuild-std=panic_abort,core,alloc,std",
        "build": "cross-env RUSTC_BOOTSTRAP=1 concurrently -m 1 'npm:build:node' 'npm:build:web-all'",
        "prepare": "cross-env RUSTC_BOOTSTRAP=1 concurrently -m 1 'npm:build:node' 'npm:build:web-all'"
    },
    "devDependencies": {
        "concurrently": "^9.2.1",
        "cross-env": "^10.1.0",
        "move-file-cli": "^3.0.0",
        "wasm-pack": "^0.14.0"
    }
}