UNPKG

tokenx

Version:

GPT token estimation and context size utilities without a full tokenizer

github.com/johannschopplich/tokenx

johannschopplich/tokenx

66 lines (65 loc) • 1.61 kB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
{ "name": "tokenx", "type": "module", "version": "0.4.1", "packageManager": "pnpm@9.14.4", "description": "GPT token estimation and context size utilities without a full tokenizer", "author": "Johann Schopplich <hello@johannschopplich.com>", "license": "MIT", "homepage": "https://github.com/johannschopplich/tokenx#readme", "repository": { "type": "git", "url": "git+https://github.com/johannschopplich/tokenx.git" }, "bugs": { "url": "https://github.com/johannschopplich/tokenx/issues" }, "keywords": [ "ai", "gpt", "token", "tiktoken" ], "sideEffects": false, "exports": { ".": { "types": "./dist/index.d.mts", "import": { "types": "./dist/index.d.mts", "default": "./dist/index.mjs" }, "require": { "types": "./dist/index.d.cts", "default": "./dist/index.cjs" }, "default": "./dist/index.mjs" } }, "main": "./dist/index.cjs", "module": "./dist/index.mjs", "types": "./dist/index.d.ts", "files": [ "dist" ], "scripts": { "build": "unbuild", "docs:generate": "tsx scripts/generateTable.ts", "dev": "unbuild --stub", "lint": "eslint .", "lint:fix": "eslint . --fix", "test": "vitest", "test:types": "tsc --noEmit", "release": "bumpp" }, "devDependencies": { "@antfu/eslint-config": "^3.11.2", "@types/node": "^22.10.1", "bumpp": "^9.8.1", "eslint": "^9.15.0", "gpt-tokenizer": "^2.7.0", "tsx": "^4.19.2", "typescript": "^5.7.2", "unbuild": "^3.0.0-rc.11", "vitest": "^2.1.6" } }