gpt-tokenizer

Version:

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

363 files

Filename	Content Type	Size
../
ada.d.ts	text/typescript	94 B
ada.js	text/javascript	1.28 kB
ada.js.map	application/json	186 B
babbage-002.d.ts	text/typescript	98 B
babbage-002.js	text/javascript	1.3 kB
babbage-002.js.map	application/json	202 B
babbage.d.ts	text/typescript	94 B
babbage.js	text/javascript	1.29 kB
babbage.js.map	application/json	194 B
chatgpt-4o-latest.d.ts	text/typescript	7.29 kB
chatgpt-4o-latest.js	text/javascript	2.95 kB
chatgpt-4o-latest.js.map	application/json	1.14 kB
code-cushman-001.d.ts	text/typescript	94 B
code-cushman-001.js	text/javascript	1.29 kB
code-cushman-001.js.map	application/json	212 B
code-cushman-002.d.ts	text/typescript	94 B
code-cushman-002.js	text/javascript	1.29 kB
code-cushman-002.js.map	application/json	212 B
code-davinci-001.d.ts	text/typescript	94 B
code-davinci-001.js	text/javascript	1.29 kB
code-davinci-001.js.map	application/json	212 B
code-davinci-002.d.ts	text/typescript	94 B
code-davinci-002.js	text/javascript	1.29 kB
code-davinci-002.js.map	application/json	212 B
code-davinci-edit-001.d.ts	text/typescript	94 B
code-davinci-edit-001.js	text/javascript	1.3 kB
code-davinci-edit-001.js.map	application/json	222 B
code-search-ada-code-001.d.ts	text/typescript	94 B
code-search-ada-code-001.js	text/javascript	1.3 kB
code-search-ada-code-001.js.map	application/json	228 B
code-search-ada-text-001.d.ts	text/typescript	94 B
code-search-ada-text-001.js	text/javascript	1.3 kB
code-search-ada-text-001.js.map	application/json	228 B
code-search-babbage-code-001.d.ts	text/typescript	94 B
code-search-babbage-code-001.js	text/javascript	1.31 kB
code-search-babbage-code-001.js.map	application/json	236 B
code-search-babbage-text-001.d.ts	text/typescript	94 B
code-search-babbage-text-001.js	text/javascript	1.31 kB
code-search-babbage-text-001.js.map	application/json	236 B
codex-mini-latest.d.ts	text/typescript	7.29 kB
codex-mini-latest.js	text/javascript	2.95 kB
codex-mini-latest.js.map	application/json	1.1 kB
computer-use-preview-2025-03-11.d.ts	text/typescript	7.29 kB
computer-use-preview-2025-03-11.js	text/javascript	2.99 kB
computer-use-preview-2025-03-11.js.map	application/json	1.21 kB
computer-use-preview.d.ts	text/typescript	7.29 kB
computer-use-preview.js	text/javascript	2.97 kB
computer-use-preview.js.map	application/json	1.19 kB
curie.d.ts	text/typescript	94 B
curie.js	text/javascript	1.28 kB
curie.js.map	application/json	190 B
cushman-codex.d.ts	text/typescript	94 B
cushman-codex.js	text/javascript	1.29 kB
cushman-codex.js.map	application/json	206 B
dall-e-2.d.ts	text/typescript	96 B
dall-e-2.js	text/javascript	1.29 kB
dall-e-2.js.map	application/json	196 B
dall-e-3.d.ts	text/typescript	96 B
dall-e-3.js	text/javascript	1.29 kB
dall-e-3.js.map	application/json	196 B
davinci-002.d.ts	text/typescript	98 B
davinci-002.js	text/javascript	1.3 kB
davinci-002.js.map	application/json	202 B
davinci-codex.d.ts	text/typescript	94 B
davinci-codex.js	text/javascript	1.29 kB
davinci-codex.js.map	application/json	206 B
davinci.d.ts	text/typescript	94 B
davinci.js	text/javascript	1.29 kB
davinci.js.map	application/json	194 B
gpt-3.5-0301.d.ts	text/typescript	7.29 kB
gpt-3.5-0301.js	text/javascript	2.92 kB
gpt-3.5-0301.js.map	application/json	1.16 kB
gpt-3.5-turbo-0125.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-0125.js	text/javascript	2.97 kB
gpt-3.5-turbo-0125.js.map	application/json	1.19 kB
gpt-3.5-turbo-0613.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-0613.js	text/javascript	2.95 kB
gpt-3.5-turbo-0613.js.map	application/json	1.18 kB
gpt-3.5-turbo-1106.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-1106.js	text/javascript	2.96 kB
gpt-3.5-turbo-1106.js.map	application/json	1.19 kB
gpt-3.5-turbo-16k-0613.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-16k-0613.js	text/javascript	2.97 kB
gpt-3.5-turbo-16k-0613.js.map	application/json	1.19 kB
gpt-3.5-turbo-instruct.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-instruct.js	text/javascript	2.92 kB
gpt-3.5-turbo-instruct.js.map	application/json	1.12 kB
gpt-3.5-turbo.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo.js	text/javascript	2.96 kB
gpt-3.5-turbo.js.map	application/json	1.18 kB
gpt-3.5.d.ts	text/typescript	7.29 kB
gpt-3.5.js	text/javascript	2.91 kB
gpt-3.5.js.map	application/json	1.15 kB
gpt-4-0125-preview.d.ts	text/typescript	7.29 kB
gpt-4-0125-preview.js	text/javascript	2.96 kB
gpt-4-0125-preview.js.map	application/json	1.18 kB
gpt-4-0314.d.ts	text/typescript	7.29 kB
gpt-4-0314.js	text/javascript	2.94 kB
gpt-4-0314.js.map	application/json	1.17 kB
gpt-4-0613.d.ts	text/typescript	7.29 kB
gpt-4-0613.js	text/javascript	2.96 kB
gpt-4-0613.js.map	application/json	1.19 kB
gpt-4-1106-preview.d.ts	text/typescript	7.29 kB
gpt-4-1106-preview.js	text/javascript	2.96 kB
gpt-4-1106-preview.js.map	application/json	1.18 kB
gpt-4-1106-vision-preview.d.ts	text/typescript	7.29 kB
gpt-4-1106-vision-preview.js	text/javascript	3.02 kB
gpt-4-1106-vision-preview.js.map	application/json	1.24 kB
gpt-4-32k.d.ts	text/typescript	7.29 kB
gpt-4-32k.js	text/javascript	2.94 kB
gpt-4-32k.js.map	application/json	1.17 kB
gpt-4-turbo-2024-04-09.d.ts	text/typescript	7.29 kB
gpt-4-turbo-2024-04-09.js	text/javascript	3.02 kB
gpt-4-turbo-2024-04-09.js.map	application/json	1.24 kB
gpt-4-turbo-preview.d.ts	text/typescript	7.29 kB
gpt-4-turbo-preview.js	text/javascript	2.96 kB
gpt-4-turbo-preview.js.map	application/json	1.19 kB
gpt-4-turbo.d.ts	text/typescript	7.29 kB
gpt-4-turbo.js	text/javascript	3 kB
gpt-4-turbo.js.map	application/json	1.21 kB
gpt-4.1-2025-04-14.d.ts	text/typescript	7.29 kB
gpt-4.1-2025-04-14.js	text/javascript	3.16 kB
gpt-4.1-2025-04-14.js.map	application/json	1.32 kB
gpt-4.1-mini-2025-04-14.d.ts	text/typescript	7.29 kB
gpt-4.1-mini-2025-04-14.js	text/javascript	3.04 kB
gpt-4.1-mini-2025-04-14.js.map	application/json	1.16 kB
gpt-4.1-mini.d.ts	text/typescript	7.29 kB
gpt-4.1-mini.js	text/javascript	3.02 kB
gpt-4.1-mini.js.map	application/json	1.14 kB
gpt-4.1-nano-2025-04-14.d.ts	text/typescript	7.29 kB
gpt-4.1-nano-2025-04-14.js	text/javascript	3.05 kB
gpt-4.1-nano-2025-04-14.js.map	application/json	1.16 kB
gpt-4.1-nano.d.ts	text/typescript	7.29 kB
gpt-4.1-nano.js	text/javascript	3.02 kB
gpt-4.1-nano.js.map	application/json	1.14 kB
gpt-4.1.d.ts	text/typescript	7.29 kB
gpt-4.1.js	text/javascript	3.14 kB
gpt-4.1.js.map	application/json	1.3 kB
gpt-4.5-preview-2025-02-27.d.ts	text/typescript	7.29 kB
gpt-4.5-preview-2025-02-27.js	text/javascript	3.02 kB
gpt-4.5-preview-2025-02-27.js.map	application/json	1.15 kB
gpt-4.5-preview.d.ts	text/typescript	7.29 kB
gpt-4.5-preview.js	text/javascript	3 kB
gpt-4.5-preview.js.map	application/json	1.13 kB
gpt-4.d.ts	text/typescript	7.29 kB
gpt-4.js	text/javascript	2.95 kB
gpt-4.js.map	application/json	1.18 kB
gpt-4o-2024-05-13.d.ts	text/typescript	7.29 kB
gpt-4o-2024-05-13.js	text/javascript	2.98 kB
gpt-4o-2024-05-13.js.map	application/json	1.13 kB
gpt-4o-2024-08-06.d.ts	text/typescript	7.29 kB
gpt-4o-2024-08-06.js	text/javascript	3.14 kB
gpt-4o-2024-08-06.js.map	application/json	1.31 kB
gpt-4o-2024-11-20.d.ts	text/typescript	7.29 kB
gpt-4o-2024-11-20.js	text/javascript	3.11 kB
gpt-4o-2024-11-20.js.map	application/json	1.29 kB
gpt-4o-audio-preview-2024-10-01.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview-2024-10-01.js	text/javascript	3.02 kB
gpt-4o-audio-preview-2024-10-01.js.map	application/json	1.22 kB
gpt-4o-audio-preview-2024-12-17.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview-2024-12-17.js	text/javascript	3.02 kB
gpt-4o-audio-preview-2024-12-17.js.map	application/json	1.22 kB
gpt-4o-audio-preview-2025-06-03.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview-2025-06-03.js	text/javascript	3.02 kB
gpt-4o-audio-preview-2025-06-03.js.map	application/json	1.22 kB
gpt-4o-audio-preview.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview.js	text/javascript	2.99 kB
gpt-4o-audio-preview.js.map	application/json	1.2 kB
gpt-4o-mini-2024-07-18.d.ts	text/typescript	7.29 kB
gpt-4o-mini-2024-07-18.js	text/javascript	3.04 kB
gpt-4o-mini-2024-07-18.js.map	application/json	1.16 kB
gpt-4o-mini-audio-preview-2024-12-17.d.ts	text/typescript	7.29 kB
gpt-4o-mini-audio-preview-2024-12-17.js	text/javascript	3.04 kB
gpt-4o-mini-audio-preview-2024-12-17.js.map	application/json	1.23 kB
gpt-4o-mini-audio-preview.d.ts	text/typescript	7.29 kB
gpt-4o-mini-audio-preview.js	text/javascript	3.01 kB
gpt-4o-mini-audio-preview.js.map	application/json	1.21 kB
gpt-4o-mini-realtime-preview-2024-12-17.d.ts	text/typescript	96 B
gpt-4o-mini-realtime-preview-2024-12-17.js	text/javascript	1.32 kB
gpt-4o-mini-realtime-preview-2024-12-17.js.map	application/json	258 B
gpt-4o-mini-realtime-preview.d.ts	text/typescript	96 B
gpt-4o-mini-realtime-preview.js	text/javascript	1.31 kB
gpt-4o-mini-realtime-preview.js.map	application/json	236 B
gpt-4o-mini-search-preview-2025-03-11.d.ts	text/typescript	7.29 kB
gpt-4o-mini-search-preview-2025-03-11.js	text/javascript	2.96 kB
gpt-4o-mini-search-preview-2025-03-11.js.map	application/json	1.08 kB
gpt-4o-mini-search-preview.d.ts	text/typescript	7.29 kB
gpt-4o-mini-search-preview.js	text/javascript	2.94 kB
gpt-4o-mini-search-preview.js.map	application/json	1.06 kB
gpt-4o-mini-transcribe.d.ts	text/typescript	96 B
gpt-4o-mini-transcribe.js	text/javascript	1.3 kB
gpt-4o-mini-transcribe.js.map	application/json	224 B
gpt-4o-mini-tts.d.ts	text/typescript	96 B
gpt-4o-mini-tts.js	text/javascript	1.3 kB
gpt-4o-mini-tts.js.map	application/json	210 B
gpt-4o-mini.d.ts	text/typescript	7.29 kB
gpt-4o-mini.js	text/javascript	3.02 kB
gpt-4o-mini.js.map	application/json	1.13 kB
gpt-4o-realtime-preview-2024-10-01.d.ts	text/typescript	96 B
gpt-4o-realtime-preview-2024-10-01.js	text/javascript	1.32 kB
gpt-4o-realtime-preview-2024-10-01.js.map	application/json	248 B
gpt-4o-realtime-preview-2024-12-17.d.ts	text/typescript	96 B
gpt-4o-realtime-preview-2024-12-17.js	text/javascript	1.32 kB
gpt-4o-realtime-preview-2024-12-17.js.map	application/json	248 B
gpt-4o-realtime-preview-2025-06-03.d.ts	text/typescript	96 B
gpt-4o-realtime-preview-2025-06-03.js	text/javascript	1.32 kB
gpt-4o-realtime-preview-2025-06-03.js.map	application/json	248 B
gpt-4o-realtime-preview.d.ts	text/typescript	96 B
gpt-4o-realtime-preview.js	text/javascript	1.31 kB
gpt-4o-realtime-preview.js.map	application/json	226 B
gpt-4o-search-preview-2025-03-11.d.ts	text/typescript	7.29 kB
gpt-4o-search-preview-2025-03-11.js	text/javascript	3.02 kB
gpt-4o-search-preview-2025-03-11.js.map	application/json	1.22 kB
gpt-4o-search-preview.d.ts	text/typescript	7.29 kB
gpt-4o-search-preview.js	text/javascript	3 kB
gpt-4o-search-preview.js.map	application/json	1.19 kB
gpt-4o-transcribe.d.ts	text/typescript	96 B
gpt-4o-transcribe.js	text/javascript	1.3 kB
gpt-4o-transcribe.js.map	application/json	214 B
gpt-4o.d.ts	text/typescript	7.29 kB
gpt-4o.js	text/javascript	3.11 kB
gpt-4o.js.map	application/json	1.28 kB
gpt-image-1.d.ts	text/typescript	96 B
gpt-image-1.js	text/javascript	1.29 kB
gpt-image-1.js.map	application/json	202 B
o1-2024-12-17.d.ts	text/typescript	7.29 kB
o1-2024-12-17.js	text/javascript	3.06 kB
o1-2024-12-17.js.map	application/json	1.26 kB
o1-mini-2024-09-12.d.ts	text/typescript	7.29 kB
o1-mini-2024-09-12.js	text/javascript	2.97 kB
o1-mini-2024-09-12.js.map	application/json	1.19 kB
o1-mini.d.ts	text/typescript	7.29 kB
o1-mini.js	text/javascript	2.95 kB
o1-mini.js.map	application/json	1.17 kB
o1-preview-2024-09-12.d.ts	text/typescript	7.29 kB
o1-preview-2024-09-12.js	text/javascript	3.02 kB
o1-preview-2024-09-12.js.map	application/json	1.23 kB
o1-preview.d.ts	text/typescript	7.29 kB
o1-preview.js	text/javascript	3 kB
o1-preview.js.map	application/json	1.2 kB
o1-pro-2025-03-19.d.ts	text/typescript	7.29 kB
o1-pro-2025-03-19.js	text/javascript	2.98 kB
o1-pro-2025-03-19.js.map	application/json	1.2 kB
o1-pro.d.ts	text/typescript	7.29 kB
o1-pro.js	text/javascript	2.96 kB
o1-pro.js.map	application/json	1.18 kB
o1.d.ts	text/typescript	7.29 kB
o1.js	text/javascript	3.04 kB
o1.js.map	application/json	1.24 kB
o3-2025-04-16.d.ts	text/typescript	7.29 kB
o3-2025-04-16.js	text/javascript	3.09 kB
o3-2025-04-16.js.map	application/json	1.29 kB
o3-mini-2025-01-31.d.ts	text/typescript	7.29 kB
o3-mini-2025-01-31.js	text/javascript	2.95 kB
o3-mini-2025-01-31.js.map	application/json	1.1 kB
o3-mini.d.ts	text/typescript	7.29 kB
o3-mini.js	text/javascript	2.93 kB
o3-mini.js.map	application/json	1.07 kB
o3-pro-2025-06-10.d.ts	text/typescript	7.29 kB
o3-pro-2025-06-10.js	text/javascript	2.98 kB
o3-pro-2025-06-10.js.map	application/json	1.2 kB
o3-pro.d.ts	text/typescript	7.29 kB
o3-pro.js	text/javascript	2.96 kB
o3-pro.js.map	application/json	1.18 kB
o3.d.ts	text/typescript	7.29 kB
o3.js	text/javascript	3.07 kB
o3.js.map	application/json	1.26 kB
o4-mini-2025-04-16.d.ts	text/typescript	7.29 kB
o4-mini-2025-04-16.js	text/javascript	3.04 kB
o4-mini-2025-04-16.js.map	application/json	1.16 kB
o4-mini.d.ts	text/typescript	7.29 kB
o4-mini.js	text/javascript	3.02 kB
o4-mini.js.map	application/json	1.14 kB
omni-moderation-2024-09-26.d.ts	text/typescript	96 B
omni-moderation-2024-09-26.js	text/javascript	1.31 kB
omni-moderation-2024-09-26.js.map	application/json	232 B
omni-moderation-latest.d.ts	text/typescript	96 B
omni-moderation-latest.js	text/javascript	1.3 kB
omni-moderation-latest.js.map	application/json	224 B
text-ada-001.d.ts	text/typescript	94 B
text-ada-001.js	text/javascript	1.29 kB
text-ada-001.js.map	application/json	204 B
text-babbage-001.d.ts	text/typescript	94 B
text-babbage-001.js	text/javascript	1.29 kB
text-babbage-001.js.map	application/json	212 B
text-curie-001.d.ts	text/typescript	94 B
text-curie-001.js	text/javascript	1.29 kB
text-curie-001.js.map	application/json	208 B
text-davinci-001.d.ts	text/typescript	94 B
text-davinci-001.js	text/javascript	1.29 kB
text-davinci-001.js.map	application/json	212 B
text-davinci-002.d.ts	text/typescript	94 B
text-davinci-002.js	text/javascript	1.29 kB
text-davinci-002.js.map	application/json	212 B
text-davinci-003.d.ts	text/typescript	94 B
text-davinci-003.js	text/javascript	1.29 kB
text-davinci-003.js.map	application/json	212 B
text-davinci-edit-001.d.ts	text/typescript	94 B
text-davinci-edit-001.js	text/javascript	1.3 kB
text-davinci-edit-001.js.map	application/json	222 B
text-embedding-3-large.d.ts	text/typescript	98 B
text-embedding-3-large.js	text/javascript	1.31 kB
text-embedding-3-large.js.map	application/json	224 B
text-embedding-3-small.d.ts	text/typescript	98 B
text-embedding-3-small.js	text/javascript	1.31 kB
text-embedding-3-small.js.map	application/json	224 B
text-embedding-ada-002.d.ts	text/typescript	98 B
text-embedding-ada-002.js	text/javascript	1.31 kB
text-embedding-ada-002.js.map	application/json	224 B
text-moderation-007.d.ts	text/typescript	96 B
text-moderation-007.js	text/javascript	1.3 kB
text-moderation-007.js.map	application/json	218 B
text-moderation-latest.d.ts	text/typescript	96 B
text-moderation-latest.js	text/javascript	1.3 kB
text-moderation-latest.js.map	application/json	224 B
text-moderation-stable.d.ts	text/typescript	96 B
text-moderation-stable.js	text/javascript	1.3 kB
text-moderation-stable.js.map	application/json	224 B
text-search-ada-doc-001.d.ts	text/typescript	94 B
text-search-ada-doc-001.js	text/javascript	1.3 kB
text-search-ada-doc-001.js.map	application/json	226 B
text-search-ada-query-001.d.ts	text/typescript	94 B
text-search-ada-query-001.js	text/javascript	1.3 kB
text-search-ada-query-001.js.map	application/json	230 B
text-search-babbage-doc-001.d.ts	text/typescript	94 B
text-search-babbage-doc-001.js	text/javascript	1.31 kB
text-search-babbage-doc-001.js.map	application/json	234 B
text-search-babbage-query-001.d.ts	text/typescript	94 B
text-search-babbage-query-001.js	text/javascript	1.31 kB
text-search-babbage-query-001.js.map	application/json	238 B
text-search-curie-doc-001.d.ts	text/typescript	94 B
text-search-curie-doc-001.js	text/javascript	1.3 kB
text-search-curie-doc-001.js.map	application/json	230 B
text-search-curie-query-001.d.ts	text/typescript	94 B
text-search-curie-query-001.js	text/javascript	1.31 kB
text-search-curie-query-001.js.map	application/json	234 B
text-search-davinci-doc-001.d.ts	text/typescript	94 B
text-search-davinci-doc-001.js	text/javascript	1.31 kB
text-search-davinci-doc-001.js.map	application/json	234 B
text-search-davinci-query-001.d.ts	text/typescript	94 B
text-search-davinci-query-001.js	text/javascript	1.31 kB
text-search-davinci-query-001.js.map	application/json	238 B
text-similarity-ada-001.d.ts	text/typescript	94 B
text-similarity-ada-001.js	text/javascript	1.3 kB
text-similarity-ada-001.js.map	application/json	226 B
text-similarity-babbage-001.d.ts	text/typescript	94 B
text-similarity-babbage-001.js	text/javascript	1.31 kB
text-similarity-babbage-001.js.map	application/json	234 B
text-similarity-curie-001.d.ts	text/typescript	94 B
text-similarity-curie-001.js	text/javascript	1.3 kB
text-similarity-curie-001.js.map	application/json	230 B
text-similarity-davinci-001.d.ts	text/typescript	94 B
text-similarity-davinci-001.js	text/javascript	1.31 kB
text-similarity-davinci-001.js.map	application/json	234 B
tts-1-hd.d.ts	text/typescript	96 B
tts-1-hd.js	text/javascript	1.29 kB
tts-1-hd.js.map	application/json	196 B
tts-1.d.ts	text/typescript	96 B
tts-1.js	text/javascript	1.29 kB
tts-1.js.map	application/json	190 B
whisper-1.d.ts	text/typescript	96 B
whisper-1.js	text/javascript	1.29 kB
whisper-1.js.map	application/json	198 B