gpt-tokenizer

Version:

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

363 files

Filename	Content Type	Size
../
ada.d.ts	text/typescript	94 B
ada.js	text/javascript	201 B
ada.js.map	application/json	184 B
babbage-002.d.ts	text/typescript	98 B
babbage-002.js	text/javascript	213 B
babbage-002.js.map	application/json	200 B
babbage.d.ts	text/typescript	94 B
babbage.js	text/javascript	205 B
babbage.js.map	application/json	192 B
chatgpt-4o-latest.d.ts	text/typescript	7.29 kB
chatgpt-4o-latest.js	text/javascript	1.31 kB
chatgpt-4o-latest.js.map	application/json	1.2 kB
code-cushman-001.d.ts	text/typescript	94 B
code-cushman-001.js	text/javascript	214 B
code-cushman-001.js.map	application/json	210 B
code-cushman-002.d.ts	text/typescript	94 B
code-cushman-002.js	text/javascript	214 B
code-cushman-002.js.map	application/json	210 B
code-davinci-001.d.ts	text/typescript	94 B
code-davinci-001.js	text/javascript	214 B
code-davinci-001.js.map	application/json	210 B
code-davinci-002.d.ts	text/typescript	94 B
code-davinci-002.js	text/javascript	214 B
code-davinci-002.js.map	application/json	210 B
code-davinci-edit-001.d.ts	text/typescript	94 B
code-davinci-edit-001.js	text/javascript	219 B
code-davinci-edit-001.js.map	application/json	220 B
code-search-ada-code-001.d.ts	text/typescript	94 B
code-search-ada-code-001.js	text/javascript	222 B
code-search-ada-code-001.js.map	application/json	226 B
code-search-ada-text-001.d.ts	text/typescript	94 B
code-search-ada-text-001.js	text/javascript	222 B
code-search-ada-text-001.js.map	application/json	226 B
code-search-babbage-code-001.d.ts	text/typescript	94 B
code-search-babbage-code-001.js	text/javascript	226 B
code-search-babbage-code-001.js.map	application/json	234 B
code-search-babbage-text-001.d.ts	text/typescript	94 B
code-search-babbage-text-001.js	text/javascript	226 B
code-search-babbage-text-001.js.map	application/json	234 B
codex-mini-latest.d.ts	text/typescript	7.29 kB
codex-mini-latest.js	text/javascript	1.31 kB
codex-mini-latest.js.map	application/json	1.16 kB
computer-use-preview-2025-03-11.d.ts	text/typescript	7.29 kB
computer-use-preview-2025-03-11.js	text/javascript	1.35 kB
computer-use-preview-2025-03-11.js.map	application/json	1.27 kB
computer-use-preview.d.ts	text/typescript	7.29 kB
computer-use-preview.js	text/javascript	1.33 kB
computer-use-preview.js.map	application/json	1.25 kB
curie.d.ts	text/typescript	94 B
curie.js	text/javascript	203 B
curie.js.map	application/json	188 B
cushman-codex.d.ts	text/typescript	94 B
cushman-codex.js	text/javascript	211 B
cushman-codex.js.map	application/json	204 B
dall-e-2.d.ts	text/typescript	96 B
dall-e-2.js	text/javascript	208 B
dall-e-2.js.map	application/json	194 B
dall-e-3.d.ts	text/typescript	96 B
dall-e-3.js	text/javascript	208 B
dall-e-3.js.map	application/json	194 B
davinci-002.d.ts	text/typescript	98 B
davinci-002.js	text/javascript	213 B
davinci-002.js.map	application/json	200 B
davinci-codex.d.ts	text/typescript	94 B
davinci-codex.js	text/javascript	211 B
davinci-codex.js.map	application/json	204 B
davinci.d.ts	text/typescript	94 B
davinci.js	text/javascript	205 B
davinci.js.map	application/json	192 B
gpt-3.5-0301.d.ts	text/typescript	7.29 kB
gpt-3.5-0301.js	text/javascript	1.28 kB
gpt-3.5-0301.js.map	application/json	1.21 kB
gpt-3.5-turbo-0125.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-0125.js	text/javascript	1.33 kB
gpt-3.5-turbo-0125.js.map	application/json	1.25 kB
gpt-3.5-turbo-0613.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-0613.js	text/javascript	1.31 kB
gpt-3.5-turbo-0613.js.map	application/json	1.24 kB
gpt-3.5-turbo-1106.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-1106.js	text/javascript	1.32 kB
gpt-3.5-turbo-1106.js.map	application/json	1.25 kB
gpt-3.5-turbo-16k-0613.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-16k-0613.js	text/javascript	1.32 kB
gpt-3.5-turbo-16k-0613.js.map	application/json	1.25 kB
gpt-3.5-turbo-instruct.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo-instruct.js	text/javascript	1.28 kB
gpt-3.5-turbo-instruct.js.map	application/json	1.18 kB
gpt-3.5-turbo.d.ts	text/typescript	7.29 kB
gpt-3.5-turbo.js	text/javascript	1.32 kB
gpt-3.5-turbo.js.map	application/json	1.24 kB
gpt-3.5.d.ts	text/typescript	7.29 kB
gpt-3.5.js	text/javascript	1.27 kB
gpt-3.5.js.map	application/json	1.2 kB
gpt-4-0125-preview.d.ts	text/typescript	7.29 kB
gpt-4-0125-preview.js	text/javascript	1.31 kB
gpt-4-0125-preview.js.map	application/json	1.24 kB
gpt-4-0314.d.ts	text/typescript	7.29 kB
gpt-4-0314.js	text/javascript	1.29 kB
gpt-4-0314.js.map	application/json	1.23 kB
gpt-4-0613.d.ts	text/typescript	7.29 kB
gpt-4-0613.js	text/javascript	1.32 kB
gpt-4-0613.js.map	application/json	1.25 kB
gpt-4-1106-preview.d.ts	text/typescript	7.29 kB
gpt-4-1106-preview.js	text/javascript	1.31 kB
gpt-4-1106-preview.js.map	application/json	1.24 kB
gpt-4-1106-vision-preview.d.ts	text/typescript	7.29 kB
gpt-4-1106-vision-preview.js	text/javascript	1.38 kB
gpt-4-1106-vision-preview.js.map	application/json	1.3 kB
gpt-4-32k.d.ts	text/typescript	7.29 kB
gpt-4-32k.js	text/javascript	1.29 kB
gpt-4-32k.js.map	application/json	1.23 kB
gpt-4-turbo-2024-04-09.d.ts	text/typescript	7.29 kB
gpt-4-turbo-2024-04-09.js	text/javascript	1.38 kB
gpt-4-turbo-2024-04-09.js.map	application/json	1.29 kB
gpt-4-turbo-preview.d.ts	text/typescript	7.29 kB
gpt-4-turbo-preview.js	text/javascript	1.32 kB
gpt-4-turbo-preview.js.map	application/json	1.24 kB
gpt-4-turbo.d.ts	text/typescript	7.29 kB
gpt-4-turbo.js	text/javascript	1.36 kB
gpt-4-turbo.js.map	application/json	1.27 kB
gpt-4.1-2025-04-14.d.ts	text/typescript	7.29 kB
gpt-4.1-2025-04-14.js	text/javascript	1.52 kB
gpt-4.1-2025-04-14.js.map	application/json	1.38 kB
gpt-4.1-mini-2025-04-14.d.ts	text/typescript	7.29 kB
gpt-4.1-mini-2025-04-14.js	text/javascript	1.4 kB
gpt-4.1-mini-2025-04-14.js.map	application/json	1.22 kB
gpt-4.1-mini.d.ts	text/typescript	7.29 kB
gpt-4.1-mini.js	text/javascript	1.38 kB
gpt-4.1-mini.js.map	application/json	1.19 kB
gpt-4.1-nano-2025-04-14.d.ts	text/typescript	7.29 kB
gpt-4.1-nano-2025-04-14.js	text/javascript	1.41 kB
gpt-4.1-nano-2025-04-14.js.map	application/json	1.22 kB
gpt-4.1-nano.d.ts	text/typescript	7.29 kB
gpt-4.1-nano.js	text/javascript	1.38 kB
gpt-4.1-nano.js.map	application/json	1.2 kB
gpt-4.1.d.ts	text/typescript	7.29 kB
gpt-4.1.js	text/javascript	1.5 kB
gpt-4.1.js.map	application/json	1.36 kB
gpt-4.5-preview-2025-02-27.d.ts	text/typescript	7.29 kB
gpt-4.5-preview-2025-02-27.js	text/javascript	1.38 kB
gpt-4.5-preview-2025-02-27.js.map	application/json	1.21 kB
gpt-4.5-preview.d.ts	text/typescript	7.29 kB
gpt-4.5-preview.js	text/javascript	1.36 kB
gpt-4.5-preview.js.map	application/json	1.19 kB
gpt-4.d.ts	text/typescript	7.29 kB
gpt-4.js	text/javascript	1.31 kB
gpt-4.js.map	application/json	1.24 kB
gpt-4o-2024-05-13.d.ts	text/typescript	7.29 kB
gpt-4o-2024-05-13.js	text/javascript	1.34 kB
gpt-4o-2024-05-13.js.map	application/json	1.18 kB
gpt-4o-2024-08-06.d.ts	text/typescript	7.29 kB
gpt-4o-2024-08-06.js	text/javascript	1.5 kB
gpt-4o-2024-08-06.js.map	application/json	1.37 kB
gpt-4o-2024-11-20.d.ts	text/typescript	7.29 kB
gpt-4o-2024-11-20.js	text/javascript	1.47 kB
gpt-4o-2024-11-20.js.map	application/json	1.35 kB
gpt-4o-audio-preview-2024-10-01.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview-2024-10-01.js	text/javascript	1.38 kB
gpt-4o-audio-preview-2024-10-01.js.map	application/json	1.28 kB
gpt-4o-audio-preview-2024-12-17.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview-2024-12-17.js	text/javascript	1.38 kB
gpt-4o-audio-preview-2024-12-17.js.map	application/json	1.28 kB
gpt-4o-audio-preview-2025-06-03.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview-2025-06-03.js	text/javascript	1.38 kB
gpt-4o-audio-preview-2025-06-03.js.map	application/json	1.28 kB
gpt-4o-audio-preview.d.ts	text/typescript	7.29 kB
gpt-4o-audio-preview.js	text/javascript	1.35 kB
gpt-4o-audio-preview.js.map	application/json	1.26 kB
gpt-4o-mini-2024-07-18.d.ts	text/typescript	7.29 kB
gpt-4o-mini-2024-07-18.js	text/javascript	1.4 kB
gpt-4o-mini-2024-07-18.js.map	application/json	1.22 kB
gpt-4o-mini-audio-preview-2024-12-17.d.ts	text/typescript	7.29 kB
gpt-4o-mini-audio-preview-2024-12-17.js	text/javascript	1.4 kB
gpt-4o-mini-audio-preview-2024-12-17.js.map	application/json	1.29 kB
gpt-4o-mini-audio-preview.d.ts	text/typescript	7.29 kB
gpt-4o-mini-audio-preview.js	text/javascript	1.37 kB
gpt-4o-mini-audio-preview.js.map	application/json	1.27 kB
gpt-4o-mini-realtime-preview-2024-12-17.d.ts	text/typescript	96 B
gpt-4o-mini-realtime-preview-2024-12-17.js	text/javascript	239 B
gpt-4o-mini-realtime-preview-2024-12-17.js.map	application/json	256 B
gpt-4o-mini-realtime-preview.d.ts	text/typescript	96 B
gpt-4o-mini-realtime-preview.js	text/javascript	228 B
gpt-4o-mini-realtime-preview.js.map	application/json	234 B
gpt-4o-mini-search-preview-2025-03-11.d.ts	text/typescript	7.29 kB
gpt-4o-mini-search-preview-2025-03-11.js	text/javascript	1.32 kB
gpt-4o-mini-search-preview-2025-03-11.js.map	application/json	1.14 kB
gpt-4o-mini-search-preview.d.ts	text/typescript	7.29 kB
gpt-4o-mini-search-preview.js	text/javascript	1.29 kB
gpt-4o-mini-search-preview.js.map	application/json	1.12 kB
gpt-4o-mini-transcribe.d.ts	text/typescript	96 B
gpt-4o-mini-transcribe.js	text/javascript	222 B
gpt-4o-mini-transcribe.js.map	application/json	222 B
gpt-4o-mini-tts.d.ts	text/typescript	96 B
gpt-4o-mini-tts.js	text/javascript	215 B
gpt-4o-mini-tts.js.map	application/json	208 B
gpt-4o-mini.d.ts	text/typescript	7.29 kB
gpt-4o-mini.js	text/javascript	1.38 kB
gpt-4o-mini.js.map	application/json	1.19 kB
gpt-4o-realtime-preview-2024-10-01.d.ts	text/typescript	96 B
gpt-4o-realtime-preview-2024-10-01.js	text/javascript	234 B
gpt-4o-realtime-preview-2024-10-01.js.map	application/json	246 B
gpt-4o-realtime-preview-2024-12-17.d.ts	text/typescript	96 B
gpt-4o-realtime-preview-2024-12-17.js	text/javascript	234 B
gpt-4o-realtime-preview-2024-12-17.js.map	application/json	246 B
gpt-4o-realtime-preview-2025-06-03.d.ts	text/typescript	96 B
gpt-4o-realtime-preview-2025-06-03.js	text/javascript	234 B
gpt-4o-realtime-preview-2025-06-03.js.map	application/json	246 B
gpt-4o-realtime-preview.d.ts	text/typescript	96 B
gpt-4o-realtime-preview.js	text/javascript	223 B
gpt-4o-realtime-preview.js.map	application/json	224 B
gpt-4o-search-preview-2025-03-11.d.ts	text/typescript	7.29 kB
gpt-4o-search-preview-2025-03-11.js	text/javascript	1.38 kB
gpt-4o-search-preview-2025-03-11.js.map	application/json	1.27 kB
gpt-4o-search-preview.d.ts	text/typescript	7.29 kB
gpt-4o-search-preview.js	text/javascript	1.36 kB
gpt-4o-search-preview.js.map	application/json	1.25 kB
gpt-4o-transcribe.d.ts	text/typescript	96 B
gpt-4o-transcribe.js	text/javascript	217 B
gpt-4o-transcribe.js.map	application/json	212 B
gpt-4o.d.ts	text/typescript	7.29 kB
gpt-4o.js	text/javascript	1.47 kB
gpt-4o.js.map	application/json	1.34 kB
gpt-image-1.d.ts	text/typescript	96 B
gpt-image-1.js	text/javascript	211 B
gpt-image-1.js.map	application/json	200 B
o1-2024-12-17.d.ts	text/typescript	7.29 kB
o1-2024-12-17.js	text/javascript	1.42 kB
o1-2024-12-17.js.map	application/json	1.32 kB
o1-mini-2024-09-12.d.ts	text/typescript	7.29 kB
o1-mini-2024-09-12.js	text/javascript	1.33 kB
o1-mini-2024-09-12.js.map	application/json	1.25 kB
o1-mini.d.ts	text/typescript	7.29 kB
o1-mini.js	text/javascript	1.31 kB
o1-mini.js.map	application/json	1.23 kB
o1-preview-2024-09-12.d.ts	text/typescript	7.29 kB
o1-preview-2024-09-12.js	text/javascript	1.38 kB
o1-preview-2024-09-12.js.map	application/json	1.28 kB
o1-preview.d.ts	text/typescript	7.29 kB
o1-preview.js	text/javascript	1.36 kB
o1-preview.js.map	application/json	1.26 kB
o1-pro-2025-03-19.d.ts	text/typescript	7.29 kB
o1-pro-2025-03-19.js	text/javascript	1.34 kB
o1-pro-2025-03-19.js.map	application/json	1.26 kB
o1-pro.d.ts	text/typescript	7.29 kB
o1-pro.js	text/javascript	1.32 kB
o1-pro.js.map	application/json	1.24 kB
o1.d.ts	text/typescript	7.29 kB
o1.js	text/javascript	1.4 kB
o1.js.map	application/json	1.3 kB
o3-2025-04-16.d.ts	text/typescript	7.29 kB
o3-2025-04-16.js	text/javascript	1.45 kB
o3-2025-04-16.js.map	application/json	1.34 kB
o3-mini-2025-01-31.d.ts	text/typescript	7.29 kB
o3-mini-2025-01-31.js	text/javascript	1.31 kB
o3-mini-2025-01-31.js.map	application/json	1.16 kB
o3-mini.d.ts	text/typescript	7.29 kB
o3-mini.js	text/javascript	1.29 kB
o3-mini.js.map	application/json	1.13 kB
o3-pro-2025-06-10.d.ts	text/typescript	7.29 kB
o3-pro-2025-06-10.js	text/javascript	1.34 kB
o3-pro-2025-06-10.js.map	application/json	1.26 kB
o3-pro.d.ts	text/typescript	7.29 kB
o3-pro.js	text/javascript	1.32 kB
o3-pro.js.map	application/json	1.24 kB
o3.d.ts	text/typescript	7.29 kB
o3.js	text/javascript	1.43 kB
o3.js.map	application/json	1.32 kB
o4-mini-2025-04-16.d.ts	text/typescript	7.29 kB
o4-mini-2025-04-16.js	text/javascript	1.4 kB
o4-mini-2025-04-16.js.map	application/json	1.22 kB
o4-mini.d.ts	text/typescript	7.29 kB
o4-mini.js	text/javascript	1.38 kB
o4-mini.js.map	application/json	1.2 kB
omni-moderation-2024-09-26.d.ts	text/typescript	96 B
omni-moderation-2024-09-26.js	text/javascript	226 B
omni-moderation-2024-09-26.js.map	application/json	230 B
omni-moderation-latest.d.ts	text/typescript	96 B
omni-moderation-latest.js	text/javascript	222 B
omni-moderation-latest.js.map	application/json	222 B
text-ada-001.d.ts	text/typescript	94 B
text-ada-001.js	text/javascript	210 B
text-ada-001.js.map	application/json	202 B
text-babbage-001.d.ts	text/typescript	94 B
text-babbage-001.js	text/javascript	214 B
text-babbage-001.js.map	application/json	210 B
text-curie-001.d.ts	text/typescript	94 B
text-curie-001.js	text/javascript	212 B
text-curie-001.js.map	application/json	206 B
text-davinci-001.d.ts	text/typescript	94 B
text-davinci-001.js	text/javascript	214 B
text-davinci-001.js.map	application/json	210 B
text-davinci-002.d.ts	text/typescript	94 B
text-davinci-002.js	text/javascript	214 B
text-davinci-002.js.map	application/json	210 B
text-davinci-003.d.ts	text/typescript	94 B
text-davinci-003.js	text/javascript	214 B
text-davinci-003.js.map	application/json	210 B
text-davinci-edit-001.d.ts	text/typescript	94 B
text-davinci-edit-001.js	text/javascript	219 B
text-davinci-edit-001.js.map	application/json	220 B
text-embedding-3-large.d.ts	text/typescript	98 B
text-embedding-3-large.js	text/javascript	224 B
text-embedding-3-large.js.map	application/json	222 B
text-embedding-3-small.d.ts	text/typescript	98 B
text-embedding-3-small.js	text/javascript	224 B
text-embedding-3-small.js.map	application/json	222 B
text-embedding-ada-002.d.ts	text/typescript	98 B
text-embedding-ada-002.js	text/javascript	224 B
text-embedding-ada-002.js.map	application/json	222 B
text-moderation-007.d.ts	text/typescript	96 B
text-moderation-007.js	text/javascript	219 B
text-moderation-007.js.map	application/json	216 B
text-moderation-latest.d.ts	text/typescript	96 B
text-moderation-latest.js	text/javascript	222 B
text-moderation-latest.js.map	application/json	222 B
text-moderation-stable.d.ts	text/typescript	96 B
text-moderation-stable.js	text/javascript	222 B
text-moderation-stable.js.map	application/json	222 B
text-search-ada-doc-001.d.ts	text/typescript	94 B
text-search-ada-doc-001.js	text/javascript	221 B
text-search-ada-doc-001.js.map	application/json	224 B
text-search-ada-query-001.d.ts	text/typescript	94 B
text-search-ada-query-001.js	text/javascript	223 B
text-search-ada-query-001.js.map	application/json	228 B
text-search-babbage-doc-001.d.ts	text/typescript	94 B
text-search-babbage-doc-001.js	text/javascript	225 B
text-search-babbage-doc-001.js.map	application/json	232 B
text-search-babbage-query-001.d.ts	text/typescript	94 B
text-search-babbage-query-001.js	text/javascript	227 B
text-search-babbage-query-001.js.map	application/json	236 B
text-search-curie-doc-001.d.ts	text/typescript	94 B
text-search-curie-doc-001.js	text/javascript	223 B
text-search-curie-doc-001.js.map	application/json	228 B
text-search-curie-query-001.d.ts	text/typescript	94 B
text-search-curie-query-001.js	text/javascript	225 B
text-search-curie-query-001.js.map	application/json	232 B
text-search-davinci-doc-001.d.ts	text/typescript	94 B
text-search-davinci-doc-001.js	text/javascript	225 B
text-search-davinci-doc-001.js.map	application/json	232 B
text-search-davinci-query-001.d.ts	text/typescript	94 B
text-search-davinci-query-001.js	text/javascript	227 B
text-search-davinci-query-001.js.map	application/json	236 B
text-similarity-ada-001.d.ts	text/typescript	94 B
text-similarity-ada-001.js	text/javascript	221 B
text-similarity-ada-001.js.map	application/json	224 B
text-similarity-babbage-001.d.ts	text/typescript	94 B
text-similarity-babbage-001.js	text/javascript	225 B
text-similarity-babbage-001.js.map	application/json	232 B
text-similarity-curie-001.d.ts	text/typescript	94 B
text-similarity-curie-001.js	text/javascript	223 B
text-similarity-curie-001.js.map	application/json	228 B
text-similarity-davinci-001.d.ts	text/typescript	94 B
text-similarity-davinci-001.js	text/javascript	225 B
text-similarity-davinci-001.js.map	application/json	232 B
tts-1-hd.d.ts	text/typescript	96 B
tts-1-hd.js	text/javascript	208 B
tts-1-hd.js.map	application/json	194 B
tts-1.d.ts	text/typescript	96 B
tts-1.js	text/javascript	205 B
tts-1.js.map	application/json	188 B
whisper-1.d.ts	text/typescript	96 B
whisper-1.js	text/javascript	209 B
whisper-1.js.map	application/json	196 B