gpt-tokenizer

Version:

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

121 files

Filename	Content Type	Size
../
ada.ts	text/typescript	168 B
babbage-002.ts	text/typescript	172 B
babbage.ts	text/typescript	168 B
chatgpt-4o-latest.ts	text/typescript	1.26 kB
code-cushman-001.ts	text/typescript	168 B
code-cushman-002.ts	text/typescript	168 B
code-davinci-001.ts	text/typescript	168 B
code-davinci-002.ts	text/typescript	168 B
code-davinci-edit-001.ts	text/typescript	168 B
code-search-ada-code-001.ts	text/typescript	168 B
code-search-ada-text-001.ts	text/typescript	168 B
code-search-babbage-code-001.ts	text/typescript	168 B
code-search-babbage-text-001.ts	text/typescript	168 B
codex-mini-latest.ts	text/typescript	1.27 kB
computer-use-preview-2025-03-11.ts	text/typescript	1.29 kB
computer-use-preview.ts	text/typescript	1.28 kB
curie.ts	text/typescript	168 B
cushman-codex.ts	text/typescript	168 B
dall-e-2.ts	text/typescript	170 B
dall-e-3.ts	text/typescript	170 B
davinci-002.ts	text/typescript	172 B
davinci-codex.ts	text/typescript	168 B
davinci.ts	text/typescript	168 B
gpt-3.5-0301.ts	text/typescript	1.24 kB
gpt-3.5-turbo-0125.ts	text/typescript	1.28 kB
gpt-3.5-turbo-0613.ts	text/typescript	1.26 kB
gpt-3.5-turbo-1106.ts	text/typescript	1.27 kB
gpt-3.5-turbo-16k-0613.ts	text/typescript	1.27 kB
gpt-3.5-turbo-instruct.ts	text/typescript	1.24 kB
gpt-3.5-turbo.ts	text/typescript	1.27 kB
gpt-3.5.ts	text/typescript	1.23 kB
gpt-4-0125-preview.ts	text/typescript	1.27 kB
gpt-4-0314.ts	text/typescript	1.25 kB
gpt-4-0613.ts	text/typescript	1.28 kB
gpt-4-1106-preview.ts	text/typescript	1.27 kB
gpt-4-1106-vision-preview.ts	text/typescript	1.32 kB
gpt-4-32k.ts	text/typescript	1.25 kB
gpt-4-turbo-2024-04-09.ts	text/typescript	1.33 kB
gpt-4-turbo-preview.ts	text/typescript	1.27 kB
gpt-4-turbo.ts	text/typescript	1.31 kB
gpt-4.1-2025-04-14.ts	text/typescript	1.46 kB
gpt-4.1-mini-2025-04-14.ts	text/typescript	1.36 kB
gpt-4.1-mini.ts	text/typescript	1.34 kB
gpt-4.1-nano-2025-04-14.ts	text/typescript	1.36 kB
gpt-4.1-nano.ts	text/typescript	1.35 kB
gpt-4.1.ts	text/typescript	1.45 kB
gpt-4.5-preview-2025-02-27.ts	text/typescript	1.34 kB
gpt-4.5-preview.ts	text/typescript	1.33 kB
gpt-4.ts	text/typescript	1.27 kB
gpt-4o-2024-05-13.ts	text/typescript	1.3 kB
gpt-4o-2024-08-06.ts	text/typescript	1.44 kB
gpt-4o-2024-11-20.ts	text/typescript	1.41 kB
gpt-4o-audio-preview-2024-10-01.ts	text/typescript	1.31 kB
gpt-4o-audio-preview-2024-12-17.ts	text/typescript	1.31 kB
gpt-4o-audio-preview-2025-06-03.ts	text/typescript	1.31 kB
gpt-4o-audio-preview.ts	text/typescript	1.3 kB
gpt-4o-mini-2024-07-18.ts	text/typescript	1.35 kB
gpt-4o-mini-audio-preview-2024-12-17.ts	text/typescript	1.33 kB
gpt-4o-mini-audio-preview.ts	text/typescript	1.32 kB
gpt-4o-mini-realtime-preview-2024-12-17.ts	text/typescript	170 B
gpt-4o-mini-realtime-preview.ts	text/typescript	170 B
gpt-4o-mini-search-preview-2025-03-11.ts	text/typescript	1.27 kB
gpt-4o-mini-search-preview.ts	text/typescript	1.26 kB
gpt-4o-mini-transcribe.ts	text/typescript	170 B
gpt-4o-mini-tts.ts	text/typescript	170 B
gpt-4o-mini.ts	text/typescript	1.34 kB
gpt-4o-realtime-preview-2024-10-01.ts	text/typescript	170 B
gpt-4o-realtime-preview-2024-12-17.ts	text/typescript	170 B
gpt-4o-realtime-preview-2025-06-03.ts	text/typescript	170 B
gpt-4o-realtime-preview.ts	text/typescript	170 B
gpt-4o-search-preview-2025-03-11.ts	text/typescript	1.32 kB
gpt-4o-search-preview.ts	text/typescript	1.31 kB
gpt-4o-transcribe.ts	text/typescript	170 B
gpt-4o.ts	text/typescript	1.43 kB
gpt-image-1.ts	text/typescript	170 B
o1-2024-12-17.ts	text/typescript	1.37 kB
o1-mini-2024-09-12.ts	text/typescript	1.28 kB
o1-mini.ts	text/typescript	1.27 kB
o1-preview-2024-09-12.ts	text/typescript	1.33 kB
o1-preview.ts	text/typescript	1.32 kB
o1-pro-2025-03-19.ts	text/typescript	1.29 kB
o1-pro.ts	text/typescript	1.28 kB
o1.ts	text/typescript	1.36 kB
o3-2025-04-16.ts	text/typescript	1.4 kB
o3-mini-2025-01-31.ts	text/typescript	1.28 kB
o3-mini.ts	text/typescript	1.27 kB
o3-pro-2025-06-10.ts	text/typescript	1.29 kB
o3-pro.ts	text/typescript	1.28 kB
o3.ts	text/typescript	1.39 kB
o4-mini-2025-04-16.ts	text/typescript	1.36 kB
o4-mini.ts	text/typescript	1.35 kB
omni-moderation-2024-09-26.ts	text/typescript	170 B
omni-moderation-latest.ts	text/typescript	170 B
text-ada-001.ts	text/typescript	168 B
text-babbage-001.ts	text/typescript	168 B
text-curie-001.ts	text/typescript	168 B
text-davinci-001.ts	text/typescript	168 B
text-davinci-002.ts	text/typescript	168 B
text-davinci-003.ts	text/typescript	168 B
text-davinci-edit-001.ts	text/typescript	168 B
text-embedding-3-large.ts	text/typescript	172 B
text-embedding-3-small.ts	text/typescript	172 B
text-embedding-ada-002.ts	text/typescript	172 B
text-moderation-007.ts	text/typescript	170 B
text-moderation-latest.ts	text/typescript	170 B
text-moderation-stable.ts	text/typescript	170 B
text-search-ada-doc-001.ts	text/typescript	168 B
text-search-ada-query-001.ts	text/typescript	168 B
text-search-babbage-doc-001.ts	text/typescript	168 B
text-search-babbage-query-001.ts	text/typescript	168 B
text-search-curie-doc-001.ts	text/typescript	168 B
text-search-curie-query-001.ts	text/typescript	168 B
text-search-davinci-doc-001.ts	text/typescript	168 B
text-search-davinci-query-001.ts	text/typescript	168 B
text-similarity-ada-001.ts	text/typescript	168 B
text-similarity-babbage-001.ts	text/typescript	168 B
text-similarity-curie-001.ts	text/typescript	168 B
text-similarity-davinci-001.ts	text/typescript	168 B
tts-1-hd.ts	text/typescript	170 B
tts-1.ts	text/typescript	170 B
whisper-1.ts	text/typescript	170 B