gpt-tokenizer

Version:

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

92 files

Filename	Content Type	Size
../
ada.ts	text/typescript	168 B
babbage-002-finetune.ts	text/typescript	168 B
babbage-002.ts	text/typescript	168 B
babbage.ts	text/typescript	168 B
chatgpt-4o-latest.ts	text/typescript	170 B
code-cushman-001.ts	text/typescript	168 B
code-cushman-002.ts	text/typescript	168 B
code-davinci-001.ts	text/typescript	168 B
code-davinci-002.ts	text/typescript	168 B
code-davinci-edit-001.ts	text/typescript	168 B
code-search-ada-code-001.ts	text/typescript	168 B
code-search-ada-text-001.ts	text/typescript	168 B
code-search-babbage-code-001.ts	text/typescript	168 B
code-search-babbage-text-001.ts	text/typescript	168 B
curie.ts	text/typescript	168 B
cushman-codex.ts	text/typescript	168 B
davinci-002-finetune.ts	text/typescript	168 B
davinci-002.ts	text/typescript	168 B
davinci-codex.ts	text/typescript	168 B
davinci.ts	text/typescript	168 B
gpt-3.5-turbo-0125.ts	text/typescript	869 B
gpt-3.5-turbo-0301.ts	text/typescript	869 B
gpt-3.5-turbo-0613.ts	text/typescript	869 B
gpt-3.5-turbo-1106.ts	text/typescript	869 B
gpt-3.5-turbo-16k-0613.ts	text/typescript	873 B
gpt-3.5-turbo-16k.ts	text/typescript	868 B
gpt-3.5-turbo-finetune.ts	text/typescript	873 B
gpt-3.5-turbo-instruct-0914.ts	text/typescript	172 B
gpt-3.5-turbo-instruct.ts	text/typescript	172 B
gpt-3.5-turbo.ts	text/typescript	864 B
gpt-4-0125-preview.ts	text/typescript	869 B
gpt-4-0314.ts	text/typescript	861 B
gpt-4-0613.ts	text/typescript	861 B
gpt-4-1106-preview.ts	text/typescript	869 B
gpt-4-1106-vision-preview.ts	text/typescript	876 B
gpt-4-32k-0314.ts	text/typescript	865 B
gpt-4-32k-0613.ts	text/typescript	865 B
gpt-4-32k.ts	text/typescript	860 B
gpt-4-turbo-2024-04-09.ts	text/typescript	873 B
gpt-4-turbo-preview.ts	text/typescript	870 B
gpt-4-turbo.ts	text/typescript	862 B
gpt-4-vision-preview.ts	text/typescript	871 B
gpt-4.ts	text/typescript	856 B
gpt-4o-2024-05-13.ts	text/typescript	867 B
gpt-4o-2024-08-06-finetune.ts	text/typescript	876 B
gpt-4o-2024-08-06.ts	text/typescript	867 B
gpt-4o-2024-11-20.ts	text/typescript	867 B
gpt-4o-audio-preview-2024-10-01.ts	text/typescript	881 B
gpt-4o-audio-preview-2024-12-17.ts	text/typescript	881 B
gpt-4o-audio-preview.ts	text/typescript	870 B
gpt-4o-mini-2024-07-18-finetune.ts	text/typescript	881 B
gpt-4o-mini-2024-07-18.ts	text/typescript	872 B
gpt-4o-mini-audio-preview-2024-12-17.ts	text/typescript	886 B
gpt-4o-mini-audio-preview.ts	text/typescript	875 B
gpt-4o-mini-realtime-preview-2024-12-17.ts	text/typescript	889 B
gpt-4o-mini-realtime-preview.ts	text/typescript	878 B
gpt-4o-mini-training-2024-07-18.ts	text/typescript	881 B
gpt-4o-mini-training.ts	text/typescript	870 B
gpt-4o-mini.ts	text/typescript	861 B
gpt-4o-realtime-preview-2024-10-01.ts	text/typescript	884 B
gpt-4o-realtime-preview-2024-12-17.ts	text/typescript	884 B
gpt-4o-realtime-preview.ts	text/typescript	873 B
gpt-4o.ts	text/typescript	856 B
o1-2024-12-17.ts	text/typescript	170 B
o1-mini-2024-09-12.ts	text/typescript	170 B
o1-mini.ts	text/typescript	170 B
o1-preview-2024-09-12.ts	text/typescript	170 B
o1-preview.ts	text/typescript	170 B
o1.ts	text/typescript	170 B
o3-mini.ts	text/typescript	170 B
text-ada-001.ts	text/typescript	168 B
text-babbage-001.ts	text/typescript	168 B
text-curie-001.ts	text/typescript	168 B
text-davinci-001.ts	text/typescript	168 B
text-davinci-002.ts	text/typescript	168 B
text-davinci-003.ts	text/typescript	168 B
text-davinci-edit-001.ts	text/typescript	168 B
text-embedding-3-large.ts	text/typescript	172 B
text-embedding-3-small.ts	text/typescript	172 B
text-embedding-ada-002.ts	text/typescript	172 B
text-search-ada-doc-001.ts	text/typescript	168 B
text-search-ada-query-001.ts	text/typescript	168 B
text-search-babbage-doc-001.ts	text/typescript	168 B
text-search-babbage-query-001.ts	text/typescript	168 B
text-search-curie-doc-001.ts	text/typescript	168 B
text-search-curie-query-001.ts	text/typescript	168 B
text-search-davinci-doc-001.ts	text/typescript	168 B
text-search-davinci-query-001.ts	text/typescript	168 B
text-similarity-ada-001.ts	text/typescript	168 B
text-similarity-babbage-001.ts	text/typescript	168 B
text-similarity-curie-001.ts	text/typescript	168 B
text-similarity-davinci-001.ts	text/typescript	168 B