gpt-tokenizer

Version:

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

276 files

Filename	Content Type	Size
../
ada.d.ts	text/typescript	94 B
ada.js	text/javascript	201 B
ada.js.map	application/json	184 B
babbage-002-finetune.d.ts	text/typescript	94 B
babbage-002-finetune.js	text/javascript	218 B
babbage-002-finetune.js.map	application/json	218 B
babbage-002.d.ts	text/typescript	94 B
babbage-002.js	text/javascript	209 B
babbage-002.js.map	application/json	200 B
babbage.d.ts	text/typescript	94 B
babbage.js	text/javascript	205 B
babbage.js.map	application/json	192 B
chatgpt-4o-latest.d.ts	text/typescript	96 B
chatgpt-4o-latest.js	text/javascript	217 B
chatgpt-4o-latest.js.map	application/json	212 B
code-cushman-001.d.ts	text/typescript	94 B
code-cushman-001.js	text/javascript	214 B
code-cushman-001.js.map	application/json	210 B
code-cushman-002.d.ts	text/typescript	94 B
code-cushman-002.js	text/javascript	214 B
code-cushman-002.js.map	application/json	210 B
code-davinci-001.d.ts	text/typescript	94 B
code-davinci-001.js	text/javascript	214 B
code-davinci-001.js.map	application/json	210 B
code-davinci-002.d.ts	text/typescript	94 B
code-davinci-002.js	text/javascript	214 B
code-davinci-002.js.map	application/json	210 B
code-davinci-edit-001.d.ts	text/typescript	94 B
code-davinci-edit-001.js	text/javascript	219 B
code-davinci-edit-001.js.map	application/json	220 B
code-search-ada-code-001.d.ts	text/typescript	94 B
code-search-ada-code-001.js	text/javascript	222 B
code-search-ada-code-001.js.map	application/json	226 B
code-search-ada-text-001.d.ts	text/typescript	94 B
code-search-ada-text-001.js	text/javascript	222 B
code-search-ada-text-001.js.map	application/json	226 B
code-search-babbage-code-001.d.ts	text/typescript	94 B
code-search-babbage-code-001.js	text/javascript	226 B
code-search-babbage-code-001.js.map	application/json	234 B
code-search-babbage-text-001.d.ts	text/typescript	94 B
code-search-babbage-text-001.js	text/javascript	226 B
code-search-babbage-text-001.js.map	application/json	234 B
curie.d.ts	text/typescript	94 B
curie.js	text/javascript	203 B
curie.js.map	application/json	188 B
cushman-codex.d.ts	text/typescript	94 B
cushman-codex.js	text/javascript	211 B
cushman-codex.js.map	application/json	204 B
davinci-002-finetune.d.ts	text/typescript	94 B
davinci-002-finetune.js	text/javascript	218 B
davinci-002-finetune.js.map	application/json	218 B
davinci-002.d.ts	text/typescript	94 B
davinci-002.js	text/javascript	209 B
davinci-002.js.map	application/json	200 B
davinci-codex.d.ts	text/typescript	94 B
davinci-codex.js	text/javascript	211 B
davinci-codex.js.map	application/json	204 B
davinci.d.ts	text/typescript	94 B
davinci.js	text/javascript	205 B
davinci.js.map	application/json	192 B
gpt-3.5-turbo-0125.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-0125.js	text/javascript	870 B
gpt-3.5-turbo-0125.js.map	application/json	736 B
gpt-3.5-turbo-0301.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-0301.js	text/javascript	870 B
gpt-3.5-turbo-0301.js.map	application/json	736 B
gpt-3.5-turbo-0613.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-0613.js	text/javascript	870 B
gpt-3.5-turbo-0613.js.map	application/json	736 B
gpt-3.5-turbo-1106.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-1106.js	text/javascript	870 B
gpt-3.5-turbo-1106.js.map	application/json	736 B
gpt-3.5-turbo-16k-0613.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-16k-0613.js	text/javascript	878 B
gpt-3.5-turbo-16k-0613.js.map	application/json	744 B
gpt-3.5-turbo-16k.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-16k.js	text/javascript	868 B
gpt-3.5-turbo-16k.js.map	application/json	734 B
gpt-3.5-turbo-finetune.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-finetune.js	text/javascript	878 B
gpt-3.5-turbo-finetune.js.map	application/json	744 B
gpt-3.5-turbo-instruct-0914.d.ts	text/typescript	98 B
gpt-3.5-turbo-instruct-0914.js	text/javascript	229 B
gpt-3.5-turbo-instruct-0914.js.map	application/json	232 B
gpt-3.5-turbo-instruct.d.ts	text/typescript	98 B
gpt-3.5-turbo-instruct.js	text/javascript	224 B
gpt-3.5-turbo-instruct.js.map	application/json	222 B
gpt-3.5-turbo.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo.js	text/javascript	860 B
gpt-3.5-turbo.js.map	application/json	724 B
gpt-4-0125-preview.d.ts	text/typescript	8.31 kB
gpt-4-0125-preview.js	text/javascript	870 B
gpt-4-0125-preview.js.map	application/json	736 B
gpt-4-0314.d.ts	text/typescript	8.31 kB
gpt-4-0314.js	text/javascript	854 B
gpt-4-0314.js.map	application/json	718 B
gpt-4-0613.d.ts	text/typescript	8.31 kB
gpt-4-0613.js	text/javascript	854 B
gpt-4-0613.js.map	application/json	718 B
gpt-4-1106-preview.d.ts	text/typescript	8.31 kB
gpt-4-1106-preview.js	text/javascript	870 B
gpt-4-1106-preview.js.map	application/json	736 B
gpt-4-1106-vision-preview.d.ts	text/typescript	8.31 kB
gpt-4-1106-vision-preview.js	text/javascript	884 B
gpt-4-1106-vision-preview.js.map	application/json	750 B
gpt-4-32k-0314.d.ts	text/typescript	8.31 kB
gpt-4-32k-0314.js	text/javascript	862 B
gpt-4-32k-0314.js.map	application/json	728 B
gpt-4-32k-0613.d.ts	text/typescript	8.31 kB
gpt-4-32k-0613.js	text/javascript	862 B
gpt-4-32k-0613.js.map	application/json	728 B
gpt-4-32k.d.ts	text/typescript	8.31 kB
gpt-4-32k.js	text/javascript	852 B
gpt-4-32k.js.map	application/json	716 B
gpt-4-turbo-2024-04-09.d.ts	text/typescript	8.31 kB
gpt-4-turbo-2024-04-09.js	text/javascript	878 B
gpt-4-turbo-2024-04-09.js.map	application/json	744 B
gpt-4-turbo-preview.d.ts	text/typescript	8.31 kB
gpt-4-turbo-preview.js	text/javascript	872 B
gpt-4-turbo-preview.js.map	application/json	738 B
gpt-4-turbo.d.ts	text/typescript	8.31 kB
gpt-4-turbo.js	text/javascript	856 B
gpt-4-turbo.js.map	application/json	720 B
gpt-4-vision-preview.d.ts	text/typescript	8.31 kB
gpt-4-vision-preview.js	text/javascript	874 B
gpt-4-vision-preview.js.map	application/json	740 B
gpt-4.d.ts	text/typescript	8.31 kB
gpt-4.js	text/javascript	844 B
gpt-4.js.map	application/json	708 B
gpt-4o-2024-05-13.d.ts	text/typescript	8.31 kB
gpt-4o-2024-05-13.js	text/javascript	867 B
gpt-4o-2024-05-13.js.map	application/json	734 B
gpt-4o-2024-08-06-finetune.d.ts	text/typescript	8.31 kB
gpt-4o-2024-08-06-finetune.js	text/javascript	885 B
gpt-4o-2024-08-06-finetune.js.map	application/json	752 B
gpt-4o-2024-08-06.d.ts	text/typescript	8.31 kB
gpt-4o-2024-08-06.js	text/javascript	867 B
gpt-4o-2024-08-06.js.map	application/json	734 B
gpt-4o-2024-11-20.d.ts	text/typescript	8.31 kB
gpt-4o-2024-11-20.js	text/javascript	867 B
gpt-4o-2024-11-20.js.map	application/json	734 B
gpt-4o-audio-preview-2024-10-01.d.ts	text/typescript	8.31 kB
gpt-4o-audio-preview-2024-10-01.js	text/javascript	895 B
gpt-4o-audio-preview-2024-10-01.js.map	application/json	762 B
gpt-4o-audio-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-audio-preview-2024-12-17.js	text/javascript	895 B
gpt-4o-audio-preview-2024-12-17.js.map	application/json	762 B
gpt-4o-audio-preview.d.ts	text/typescript	8.31 kB
gpt-4o-audio-preview.js	text/javascript	873 B
gpt-4o-audio-preview.js.map	application/json	740 B
gpt-4o-mini-2024-07-18-finetune.d.ts	text/typescript	8.31 kB
gpt-4o-mini-2024-07-18-finetune.js	text/javascript	895 B
gpt-4o-mini-2024-07-18-finetune.js.map	application/json	762 B
gpt-4o-mini-2024-07-18.d.ts	text/typescript	8.31 kB
gpt-4o-mini-2024-07-18.js	text/javascript	877 B
gpt-4o-mini-2024-07-18.js.map	application/json	744 B
gpt-4o-mini-audio-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-mini-audio-preview-2024-12-17.js	text/javascript	905 B
gpt-4o-mini-audio-preview-2024-12-17.js.map	application/json	772 B
gpt-4o-mini-audio-preview.d.ts	text/typescript	8.31 kB
gpt-4o-mini-audio-preview.js	text/javascript	883 B
gpt-4o-mini-audio-preview.js.map	application/json	750 B
gpt-4o-mini-realtime-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-mini-realtime-preview-2024-12-17.js	text/javascript	911 B
gpt-4o-mini-realtime-preview-2024-12-17.js.map	application/json	778 B
gpt-4o-mini-realtime-preview.d.ts	text/typescript	8.31 kB
gpt-4o-mini-realtime-preview.js	text/javascript	889 B
gpt-4o-mini-realtime-preview.js.map	application/json	756 B
gpt-4o-mini-training-2024-07-18.d.ts	text/typescript	8.31 kB
gpt-4o-mini-training-2024-07-18.js	text/javascript	895 B
gpt-4o-mini-training-2024-07-18.js.map	application/json	762 B
gpt-4o-mini-training.d.ts	text/typescript	8.31 kB
gpt-4o-mini-training.js	text/javascript	873 B
gpt-4o-mini-training.js.map	application/json	740 B
gpt-4o-mini.d.ts	text/typescript	8.31 kB
gpt-4o-mini.js	text/javascript	855 B
gpt-4o-mini.js.map	application/json	720 B
gpt-4o-realtime-preview-2024-10-01.d.ts	text/typescript	8.31 kB
gpt-4o-realtime-preview-2024-10-01.js	text/javascript	901 B
gpt-4o-realtime-preview-2024-10-01.js.map	application/json	768 B
gpt-4o-realtime-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-realtime-preview-2024-12-17.js	text/javascript	901 B
gpt-4o-realtime-preview-2024-12-17.js.map	application/json	768 B
gpt-4o-realtime-preview.d.ts	text/typescript	8.31 kB
gpt-4o-realtime-preview.js	text/javascript	879 B
gpt-4o-realtime-preview.js.map	application/json	746 B
gpt-4o.d.ts	text/typescript	8.31 kB
gpt-4o.js	text/javascript	845 B
gpt-4o.js.map	application/json	710 B
o1-2024-12-17.d.ts	text/typescript	96 B
o1-2024-12-17.js	text/javascript	213 B
o1-2024-12-17.js.map	application/json	204 B
o1-mini-2024-09-12.d.ts	text/typescript	96 B
o1-mini-2024-09-12.js	text/javascript	218 B
o1-mini-2024-09-12.js.map	application/json	214 B
o1-mini.d.ts	text/typescript	96 B
o1-mini.js	text/javascript	207 B
o1-mini.js.map	application/json	192 B
o1-preview-2024-09-12.d.ts	text/typescript	96 B
o1-preview-2024-09-12.js	text/javascript	221 B
o1-preview-2024-09-12.js.map	application/json	220 B
o1-preview.d.ts	text/typescript	96 B
o1-preview.js	text/javascript	210 B
o1-preview.js.map	application/json	198 B
o1.d.ts	text/typescript	96 B
o1.js	text/javascript	202 B
o1.js.map	application/json	182 B
o3-mini.d.ts	text/typescript	96 B
o3-mini.js	text/javascript	207 B
o3-mini.js.map	application/json	192 B
text-ada-001.d.ts	text/typescript	94 B
text-ada-001.js	text/javascript	210 B
text-ada-001.js.map	application/json	202 B
text-babbage-001.d.ts	text/typescript	94 B
text-babbage-001.js	text/javascript	214 B
text-babbage-001.js.map	application/json	210 B
text-curie-001.d.ts	text/typescript	94 B
text-curie-001.js	text/javascript	212 B
text-curie-001.js.map	application/json	206 B
text-davinci-001.d.ts	text/typescript	94 B
text-davinci-001.js	text/javascript	214 B
text-davinci-001.js.map	application/json	210 B
text-davinci-002.d.ts	text/typescript	94 B
text-davinci-002.js	text/javascript	214 B
text-davinci-002.js.map	application/json	210 B
text-davinci-003.d.ts	text/typescript	94 B
text-davinci-003.js	text/javascript	214 B
text-davinci-003.js.map	application/json	210 B
text-davinci-edit-001.d.ts	text/typescript	94 B
text-davinci-edit-001.js	text/javascript	219 B
text-davinci-edit-001.js.map	application/json	220 B
text-embedding-3-large.d.ts	text/typescript	98 B
text-embedding-3-large.js	text/javascript	224 B
text-embedding-3-large.js.map	application/json	222 B
text-embedding-3-small.d.ts	text/typescript	98 B
text-embedding-3-small.js	text/javascript	224 B
text-embedding-3-small.js.map	application/json	222 B
text-embedding-ada-002.d.ts	text/typescript	98 B
text-embedding-ada-002.js	text/javascript	224 B
text-embedding-ada-002.js.map	application/json	222 B
text-search-ada-doc-001.d.ts	text/typescript	94 B
text-search-ada-doc-001.js	text/javascript	221 B
text-search-ada-doc-001.js.map	application/json	224 B
text-search-ada-query-001.d.ts	text/typescript	94 B
text-search-ada-query-001.js	text/javascript	223 B
text-search-ada-query-001.js.map	application/json	228 B
text-search-babbage-doc-001.d.ts	text/typescript	94 B
text-search-babbage-doc-001.js	text/javascript	225 B
text-search-babbage-doc-001.js.map	application/json	232 B
text-search-babbage-query-001.d.ts	text/typescript	94 B
text-search-babbage-query-001.js	text/javascript	227 B
text-search-babbage-query-001.js.map	application/json	236 B
text-search-curie-doc-001.d.ts	text/typescript	94 B
text-search-curie-doc-001.js	text/javascript	223 B
text-search-curie-doc-001.js.map	application/json	228 B
text-search-curie-query-001.d.ts	text/typescript	94 B
text-search-curie-query-001.js	text/javascript	225 B
text-search-curie-query-001.js.map	application/json	232 B
text-search-davinci-doc-001.d.ts	text/typescript	94 B
text-search-davinci-doc-001.js	text/javascript	225 B
text-search-davinci-doc-001.js.map	application/json	232 B
text-search-davinci-query-001.d.ts	text/typescript	94 B
text-search-davinci-query-001.js	text/javascript	227 B
text-search-davinci-query-001.js.map	application/json	236 B
text-similarity-ada-001.d.ts	text/typescript	94 B
text-similarity-ada-001.js	text/javascript	221 B
text-similarity-ada-001.js.map	application/json	224 B
text-similarity-babbage-001.d.ts	text/typescript	94 B
text-similarity-babbage-001.js	text/javascript	225 B
text-similarity-babbage-001.js.map	application/json	232 B
text-similarity-curie-001.d.ts	text/typescript	94 B
text-similarity-curie-001.js	text/javascript	223 B
text-similarity-curie-001.js.map	application/json	228 B
text-similarity-davinci-001.d.ts	text/typescript	94 B
text-similarity-davinci-001.js	text/javascript	225 B
text-similarity-davinci-001.js.map	application/json	232 B