gpt-tokenizer

Version:

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

276 files

Filename	Content Type	Size
../
ada.d.ts	text/typescript	94 B
ada.js	text/javascript	1.28 kB
ada.js.map	application/json	186 B
babbage-002-finetune.d.ts	text/typescript	94 B
babbage-002-finetune.js	text/javascript	1.3 kB
babbage-002-finetune.js.map	application/json	220 B
babbage-002.d.ts	text/typescript	94 B
babbage-002.js	text/javascript	1.29 kB
babbage-002.js.map	application/json	202 B
babbage.d.ts	text/typescript	94 B
babbage.js	text/javascript	1.29 kB
babbage.js.map	application/json	194 B
chatgpt-4o-latest.d.ts	text/typescript	96 B
chatgpt-4o-latest.js	text/javascript	1.3 kB
chatgpt-4o-latest.js.map	application/json	214 B
code-cushman-001.d.ts	text/typescript	94 B
code-cushman-001.js	text/javascript	1.29 kB
code-cushman-001.js.map	application/json	212 B
code-cushman-002.d.ts	text/typescript	94 B
code-cushman-002.js	text/javascript	1.29 kB
code-cushman-002.js.map	application/json	212 B
code-davinci-001.d.ts	text/typescript	94 B
code-davinci-001.js	text/javascript	1.29 kB
code-davinci-001.js.map	application/json	212 B
code-davinci-002.d.ts	text/typescript	94 B
code-davinci-002.js	text/javascript	1.29 kB
code-davinci-002.js.map	application/json	212 B
code-davinci-edit-001.d.ts	text/typescript	94 B
code-davinci-edit-001.js	text/javascript	1.3 kB
code-davinci-edit-001.js.map	application/json	222 B
code-search-ada-code-001.d.ts	text/typescript	94 B
code-search-ada-code-001.js	text/javascript	1.3 kB
code-search-ada-code-001.js.map	application/json	228 B
code-search-ada-text-001.d.ts	text/typescript	94 B
code-search-ada-text-001.js	text/javascript	1.3 kB
code-search-ada-text-001.js.map	application/json	228 B
code-search-babbage-code-001.d.ts	text/typescript	94 B
code-search-babbage-code-001.js	text/javascript	1.31 kB
code-search-babbage-code-001.js.map	application/json	236 B
code-search-babbage-text-001.d.ts	text/typescript	94 B
code-search-babbage-text-001.js	text/javascript	1.31 kB
code-search-babbage-text-001.js.map	application/json	236 B
curie.d.ts	text/typescript	94 B
curie.js	text/javascript	1.28 kB
curie.js.map	application/json	190 B
cushman-codex.d.ts	text/typescript	94 B
cushman-codex.js	text/javascript	1.29 kB
cushman-codex.js.map	application/json	206 B
davinci-002-finetune.d.ts	text/typescript	94 B
davinci-002-finetune.js	text/javascript	1.3 kB
davinci-002-finetune.js.map	application/json	220 B
davinci-002.d.ts	text/typescript	94 B
davinci-002.js	text/javascript	1.29 kB
davinci-002.js.map	application/json	202 B
davinci-codex.d.ts	text/typescript	94 B
davinci-codex.js	text/javascript	1.29 kB
davinci-codex.js.map	application/json	206 B
davinci.d.ts	text/typescript	94 B
davinci.js	text/javascript	1.29 kB
davinci.js.map	application/json	194 B
gpt-3.5-turbo-0125.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-0125.js	text/javascript	2.51 kB
gpt-3.5-turbo-0125.js.map	application/json	678 B
gpt-3.5-turbo-0301.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-0301.js	text/javascript	2.51 kB
gpt-3.5-turbo-0301.js.map	application/json	678 B
gpt-3.5-turbo-0613.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-0613.js	text/javascript	2.51 kB
gpt-3.5-turbo-0613.js.map	application/json	678 B
gpt-3.5-turbo-1106.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-1106.js	text/javascript	2.51 kB
gpt-3.5-turbo-1106.js.map	application/json	678 B
gpt-3.5-turbo-16k-0613.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-16k-0613.js	text/javascript	2.52 kB
gpt-3.5-turbo-16k-0613.js.map	application/json	686 B
gpt-3.5-turbo-16k.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-16k.js	text/javascript	2.51 kB
gpt-3.5-turbo-16k.js.map	application/json	676 B
gpt-3.5-turbo-finetune.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo-finetune.js	text/javascript	2.52 kB
gpt-3.5-turbo-finetune.js.map	application/json	686 B
gpt-3.5-turbo-instruct-0914.d.ts	text/typescript	98 B
gpt-3.5-turbo-instruct-0914.js	text/javascript	1.31 kB
gpt-3.5-turbo-instruct-0914.js.map	application/json	234 B
gpt-3.5-turbo-instruct.d.ts	text/typescript	98 B
gpt-3.5-turbo-instruct.js	text/javascript	1.31 kB
gpt-3.5-turbo-instruct.js.map	application/json	224 B
gpt-3.5-turbo.d.ts	text/typescript	8.31 kB
gpt-3.5-turbo.js	text/javascript	2.5 kB
gpt-3.5-turbo.js.map	application/json	666 B
gpt-4-0125-preview.d.ts	text/typescript	8.31 kB
gpt-4-0125-preview.js	text/javascript	2.51 kB
gpt-4-0125-preview.js.map	application/json	678 B
gpt-4-0314.d.ts	text/typescript	8.31 kB
gpt-4-0314.js	text/javascript	2.5 kB
gpt-4-0314.js.map	application/json	660 B
gpt-4-0613.d.ts	text/typescript	8.31 kB
gpt-4-0613.js	text/javascript	2.5 kB
gpt-4-0613.js.map	application/json	660 B
gpt-4-1106-preview.d.ts	text/typescript	8.31 kB
gpt-4-1106-preview.js	text/javascript	2.51 kB
gpt-4-1106-preview.js.map	application/json	678 B
gpt-4-1106-vision-preview.d.ts	text/typescript	8.31 kB
gpt-4-1106-vision-preview.js	text/javascript	2.53 kB
gpt-4-1106-vision-preview.js.map	application/json	692 B
gpt-4-32k-0314.d.ts	text/typescript	8.31 kB
gpt-4-32k-0314.js	text/javascript	2.5 kB
gpt-4-32k-0314.js.map	application/json	670 B
gpt-4-32k-0613.d.ts	text/typescript	8.31 kB
gpt-4-32k-0613.js	text/javascript	2.5 kB
gpt-4-32k-0613.js.map	application/json	670 B
gpt-4-32k.d.ts	text/typescript	8.31 kB
gpt-4-32k.js	text/javascript	2.49 kB
gpt-4-32k.js.map	application/json	658 B
gpt-4-turbo-2024-04-09.d.ts	text/typescript	8.31 kB
gpt-4-turbo-2024-04-09.js	text/javascript	2.52 kB
gpt-4-turbo-2024-04-09.js.map	application/json	686 B
gpt-4-turbo-preview.d.ts	text/typescript	8.31 kB
gpt-4-turbo-preview.js	text/javascript	2.51 kB
gpt-4-turbo-preview.js.map	application/json	680 B
gpt-4-turbo.d.ts	text/typescript	8.31 kB
gpt-4-turbo.js	text/javascript	2.5 kB
gpt-4-turbo.js.map	application/json	662 B
gpt-4-vision-preview.d.ts	text/typescript	8.31 kB
gpt-4-vision-preview.js	text/javascript	2.52 kB
gpt-4-vision-preview.js.map	application/json	682 B
gpt-4.d.ts	text/typescript	8.31 kB
gpt-4.js	text/javascript	2.49 kB
gpt-4.js.map	application/json	650 B
gpt-4o-2024-05-13.d.ts	text/typescript	8.31 kB
gpt-4o-2024-05-13.js	text/javascript	2.51 kB
gpt-4o-2024-05-13.js.map	application/json	676 B
gpt-4o-2024-08-06-finetune.d.ts	text/typescript	8.31 kB
gpt-4o-2024-08-06-finetune.js	text/javascript	2.52 kB
gpt-4o-2024-08-06-finetune.js.map	application/json	694 B
gpt-4o-2024-08-06.d.ts	text/typescript	8.31 kB
gpt-4o-2024-08-06.js	text/javascript	2.51 kB
gpt-4o-2024-08-06.js.map	application/json	676 B
gpt-4o-2024-11-20.d.ts	text/typescript	8.31 kB
gpt-4o-2024-11-20.js	text/javascript	2.51 kB
gpt-4o-2024-11-20.js.map	application/json	676 B
gpt-4o-audio-preview-2024-10-01.d.ts	text/typescript	8.31 kB
gpt-4o-audio-preview-2024-10-01.js	text/javascript	2.54 kB
gpt-4o-audio-preview-2024-10-01.js.map	application/json	704 B
gpt-4o-audio-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-audio-preview-2024-12-17.js	text/javascript	2.54 kB
gpt-4o-audio-preview-2024-12-17.js.map	application/json	704 B
gpt-4o-audio-preview.d.ts	text/typescript	8.31 kB
gpt-4o-audio-preview.js	text/javascript	2.51 kB
gpt-4o-audio-preview.js.map	application/json	682 B
gpt-4o-mini-2024-07-18-finetune.d.ts	text/typescript	8.31 kB
gpt-4o-mini-2024-07-18-finetune.js	text/javascript	2.54 kB
gpt-4o-mini-2024-07-18-finetune.js.map	application/json	704 B
gpt-4o-mini-2024-07-18.d.ts	text/typescript	8.31 kB
gpt-4o-mini-2024-07-18.js	text/javascript	2.52 kB
gpt-4o-mini-2024-07-18.js.map	application/json	686 B
gpt-4o-mini-audio-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-mini-audio-preview-2024-12-17.js	text/javascript	2.54 kB
gpt-4o-mini-audio-preview-2024-12-17.js.map	application/json	714 B
gpt-4o-mini-audio-preview.d.ts	text/typescript	8.31 kB
gpt-4o-mini-audio-preview.js	text/javascript	2.52 kB
gpt-4o-mini-audio-preview.js.map	application/json	692 B
gpt-4o-mini-realtime-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-mini-realtime-preview-2024-12-17.js	text/javascript	2.55 kB
gpt-4o-mini-realtime-preview-2024-12-17.js.map	application/json	720 B
gpt-4o-mini-realtime-preview.d.ts	text/typescript	8.31 kB
gpt-4o-mini-realtime-preview.js	text/javascript	2.53 kB
gpt-4o-mini-realtime-preview.js.map	application/json	698 B
gpt-4o-mini-training-2024-07-18.d.ts	text/typescript	8.31 kB
gpt-4o-mini-training-2024-07-18.js	text/javascript	2.54 kB
gpt-4o-mini-training-2024-07-18.js.map	application/json	704 B
gpt-4o-mini-training.d.ts	text/typescript	8.31 kB
gpt-4o-mini-training.js	text/javascript	2.51 kB
gpt-4o-mini-training.js.map	application/json	682 B
gpt-4o-mini.d.ts	text/typescript	8.31 kB
gpt-4o-mini.js	text/javascript	2.5 kB
gpt-4o-mini.js.map	application/json	662 B
gpt-4o-realtime-preview-2024-10-01.d.ts	text/typescript	8.31 kB
gpt-4o-realtime-preview-2024-10-01.js	text/javascript	2.54 kB
gpt-4o-realtime-preview-2024-10-01.js.map	application/json	710 B
gpt-4o-realtime-preview-2024-12-17.d.ts	text/typescript	8.31 kB
gpt-4o-realtime-preview-2024-12-17.js	text/javascript	2.54 kB
gpt-4o-realtime-preview-2024-12-17.js.map	application/json	710 B
gpt-4o-realtime-preview.d.ts	text/typescript	8.31 kB
gpt-4o-realtime-preview.js	text/javascript	2.52 kB
gpt-4o-realtime-preview.js.map	application/json	688 B
gpt-4o.d.ts	text/typescript	8.31 kB
gpt-4o.js	text/javascript	2.48 kB
gpt-4o.js.map	application/json	652 B
o1-2024-12-17.d.ts	text/typescript	96 B
o1-2024-12-17.js	text/javascript	1.3 kB
o1-2024-12-17.js.map	application/json	206 B
o1-mini-2024-09-12.d.ts	text/typescript	96 B
o1-mini-2024-09-12.js	text/javascript	1.3 kB
o1-mini-2024-09-12.js.map	application/json	216 B
o1-mini.d.ts	text/typescript	96 B
o1-mini.js	text/javascript	1.29 kB
o1-mini.js.map	application/json	194 B
o1-preview-2024-09-12.d.ts	text/typescript	96 B
o1-preview-2024-09-12.js	text/javascript	1.3 kB
o1-preview-2024-09-12.js.map	application/json	222 B
o1-preview.d.ts	text/typescript	96 B
o1-preview.js	text/javascript	1.29 kB
o1-preview.js.map	application/json	200 B
o1.d.ts	text/typescript	96 B
o1.js	text/javascript	1.28 kB
o1.js.map	application/json	184 B
o3-mini.d.ts	text/typescript	96 B
o3-mini.js	text/javascript	1.29 kB
o3-mini.js.map	application/json	194 B
text-ada-001.d.ts	text/typescript	94 B
text-ada-001.js	text/javascript	1.29 kB
text-ada-001.js.map	application/json	204 B
text-babbage-001.d.ts	text/typescript	94 B
text-babbage-001.js	text/javascript	1.29 kB
text-babbage-001.js.map	application/json	212 B
text-curie-001.d.ts	text/typescript	94 B
text-curie-001.js	text/javascript	1.29 kB
text-curie-001.js.map	application/json	208 B
text-davinci-001.d.ts	text/typescript	94 B
text-davinci-001.js	text/javascript	1.29 kB
text-davinci-001.js.map	application/json	212 B
text-davinci-002.d.ts	text/typescript	94 B
text-davinci-002.js	text/javascript	1.29 kB
text-davinci-002.js.map	application/json	212 B
text-davinci-003.d.ts	text/typescript	94 B
text-davinci-003.js	text/javascript	1.29 kB
text-davinci-003.js.map	application/json	212 B
text-davinci-edit-001.d.ts	text/typescript	94 B
text-davinci-edit-001.js	text/javascript	1.3 kB
text-davinci-edit-001.js.map	application/json	222 B
text-embedding-3-large.d.ts	text/typescript	98 B
text-embedding-3-large.js	text/javascript	1.31 kB
text-embedding-3-large.js.map	application/json	224 B
text-embedding-3-small.d.ts	text/typescript	98 B
text-embedding-3-small.js	text/javascript	1.31 kB
text-embedding-3-small.js.map	application/json	224 B
text-embedding-ada-002.d.ts	text/typescript	98 B
text-embedding-ada-002.js	text/javascript	1.31 kB
text-embedding-ada-002.js.map	application/json	224 B
text-search-ada-doc-001.d.ts	text/typescript	94 B
text-search-ada-doc-001.js	text/javascript	1.3 kB
text-search-ada-doc-001.js.map	application/json	226 B
text-search-ada-query-001.d.ts	text/typescript	94 B
text-search-ada-query-001.js	text/javascript	1.3 kB
text-search-ada-query-001.js.map	application/json	230 B
text-search-babbage-doc-001.d.ts	text/typescript	94 B
text-search-babbage-doc-001.js	text/javascript	1.31 kB
text-search-babbage-doc-001.js.map	application/json	234 B
text-search-babbage-query-001.d.ts	text/typescript	94 B
text-search-babbage-query-001.js	text/javascript	1.31 kB
text-search-babbage-query-001.js.map	application/json	238 B
text-search-curie-doc-001.d.ts	text/typescript	94 B
text-search-curie-doc-001.js	text/javascript	1.3 kB
text-search-curie-doc-001.js.map	application/json	230 B
text-search-curie-query-001.d.ts	text/typescript	94 B
text-search-curie-query-001.js	text/javascript	1.31 kB
text-search-curie-query-001.js.map	application/json	234 B
text-search-davinci-doc-001.d.ts	text/typescript	94 B
text-search-davinci-doc-001.js	text/javascript	1.31 kB
text-search-davinci-doc-001.js.map	application/json	234 B
text-search-davinci-query-001.d.ts	text/typescript	94 B
text-search-davinci-query-001.js	text/javascript	1.31 kB
text-search-davinci-query-001.js.map	application/json	238 B
text-similarity-ada-001.d.ts	text/typescript	94 B
text-similarity-ada-001.js	text/javascript	1.3 kB
text-similarity-ada-001.js.map	application/json	226 B
text-similarity-babbage-001.d.ts	text/typescript	94 B
text-similarity-babbage-001.js	text/javascript	1.31 kB
text-similarity-babbage-001.js.map	application/json	234 B
text-similarity-curie-001.d.ts	text/typescript	94 B
text-similarity-curie-001.js	text/javascript	1.3 kB
text-similarity-curie-001.js.map	application/json	230 B
text-similarity-davinci-001.d.ts	text/typescript	94 B
text-similarity-davinci-001.js	text/javascript	1.31 kB
text-similarity-davinci-001.js.map	application/json	234 B