kitoken
Version:
Fast and versatile tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization
15 lines (14 loc) • 960 B
TypeScript
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export const __wbg_kitoken_free: (a: number, b: number) => void;
export const kitoken_new: (a: number, b: number, c: number) => void;
export const kitoken_encode: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_encode_all: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_decode: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_decode_all: (a: number, b: number, c: number, d: number, e: number) => void;
export const __wbindgen_export_0: (a: number) => void;
export const __wbindgen_export_1: (a: number, b: number) => number;
export const __wbindgen_export_2: (a: number, b: number, c: number, d: number) => number;
export const __wbindgen_export_3: (a: number, b: number, c: number) => void;
export const __wbindgen_add_to_stack_pointer: (a: number) => number;