kitoken
Version:
Fast tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization
17 lines (16 loc) • 1.06 kB
TypeScript
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export const __wbg_kitoken_free: (a: number, b: number) => void;
export const kitoken_config: (a: number) => number;
export const kitoken_decode: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_decode_all: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_encode: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_encode_all: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_meta: (a: number) => number;
export const kitoken_new: (a: number, b: number, c: number) => void;
export const __wbindgen_export: (a: number, b: number) => number;
export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number;
export const __wbindgen_export3: (a: number) => void;
export const __wbindgen_export4: (a: number, b: number, c: number) => void;
export const __wbindgen_add_to_stack_pointer: (a: number) => number;