kitoken/dist/node_bg.wasm.d.ts

Fast and versatile tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization

/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export const __wbg_kitoken_free: (a: number, b: number) => void;
export const kitoken_new: (a: number, b: number, c: number) => void;
export const kitoken_encode: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_encode_all: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_decode: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_decode_all: (a: number, b: number, c: number, d: number, e: number) => void;
export const kitoken_definition: (a: number) => number;
export const kitoken_set_definition: (a: number, b: number, c: number) => void;
export const kitoken_config: (a: number) => number;
export const kitoken_set_config: (a: number, b: number, c: number) => void;
export const kitoken_to_bytes: (a: number, b: number) => void;
export const kitoken_from_sentencepiece: (a: number, b: number, c: number) => void;
export const kitoken_from_tiktoken: (a: number, b: number, c: number) => void;
export const kitoken_from_tokenizers: (a: number, b: number, c: number) => void;
export const kitoken_from_tekken: (a: number, b: number, c: number) => void;
export const __wbindgen_export_0: (a: number, b: number) => number;
export const __wbindgen_export_1: (a: number, b: number, c: number, d: number) => number;
export const __wbindgen_export_2: (a: number) => void;
export const __wbindgen_export_3: (a: number, b: number, c: number) => void;
export const __wbindgen_add_to_stack_pointer: (a: number) => number;