@huggingface/transformers
Version:
State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!
1,618 lines (1,610 loc) • 1.1 MB
JavaScript
var __defProp = Object.defineProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
// ignore-modules:node:fs
var node_fs_default = {};
// ignore-modules:node:path
var node_path_default = {};
// ignore-modules:node:url
var node_url_default = {};
// src/env.js
var VERSION = "4.2.0";
var HAS_SELF = typeof self !== "undefined";
var IS_FS_AVAILABLE = !isEmpty(node_fs_default);
var IS_PATH_AVAILABLE = !isEmpty(node_path_default);
var IS_WEB_CACHE_AVAILABLE = HAS_SELF && "caches" in self;
var IS_DENO_RUNTIME = typeof globalThis.Deno !== "undefined";
var IS_BUN_RUNTIME = typeof globalThis.Bun !== "undefined";
var IS_DENO_WEB_RUNTIME = IS_DENO_RUNTIME && IS_WEB_CACHE_AVAILABLE && !IS_FS_AVAILABLE;
var IS_PROCESS_AVAILABLE = typeof process !== "undefined";
var IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === "node" && !IS_DENO_WEB_RUNTIME;
var IS_BROWSER_ENV = typeof window !== "undefined" && typeof window.document !== "undefined";
var IS_WEBWORKER_ENV = HAS_SELF && ["DedicatedWorkerGlobalScope", "ServiceWorkerGlobalScope", "SharedWorkerGlobalScope"].includes(
self.constructor?.name
);
var IS_WEB_ENV = IS_BROWSER_ENV || IS_WEBWORKER_ENV || IS_DENO_WEB_RUNTIME;
var IS_WEBGPU_AVAILABLE = IS_NODE_ENV || typeof navigator !== "undefined" && "gpu" in navigator;
var IS_WEBNN_AVAILABLE = typeof navigator !== "undefined" && "ml" in navigator;
var IS_CRYPTO_AVAILABLE = typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function";
var IS_CHROME_AVAILABLE = (
// @ts-ignore - chrome may not exist in all environments
typeof chrome !== "undefined" && typeof chrome.runtime !== "undefined" && typeof chrome.runtime.id === "string"
);
var IS_SERVICE_WORKER_ENV = (
// @ts-ignore - ServiceWorkerGlobalScope may not exist in all environments
typeof ServiceWorkerGlobalScope !== "undefined" && HAS_SELF && self instanceof ServiceWorkerGlobalScope
);
var isSafari = () => {
if (typeof navigator === "undefined") {
return false;
}
const userAgent = navigator.userAgent;
const vendor = navigator.vendor || "";
const isAppleVendor = vendor.indexOf("Apple") > -1;
const notOtherBrowser = !userAgent.match(/CriOS|FxiOS|EdgiOS|OPiOS|mercury|brave/i) && !userAgent.includes("Chrome") && !userAgent.includes("Android");
return isAppleVendor && notOtherBrowser;
};
var IS_SAFARI = isSafari();
var apis = Object.freeze({
/** Whether we are running in a browser environment (and not a web worker) */
IS_BROWSER_ENV,
/** Whether we are running in a web worker environment */
IS_WEBWORKER_ENV,
/** Whether we are running in a web-like environment (browser, web worker, or Deno web runtime) */
IS_WEB_ENV,
/** Whether we are running in a service worker environment */
IS_SERVICE_WORKER_ENV,
/** Whether we are running in Deno's web runtime (CDN imports, Cache API available, no filesystem) */
IS_DENO_WEB_RUNTIME,
/** Whether the Cache API is available */
IS_WEB_CACHE_AVAILABLE,
/** Whether the WebGPU API is available */
IS_WEBGPU_AVAILABLE,
/** Whether the WebNN API is available */
IS_WEBNN_AVAILABLE,
/** Whether we are running in a Safari browser */
IS_SAFARI,
/** Whether the Node.js process API is available */
IS_PROCESS_AVAILABLE,
/** Whether we are running in a Node.js-like environment (node, deno, bun) */
IS_NODE_ENV,
/** Whether the filesystem API is available */
IS_FS_AVAILABLE,
/** Whether the path API is available */
IS_PATH_AVAILABLE,
/** Whether the crypto API is available */
IS_CRYPTO_AVAILABLE,
/** Whether the Chrome runtime API is available */
IS_CHROME_AVAILABLE
});
var RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE;
var dirname__ = "./";
if (RUNNING_LOCALLY) {
const _import_meta_url = Object(import.meta).url;
if (_import_meta_url) {
dirname__ = node_path_default.dirname(node_path_default.dirname(node_url_default.fileURLToPath(_import_meta_url)));
} else if (typeof __dirname !== "undefined") {
dirname__ = node_path_default.dirname(__dirname);
}
}
var DEFAULT_CACHE_DIR = RUNNING_LOCALLY ? node_path_default.join(dirname__, "/.cache/") : null;
var DEFAULT_LOCAL_MODEL_PATH = "/models/";
var localModelPath = RUNNING_LOCALLY ? node_path_default.join(dirname__, DEFAULT_LOCAL_MODEL_PATH) : DEFAULT_LOCAL_MODEL_PATH;
var DEFAULT_FETCH = typeof globalThis.fetch === "function" ? globalThis.fetch.bind(globalThis) : void 0;
var LogLevel = Object.freeze({
/** All messages including debug output (value: 10) */
DEBUG: 10,
/** Errors, warnings, and info messages (value: 20) */
INFO: 20,
/** Errors and warnings (value: 30) */
WARNING: 30,
/** Only error messages (value: 40) */
ERROR: 40,
/** No logging output (value: 50) */
NONE: 50
});
var logLevel = LogLevel.WARNING;
var env = {
version: VERSION,
/////////////////// Backends settings ///////////////////
// NOTE: These will be populated later by the backends themselves.
backends: {
// onnxruntime-web/onnxruntime-node
onnx: {}
},
/////////////////// Logging settings ///////////////////
get logLevel() {
return logLevel;
},
set logLevel(level) {
logLevel = level;
env.backends.onnx?.setLogLevel?.(level);
},
/////////////////// Model settings ///////////////////
allowRemoteModels: true,
remoteHost: "https://huggingface.co/",
remotePathTemplate: "{model}/resolve/{revision}/",
allowLocalModels: !(IS_BROWSER_ENV || IS_WEBWORKER_ENV || IS_DENO_WEB_RUNTIME),
// Default to true for non-web environments, false for web environments
localModelPath,
useFS: IS_FS_AVAILABLE,
/////////////////// Cache settings ///////////////////
useBrowserCache: IS_WEB_CACHE_AVAILABLE,
useFSCache: IS_FS_AVAILABLE,
cacheDir: DEFAULT_CACHE_DIR,
useCustomCache: false,
customCache: null,
useWasmCache: IS_WEB_CACHE_AVAILABLE || IS_FS_AVAILABLE,
cacheKey: "transformers-cache",
experimental_useCrossOriginStorage: false,
/////////////////// Custom fetch /////////////////////
fetch: DEFAULT_FETCH
//////////////////////////////////////////////////////
};
function isEmpty(obj) {
return Object.keys(obj).length === 0;
}
// src/utils/generic.js
var Callable = (
/** @type {any} */
class {
/**
* Creates a new instance of the Callable class.
*/
constructor() {
let closure = function(...args) {
return closure._call(...args);
};
return Object.setPrototypeOf(closure, new.target.prototype);
}
/**
* This method should be implemented in subclasses to provide the
* functionality of the callable object.
*
* @param {any[]} args
* @throws {Error} If the subclass does not implement the `_call` method.
*/
_call(...args) {
throw Error("Must implement _call method in subclass");
}
}
);
// src/utils/core.js
function dispatchCallback(progress_callback, data) {
if (progress_callback) progress_callback(data);
}
var DefaultProgressCallback = class extends Callable {
/**
* @param {ProgressCallback} callback The original callback.
* @param {FilesLoadingMap} files_loading Mutable map storing per-file progress.
*/
constructor(callback, files_loading) {
super();
this.callback = callback;
this.files_loading = files_loading;
}
/**
* @param {ProgressInfo} info
*/
_call(info) {
if (info.status === "progress") {
this.files_loading[info.file] = {
loaded: info.loaded,
total: info.total
};
const loaded = Object.values(this.files_loading).reduce((acc, curr) => acc + curr.loaded, 0);
const total = Object.values(this.files_loading).reduce((acc, curr) => acc + curr.total, 0);
const progress = total > 0 ? loaded / total * 100 : 0;
this.callback({
status: "progress_total",
name: info.name,
progress,
loaded,
total,
files: structuredClone(this.files_loading)
});
}
this.callback(info);
}
};
function isIntegralNumber(x) {
return Number.isInteger(x) || typeof x === "bigint";
}
function isNullishDimension(x) {
return x === null || x === void 0 || x === -1;
}
function calculateDimensions(arr) {
const dimensions = [];
let current = arr;
while (Array.isArray(current)) {
dimensions.push(current.length);
current = current[0];
}
return dimensions;
}
function mergeArrays(...arrs) {
return Array.prototype.concat.apply([], arrs);
}
function product(...a) {
return a.reduce((a2, b) => a2.flatMap((d) => b.map((e) => [d, e])));
}
function calculateReflectOffset(i, w) {
return Math.abs((i + w) % (2 * w) - w);
}
function pick(o, props) {
return Object.assign(
{},
...props.map((prop) => {
if (o[prop] !== void 0) {
return { [prop]: o[prop] };
}
})
);
}
function count(arr, value) {
let count2 = 0;
for (const v of arr) {
if (v === value) ++count2;
}
return count2;
}
// src/utils/logger.js
var logger = {
/**
* Log an error message. Only suppressed when logLevel is NONE.
* @param {...any} args - Arguments to log
*/
error(...args) {
if (env.logLevel <= LogLevel.ERROR) {
console.error(...args);
}
},
/**
* Log a warning message. Shown when logLevel <= WARNING.
* @param {...any} args - Arguments to log
*/
warn(...args) {
if (env.logLevel <= LogLevel.WARNING) {
console.warn(...args);
}
},
/**
* Log an info message. Shown when logLevel <= INFO.
* @param {...any} args - Arguments to log
*/
info(...args) {
if (env.logLevel <= LogLevel.INFO) {
console.log(...args);
}
},
/**
* Log a debug message. Only shown when logLevel is DEBUG.
* @param {...any} args - Arguments to log
*/
debug(...args) {
if (env.logLevel <= LogLevel.DEBUG) {
console.log(...args);
}
},
/**
* Log a message (alias for info). Shown when logLevel <= INFO.
* @param {...any} args - Arguments to log
*/
log(...args) {
this.info(...args);
}
};
// ../../node_modules/.pnpm/@huggingface+tokenizers@0.1.3/node_modules/@huggingface/tokenizers/dist/tokenizers.mjs
var DictionarySplitter = class {
/**
* @param dictionary The dictionary of words to use for splitting.
*/
constructor(dictionary) {
this.trie = this._build_trie(dictionary);
}
/**
* Builds a trie from the given dictionary.
* @param dictionary The dictionary of words to build the trie from.
* @returns The root node of the trie.
* @private
*/
_build_trie(dictionary) {
const trie = /* @__PURE__ */ Object.create(null);
for (const word of dictionary) {
let node = trie;
for (let i = 0; i < word.length; ++i) {
const char = word[i];
node = node[char] ??= /* @__PURE__ */ Object.create(null);
}
node.end = word;
}
return trie;
}
/**
* Splits the input text into tokens based on the dictionary.
* @param text The input text to split.
* @returns An array of tokens.
*/
split(text) {
const result = [];
const n = text.length;
let start = 0;
let i = 0;
while (i < n) {
let node = this.trie;
let match = null;
let j = i;
while (j < n && (node = node[text[j]])) {
if (node.end) {
match = node.end;
}
++j;
}
if (match) {
if (i > start) {
result.push(text.slice(start, i));
}
result.push(match);
i += match.length;
start = i;
} else {
++i;
}
}
if (start < n) {
result.push(text.slice(start));
}
return result;
}
};
var DictionarySplitter_default = DictionarySplitter;
var AddedToken = class {
/**
* Creates a new instance of AddedToken.
* @param config Added token configuration object.
*/
constructor(config) {
this.content = config.content;
this.id = config.id;
this.single_word = config.single_word ?? false;
this.lstrip = config.lstrip ?? false;
this.rstrip = config.rstrip ?? false;
this.special = config.special ?? false;
this.normalized = config.normalized ?? !this.special;
}
};
var AddedToken_default = AddedToken;
var BYTES_TO_UNICODE = (() => {
const bs = [
...Array.from(
{ length: "~".charCodeAt(0) - "!".charCodeAt(0) + 1 },
(_, i) => i + "!".charCodeAt(0)
),
...Array.from(
{ length: "\xAC".charCodeAt(0) - "\xA1".charCodeAt(0) + 1 },
(_, i) => i + "\xA1".charCodeAt(0)
),
...Array.from(
{ length: "\xFF".charCodeAt(0) - "\xAE".charCodeAt(0) + 1 },
(_, i) => i + "\xAE".charCodeAt(0)
)
];
const cs = bs.slice();
let n = 0;
for (let b = 0; b < 256; ++b) {
if (!bs.includes(b)) {
bs.push(b);
cs.push(256 + n);
n += 1;
}
}
const ccs = cs.map((n2) => String.fromCharCode(n2));
return Object.fromEntries(bs.map((b, i) => [b, ccs[i]]));
})();
var reverse_dictionary = (data) => Object.fromEntries(Object.entries(data).map(([key, value]) => [value, key]));
var UNICODE_TO_BYTES = reverse_dictionary(BYTES_TO_UNICODE);
var BLOOM_SPLIT_CHARS = ".,!?\u2026\u3002\uFF0C\u3001\u0964\u06D4\u060C";
var PROBLEMATIC_REGEX_MAP = /* @__PURE__ */ new Map([
// These uses the case insensitive group modifier, which is not supported in JavaScript.
// When parsing the regex, an "Invalid group" error is thrown.
[
"(?i:'s|'t|'re|'ve|'m|'ll|'d)",
"(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))"
],
[
"(?i:[sdmt]|ll|ve|re)",
"(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])"
],
// JS doesn't support possessive quantifiers (these are used in recent OpenAI tokenizers).
["[^\\r\\n\\p{L}\\p{N}]?+", "[^\\r\\n\\p{L}\\p{N}]?"],
["[^\\s\\p{L}\\p{N}]++", "[^\\s\\p{L}\\p{N}]+"],
// JS doesn't support atomic groups (these are used in AFMoE tokenizers).
["(?>\\p{Nd}{510})", "(?:\\p{Nd}{510})"],
// JS doesn't support stacking quantifiers.
// Uncaught SyntaxError: Invalid regular expression: /\p{Nd}{3}+/u: Nothing to repeat
["\\p{Nd}{3}+", "(?:\\p{Nd}{3})+"],
// \G is an invalid escape in JS, and in most cases is just used as an optimization.
// So, we can safely remove it.
["\\G", ""],
// Used to override the default (invalid) regex of the bloom pretokenizer.
// For more information, see https://github.com/huggingface/transformers.js/issues/94
[` ?[^(\\s|[${BLOOM_SPLIT_CHARS}])]+`, ` ?[^\\s${BLOOM_SPLIT_CHARS}]+`]
]);
var PUNCTUATION_REGEX = "\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E";
var clean_up_tokenization = (text) => text.replace(/ \./g, ".").replace(/ \?/g, "?").replace(/ \!/g, "!").replace(/ ,/g, ",").replace(/ \' /g, "'").replace(/ n't/g, "n't").replace(/ 'm/g, "'m").replace(/ 's/g, "'s").replace(/ 've/g, "'ve").replace(/ 're/g, "'re");
var create_pattern = (pattern, invert = true) => {
if (pattern.Regex !== void 0) {
let regex = pattern.Regex.replace(/\\([#&~])/g, "$1");
regex = regex.replace(/\\A/g, "^").replace(/\\z/g, "$").replace(/\\Z/g, "(?=\\r?\\n?$)");
for (const [key, value] of PROBLEMATIC_REGEX_MAP) {
regex = regex.replaceAll(key, value);
}
try {
return new RegExp(regex, "gu");
} catch (error) {
if (!(error instanceof SyntaxError) || !error.message.toLowerCase().includes("invalid property name"))
throw error;
let changed = false;
const fixed = regex.replace(/(\\[pP])\{([^}=]+)\}/g, (_, p, n) => {
try {
new RegExp(`\\p{${n}}`, "u");
return `${p}{${n}}`;
} catch {
changed = true;
return `${p}{Script=${n}}`;
}
});
if (!changed) throw error;
try {
return new RegExp(fixed, "gu");
} catch (e) {
throw error;
}
}
} else if (pattern.String !== void 0) {
const escaped = escape_reg_exp(pattern.String);
return new RegExp(invert ? escaped : `(${escaped})`, "gu");
} else {
console.warn("Unknown pattern type:", pattern);
return null;
}
};
var escape_reg_exp = (string) => string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
var fuse_unk = (arr, tokens_to_ids, unk_token_id) => {
const fused = [];
let i = 0;
while (i < arr.length) {
fused.push(arr[i]);
const token_id = tokens_to_ids.get(arr[i]) ?? unk_token_id;
if (token_id !== unk_token_id) {
++i;
continue;
}
while (++i < arr.length && (tokens_to_ids.get(arr[i]) ?? unk_token_id) === unk_token_id) {
if (tokens_to_ids.get(fused.at(-1)) !== unk_token_id) {
fused[fused.length - 1] += arr[i];
}
}
}
return fused;
};
var is_chinese_char = (cp) => cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
var is_integral_number = (x) => Number.isInteger(x) || typeof x === "bigint";
var len = (s) => {
let length = 0;
for (const c of s) ++length;
return length;
};
var lowercase_and_remove_accents = (text) => remove_accents(text.toLowerCase());
var merge_arrays = (...arrs) => Array.prototype.concat.apply([], arrs);
var object_to_map = (obj) => new Map(Object.entries(obj));
var regex_split = (text, regex) => {
const result = [];
let prev = 0;
for (const match of text.matchAll(regex)) {
const full_match = match[0];
if (prev < match.index) {
result.push(text.slice(prev, match.index));
}
if (full_match.length > 0) {
result.push(full_match);
}
prev = match.index + full_match.length;
}
if (prev < text.length) {
result.push(text.slice(prev));
}
return result;
};
var remove_accents = (text) => text.replace(/\p{M}/gu, "");
var validate_object = (obj, name, required_keys = []) => {
if (!obj || Array.isArray(obj) || typeof obj !== "object") {
return `${name} must be a valid object`;
}
for (const key of required_keys) {
if (!(key in obj)) {
return `${name} must contain a "${key}" property`;
}
}
return null;
};
var whitespace_split = (text) => text.match(/\S+/g) || [];
var Callable2 = class {
/**
* Creates a new instance of the Callable class.
*/
constructor() {
const closure = function(...args) {
return closure._call(...args);
};
return Object.setPrototypeOf(closure, new.target.prototype);
}
};
var Callable_default = Callable2;
var Normalizer = class extends Callable_default {
/**
* @param config The configuration object for the normalizer.
*/
constructor(config) {
super();
this.config = config;
}
/**
* Alias for {@link Normalizer#normalize}.
* @param text The text to normalize.
* @returns The normalized text.
*/
_call(text) {
return this.normalize(text);
}
};
var Normalizer_default = Normalizer;
var BertNormalizer = class extends Normalizer_default {
/**
* Adds whitespace around any CJK (Chinese, Japanese, or Korean) character in the input text.
*
* @param text The input text to tokenize.
* @returns The tokenized text with whitespace added around CJK characters.
*/
tokenize_chinese_chars(text) {
const output = [];
for (let i = 0; i < text.length; ++i) {
const char = text[i];
const cp = char.charCodeAt(0);
if (is_chinese_char(cp)) {
output.push(" ");
output.push(char);
output.push(" ");
} else {
output.push(char);
}
}
return output.join("");
}
/**
* Strips accents from the given text.
* @param text The text to strip accents from.
* @returns The text with accents removed.
*/
strip_accents(text) {
return text.normalize("NFD").replace(/\p{Mn}/gu, "");
}
/**
* Checks whether `char` is a control character.
* @param char The character to check.
* @returns Whether `char` is a control character.
*/
is_control(char) {
switch (char) {
case " ":
case "\n":
case "\r":
return false;
default:
return /^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(char);
}
}
/**
* Performs invalid character removal and whitespace cleanup on text.
* @param text The text to clean.
* @returns The cleaned text.
*/
clean_text(text) {
const output = [];
for (const char of text) {
const cp = char.charCodeAt(0);
if (cp === 0 || cp === 65533 || this.is_control(char)) {
continue;
}
if (/^\s$/.test(char)) {
output.push(" ");
} else {
output.push(char);
}
}
return output.join("");
}
/**
* Normalizes the given text based on the configuration.
* @param text The text to normalize.
* @returns The normalized text.
*/
normalize(text) {
if (this.config.clean_text) {
text = this.clean_text(text);
}
if (this.config.handle_chinese_chars) {
text = this.tokenize_chinese_chars(text);
}
if (this.config.lowercase) {
text = text.toLowerCase();
if (this.config.strip_accents !== false) {
text = this.strip_accents(text);
}
} else if (this.config.strip_accents) {
text = this.strip_accents(text);
}
return text;
}
};
var BertNormalizer_default = BertNormalizer;
var Precompiled = class extends Normalizer_default {
/**
* Create a new instance of Precompiled normalizer.
* @param config The configuration object.
*/
constructor(config) {
super(config);
this.charsmap = config.precompiled_charsmap ?? null;
}
/**
* Normalizes the given text by applying the precompiled charsmap.
* @param text The text to normalize.
* @returns The normalized text.
*/
normalize(text) {
text = text.replace(
/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,
""
);
text = text.replace(
/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm,
" "
);
if (text.includes("\uFF5E")) {
const parts = text.split("\uFF5E");
text = parts.map((part) => part.normalize("NFKC")).join("\uFF5E");
} else {
text = text.normalize("NFKC");
}
return text;
}
};
var Precompiled_default = Precompiled;
var Sequence = class extends Normalizer_default {
/**
* Create a new instance of NormalizerSequence.
* @param config The configuration object.
*/
constructor(config) {
super(config);
this.normalizers = (config.normalizers ?? []).map(
(x) => create_normalizer_default(x)
);
}
/**
* Apply a sequence of Normalizers to the input text.
* @param text The text to normalize.
* @returns The normalized text.
*/
normalize(text) {
return this.normalizers.reduce((t, normalizer) => {
return normalizer ? normalizer.normalize(t) : t;
}, text);
}
};
var Sequence_default = Sequence;
var Replace = class extends Normalizer_default {
/**
* Normalize the input text by replacing the pattern with the content.
* @param text The input text to be normalized.
* @returns The normalized text after replacing the pattern with the content.
*/
normalize(text) {
const pattern = create_pattern(this.config.pattern ?? {});
return pattern === null ? text : text.replaceAll(pattern, this.config.content ?? "");
}
};
var Replace_default = Replace;
var UnicodeNormalizer = class extends Normalizer_default {
constructor() {
super(...arguments);
this.form = "NFC";
}
/**
* Normalize the input text by applying Unicode normalization.
* @param text The input text to be normalized.
* @returns The normalized text.
*/
normalize(text) {
text = text.normalize(this.form);
return text;
}
};
var UnicodeNormalizer_default = UnicodeNormalizer;
var NFC = class extends UnicodeNormalizer_default {
constructor() {
super(...arguments);
this.form = "NFC";
}
};
var NFC_default = NFC;
var NFD = class extends UnicodeNormalizer_default {
constructor() {
super(...arguments);
this.form = "NFD";
}
};
var NFD_default = NFD;
var NFKC = class extends UnicodeNormalizer_default {
constructor() {
super(...arguments);
this.form = "NFKC";
}
};
var NFKC_default = NFKC;
var NFKD = class extends UnicodeNormalizer_default {
constructor() {
super(...arguments);
this.form = "NFKD";
}
};
var NFKD_default = NFKD;
var Strip = class extends Normalizer_default {
/**
* Strip leading and/or trailing whitespace from the input text.
* @param text The input text.
* @returns The normalized text.
*/
normalize(text) {
if (this.config.strip_left && this.config.strip_right) {
text = text.trim();
} else {
if (this.config.strip_left) {
text = text.trimStart();
}
if (this.config.strip_right) {
text = text.trimEnd();
}
}
return text;
}
};
var Strip_default = Strip;
var StripAccents = class extends Normalizer_default {
/**
* Remove all accents from the text.
* @param text The input text.
* @returns The normalized text without accents.
*/
normalize(text) {
return remove_accents(text);
}
};
var StripAccents_default = StripAccents;
var Lowercase = class extends Normalizer_default {
/**
* Lowercases the input string.
* @param {string} text The text to normalize.
* @returns {string} The normalized text.
*/
normalize(text) {
return text.toLowerCase();
}
};
var Lowercase_default = Lowercase;
var Prepend = class extends Normalizer_default {
/**
* Prepends the input string.
* @param text The text to normalize.
* @returns The normalized text.
*/
normalize(text) {
text = this.config.prepend + text;
return text;
}
};
var Prepend_default = Prepend;
function create_normalizer(config) {
if (config === null) return null;
switch (config.type) {
case "BertNormalizer":
return new BertNormalizer_default(config);
case "Precompiled":
return new Precompiled_default(config);
case "Sequence":
return new Sequence_default(config);
case "Replace":
return new Replace_default(config);
case "NFC":
return new NFC_default(config);
case "NFD":
return new NFD_default(config);
case "NFKC":
return new NFKC_default(config);
case "NFKD":
return new NFKD_default(config);
case "Strip":
return new Strip_default(config);
case "StripAccents":
return new StripAccents_default(config);
case "Lowercase":
return new Lowercase_default(config);
case "Prepend":
return new Prepend_default(config);
default:
throw new Error(`Unknown Normalizer type: ${config.type}`);
}
}
var create_normalizer_default = create_normalizer;
var PreTokenizer = class extends Callable_default {
/**
* Tokenizes the given text into pre-tokens.
* @param text The text or array of texts to pre-tokenize.
* @param options Additional options for the pre-tokenization logic.
* @returns An array of pre-tokens.
*/
pre_tokenize(text, options) {
return (Array.isArray(text) ? text.map((x) => this.pre_tokenize_text(x, options)) : this.pre_tokenize_text(text, options)).flat();
}
/**
* Alias for {@link PreTokenizer#pre_tokenize}.
* @param text The text or array of texts to pre-tokenize.
* @param options Additional options for the pre-tokenization logic.
* @returns An array of pre-tokens.
*/
_call(text, options) {
return this.pre_tokenize(text, options);
}
};
var PreTokenizer_default = PreTokenizer;
var ByteLevel = class extends PreTokenizer_default {
/**
* Creates a new instance of the `ByteLevelPreTokenizer` class.
* @param config The configuration object.
*/
constructor(config) {
super();
this.config = config;
this.add_prefix_space = this.config.add_prefix_space ?? false;
this.trim_offsets = this.config.trim_offsets ?? false;
this.use_regex = this.config.use_regex ?? true;
this.pattern = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
this.byte_encoder = BYTES_TO_UNICODE;
this.text_encoder = new TextEncoder();
}
/**
* Tokenizes a single piece of text using byte-level tokenization.
* @param text The text to tokenize.
* @param options Additional options for the pre-tokenization logic.
* @returns An array of tokens.
*/
pre_tokenize_text(text, options) {
if (this.add_prefix_space && !text.startsWith(" ")) {
text = " " + text;
}
const tokens = this.use_regex ? text.match(this.pattern) || [] : [text];
return tokens.map(
(token) => Array.from(
this.text_encoder.encode(token),
(byte) => this.byte_encoder[byte]
).join("")
);
}
};
var ByteLevel_default = ByteLevel;
var Whitespace = class extends PreTokenizer_default {
/**
* Pre-tokenizes the input text by splitting it on word boundaries.
* @param text The text to be pre-tokenized.
* @param options Additional options for the pre-tokenization logic.
* @returns An array of tokens produced by splitting the input text on whitespace.
*/
pre_tokenize_text(text, options) {
return text.match(/\w+|[^\w\s]+/g) || [];
}
};
var Whitespace_default = Whitespace;
var Metaspace = class extends PreTokenizer_default {
/**
* @param config The configuration object for the MetaspacePreTokenizer.
*/
constructor(config) {
super();
this.replacement = config.replacement ?? "\u2581";
this.str_rep = config.str_rep || this.replacement;
this.prepend_scheme = config.prepend_scheme ?? "always";
}
/**
* This method takes a string, replaces spaces with the replacement character,
* adds a prefix space if requested, and returns a new list of tokens.
* @param text The text to pre-tokenize.
* @param options The options for the pre-tokenization.
* @returns A new list of pre-tokenized tokens.
*/
pre_tokenize_text(text, options) {
const { section_index = void 0 } = options ?? {};
let normalized = text.replaceAll(" ", this.str_rep);
if (
// We add a prefix space if:
// (1) The normalized token does not already start with the replacement character.
!normalized.startsWith(this.replacement) && // and (2) either:
// (a) prepend_scheme is 'always'
// (b) prepend_scheme is 'first' and this is the first section
(this.prepend_scheme === "always" || this.prepend_scheme === "first" && section_index === 0)
) {
normalized = this.str_rep + normalized;
}
return [normalized];
}
};
var Metaspace_default = Metaspace;
var Split = class extends PreTokenizer_default {
/**
* @param config The configuration options for the pre-tokenizer.
*/
constructor(config) {
super();
this.config = config;
this.pattern = create_pattern(
this.config.pattern ?? {},
this.config.invert ?? true
);
}
/**
* Tokenizes text by splitting it using the given pattern.
* @param text The text to tokenize.
* @returns An array of tokens.
*/
pre_tokenize_text(text) {
if (this.pattern === null) {
return [];
}
if (this.config.invert) {
return text.match(this.pattern) || [];
} else if (this.config.behavior?.toLowerCase() === "removed") {
return text.split(this.pattern).filter((x) => x);
} else {
return regex_split(text, this.pattern);
}
}
};
var Split_default = Split;
var Punctuation = class extends PreTokenizer_default {
/**
* @param config The configuration options for the pre-tokenizer.
*/
constructor(config) {
super();
this.config = config;
this.pattern = new RegExp(
`[^${PUNCTUATION_REGEX}]+|[${PUNCTUATION_REGEX}]+`,
"gu"
);
}
/**
* Tokenizes text by splitting it using the given pattern.
* @param text The text to tokenize.
* @returns An array of tokens.
*/
pre_tokenize_text(text) {
return text.match(this.pattern) || [];
}
};
var Punctuation_default = Punctuation;
var Digits = class extends PreTokenizer_default {
/**
* @param config The configuration options for the pre-tokenizer.
*/
constructor(config) {
super();
this.config = config;
const digit_pattern = `[^\\d]+|\\d${this.config.individual_digits ? "" : "+"}`;
this.pattern = new RegExp(digit_pattern, "gu");
}
/**
* Tokenizes text by splitting it using the given pattern.
* @param text The text to tokenize.
* @returns An array of tokens.
*/
pre_tokenize_text(text) {
return text.match(this.pattern) || [];
}
};
var Digits_default = Digits;
var BertPreTokenizer = class extends PreTokenizer_default {
/**
* A PreTokenizer that splits text into wordpieces using a basic tokenization scheme
* similar to that used in the original implementation of BERT.
*/
constructor() {
super();
this.pattern = new RegExp(
`[^\\s${PUNCTUATION_REGEX}]+|[${PUNCTUATION_REGEX}]`,
"gu"
);
}
/**
* Tokenizes a single text using the BERT pre-tokenization scheme.
*
* @param text The text to tokenize.
* @param options Additional options for the pre-tokenization logic.
* @returns An array of tokens.
*/
pre_tokenize_text(text, options) {
return text.trim().match(this.pattern) || [];
}
};
var BertPreTokenizer_default = BertPreTokenizer;
var Replace2 = class extends PreTokenizer_default {
/**
* @param config The configuration options for the pre-tokenizer.
*/
constructor(config) {
super();
this.config = config;
this.pattern = create_pattern(this.config.pattern ?? {});
this.content = this.config.content ?? "";
}
/**
* Pre-tokenizes the input text by replacing certain characters.
* @param text The text to be pre-tokenized.
* @returns An array of tokens produced by replacing certain characters.
*/
pre_tokenize_text(text) {
if (this.pattern === null) {
return [text];
}
return [text.replaceAll(this.pattern, this.config.content ?? "")];
}
};
var Replace_default2 = Replace2;
var Sequence2 = class extends PreTokenizer_default {
/**
* Creates an instance of PreTokenizerSequence.
* @param config The configuration object for the pre-tokenizer sequence.
*/
constructor(config) {
super();
this.tokenizers = (config.pretokenizers ?? []).map(
(x) => create_pre_tokenizer_default(x)
);
}
/**
* Applies each pre-tokenizer in the sequence to the input text in turn.
* @param text The text to pre-tokenize.
* @param options Additional options for the pre-tokenization logic.
* @returns The pre-tokenized text.
*/
pre_tokenize_text(text, options) {
return this.tokenizers.reduce(
(pre_tokenized_text, tokenizer) => {
return tokenizer ? tokenizer.pre_tokenize(pre_tokenized_text, options) : pre_tokenized_text;
},
[text]
);
}
};
var Sequence_default2 = Sequence2;
var WhitespaceSplit = class extends PreTokenizer_default {
/**
* Pre-tokenizes the input text by splitting it on whitespace characters.
* @param text The text to be pre-tokenized.
* @returns An array of tokens produced by splitting the input text on whitespace.
*/
pre_tokenize_text(text) {
return whitespace_split(text);
}
};
var WhitespaceSplit_default = WhitespaceSplit;
var FixedLength = class extends PreTokenizer_default {
/**
* @param config The configuration options for the pre-tokenizer.
*/
constructor(config) {
super();
this.config = config;
this._length = config.length;
}
/**
* Pre-tokenizes the input text by splitting it into fixed-length tokens.
* @param text The text to be pre-tokenized.
* @returns An array of tokens produced by splitting the input text into fixed-length tokens.
*/
pre_tokenize_text(text) {
const tokens = [];
for (let i = 0; i < text.length; i += this._length) {
tokens.push(text.slice(i, i + this._length));
}
return tokens;
}
};
var FixedLength_default = FixedLength;
function create_pre_tokenizer(config) {
if (config === null) return null;
switch (config.type) {
case "BertPreTokenizer":
return new BertPreTokenizer_default();
case "Sequence":
return new Sequence_default2(config);
case "Whitespace":
return new Whitespace_default();
case "WhitespaceSplit":
return new WhitespaceSplit_default();
case "Metaspace":
return new Metaspace_default(config);
case "ByteLevel":
return new ByteLevel_default(config);
case "Split":
return new Split_default(config);
case "Punctuation":
return new Punctuation_default(config);
case "Digits":
return new Digits_default(config);
case "Replace":
return new Replace_default2(config);
case "FixedLength":
return new FixedLength_default(config);
default:
throw new Error(`Unknown PreTokenizer type: ${config.type}`);
}
}
var create_pre_tokenizer_default = create_pre_tokenizer;
var TokenizerModel = class extends Callable_default {
/**
* Creates a new instance of TokenizerModel.
* @param config The configuration object for the TokenizerModel.
*/
constructor(config) {
super();
this.config = config;
this.vocab = [];
this.tokens_to_ids = /* @__PURE__ */ new Map();
this.unk_token_id = void 0;
this.unk_token = void 0;
this.end_of_word_suffix = void 0;
this.fuse_unk = this.config.fuse_unk ?? false;
}
/**
* Internal function to call the TokenizerModel instance.
* @param tokens The tokens to encode.
* @returns The encoded tokens.
*/
_call(tokens) {
let result = this.encode(tokens);
if (this.fuse_unk) {
result = fuse_unk(result, this.tokens_to_ids, this.unk_token_id);
}
return result;
}
};
var TokenizerModel_default = TokenizerModel;
var WordPieceTokenizer = class extends TokenizerModel_default {
/**
* @param config The configuration object.
*/
constructor(config) {
super(config);
this.max_input_chars_per_word = 100;
this.tokens_to_ids = object_to_map(config.vocab);
this.unk_token_id = this.tokens_to_ids.get(config.unk_token);
this.unk_token = config.unk_token;
this.max_input_chars_per_word = config.max_input_chars_per_word ?? 100;
this.vocab = new Array(this.tokens_to_ids.size);
for (const [key, value] of this.tokens_to_ids) {
this.vocab[value] = key;
}
}
/**
* Encodes an array of tokens using WordPiece encoding.
* @param tokens The tokens to encode.
* @returns An array of encoded tokens.
*/
encode(tokens) {
const output_tokens = [];
for (const token of tokens) {
const chars = [...token];
if (chars.length > this.max_input_chars_per_word) {
output_tokens.push(this.unk_token);
continue;
}
let is_unknown = false;
let start = 0;
const sub_tokens = [];
while (start < chars.length) {
let end = chars.length;
let current_substring = null;
while (start < end) {
let substr = chars.slice(start, end).join("");
if (start > 0) {
substr = this.config.continuing_subword_prefix + substr;
}
if (this.tokens_to_ids.has(substr)) {
current_substring = substr;
break;
}
--end;
}
if (current_substring === null) {
is_unknown = true;
break;
}
sub_tokens.push(current_substring);
start = end;
}
if (is_unknown) {
output_tokens.push(this.unk_token);
} else {
output_tokens.push(...sub_tokens);
}
}
return output_tokens;
}
};
var WordPiece_default = WordPieceTokenizer;
var CharTrieNode = class _CharTrieNode {
/**
* Create a new CharTrieNode.
* @param is_leaf Whether the node is a leaf node or not.
* @param children A map containing the node's children, where the key is a character and the value is a `CharTrieNode`.
*/
constructor(is_leaf, children) {
this.is_leaf = is_leaf;
this.children = children;
}
/**
* Returns a new `CharTrieNode` instance with default values.
* @returns A new `CharTrieNode` instance with `is_leaf` set to `false` and an empty `children` map.
*/
static default() {
return new _CharTrieNode(false, /* @__PURE__ */ new Map());
}
};
var CharTrie = class {
constructor() {
this.root = CharTrieNode.default();
}
/**
* Adds one or more `texts` to the trie.
* @param texts The strings to add to the trie.
*/
extend(texts) {
for (const text of texts) {
this.push(text);
}
}
/**
* Adds text to the trie.
* @param text The string to add to the trie.
*/
push(text) {
let node = this.root;
for (const ch of text) {
let child = node.children.get(ch);
if (child === void 0) {
child = CharTrieNode.default();
node.children.set(ch, child);
}
node = child;
}
node.is_leaf = true;
}
/**
* Searches the trie for all strings with a common prefix of `text`.
* @param text The common prefix to search for.
* @yields Each string in the trie that has `text` as a prefix.
*/
*common_prefix_search(text) {
let node = this.root;
if (node === void 0) return;
let prefix = "";
for (const ch of text) {
prefix += ch;
node = node.children.get(ch);
if (node === void 0) return;
if (node.is_leaf) {
yield prefix;
}
}
}
};
var CharTrie_default = CharTrie;
var TokenLatticeNode = class _TokenLatticeNode {
/**
* Represents a node in a token lattice for a given sentence.
* @param token_id The ID of the token associated with this node.
* @param node_id The ID of this node.
* @param pos The starting position of the token in the sentence.
* @param length The length of the token.
* @param score The score associated with the token.
*/
constructor(token_id, node_id, pos, length, score) {
this.token_id = token_id;
this.node_id = node_id;
this.pos = pos;
this.length = length;
this.score = score;
this.prev = null;
this.backtrace_score = 0;
}
/**
* Returns a clone of this node.
* @returns A clone of this node.
*/
clone() {
const n = new _TokenLatticeNode(
this.token_id,
this.node_id,
this.pos,
this.length,
this.score
);
n.prev = this.prev;
n.backtrace_score = this.backtrace_score;
return n;
}
};
var TokenLattice = class {
/**
* Creates a new TokenLattice instance.
*
* @param sentence The input sentence to be tokenized.
* @param bos_token_id The beginning-of-sequence token ID.
* @param eos_token_id The end-of-sequence token ID.
*/
constructor(sentence, bos_token_id, eos_token_id) {
this.chars = Array.from(sentence);
this.len = this.chars.length;
this.bos_token_id = bos_token_id;
this.eos_token_id = eos_token_id;
this.nodes = [];
this.begin_nodes = Array.from(
{ length: this.len + 1 },
() => []
);
this.end_nodes = Array.from({ length: this.len + 1 }, () => []);
const bos = new TokenLatticeNode(this.bos_token_id ?? 0, 0, 0, 0, 0);
const eos = new TokenLatticeNode(
this.eos_token_id ?? 0,
1,
this.len,
0,
0
);
this.nodes.push(bos.clone());
this.nodes.push(eos.clone());
this.begin_nodes[this.len].push(eos);
this.end_nodes[0].push(bos);
}
/**
* Inserts a new token node into the token lattice.
*
* @param pos The starting position of the token.
* @param length The length of the token.
* @param score The score of the token.
* @param token_id The token ID of the token.
*/
insert(pos, length, score, token_id) {
const node_id = this.nodes.length;
const node = new TokenLatticeNode(token_id, node_id, pos, length, score);
this.begin_nodes[pos].push(node);
this.end_nodes[pos + length].push(node);
this.nodes.push(node);
}
/**
* Implements the Viterbi algorithm to compute the most likely sequence of tokens.
*
* @returns The most likely sequence of tokens.
*/
viterbi() {
const len2 = this.len;
let pos = 0;
while (pos <= len2) {
if (this.begin_nodes[pos].length == 0) {
return [];
}
for (let rnode of this.begin_nodes[pos]) {
rnode.prev = null;
let best_score = 0;
let best_node = null;
for (let lnode of this.end_nodes[pos]) {
const score = lnode.backtrace_score + rnode.score;
if (best_node === null || score > best_score) {
best_node = lnode.clone();
best_score = score;
}
}
if (best_node !== null) {
rnode.prev = best_node;
rnode.backtrace_score = best_score;
} else {
return [];
}
}
++pos;
}
const results = [];
const root = this.begin_nodes[len2][0];
const prev = root.prev;
if (prev === null) {
return [];
}
let node = prev.clone();
while (node.prev !== null) {
results.push(node.clone());
const n = node.clone();
node = n.prev.clone();
}
results.reverse();
return results;
}
/**
* Get the text piece for a given node.
* @param node The node to get the piece for.
* @returns The array of nodes representing the most likely sequence of tokens.
*/
piece(node) {
return this.chars.slice(node.pos, node.pos + node.length).join("");
}
/**
* @returns The most likely sequence of tokens.
*/
tokens() {
const nodes = this.viterbi();
return nodes.map((x) => this.piece(x));
}
/**
* @returns The most likely sequence of token ids.
*/
token_ids() {
const nodes = this.viterbi();
return nodes.map((x) => x.token_id);
}
};
var TokenLattice_default = TokenLattice;
function min(arr) {
if (arr.length === 0) throw new Error("Array must not be empty");
let min_value = arr[0];
let index_of_min = 0;
for (let i = 1; i < arr.length; ++i) {
if (arr[i] < min_value) {
min_value = arr[i];
index_of_min = i;
}
}
return [min_value, index_of_min];
}
var Unigram = class extends TokenizerModel_default {
/**
* Create a new Unigram tokenizer model.
* @param config The configuration object for the Unigram model.
* @param eos_token
*/
constructor(config, eos_token) {
super(config);
const vocab_size = config.vocab.length;
this.vocab = new Array(vocab_size);
this.scores = new Array(vocab_size);
for (let i = 0; i < vocab_size; ++i) {
[this.vocab[i], this.scores[i]] = config.vocab[i];
}
this.unk_token_id = config.unk_id;
this.unk_token = this.vocab[config.unk_id];
this.tokens_to_ids = new Map(this.vocab.map((x, i) => [x, i]));
this.bos_token = " ";
this.bos_token_id = this.tokens_to_ids.get(this.bos_token);
this.eos_token = eos_token;
this.eos_token_id = this.tokens_to_ids.get(this.eos_token);
this.unk_token = this.vocab[this.unk_token_id];
this.min_score = min(this.scores)[0];
this.unk_score = this.min_score - 10;
this.scores[this.unk_token_id] = this.unk_score;
this.trie = new CharTrie_default();
this.trie.extend(this.vocab);
this.fuse_unk = true;
}
/**
* Populates lattice nodes.
* @param lattice The token lattice to populate with nodes.
*/
populate_nodes(lattice) {
const chars = lattice.chars;
const mblen = 1;
let begin_pos = 0;
while (begin_pos < chars.length) {
let has_single_node = false;
const tokens = [];
const sliced = chars.slice(begin_pos).join("");
const prefixed_tokens = this.trie.common_prefix_search(sliced);
for (const token of prefixed_tokens) {
tokens.push(token);
const token_id = this.tokens_to_ids.get(token);
const token_score = this.scores[token_id];
const n = len(token);
lattice.insert(begin_pos, n, token_score, token_id);
if (!has_single_node && n === mblen) {
has_single_node = true;
}
}
if (!has_single_node) {
lattice.insert(begin_pos, mblen, this.unk_score, this.unk_token_id);
}
begin_pos += mblen;
}
}
/**
* Encodes an array of tokens into an array of subtokens using the unigram model.
*
* @param normalized The normalized string.
* @returns An array of subtokens obtained by encoding the input tokens using the unigram model.
*/
tokenize(normalized) {
const lattice = new TokenLattice_default(
normalized,
this.bos_token_id,
this.eos_token_id
);
this.populate_nodes(lattice);
return lattice.tokens();
}
/**
* Encodes an array of tokens using Unigram encoding.
* @param tokens The tokens to encode.
* @returns An array of encoded tokens.
*/
encode(tokens) {
const to_return = [];
for (const token of tokens) {
const tokenized = this.tokenize(token);
to_return.push(...tokenized);
}
return to_return;
}
};
var Unigram_default = Unigram;
var PriorityQueue = class {
/**
* Create a new PriorityQueue.
* @param comparator Comparator function to determine priority. Defaults to a MaxHeap.
* @param max_size Maximum size of the queue. Defaults to Infinity.
*/
constructor(comparator = (a, b) => a > b, max_size = Infinity) {
this._heap = [];
this._comparator = comparator;
this._max_size = max_size;
}
/**
* The size of the queue
*/
get size() {
return this._heap.length;
}
/**
* Check if the queue is empty.
* @returns `true` if the queue is empty, `false` otherwise.
*/
is_empty() {
return this.size === 0;
}
/**
* Return the element with the highest priority in the queue.
* @returns The highest priority element in the queue.
*/
peek() {
return this._heap[0];
}
/**
* Add one or more elements to the queue.
* @param values Th