@agnai/web-tokenizers
Version:
| [NPM Package](https://www.npmjs.com/package/@mlc-ai/web-tokenizers) | [WebLLM](https://github.com/mlc-ai/web-llm) |
319 lines (312 loc) • 4.18 MB
JavaScript
'use strict'
/******************************************************************************
Copyright (c) Microsoft Corporation.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
***************************************************************************** */
function __awaiter(thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value)
})
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value))
} catch (e) {
reject(e)
}
}
function rejected(value) {
try {
step(generator['throw'](value))
} catch (e) {
reject(e)
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected)
}
step((generator = generator.apply(thisArg, _arguments || [])).next())
})
}
function __generator(thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1]
return t[1]
},
trys: [],
ops: [],
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype)
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this
}),
g
)
function verb(n) {
return function (v) {
return step([n, v])
}
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.')
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t =
op[0] & 2
? y['return']
: op[0]
? y['throw'] || ((t = y['return']) && t.call(y), 0)
: y.next) &&
!(t = t.call(y, op[1])).done)
)
return t
if (((y = 0), t)) op = [op[0] & 2, t.value]
switch (op[0]) {
case 0:
case 1:
t = op
break
case 4:
_.label++
return { value: op[1], done: false }
case 5:
_.label++
y = op[1]
op = [0]
continue
case 7:
op = _.ops.pop()
_.trys.pop()
continue
default:
if (
!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) &&
(op[0] === 6 || op[0] === 2)
) {
_ = 0
continue
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1]
break
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1]
t = op
break
}
if (t && _.label < t[2]) {
_.label = t[2]
_.ops.push(op)
break
}
if (t[2]) _.ops.pop()
_.trys.pop()
continue
}
op = body.call(thisArg, _)
} catch (e) {
op = [6, e]
y = 0
} finally {
f = t = 0
}
if (op[0] & 5) throw op[1]
return { value: op[0] ? op[1] : void 0, done: true }
}
}
typeof SuppressedError === 'function'
? SuppressedError
: function (error, suppressed, message) {
var e = new Error(message)
return (e.name = 'SuppressedError'), (e.error = error), (e.suppressed = suppressed), e
}
async function Module(moduleArg = {}) {
var moduleRtn
var Module = moduleArg
var ENVIRONMENT_IS_WEB = !!globalThis.window
var ENVIRONMENT_IS_WORKER = !!globalThis.WorkerGlobalScope
var ENVIRONMENT_IS_NODE =
globalThis.process?.versions?.node && globalThis.process?.type != 'renderer'
if (ENVIRONMENT_IS_NODE) {
const { createRequire } = await import('module')
var require$1 = createRequire(
typeof document === 'undefined' && typeof location === 'undefined'
? new (require('u' + 'rl').URL)('file:' + __filename).href
: typeof document === 'undefined'
? location.href
: (document.currentScript &&
document.currentScript.tagName.toUpperCase() === 'SCRIPT' &&
document.currentScript.src) ||
new URL('index.js', document.baseURI).href
)
}
var thisProgram = './this.program'
var quit_ = (status, toThrow) => {
throw toThrow
}
var _scriptName =
typeof document === 'undefined' && typeof location === 'undefined'
? new (require('u' + 'rl').URL)('file:' + __filename).href
: typeof document === 'undefined'
? location.href
: (document.currentScript &&
document.currentScript.tagName.toUpperCase() === 'SCRIPT' &&
document.currentScript.src) ||
new URL('index.js', document.baseURI).href
var scriptDirectory = ''
var readAsync, readBinary
if (ENVIRONMENT_IS_NODE) {
var fs = require$1('fs')
if (_scriptName.startsWith('file:')) {
scriptDirectory = require$1('path').dirname(require$1('url').fileURLToPath(_scriptName)) + '/'
}
readBinary = (filename) => {
filename = isFileURI(filename) ? new URL(filename) : filename
var ret = fs.readFileSync(filename)
return ret
}
readAsync = async (filename, binary = true) => {
filename = isFileURI(filename) ? new URL(filename) : filename
var ret = fs.readFileSync(filename, binary ? undefined : 'utf8')
return ret
}
if (process.argv.length > 1) {
thisProgram = process.argv[1].replace(/\\/g, '/')
}
process.argv.slice(2)
quit_ = (status, toThrow) => {
process.exitCode = status
throw toThrow
}
} else if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) {
try {
scriptDirectory = new URL('.', _scriptName).href
} catch {}
{
if (ENVIRONMENT_IS_WORKER) {
readBinary = (url) => {
var xhr = new XMLHttpRequest()
xhr.open('GET', url, false)
xhr.responseType = 'arraybuffer'
xhr.send(null)
return new Uint8Array(xhr.response)
}
}
readAsync = async (url) => {
if (isFileURI(url)) {
return new Promise((resolve, reject) => {
var xhr = new XMLHttpRequest()
xhr.open('GET', url, true)
xhr.responseType = 'arraybuffer'
xhr.onload = () => {
if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) {
resolve(xhr.response)
return
}
reject(xhr.status)
}
xhr.onerror = reject
xhr.send(null)
})
}
var response = await fetch(url, { credentials: 'same-origin' })
if (response.ok) {
return response.arrayBuffer()
}
throw new Error(response.status + ' : ' + response.url)
}
}
} else;
var out = console.log.bind(console)
var err = console.error.bind(console)
var wasmBinary
var ABORT = false
var isFileURI = (filename) => filename.startsWith('file://')
function binaryDecode(bin) {
for (var i = 0, l = bin.length, o = new Uint8Array(l), c; i < l; ++i) {
c = bin.charCodeAt(i)
o[i] = (~c >> 8) & c
}
return o
}
var readyPromiseResolve, readyPromiseReject
var HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64
var HEAP64, HEAPU64
var runtimeInitialized = false
function updateMemoryViews() {
var b = wasmMemory.buffer
HEAP8 = new Int8Array(b)
HEAP16 = new Int16Array(b)
HEAPU8 = new Uint8Array(b)
HEAPU16 = new Uint16Array(b)
HEAP32 = new Int32Array(b)
HEAPU32 = new Uint32Array(b)
HEAPF32 = new Float32Array(b)
HEAPF64 = new Float64Array(b)
HEAP64 = new BigInt64Array(b)
HEAPU64 = new BigUint64Array(b)
}
function preRun() {
if (Module['preRun']) {
if (typeof Module['preRun'] == 'function') Module['preRun'] = [Module['preRun']]
while (Module['preRun'].length) {
addOnPreRun(Module['preRun'].shift())
}
}
callRuntimeCallbacks(onPreRuns)
}
function initRuntime() {
runtimeInitialized = true
if (!Module['noFSInit'] && !FS.initialized) FS.init()
wasmExports['va']()
FS.ignorePermissions = false
}
function postRun() {
if (Module['postRun']) {
if (typeof Module['postRun'] == 'function') Module['postRun'] = [Module['postRun']]
while (Module['postRun'].length) {
addOnPostRun(Module['postRun'].shift())
}
}
callRuntimeCallbacks(onPostRuns)
}
function abort(what) {
Module['onAbort']?.(what)
what = 'Aborted(' + what + ')'
err(what)
ABORT = true
what += '. Build with -sASSERTIONS for more info.'
var e = new WebAssembly.RuntimeError(what)
readyPromiseReject?.(e)
throw e
}
var wasmBinaryFile
function findWasmBinary() {
return binaryDecode(
'