@loaders.gl/worker-utils
Version:
Utilities for running tasks on worker threads
4 lines • 74.4 kB
Source Map (JSON)
{
"version": 3,
"sources": ["../src/index.ts", "../src/lib/npm-tag.ts", "../src/lib/env-utils/version.ts", "../src/lib/env-utils/assert.ts", "../src/lib/env-utils/globals.ts", "../src/lib/worker-farm/worker-job.ts", "../src/lib/node/worker_threads-browser.ts", "../src/lib/worker-utils/get-loadable-worker-url.ts", "../src/lib/worker-utils/get-transfer-list.ts", "../src/lib/worker-farm/worker-thread.ts", "../src/lib/worker-farm/worker-pool.ts", "../src/lib/worker-farm/worker-farm.ts", "../src/lib/worker-farm/worker-body.ts", "../src/lib/worker-api/get-worker-url.ts", "../src/lib/worker-api/process-on-worker.ts", "../src/lib/async-queue/async-queue.ts", "../src/lib/worker-api/create-worker.ts", "../src/lib/worker-api/validate-worker-version.ts", "../src/lib/library-utils/library-utils.ts", "../src/lib/process-utils/child-process-proxy.browser.ts"],
"sourcesContent": ["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {WorkerObject} from './types';\nimport {VERSION} from './lib/env-utils/version';\n\n// TYPES\nexport type {\n WorkerObject,\n WorkerOptions,\n // Protocol\n WorkerMessage,\n WorkerMessageType,\n WorkerMessageData,\n WorkerMessagePayload\n} from './types';\n\n// GENERAL UTILS\nexport {assert} from './lib/env-utils/assert';\nexport {isBrowser, isWorker} from './lib/env-utils/globals';\n\n// WORKER UTILS - TYPES\nexport {default as WorkerJob} from './lib/worker-farm/worker-job';\nexport {default as WorkerThread} from './lib/worker-farm/worker-thread';\n\n// WORKER FARMS\nexport {default as WorkerFarm} from './lib/worker-farm/worker-farm';\nexport {default as WorkerPool} from './lib/worker-farm/worker-pool';\nexport {default as WorkerBody} from './lib/worker-farm/worker-body';\n\n// PROCESS ON WORKER\nexport type {ProcessOnWorkerOptions} from './lib/worker-api/process-on-worker';\nexport {processOnWorker, canProcessOnWorker} from './lib/worker-api/process-on-worker';\nexport {createWorker} from './lib/worker-api/create-worker';\n\n// WORKER UTILS - EXPORTS\nexport {getWorkerURL} from './lib/worker-api/get-worker-url';\nexport {validateWorkerVersion} from './lib/worker-api/validate-worker-version';\nexport {getTransferList, getTransferListForWriter} from './lib/worker-utils/get-transfer-list';\n\n// LIBRARY UTILS\nexport {\n extractLoadLibraryOptions,\n getLibraryUrl,\n loadLibrary,\n type LoadLibraryOptions\n} from './lib/library-utils/library-utils';\n\n// PARSER UTILS\nexport {default as AsyncQueue} from './lib/async-queue/async-queue';\n\n// PROCESS UTILS\nexport {default as ChildProcessProxy} from './lib/process-utils/child-process-proxy';\n\n// WORKER OBJECTS\n\n/** A null worker to test that worker processing is functional */\nexport const NullWorker: WorkerObject = {\n id: 'null',\n name: 'null',\n module: 'worker-utils',\n version: VERSION,\n options: {\n null: {}\n }\n};\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n/**\n * NPM tag to use when loading modules from unpkg.com\n * 'beta' on beta branch, 'latest' on prod branch\n * @note Change between 'beta' and 'latest' depending on whether publishing alpha or prod releases\n * @todo - unpkg.com doesn't seem to have a `latest` specifier for alpha releases...\n */\nexport const NPM_TAG = 'latest';\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport {NPM_TAG} from '../npm-tag';\n\n// Version constant cannot be imported, it needs to correspond to the build version of **this** module.\ndeclare let __VERSION__: string;\n\nlet warningIssued = false;\n\nfunction getVersion() {\n if (!globalThis._loadersgl_?.version) {\n globalThis._loadersgl_ = globalThis._loadersgl_ || {};\n // __VERSION__ is injected by babel-plugin-version-inline\n if (typeof __VERSION__ === 'undefined' && !warningIssued) {\n // eslint-disable-next-line\n console.warn(\n 'loaders.gl: The __VERSION__ variable is not injected using babel plugin. Latest unstable workers would be fetched from the CDN.'\n );\n globalThis._loadersgl_.version = NPM_TAG;\n warningIssued = true;\n } else {\n globalThis._loadersgl_.version = __VERSION__;\n }\n }\n\n return globalThis._loadersgl_.version;\n}\n\nexport const VERSION = getVersion();\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// Replacement for the external assert method to reduce bundle size\n// Note: We don't use the second \"message\" argument in calling code,\n// so no need to support it here\n\n/** Throws an `Error` with the optional `message` if `condition` is falsy */\nexport function assert(condition: any, message?: string): void {\n if (!condition) {\n throw new Error(message || 'loaders.gl assertion failed.');\n }\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// Purpose: include this in your module to avoids adding dependencies on\n// micro modules like 'global' and 'is-browser';\n\n/* eslint-disable no-restricted-globals */\nconst globals = {\n self: typeof self !== 'undefined' && self,\n window: typeof window !== 'undefined' && window,\n global: typeof global !== 'undefined' && global,\n document: typeof document !== 'undefined' && document\n};\n\nconst self_: {[key: string]: any} = globals.self || globals.window || globals.global || {};\nconst window_: {[key: string]: any} = globals.window || globals.self || globals.global || {};\nconst global_: {[key: string]: any} = globals.global || globals.self || globals.window || {};\nconst document_: {[key: string]: any} = globals.document || {};\n\nexport {self_ as self, window_ as window, global_ as global, document_ as document};\n\n/** true if running in the browser, false if running in Node.js */\nexport const isBrowser: boolean =\n // @ts-ignore process.browser\n typeof process !== 'object' || String(process) !== '[object process]' || process.browser;\n\n/** true if running on a worker thread */\nexport const isWorker: boolean = typeof importScripts === 'function';\n\n/** true if running on a mobile device */\nexport const isMobile: boolean =\n typeof window !== 'undefined' && typeof window.orientation !== 'undefined';\n\n// Extract node major version\nconst matches =\n typeof process !== 'undefined' && process.version && /v([0-9]*)/.exec(process.version);\n\n/** Version of Node.js if running under Node, otherwise 0 */\nexport const nodeVersion: number = (matches && parseFloat(matches[1])) || 0;\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {WorkerMessageType, WorkerMessagePayload} from '../../types';\nimport WorkerThread from './worker-thread';\nimport {assert} from '../env-utils/assert';\n\n/**\n * Represents one Job handled by a WorkerPool or WorkerFarm\n */\nexport default class WorkerJob {\n readonly name: string;\n readonly workerThread: WorkerThread;\n isRunning: boolean = true;\n /** Promise that resolves when Job is done */\n readonly result: Promise<any>;\n\n private _resolve: (value: any) => void = () => {};\n private _reject: (reason?: any) => void = () => {};\n\n constructor(jobName: string, workerThread: WorkerThread) {\n this.name = jobName;\n this.workerThread = workerThread;\n this.result = new Promise((resolve, reject) => {\n this._resolve = resolve;\n this._reject = reject;\n });\n }\n\n /**\n * Send a message to the job's worker thread\n * @param data any data structure, ideally consisting mostly of transferrable objects\n */\n postMessage(type: WorkerMessageType, payload: WorkerMessagePayload): void {\n this.workerThread.postMessage({\n source: 'loaders.gl', // Lets worker ignore unrelated messages\n type,\n payload\n });\n }\n\n /**\n * Call to resolve the `result` Promise with the supplied value\n */\n done(value: any): void {\n assert(this.isRunning);\n this.isRunning = false;\n this._resolve(value);\n }\n\n /**\n * Call to reject the `result` Promise with the supplied error\n */\n error(error: Error): void {\n assert(this.isRunning);\n this.isRunning = false;\n this._reject(error);\n }\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n/** Browser polyfill for Node.js built-in `worker_threads` module.\n * These fills are non-functional, and just intended to ensure that\n * `import 'worker_threads` doesn't break browser builds.\n * The replacement is done in package.json browser field\n */\nexport class NodeWorker {\n terminate() {}\n}\n\nexport type {NodeWorker as NodeWorkerType};\n\nexport const parentPort = null;\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport {assert} from '../env-utils/assert';\n\nconst workerURLCache = new Map();\n\n/**\n * Creates a loadable URL from worker source or URL\n * that can be used to create `Worker` instances.\n * Due to CORS issues it may be necessary to wrap a URL in a small importScripts\n * @param props\n * @param props.source Worker source\n * @param props.url Worker URL\n * @returns loadable url\n */\nexport function getLoadableWorkerURL(props: {source?: string; url?: string}) {\n assert((props.source && !props.url) || (!props.source && props.url)); // Either source or url must be defined\n\n let workerURL = workerURLCache.get(props.source || props.url);\n if (!workerURL) {\n // Differentiate worker urls from worker source code\n if (props.url) {\n workerURL = getLoadableWorkerURLFromURL(props.url);\n workerURLCache.set(props.url, workerURL);\n }\n\n if (props.source) {\n workerURL = getLoadableWorkerURLFromSource(props.source);\n workerURLCache.set(props.source, workerURL);\n }\n }\n\n assert(workerURL);\n return workerURL;\n}\n\n/**\n * Build a loadable worker URL from worker URL\n * @param url\n * @returns loadable URL\n */\nfunction getLoadableWorkerURLFromURL(url: string): string {\n // A local script url, we can use it to initialize a Worker directly\n if (!url.startsWith('http')) {\n return url;\n }\n\n // A remote script, we need to use `importScripts` to load from different origin\n const workerSource = buildScriptSource(url);\n return getLoadableWorkerURLFromSource(workerSource);\n}\n\n/**\n * Build a loadable worker URL from worker source\n * @param workerSource\n * @returns loadable url\n */\nfunction getLoadableWorkerURLFromSource(workerSource: string): string {\n const blob = new Blob([workerSource], {type: 'application/javascript'});\n return URL.createObjectURL(blob);\n}\n\n/**\n * Per spec, worker cannot be initialized with a script from a different origin\n * However a local worker script can still import scripts from other origins,\n * so we simply build a wrapper script.\n *\n * @param workerUrl\n * @returns source\n */\nfunction buildScriptSource(workerUrl: string): string {\n return `\\\ntry {\n importScripts('${workerUrl}');\n} catch (error) {\n console.error(error);\n throw error;\n}`;\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// NOTE - there is a copy of this function is both in core and loader-utils\n// core does not need all the utils in loader-utils, just this one.\n\n/**\n * Returns an array of Transferrable objects that can be used with postMessage\n * https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage\n * @param object data to be sent via postMessage\n * @param recursive - not for application use\n * @param transfers - not for application use\n * @returns a transfer list that can be passed to postMessage\n */\nexport function getTransferList(\n object: any,\n recursive: boolean = true,\n transfers?: Set<any>\n): Transferable[] {\n // Make sure that items in the transfer list is unique\n const transfersSet = transfers || new Set();\n\n if (!object) {\n // ignore\n } else if (isTransferable(object)) {\n transfersSet.add(object);\n } else if (isTransferable(object.buffer)) {\n // Typed array\n transfersSet.add(object.buffer);\n } else if (ArrayBuffer.isView(object)) {\n // object is a TypeArray viewing into a SharedArrayBuffer (not transferable)\n // Do not iterate through the content in this case\n } else if (recursive && typeof object === 'object') {\n for (const key in object) {\n // Avoid perf hit - only go one level deep\n getTransferList(object[key], recursive, transfersSet);\n }\n }\n\n // If transfers is defined, is internal recursive call\n // Otherwise it's called by the user\n return transfers === undefined ? Array.from(transfersSet) : [];\n}\n\n// https://developer.mozilla.org/en-US/docs/Web/API/Transferable\nfunction isTransferable(object: unknown) {\n if (!object) {\n return false;\n }\n if (object instanceof ArrayBuffer) {\n return true;\n }\n if (typeof MessagePort !== 'undefined' && object instanceof MessagePort) {\n return true;\n }\n if (typeof ImageBitmap !== 'undefined' && object instanceof ImageBitmap) {\n return true;\n }\n // @ts-ignore\n if (typeof OffscreenCanvas !== 'undefined' && object instanceof OffscreenCanvas) {\n return true;\n }\n return false;\n}\n\n/**\n * Recursively drop non serializable values like functions and regexps.\n * @param object\n */\nexport function getTransferListForWriter(object: object | null): object {\n if (object === null) {\n return {};\n }\n const clone = Object.assign({}, object);\n\n Object.keys(clone).forEach((key) => {\n // Typed Arrays and Arrays are passed with no change\n if (\n typeof object[key] === 'object' &&\n !ArrayBuffer.isView(object[key]) &&\n !(object[key] instanceof Array)\n ) {\n clone[key] = getTransferListForWriter(object[key]);\n } else if (typeof clone[key] === 'function' || clone[key] instanceof RegExp) {\n clone[key] = {};\n } else {\n clone[key] = object[key];\n }\n });\n\n return clone;\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport {NodeWorker, NodeWorkerType} from '../node/worker_threads';\nimport {isBrowser} from '../env-utils/globals';\nimport {assert} from '../env-utils/assert';\nimport {getLoadableWorkerURL} from '../worker-utils/get-loadable-worker-url';\nimport {getTransferList} from '../worker-utils/get-transfer-list';\n\nconst NOOP = () => {};\n\nexport type WorkerThreadProps = {\n name: string;\n source?: string;\n url?: string;\n};\n\n/**\n * Represents one worker thread\n */\nexport default class WorkerThread {\n readonly name: string;\n readonly source: string | undefined;\n readonly url: string | undefined;\n terminated: boolean = false;\n worker: Worker | NodeWorkerType;\n onMessage: (message: any) => void;\n onError: (error: Error) => void;\n\n private _loadableURL: string = '';\n\n /** Checks if workers are supported on this platform */\n static isSupported(): boolean {\n return (\n (typeof Worker !== 'undefined' && isBrowser) ||\n (typeof NodeWorker !== 'undefined' && !isBrowser)\n );\n }\n\n constructor(props: WorkerThreadProps) {\n const {name, source, url} = props;\n assert(source || url); // Either source or url must be defined\n this.name = name;\n this.source = source;\n this.url = url;\n this.onMessage = NOOP;\n this.onError = (error) => console.log(error); // eslint-disable-line\n\n this.worker = isBrowser ? this._createBrowserWorker() : this._createNodeWorker();\n }\n\n /**\n * Terminate this worker thread\n * @note Can free up significant memory\n */\n destroy(): void {\n this.onMessage = NOOP;\n this.onError = NOOP;\n this.worker.terminate(); // eslint-disable-line @typescript-eslint/no-floating-promises\n this.terminated = true;\n }\n\n get isRunning() {\n return Boolean(this.onMessage);\n }\n\n /**\n * Send a message to this worker thread\n * @param data any data structure, ideally consisting mostly of transferrable objects\n * @param transferList If not supplied, calculated automatically by traversing data\n */\n postMessage(data: any, transferList?: any[]): void {\n transferList = transferList || getTransferList(data);\n // @ts-ignore\n this.worker.postMessage(data, transferList);\n }\n\n // PRIVATE\n\n /**\n * Generate a standard Error from an ErrorEvent\n * @param event\n */\n _getErrorFromErrorEvent(event: ErrorEvent): Error {\n // Note Error object does not have the expected fields if loading failed completely\n // https://developer.mozilla.org/en-US/docs/Web/API/Worker#Event_handlers\n // https://developer.mozilla.org/en-US/docs/Web/API/ErrorEvent\n let message = 'Failed to load ';\n message += `worker ${this.name} from ${this.url}. `;\n if (event.message) {\n message += `${event.message} in `;\n }\n // const hasFilename = event.filename && !event.filename.startsWith('blob:');\n // message += hasFilename ? event.filename : this.source.slice(0, 100);\n if (event.lineno) {\n message += `:${event.lineno}:${event.colno}`;\n }\n return new Error(message);\n }\n\n /**\n * Creates a worker thread on the browser\n */\n _createBrowserWorker(): Worker {\n this._loadableURL = getLoadableWorkerURL({source: this.source, url: this.url});\n const worker = new Worker(this._loadableURL, {name: this.name});\n\n worker.onmessage = (event) => {\n if (!event.data) {\n this.onError(new Error('No data received'));\n } else {\n this.onMessage(event.data);\n }\n };\n // This callback represents an uncaught exception in the worker thread\n worker.onerror = (error: ErrorEvent): void => {\n this.onError(this._getErrorFromErrorEvent(error));\n this.terminated = true;\n };\n // TODO - not clear when this would be called, for now just log in case it happens\n worker.onmessageerror = (event) => console.error(event); // eslint-disable-line\n\n return worker;\n }\n\n /**\n * Creates a worker thread in node.js\n * @todo https://nodejs.org/api/async_hooks.html#async-resource-worker-pool\n */\n _createNodeWorker(): NodeWorkerType {\n let worker: NodeWorkerType;\n if (this.url) {\n // Make sure relative URLs start with './'\n const absolute = this.url.includes(':/') || this.url.startsWith('/');\n const url = absolute ? this.url : `./${this.url}`;\n const type = this.url.endsWith('.ts') || this.url.endsWith('.mjs') ? 'module' : 'commonjs';\n // console.log('Starting work from', url);\n // @ts-expect-error type is not known\n worker = new NodeWorker(url, {eval: false, type});\n } else if (this.source) {\n worker = new NodeWorker(this.source, {eval: true});\n } else {\n throw new Error('no worker');\n }\n worker.on('message', (data) => {\n // console.error('message', data);\n this.onMessage(data);\n });\n worker.on('error', (error) => {\n this.onError(error as Error);\n });\n worker.on('exit', (code) => {\n // console.error('exit', code);\n });\n return worker;\n }\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {WorkerMessageType, WorkerMessagePayload} from '../../types';\nimport {isMobile, isBrowser} from '../env-utils/globals';\nimport WorkerThread from './worker-thread';\nimport WorkerJob from './worker-job';\n\n/** WorkerPool onDebug Callback Parameters */\ntype OnDebugParameters = {\n message: string;\n worker: string;\n name: string;\n job: string;\n backlog: number;\n workerThread: WorkerThread;\n};\n\n/** WorkerPool Properties */\nexport type WorkerPoolProps = {\n name?: string;\n source?: string; // | Function;\n url?: string;\n maxConcurrency?: number;\n maxMobileConcurrency?: number;\n onDebug?: (options: OnDebugParameters) => any;\n reuseWorkers?: boolean;\n};\n\n/** Private helper types */\ntype OnMessage = (job: WorkerJob, type: WorkerMessageType, payload: WorkerMessagePayload) => void;\ntype OnError = (job: WorkerJob, error: Error) => void;\n\ntype QueuedJob = {\n name: string;\n onMessage: OnMessage;\n onError: OnError;\n onStart: (value: any) => void; // Resolve job start promise\n};\n\n/**\n * Process multiple data messages with small pool of identical workers\n */\nexport default class WorkerPool {\n name: string = 'unnamed';\n source?: string; // | Function;\n url?: string;\n maxConcurrency: number = 1;\n maxMobileConcurrency: number = 1;\n onDebug: (options: OnDebugParameters) => any = () => {};\n reuseWorkers: boolean = true;\n\n private props: WorkerPoolProps = {};\n private jobQueue: QueuedJob[] = [];\n private idleQueue: WorkerThread[] = [];\n private count = 0;\n private isDestroyed = false;\n\n /** Checks if workers are supported on this platform */\n static isSupported(): boolean {\n return WorkerThread.isSupported();\n }\n\n /**\n * @param processor - worker function\n * @param maxConcurrency - max count of workers\n */\n constructor(props: WorkerPoolProps) {\n this.source = props.source;\n this.url = props.url;\n this.setProps(props);\n }\n\n /**\n * Terminates all workers in the pool\n * @note Can free up significant memory\n */\n destroy(): void {\n // Destroy idle workers, active Workers will be destroyed on completion\n this.idleQueue.forEach((worker) => worker.destroy());\n this.isDestroyed = true;\n }\n\n setProps(props: WorkerPoolProps) {\n this.props = {...this.props, ...props};\n\n if (props.name !== undefined) {\n this.name = props.name;\n }\n if (props.maxConcurrency !== undefined) {\n this.maxConcurrency = props.maxConcurrency;\n }\n if (props.maxMobileConcurrency !== undefined) {\n this.maxMobileConcurrency = props.maxMobileConcurrency;\n }\n if (props.reuseWorkers !== undefined) {\n this.reuseWorkers = props.reuseWorkers;\n }\n if (props.onDebug !== undefined) {\n this.onDebug = props.onDebug;\n }\n }\n\n async startJob(\n name: string,\n onMessage: OnMessage = (job, type, data) => job.done(data),\n onError: OnError = (job, error) => job.error(error)\n ): Promise<WorkerJob> {\n // Promise resolves when thread starts working on this job\n const startPromise = new Promise<WorkerJob>((onStart) => {\n // Promise resolves when thread completes or fails working on this job\n this.jobQueue.push({name, onMessage, onError, onStart});\n return this;\n });\n this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises\n return await startPromise;\n }\n\n // PRIVATE\n\n /**\n * Starts first queued job if worker is available or can be created\n * Called when job is started and whenever a worker returns to the idleQueue\n */\n async _startQueuedJob(): Promise<void> {\n if (!this.jobQueue.length) {\n return;\n }\n\n const workerThread = this._getAvailableWorker();\n if (!workerThread) {\n return;\n }\n\n // We have a worker, dequeue and start the job\n const queuedJob = this.jobQueue.shift();\n if (queuedJob) {\n // Emit a debug event\n // @ts-ignore\n this.onDebug({\n message: 'Starting job',\n name: queuedJob.name,\n workerThread,\n backlog: this.jobQueue.length\n });\n\n // Create a worker job to let the app access thread and manage job completion\n const job = new WorkerJob(queuedJob.name, workerThread);\n\n // Set the worker thread's message handlers\n workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);\n workerThread.onError = (error) => queuedJob.onError(job, error);\n\n // Resolve the start promise so that the app can start sending messages to worker\n queuedJob.onStart(job);\n\n // Wait for the app to signal that the job is complete, then return worker to queue\n try {\n await job.result;\n } catch (error) {\n // eslint-disable-next-line no-console\n console.error(`Worker exception: ${error}`);\n } finally {\n this.returnWorkerToQueue(workerThread);\n }\n }\n }\n\n /**\n * Returns a worker to the idle queue\n * Destroys the worker if\n * - pool is destroyed\n * - if this pool doesn't reuse workers\n * - if maxConcurrency has been lowered\n * @param worker\n */\n returnWorkerToQueue(worker: WorkerThread) {\n const shouldDestroyWorker =\n // Workers on Node.js prevent the process from exiting.\n // Until we figure out how to close them before exit, we always destroy them\n !isBrowser ||\n // If the pool is destroyed, there is no reason to keep the worker around\n this.isDestroyed ||\n // If the app has disabled worker reuse, any completed workers should be destroyed\n !this.reuseWorkers ||\n // If concurrency has been lowered, this worker might be surplus to requirements\n this.count > this._getMaxConcurrency();\n\n if (shouldDestroyWorker) {\n worker.destroy();\n this.count--;\n } else {\n this.idleQueue.push(worker);\n }\n\n if (!this.isDestroyed) {\n this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises\n }\n }\n\n /**\n * Returns idle worker or creates new worker if maxConcurrency has not been reached\n */\n _getAvailableWorker(): WorkerThread | null {\n // If a worker has completed and returned to the queue, it can be used\n if (this.idleQueue.length > 0) {\n return this.idleQueue.shift() || null;\n }\n\n // Create fresh worker if we haven't yet created the max amount of worker threads for this worker source\n if (this.count < this._getMaxConcurrency()) {\n this.count++;\n const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;\n return new WorkerThread({name, source: this.source, url: this.url});\n }\n\n // No worker available, have to wait\n return null;\n }\n\n _getMaxConcurrency() {\n return isMobile ? this.maxMobileConcurrency : this.maxConcurrency;\n }\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport WorkerPool from './worker-pool';\nimport WorkerThread from './worker-thread';\n\n/**\n * @param maxConcurrency - max count of workers\n * @param maxMobileConcurrency - max count of workers on mobile\n * @param maxConcurrency - max count of workers\n * @param reuseWorkers - if false, destroys workers when task is completed\n * @param onDebug - callback intended to allow application to log worker pool activity\n */\nexport type WorkerFarmProps = {\n maxConcurrency?: number;\n maxMobileConcurrency?: number;\n reuseWorkers?: boolean;\n onDebug?: () => void;\n};\n\nconst DEFAULT_PROPS: Required<WorkerFarmProps> = {\n maxConcurrency: 3,\n maxMobileConcurrency: 1,\n reuseWorkers: true,\n onDebug: () => {}\n};\n\n/**\n * Process multiple jobs with a \"farm\" of different workers in worker pools.\n */\nexport default class WorkerFarm {\n private props: WorkerFarmProps;\n private workerPools = new Map<string, WorkerPool>();\n // singleton\n private static _workerFarm?: WorkerFarm;\n\n /** Checks if workers are supported on this platform */\n static isSupported(): boolean {\n return WorkerThread.isSupported();\n }\n\n /** Get the singleton instance of the global worker farm */\n static getWorkerFarm(props: WorkerFarmProps = {}): WorkerFarm {\n WorkerFarm._workerFarm = WorkerFarm._workerFarm || new WorkerFarm({});\n WorkerFarm._workerFarm.setProps(props);\n return WorkerFarm._workerFarm;\n }\n\n /** get global instance with WorkerFarm.getWorkerFarm() */\n private constructor(props: WorkerFarmProps) {\n this.props = {...DEFAULT_PROPS};\n this.setProps(props);\n /** @type Map<string, WorkerPool>} */\n this.workerPools = new Map();\n }\n\n /**\n * Terminate all workers in the farm\n * @note Can free up significant memory\n */\n destroy(): void {\n for (const workerPool of this.workerPools.values()) {\n workerPool.destroy();\n }\n this.workerPools = new Map<string, WorkerPool>();\n }\n\n /**\n * Set props used when initializing worker pools\n * @param props\n */\n setProps(props: WorkerFarmProps): void {\n this.props = {...this.props, ...props};\n // Update worker pool props\n for (const workerPool of this.workerPools.values()) {\n workerPool.setProps(this._getWorkerPoolProps());\n }\n }\n\n /**\n * Returns a worker pool for the specified worker\n * @param options - only used first time for a specific worker name\n * @param options.name - the name of the worker - used to identify worker pool\n * @param options.url -\n * @param options.source -\n * @example\n * const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);\n */\n getWorkerPool(options: {name: string; source?: string; url?: string}): WorkerPool {\n const {name, source, url} = options;\n let workerPool = this.workerPools.get(name);\n if (!workerPool) {\n workerPool = new WorkerPool({\n name,\n source,\n url\n });\n workerPool.setProps(this._getWorkerPoolProps());\n this.workerPools.set(name, workerPool);\n }\n return workerPool;\n }\n\n _getWorkerPoolProps() {\n return {\n maxConcurrency: this.props.maxConcurrency,\n maxMobileConcurrency: this.props.maxMobileConcurrency,\n reuseWorkers: this.props.reuseWorkers,\n onDebug: this.props.onDebug\n };\n }\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {WorkerMessageData, WorkerMessageType, WorkerMessagePayload} from '../../types';\nimport {getTransferList} from '../worker-utils/get-transfer-list';\n// import type {TransferListItem} from '../node/worker_threads';\nimport {parentPort} from '../node/worker_threads';\n\ntype TransferListItem = any;\n\n/** Vile hack to defeat over-zealous bundlers from stripping out the require */\nasync function getParentPort() {\n // const isNode = globalThis.process;\n // let parentPort;\n // try {\n // // prettier-ignore\n // eval('globalThis.parentPort = require(\\'worker_threads\\').parentPort'); // eslint-disable-line no-eval\n // parentPort = globalThis.parentPort;\n // } catch {\n // try {\n // // prettier-ignore\n // eval('globalThis.workerThreadsPromise = import(\\'worker_threads\\')'); // eslint-disable-line no-eval\n // const workerThreads = await globalThis.workerThreadsPromise;\n // parentPort = workerThreads.parentPort;\n // } catch (error) {\n // console.error((error as Error).message); // eslint-disable-line no-console\n // }\n // }\n return parentPort;\n}\n\nconst onMessageWrapperMap = new Map();\n\n/**\n * Type safe wrapper for worker code\n */\nexport default class WorkerBody {\n /** Check that we are actually in a worker thread */\n static async inWorkerThread(): Promise<boolean> {\n return typeof self !== 'undefined' || Boolean(await getParentPort());\n }\n\n /*\n * (type: WorkerMessageType, payload: WorkerMessagePayload) => any\n */\n static set onmessage(onMessage: (type: WorkerMessageType, payload: WorkerMessagePayload) => any) {\n async function handleMessage(message) {\n const parentPort = await getParentPort();\n // Confusingly the message itself also has a 'type' field which is always set to 'message'\n const {type, payload} = parentPort ? message : message.data;\n // if (!isKnownMessage(message)) {\n // return;\n // }\n onMessage(type, payload);\n }\n\n getParentPort().then((parentPort) => {\n if (parentPort) {\n parentPort.on('message', (message) => {\n handleMessage(message);\n });\n // if (message == 'exit') { parentPort.unref(); }\n // eslint-disable-next-line\n parentPort.on('exit', () => console.debug('Node worker closing'));\n } else {\n // eslint-disable-next-line no-restricted-globals\n globalThis.onmessage = handleMessage;\n }\n });\n }\n\n static async addEventListener(\n onMessage: (type: WorkerMessageType, payload: WorkerMessagePayload) => any\n ) {\n let onMessageWrapper = onMessageWrapperMap.get(onMessage);\n\n if (!onMessageWrapper) {\n onMessageWrapper = async (message: MessageEvent<any>) => {\n if (!isKnownMessage(message)) {\n return;\n }\n\n const parentPort = await getParentPort();\n // Confusingly in the browser, the message itself also has a 'type' field which is always set to 'message'\n const {type, payload} = parentPort ? message : message.data;\n onMessage(type, payload);\n };\n }\n\n const parentPort = await getParentPort();\n if (parentPort) {\n console.error('not implemented'); // eslint-disable-line\n } else {\n globalThis.addEventListener('message', onMessageWrapper);\n }\n }\n\n static async removeEventListener(\n onMessage: (type: WorkerMessageType, payload: WorkerMessagePayload) => any\n ) {\n const onMessageWrapper = onMessageWrapperMap.get(onMessage);\n onMessageWrapperMap.delete(onMessage);\n const parentPort = await getParentPort();\n if (parentPort) {\n console.error('not implemented'); // eslint-disable-line\n } else {\n globalThis.removeEventListener('message', onMessageWrapper);\n }\n }\n\n /**\n * Send a message from a worker to creating thread (main thread)\n * @param type\n * @param payload\n */\n static async postMessage(type: WorkerMessageType, payload: WorkerMessagePayload): Promise<void> {\n const data: WorkerMessageData = {source: 'loaders.gl', type, payload};\n // console.log('posting message', data);\n\n // Cast to Node compatible transfer list\n const transferList = getTransferList(payload) as TransferListItem[];\n\n const parentPort = await getParentPort();\n if (parentPort) {\n parentPort.postMessage(data, transferList);\n // console.log('posted message', data);\n } else {\n // @ts-expect-error Outside of worker scopes this call has a third parameter\n globalThis.postMessage(data, transferList);\n }\n }\n}\n\n// Filter out noise messages sent to workers\nfunction isKnownMessage(message: MessageEvent<any>) {\n const {type, data} = message;\n return (\n type === 'message' &&\n data &&\n typeof data.source === 'string' &&\n data.source.startsWith('loaders.gl')\n );\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {WorkerObject, WorkerOptions} from '../../types';\nimport {assert} from '../env-utils/assert';\nimport {isBrowser} from '../env-utils/globals';\nimport {VERSION} from '../env-utils/version';\nimport {NPM_TAG} from '../npm-tag';\n\n/**\n * Gets worker object's name (for debugging in Chrome thread inspector window)\n */\nexport function getWorkerName(worker: WorkerObject): string {\n const warning = worker.version !== VERSION ? ` (worker-utils@${VERSION})` : '';\n return `${worker.name}@${worker.version}${warning}`;\n}\n\n/**\n * Generate a worker URL based on worker object and options\n * @returns A URL to one of the following:\n * - a published worker on unpkg CDN\n * - a local test worker\n * - a URL provided by the user in options\n */\nexport function getWorkerURL(worker: WorkerObject, options: WorkerOptions = {}): string {\n const workerOptions = options[worker.id] || {};\n\n const workerFile = isBrowser ? `${worker.id}-worker.js` : `${worker.id}-worker-node.js`;\n\n let url = workerOptions.workerUrl;\n\n // HACK: Allow for non-nested workerUrl for the CompressionWorker.\n // For the compression worker, workerOptions is currently not nested correctly. For most loaders,\n // you'd have options within an object, i.e. `{mvt: {coordinates: ...}}` but the CompressionWorker\n // puts options at the top level, not within a `compression` key (its `id`). For this reason, the\n // above `workerOptions` will always be a string (i.e. `'gzip'`) for the CompressionWorker. To not\n // break backwards compatibility, we allow the CompressionWorker to have options at the top level.\n if (!url && worker.id === 'compression') {\n url = options.workerUrl;\n }\n\n // If URL is test, generate local loaders.gl url\n // @ts-ignore _workerType\n const workerType = (options as any)._workerType || (options as any)?.core?._workerType;\n if (workerType === 'test') {\n if (isBrowser) {\n url = `modules/${worker.module}/dist/${workerFile}`;\n } else {\n // In the test environment the ts-node loader requires TypeScript code\n url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;\n }\n }\n\n // If url override is not provided, generate a URL to published version on npm CDN unpkg.com\n if (!url) {\n // GENERATE\n let version = worker.version;\n // On master we need to load npm alpha releases published with the `beta` tag\n if (version === 'latest') {\n // throw new Error('latest worker version specified');\n version = NPM_TAG;\n }\n const versionTag = version ? `@${version}` : '';\n url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;\n }\n\n assert(url);\n\n // Allow user to override location\n return url;\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {\n WorkerObject,\n WorkerOptions,\n WorkerContext,\n WorkerMessageType,\n WorkerMessagePayload\n} from '../../types';\nimport type WorkerJob from '../worker-farm/worker-job';\nimport WorkerFarm from '../worker-farm/worker-farm';\nimport {getWorkerURL, getWorkerName} from './get-worker-url';\nimport {getTransferListForWriter} from '../worker-utils/get-transfer-list';\n\n/** Options for worker processing */\nexport type ProcessOnWorkerOptions = WorkerOptions & {\n jobName?: string;\n [key: string]: any;\n};\n\n/**\n * Determines if we can parse with worker\n * @param loader\n * @param data\n * @param options\n */\nexport function canProcessOnWorker(worker: WorkerObject, options?: WorkerOptions) {\n if (!WorkerFarm.isSupported()) {\n return false;\n }\n\n return worker.worker && options?.worker;\n}\n\n/**\n * This function expects that the worker thread sends certain messages,\n * Creating such a worker can be automated if the worker is wrapper by a call to\n * createWorker in @loaders.gl/worker-utils.\n */\nexport async function processOnWorker(\n worker: WorkerObject,\n data: any,\n options: ProcessOnWorkerOptions = {},\n context: WorkerContext = {}\n): Promise<any> {\n const name = getWorkerName(worker);\n\n const workerFarm = WorkerFarm.getWorkerFarm(options);\n const {source} = options;\n const workerPoolProps: {name: string; source?: string; url?: string} = {name, source};\n if (!source) {\n workerPoolProps.url = getWorkerURL(worker, options);\n }\n const workerPool = workerFarm.getWorkerPool(workerPoolProps);\n\n const jobName = options.jobName || worker.name;\n const job = await workerPool.startJob(\n jobName,\n // eslint-disable-next-line\n onMessage.bind(null, context)\n );\n\n // Kick off the processing in the worker\n const transferableOptions = getTransferListForWriter(options);\n job.postMessage('process', {input: data, options: transferableOptions});\n\n const result = await job.result;\n return result.result;\n}\n\n/**\n * Job completes when we receive the result\n * @param job\n * @param message\n */\nasync function onMessage(\n context: WorkerContext,\n job: WorkerJob,\n type: WorkerMessageType,\n payload: WorkerMessagePayload\n) {\n switch (type) {\n case 'done':\n // Worker is done\n job.done(payload);\n break;\n\n case 'error':\n // Worker encountered an error\n job.error(new Error(payload.error));\n break;\n\n case 'process':\n // Worker is asking for us (main thread) to process something\n const {id, input, options} = payload;\n try {\n if (!context.process) {\n job.postMessage('error', {id, error: 'Worker not set up to process on main thread'});\n return;\n }\n const result = await context.process(input, options);\n job.postMessage('done', {id, result});\n } catch (error) {\n const message = error instanceof Error ? error.message : 'unknown error';\n job.postMessage('error', {id, error: message});\n }\n break;\n\n default:\n // eslint-disable-next-line\n console.warn(`process-on-worker: unknown message ${type}`);\n }\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// From https://github.com/rauschma/async-iter-demo/tree/master/src under MIT license\n// http://2ality.com/2016/10/asynchronous-iteration.html\n\n/**\n * Async Queue\n * - AsyncIterable: An async iterator can be\n * - Values can be pushed onto the queue\n * @example\n * const asyncQueue = new AsyncQueue();\n * setTimeout(() => asyncQueue.enqueue('tick'), 1000);\n * setTimeout(() => asyncQueue.enqueue(new Error('done')), 10000);\n * for await (const value of asyncQueue) {\n * console.log(value); // tick\n * }\n */\nexport default class AsyncQueue<T> {\n private _values: any[];\n private _settlers: any[];\n private _closed: boolean;\n\n constructor() {\n this._values = []; // enqueues > dequeues\n this._settlers = []; // dequeues > enqueues\n this._closed = false;\n }\n\n /** Return an async iterator for this queue */\n [Symbol.asyncIterator](): AsyncIterator<T> {\n return this;\n }\n\n /** Push a new value - the async iterator will yield a promise resolved to this value */\n push(value: T): void {\n return this.enqueue(value);\n }\n\n /**\n * Push a new value - the async iterator will yield a promise resolved to this value\n * Add an error - the async iterator will yield a promise rejected with this value\n */\n enqueue(value: T | Error): void {\n if (this._closed) {\n throw new Error('Closed');\n }\n\n if (this._settlers.length > 0) {\n if (this._values.length > 0) {\n throw new Error('Illegal internal state');\n }\n const settler = this._settlers.shift();\n if (value instanceof Error) {\n settler.reject(value);\n } else {\n settler.resolve({value});\n }\n } else {\n this._values.push(value);\n }\n }\n\n /** Indicate that we not waiting for more values - The async iterator will be done */\n close(): void {\n while (this._settlers.length > 0) {\n const settler = this._settlers.shift();\n settler.resolve({done: true});\n }\n this._closed = true;\n }\n\n // ITERATOR IMPLEMENTATION\n\n /** @returns a Promise for an IteratorResult */\n next(): Promise<IteratorResult<T, any>> {\n // If values in queue, yield the first value\n if (this._values.length > 0) {\n const value = this._values.shift();\n if (value instanceof Error) {\n return Promise.reject(value);\n }\n return Promise.resolve({done: false, value});\n }\n\n // If queue is closed, the iterator is done\n if (this._closed) {\n if (this._settlers.length > 0) {\n throw new Error('Illegal internal state');\n }\n return Promise.resolve({done: true, value: undefined});\n }\n\n // Yield a promise that waits for new values to be enqueued\n return new Promise((resolve, reject) => {\n this._settlers.push({resolve, reject});\n });\n }\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {\n WorkerMessageType,\n WorkerMessagePayload,\n WorkerContext,\n Process,\n ProcessInBatches\n} from '../../types';\nimport AsyncQueue from '../async-queue/async-queue';\nimport WorkerBody from '../worker-farm/worker-body';\n// import {validateWorkerVersion} from './validate-worker-version';\n\n/** Counter for jobs */\nlet requestId = 0;\nlet inputBatches: AsyncQueue<any>;\nlet options: {[key: string]: any};\n\nexport type ProcessOnMainThread = (\n data: any,\n options?: {[key: string]: any},\n context?: WorkerContext\n) => any;\n\n/**\n * Set up a WebWorkerGlobalScope to talk with the main thread\n */\nexport async function createWorker(\n process: Process,\n processInBatches?: ProcessInBatches\n): Promise<void> {\n if (!(await WorkerBody.inWorkerThread())) {\n return;\n }\n\n const context: WorkerContext = {\n process: processOnMainThread\n };\n\n // eslint-disable-next-line complexity\n WorkerBody.onmessage = async (type: WorkerMessageType, payload: WorkerMessagePayload) => {\n try {\n switch (type) {\n case 'process':\n if (!process) {\n throw new Error('Worker does not support atomic processing');\n }\n const result = await process(payload.input, payload.options || {}, context);\n WorkerBody.postMessage('done', {result});\n break;\n\n case 'process-in-batches':\n if (!processInBatches) {\n throw new Error('Worker does not support batched processing');\n }\n inputBatches = new AsyncQueue<any>();\n options = payload.options || {};\n const resultIterator = processInBatches(inputBatches, options, context);\n for await (const batch of resultIterator) {\n WorkerBody.postMessage('output-batch', {result: batch});\n }\n WorkerBody.postMessage('done', {});\n break;\n\n case 'input-batch':\n inputBatches.push(payload.input);\n break;\n\n case 'input-done':\n inputBatches.close();\n break;\n\n default:\n }\n } catch (error) {\n const message = error instanceof Error ? error.message : '';\n WorkerBody.postMessage('error', {error: message});\n }\n };\n}\n\nfunction processOnMainThread(arrayBuffer: ArrayBuffer, options = {}) {\n return new Promise((resolve, reject) => {\n const id = requestId++;\n\n /**\n */\n const onMessage = (type: string, payload: WorkerMessagePayload) => {\n if (payload.id !== id) {\n // not ours\n return;\n }\n\n switch (type) {\n case 'done':\n WorkerBody.removeEventListener(onMessage);\n resolve(payload.result);\n break;\n\n case 'error':\n WorkerBody.removeEventListener(onMessage);\n reject(payload.error);\n break;\n\n default:\n // ignore\n }\n };\n\n WorkerBody.addEventListener(onMessage);\n\n // Ask the main thread to decode data\n const payload = {id, input: arrayBuffer, options};\n WorkerBody.postMessage('process', payload);\n });\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {WorkerObject} from '../../types';\nimport {assert} from '../env-utils/assert';\nimport {VERSION} from '../env-utils/version';\n\n/**\n * Check if worker is compatible with this library version\n * @param worker\n * @param libVersion\n * @returns `true` if the two versions are compatible\n */\nexport function validateWorkerVersion(\n worker: WorkerObject,\n coreVersion: string = VERSION\n): boolean {\n assert(worker, 'no worker provided');\n\n const workerVersion = worker.version;\n if (!coreVersion || !workerVersion) {\n return false;\n }\n\n // TODO enable when fix the __version__ injection\n // const coreVersions = parseVersion(coreVersion);\n // const workerVersions = parseVersion(workerVersion);\n // assert(\n // coreVersion.major === workerVersion.major && coreVersion.minor <= workerVersion.minor,\n // `worker: ${worker.name} is not compatible. ${coreVersion.major}.${\n // coreVersion.minor\n // }+ is required.`\n // );\n\n return true;\n}\n\n// @ts-ignore\n// eslint-disable-next-line @typescript-eslint/no-unused-vars\nfunction parseVersion(version) {\n const parts = version.split('.').map(Number);\n return {major: parts[0], minor: parts[1]};\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n/* global importScripts */\nimport {isBrowser, isWorker} from '../env-utils/globals';\nimport {assert} from '../env-utils/assert';\nimport {VERSION} from '../env-utils/version';\n\nexport type LoadLibraryOptions<ModulesT extends Record<string, any> = Record<string, any>> = {\n useLocalLibraries?: boolean;\n CDN?: string | null;\n modules?: ModulesT;\n // Core must not be supplied\n core?: never;\n};\n\ntype ExtractableLoadLibraryOptions<ModulesT extends Record<string, any> = Record<string, any>> = {\n useLocalLibraries?: boolean;\n CDN?: string | null;\n modules?: ModulesT;\n core?: {\n useLocalLibraries?: boolean;\n CDN?: string | null;\n } | null;\n};\n\nconst loadLibraryPromises: Record<string, Promise<any>> = {}; // promises\n\nexport function extractLoadLibraryOptions<\n ModulesT extends Record<string, any> = Record<string, any>\n>(options: ExtractableLoadLibraryOptions<ModulesT> = {}): LoadLibraryOptions<ModulesT> {\n const useLocalLibraries = options.useLocalLibraries ?? options.core?.useLocalLibraries;\n const CDN = options.CDN ?? options.core?.CDN;\n const modules = options.modules;\n\n return {\n ...(useLocalLibraries !== undefined ? {useLocalLibraries} : {}),\n ...(CDN !== undefined ? {CDN} : {}),\n ...(modules !== undefined ? {modules} : {})\n };\n}\n\n/**\n * Dynamically loads a library (\"module\")\n