@dekkai/data-source
Version:
Data source wrapper for local and remote files. Works on browsers, node.js and deno.
961 lines (947 loc) • 36.6 kB
JavaScript
/**
* [[DataSource]] that represents a section (chunk) of a larger data source (parent). Useful when chunking a data source to
* parallelize processing.
*/
class DataChunk {
/**
* @param source - The parent data source for this chunk
* @param start - The start of this chunk, in bytes, within the parent data source
* @param end - The end of this chunk, in bytes, within the parent data source
*/
constructor(source, start, end) {
/**
* Variable to store the loaded data for this chunk.
*/
this._buffer = null;
this.source = source;
this.start = start;
this.end = end;
}
/**
* When this chunk is loaded, returns the buffer containing the data for this chunk, `null` otherwise.
*/
get buffer() {
return this._buffer;
}
/**
* The total byte length this chunk represents.
*
* NOTE: This value can change after a chunk is loaded for the first time if the chunk belongs to a remote data
* source, if the total size is unknown, the value will be -1.
*/
get byteLength() {
return Promise.resolve(this.end - this.start);
}
/**
* Is this chunk loaded in memory.
*/
get loaded() {
return Boolean(this._buffer);
}
/**
* Loads this chunk into memory.
*
* NOTE: If this chunk belongs to a remote [[DataSource]], this function waits until the data for this chunk has been
* transferred from the remote and into memory. Also, if the final size for the remote [[DataSource]] is unknown,
* the [[byteLength]] of this chunk could change after it finishes loading.
*/
async load() {
if (!this._buffer) {
this._buffer = await this.loadData();
// if we don't know the total size of remote files, the actual size of the chunk could change
if (this._buffer === null) { // the chunk could not be loaded
this.start = 0;
this.end = 0;
}
else if (this.end - this.start > this._buffer.byteLength) { // the actual data is smaller than the requested size
this.end -= this.end - this.start - this._buffer.byteLength;
}
}
}
/**
* Unloads this chunk from memory.
*/
unload() {
this._buffer = null;
}
/**
* Slices this chunk and returns a new data chunk pointing at the data within the specified boundaries.
* @param start - Pointer to the start of the data in bytes
* @param end - Pointer to the end of the data in bytes
*/
slice(start, end) {
return new DataChunk(this, start, end);
}
/**
* Loads the data source into an ArrayBuffer. Optionally a `start` and `end` can be specified to load a part of the
* data.
* @param start - The offset at which the data will start loading
* @param end - The offset at which the data will stop loading
*/
loadData(start = 0, end = (this.end - this.start)) {
return this.source.loadData(this.start + start, this.start + end);
}
}
class LocalDataFile {
/**
* Slices the file and returns a data chunk pointing at the data within the specified boundaries.
* @param start - Pointer to the start of the data in bytes
* @param end - Pointer to the end of the data in bytes
*/
slice(start, end) {
return new DataChunk(this, start, end);
}
}
/**
* Caches the result of a NodeJS environment check.
* @internal
*/
const kIsNodeJS = Object.prototype.toString.call(typeof process !== 'undefined' ? process : 0) === '[object process]';
/**
* Checks if the current environment is NodeJS.
*/
function isNodeJS() {
return kIsNodeJS;
}
/**
* Checks if the current environment supports dynamic imports.
* @internal
*/
function checkDynamicImport() {
try {
import(/* webpackIgnore: true */ `${null}`).catch(() => false);
return true;
}
catch {
return false;
}
}
/**
* Caches the result of a dynamic imports check.
* @internal
*/
const kSupportsDynamicImport = checkDynamicImport();
// eslint-disable-next-line no-new-func
const requireFunc = new Function('mod', 'return require(mod)');
/**
* Detects the environment and loads a module using either `require` or `import`.
* @param mod - The name or path to the module to load.
*/
async function loadModule(mod) {
if (kSupportsDynamicImport) {
return await import(/* webpackIgnore: true */ mod.toString());
}
else if (isNodeJS()) {
// return typeof module !== 'undefined' && typeof module.require === 'function' && module.require(mod.toString()) ||
// // eslint-disable-next-line camelcase
// typeof __non_webpack_require__ === 'function' && __non_webpack_require__(mod.toString()) ||
// typeof require === 'function' && require(mod.toString()); // eslint-disable-line
return requireFunc(mod);
}
// not supported, a dynamic loader could be created for browser environments here, all modern browsers support
// dynamic imports though so not implemented for now.
throw 'ERROR: Can\'t load modules dynamically on this platform';
}
/**
* Cached [`fs`](https://nodejs.org/api/fs.html) module in node and `null` in every other platform. If this in null in
* node, `await` for [[kFsPromise]] to finish.
* @internal
*/
let gFS = null;
/**
* Promise that resolves to the [`fs`](https://nodejs.org/api/fs.html) module in node and `null` in every other platform.
* @internal
*/
const kFsPromise = (isNodeJS() ? loadModule('fs') : Promise.resolve(null)).then(fs => (gFS = fs));
/**
* Represents a data file on the node platform.
*/
class LocalDataFileNode extends LocalDataFile {
/**
* @param handle - A node file handle
* @param stats - Stats for the file the handle points at
*/
constructor(handle, stats) {
super();
this.handle = handle;
this.stats = stats;
}
/**
* Utility function to wrap a file as a [[DataFile]] for this platform.
* @param source - The file to wrap
*/
static async fromSource(source) {
// wait for `fs` to be loaded
await kFsPromise;
let handle;
if (source instanceof URL || typeof source === 'string') {
handle = gFS.openSync(source);
}
else if (typeof source === 'number') {
handle = source;
}
else {
throw `A LocalDataFileNode cannot be created from a ${typeof source} instance`;
}
const stats = gFS.fstatSync(handle);
return new LocalDataFileNode(handle, stats);
}
/**
* The total length, in bytes, of the file this instance represents.
*/
get byteLength() {
return Promise.resolve(this.stats.size);
}
/**
* Closes the local file handle for the current platform. After this function is called all subsequent operations
* on this file, or any other data sources depending on this file, will fail.
*/
close() {
const handle = this.handle;
kFsPromise.then(() => gFS.closeSync(handle));
this.handle = null;
this.stats = null;
}
/**
* Loads the file into an ArrayBuffer. Optionally a `start` and `end` can be specified to load a part of the file.
* @param start - The offset at which the data will start loading
* @param end - The offset at which the data will stop loading
*/
async loadData(start = 0, end = this.stats.size) {
// wait for `fs` to be loaded
await kFsPromise;
const normalizedEnd = Math.min(end, this.stats.size);
const length = normalizedEnd - start;
const result = new Uint8Array(length);
let loaded = 0;
while (loaded < length) {
loaded += await this.loadDataIntoBuffer(result, loaded, start + loaded, normalizedEnd);
}
return result.buffer;
}
/**
* Loads data into a buffer with the specified parameters.
* @param buffer - The buffer in which the data will be loaded. It must be large enough to fit the data requested
* @param offset - The byte offset within the buffer at which the data will be written
* @param start - The byte offset within the file where data will be read
* @param end - The byte offset within the file at which the data will stop being read
*/
loadDataIntoBuffer(buffer, offset, start, end) {
return new Promise((resolve, reject) => {
const length = end - start;
gFS.read(this.handle, buffer, offset, length, start, (err, bytesRead) => {
if (err) {
reject(err);
}
else {
resolve(bytesRead);
}
});
});
}
}
/**
* Represents a data file on the browser platform.
*/
class LocalDataFileBrowser extends LocalDataFile {
/**
* @param blob - Container of the file
*/
constructor(blob) {
super();
this.blob = blob;
}
/**
* Utility function to wrap a file as a [[DataFile]] for this platform.
* @param source - The file to wrap
*/
static async fromSource(source) {
return new LocalDataFileBrowser(source);
}
/**
* The total length, in bytes, of the file this instance represents.
*/
get byteLength() {
return Promise.resolve(this.blob.size);
}
/**
* Closes the local file handle for the current platform. After this function is called all subsequent operations
* on this file, or any other data sources depending on this file, will fail.
*/
close() {
this.blob = null;
}
/**
* Loads the file into an ArrayBuffer. Optionally a `start` and `end` can be specified to load a part of the file.
* @param start - The offset at which the data will start loading
* @param end - The offset at which the data will stop loading
*/
async loadData(start = 0, end = this.blob.size) {
const slice = this.blob.slice(start, Math.min(end, this.blob.size));
return await this.loadBlob(slice);
}
/**
* Loads the specified blob into an array buffer.
* @param blob - The blob to load
*/
loadBlob(blob) {
return new Promise(resolve => {
const reader = new FileReader();
reader.onload = () => {
resolve(reader.result);
};
reader.readAsArrayBuffer(blob);
});
}
}
/**
* Represents a data file on the deno platform.
*/
class LocalDataFileDeno extends LocalDataFile {
/**
* @param file - A deno file instance
* @param info - Info for the file instance
*/
constructor(file, info) {
super();
this.file = file;
this.info = info;
}
/**
* Utility function to wrap a file as a [[DataFile]] for this platform.
* @param source - The file to wrap
*/
static async fromSource(source) {
if (!(source instanceof URL) && typeof source !== 'string') {
throw `A LocalDataFileDeno cannot be created from a ${typeof source} instance`;
}
const stats = await Deno.stat(source);
if (!stats.isFile) {
throw `The path "${source} does not point to a file"`;
}
const file = await Deno.open(source, { read: true, write: false });
return new LocalDataFileDeno(file, stats);
}
/**
* The total length, in bytes, of the file this instance represents.
*/
get byteLength() {
return Promise.resolve(this.info.size);
}
/**
* Closes the local file handle for the current platform. After this function is called all subsequent operations
* on this file, or any other data sources depending on this file, will fail.
*/
close() {
Deno.close(this.file.rid);
this.file = null;
this.info = null;
}
/**
* Loads the file into an ArrayBuffer. Optionally a `start` and `end` can be specified to load a part of the file.
* @param start - The offset at which the data will start loading
* @param end - The offset at which the data will stop loading
*/
async loadData(start = 0, end = this.info.size) {
const normalizedEnd = Math.min(end, this.info.size);
const length = normalizedEnd - start;
const result = new Uint8Array(length);
let loaded = 0;
while (loaded < length) {
loaded += await this.loadDataIntoBuffer(result, loaded, start + loaded, normalizedEnd);
}
return result.buffer;
}
/**
* Loads data into a buffer with the specified parameters.
* @param buffer - The buffer in which the data will be loaded. It must be large enough to fit the data requested
* @param offset - The byte offset within the buffer at which the data will be written
* @param start - The byte offset within the file where data will be read
* @param end - The byte offset within the file at which the data will stop being read
*/
async loadDataIntoBuffer(buffer, offset, start, end) {
const cursorPosition = await this.file.seek(start, Deno.SeekMode.Start);
if (cursorPosition !== start) {
throw 'ERROR: Cannot seek to the desired position';
}
const result = new Uint8Array(end - start);
const bytesRead = await this.file.read(result);
buffer.set(result, offset);
return bytesRead;
}
}
/**
* A symbol that used to register omni-listeners.
* @internal
*/
const kOmniEvent = Symbol('EventEmitter::omni::event');
/**
* @internal
*/
class EventEmitterMixin {
/**
* Mixin method that holds the EventEmitter implementation, this function is exposed through `EventEmitter.mixin`.
* @param Parent - The parent class into which the EventEmitter implementation will be mixed in.
*/
static mixin(Parent) {
const ParentConstructor = Parent; // eslint-disable-line @typescript-eslint/ban-types
class EventEmitter extends ParentConstructor {
constructor() {
super(...arguments);
/**
* Map of registered event listeners with this instance.
* @private
*/
this.listeners = new Map();
}
/**
* Returns a symbol that can be used to register an omni-listener.
*/
static get omniEvent() {
return kOmniEvent;
}
/**
* Register an event listener callback for the specified event.
*
* NOTE: Pass `*` as the event type to listen to all event emitted by this instance.
* @param type - The event to listen for
* @param callback - Called when the event is emitted by this instance
*/
on(type, callback) {
const queue = this.listeners.get(type);
if (queue) {
queue.add(callback);
}
else {
this.listeners.set(type, new Set([callback]));
}
}
/**
* Unregister an event listener callback.
* @param type - The event to unregister from
* @param callback - Callback function to remove
*/
off(type, callback) {
const queue = this.listeners.get(type);
if (queue) {
queue.delete(callback);
}
}
/**
* Emit an event to all event listeners register for that specific event and omni-listeners (listeners registered
* wising `*` as the event type).
* @param type - The event to emit, cannot be `*`
* @param args - Parameters to pass to the callback functions registered for this event
*/
emit(type, ...args) {
if (type === kOmniEvent) {
return;
}
if (this.listeners.has(type)) {
const stack = new Set(this.listeners.get(type));
for (const callback of stack) {
callback.call(this, type, ...args);
}
}
if (this.listeners.has(kOmniEvent)) {
const omni = new Set(this.listeners.get(kOmniEvent));
for (const callback of omni) {
callback.call(this, type, ...args);
}
}
}
}
return EventEmitter;
}
}
/**
* Simple event emitter class.
*
* Supports "omni-listeners" through its `omniEvent` static property. An omni-listener is added/removed as any other
* listener but it will be triggered with ANY event rather than with a specific one.
*
* Events can be strings or symbols. Internally, uses Map and Set instances to deal with events and listeners so in
* theory anything that can be used as a Map key can be used as an event, only strings and symbols are guaranteed to
* work however.
*/
class EventEmitter extends EventEmitterMixin.mixin(EventEmitterMixin) {
}
class RemoteDataFile extends EventEmitter {
/**
* Slices the file and returns a data chunk pointing at the data within the specified boundaries.
* @param start - Pointer to the start of the data in bytes
* @param end - Pointer to the end of the data in bytes
*/
slice(start, end) {
return new DataChunk(this, start, end);
}
}
/**
* Fired when data loading progresses. Not fired when data loading finishes.
* @event
*/
RemoteDataFile.LOADING_START = Symbol('DataFileEvents::LoadingStart');
/**
* Fired when data loading progresses. Not fired when data loading finishes.
* @event
*/
RemoteDataFile.LOADING_PROGRESS = Symbol('DataFileEvents::LoadingProgress');
/**
* Fired when the data loading finishes.
* @event
*/
RemoteDataFile.LOADING_COMPLETE = Symbol('DataFileEvents::LoadingComplete');
/**
* The byte size of 4MB.
* @internal
*/
const kSizeOf4MB$1 = 1024 * 1024 * 4;
/**
* Represents a remote data file on the browser platform.
*/
class RemoteDataFileBrowser extends RemoteDataFile {
/**
* @param source - The source from where this instance should load its file contents.
*/
constructor(source) {
super();
/**
* Variable to hold the byte length of the loaded file.
*/
this._byteLength = null;
/**
* Variable to hold the bytes loaded so far from the remote file.
*/
this._bytesLoaded = 0;
/**
* Variable that holds a promise that resolves when the file finishes loading
*/
this._onLoadingComplete = null;
/**
* Variable that holds a boolean describing if the loading is complete or not.
*/
this._isLoadingComplete = false;
/**
* An ArrayBuffer instance that holds the data loaded for this file, do not keep a local copy of this variable as
* it could be replaced as the file loads into memory.
*/
this.buffer = null;
this.source = source;
this._onLoadingComplete = {
promise: null,
resolve: null,
reject: null,
started: false,
};
this._onLoadingComplete.promise = new Promise((resolve, reject) => {
this._onLoadingComplete.resolve = resolve;
this._onLoadingComplete.reject = reject;
});
}
/**
* Utility function to wrap a file as a [[DataFile]] for this platform.
* NOTE: This function calls `startDownloading` on the file.
* @param source - The file to wrap
*/
static async fromSource(source) {
const result = new RemoteDataFileBrowser(source);
await result.startDownloading();
return result;
}
/**
* The total length, in bytes, of the file this instance represents.
*/
get byteLength() {
if (this._byteLength === null) {
return new Promise(resolve => {
const handleEvent = (e, byteLength) => {
this.off(RemoteDataFile.LOADING_START, handleEvent);
this._byteLength = byteLength;
resolve(byteLength);
};
this.on(RemoteDataFile.LOADING_START, handleEvent);
});
}
return Promise.resolve(this._byteLength);
}
/**
* Bytes loaded for this file, useful when parsing streaming files.
*/
get bytesLoaded() {
return this._bytesLoaded;
}
/**
* Promise that resolves when this file has finished downloading from the remote server.
*/
get onLoadingComplete() {
return this._onLoadingComplete.promise;
}
/**
* Has the file finished downloading from the remote server.
*/
get isLoadingComplete() {
return this._isLoadingComplete;
}
/**
* This function must ba called in order to start downloading the file, if this function fail the file cannot be
* fetched from the server.
*/
async startDownloading() {
if (!this._onLoadingComplete.started) {
this._onLoadingComplete.started = true;
let response;
try {
response = await fetch(this.source);
}
catch (e) {
this._onLoadingComplete.reject(e);
throw e;
}
if (!response.ok) {
const notOK = new Error('Network response was not ok');
this._onLoadingComplete.reject(notOK);
throw notOK;
}
// allow for the calling script to register events, etc
setTimeout(() => this.readFileStream(response));
}
}
/**
* Loads the file into an ArrayBuffer. Optionally a `start` and `end` can be specified to load a part of the file.
* @param start - The offset at which the data will start loading
* @param end - The offset at which the data will stop loading
*/
async loadData(start = 0, end = this._byteLength) {
if (this._isLoadingComplete && start >= this._byteLength) {
return new ArrayBuffer(0);
}
if (this._bytesLoaded >= end || this._isLoadingComplete) {
return this.buffer.slice(start, Math.min(end, this._bytesLoaded));
}
return new Promise(resolve => {
const handleEvent = (e, loaded) => {
if (loaded >= end || e === RemoteDataFile.LOADING_COMPLETE) {
this.off(RemoteDataFile.LOADING_PROGRESS, handleEvent);
this.off(RemoteDataFile.LOADING_COMPLETE, handleEvent);
resolve(this.buffer.slice(start, Math.min(end, loaded)));
}
};
this.on(RemoteDataFile.LOADING_PROGRESS, handleEvent);
this.on(RemoteDataFile.LOADING_COMPLETE, handleEvent);
});
}
/**
* Reads the data from a remote response using streams, this allows for data to be processed even if the file has
* not been completely loaded.
* @param response - A response object returned by `fetch`. This object will be used to retrieve the read stream.
*/
async readFileStream(response) {
const contentLength = response.headers.get('content-length');
if (contentLength !== null) {
this._byteLength = parseInt(contentLength, 10);
this.buffer = new ArrayBuffer(this._byteLength);
}
else {
this._byteLength = -1;
this.buffer = new ArrayBuffer(kSizeOf4MB$1);
}
this._bytesLoaded = 0;
this.emit(RemoteDataFile.LOADING_START, this._byteLength);
if (this._byteLength === 0) {
this.emit(RemoteDataFile.LOADING_PROGRESS, this._bytesLoaded, this._byteLength);
this._isLoadingComplete = true;
this.emit(RemoteDataFile.LOADING_COMPLETE, this._byteLength);
this._onLoadingComplete.resolve(this._byteLength);
}
else {
const reader = response.body.getReader();
let view = new Uint8Array(this.buffer);
while (true) {
try {
const result = await reader.read();
if (result.done) {
this._byteLength = this._bytesLoaded;
this._isLoadingComplete = true;
this.emit(RemoteDataFile.LOADING_COMPLETE, this._byteLength);
this._onLoadingComplete.resolve(this._byteLength);
break;
}
if (this.buffer.byteLength < this._bytesLoaded + result.value.byteLength) {
const oldView = view;
this.buffer = new ArrayBuffer(this._bytesLoaded + Math.max(result.value.byteLength, kSizeOf4MB$1));
view = new Uint8Array(this.buffer);
view.set(oldView, 0);
}
view.set(result.value, this._bytesLoaded);
this.emit(RemoteDataFile.LOADING_PROGRESS, this._bytesLoaded, this._byteLength);
this._bytesLoaded += result.value.length;
}
catch (e) {
this._onLoadingComplete.reject(e);
throw e;
}
}
}
}
}
/**
* Caches the result of a NodeJS environment check.
* @internal
*/
const kIsDeno = Boolean(typeof Deno !== 'undefined');
/**
* Checks if the current environment is Deno.
*/
function isDeno() {
return kIsDeno;
}
/**
* Cached [`http`](https://nodejs.org/api/http.html) module in node and `null` in every other platform. If this in null
* in node, `await` for [[kLibPromise]] to finish.
* @internal
*/
let gHTTP = null;
/**
* Cached [`https`](https://nodejs.org/api/https.html) module in node and `null` in every other platform. If this in
* null in node, `await` for [[kLibPromise]] to finish.
* @internal
*/
let gHTTPS = null;
/**
* Cached [`url`](https://nodejs.org/api/url.html) module in node and `null` in every other platform. If this in
* null in node, `await` for [[kLibPromise]] to finish.
* @internal
*/
let gURL = null;
/**
* Promise that resolves to the [`http`](https://nodejs.org/api/fs.html) and [`https`](https://nodejs.org/api/https.html)
* modules in node and `null` in every other platform.
* @internal
*/
const kLibPromise = (isNodeJS() ?
Promise.all([loadModule('http'), loadModule('https'), loadModule('url')]) :
Promise.resolve([null, null])).then(libs => {
gHTTP = libs[0];
gHTTPS = libs[1];
gURL = libs[2];
});
/**
* The byte size of 4MB.
* @internal
*/
const kSizeOf4MB = 1024 * 1024 * 4;
/**
* Represents a remote data file on node.js.
*/
class RemoteDataFileNode extends RemoteDataFile {
constructor(source) {
super();
/**
* Variable to hold the byte length of the loaded file.
*/
this._byteLength = null;
/**
* Variable to hold the bytes loaded so far from the remote file.
*/
this._bytesLoaded = null;
/**
* Variable that holds a promise that resolves when the file finishes loading
*/
this._onLoadingComplete = null;
/**
* Variable that holds a boolean describing if the loading is complete or not.
*/
this._isLoadingComplete = false;
/**
* An ArrayBuffer instance that holds the data loaded for this file, do not keep a local copy of this variable as
* it could be replaced as the file loads into memory.
*/
this.buffer = null;
this.source = source;
this._onLoadingComplete = {
promise: null,
resolve: null,
reject: null,
started: false,
};
this._onLoadingComplete.promise = new Promise((resolve, reject) => {
this._onLoadingComplete.resolve = resolve;
this._onLoadingComplete.reject = reject;
});
}
/**
* Utility function to wrap a file as a [[DataFile]] for this platform.
* NOTE: This function calls `startDownloading` on the file.
* @param source - The file to wrap
*/
static async fromSource(source) {
const result = new RemoteDataFileNode(source);
await result.startDownloading();
return result;
}
/**
* The total length, in bytes, of the file this instance represents.
*/
get byteLength() {
if (this._byteLength === null) {
return new Promise(resolve => {
const handleEvent = (e, byteLength) => {
this.off(RemoteDataFile.LOADING_START, handleEvent);
this._byteLength = byteLength;
resolve(byteLength);
};
this.on(RemoteDataFile.LOADING_START, handleEvent);
});
}
return Promise.resolve(this._byteLength);
}
/**
* Bytes loaded for this file, useful when parsing streaming files.
*/
get bytesLoaded() {
return this._bytesLoaded;
}
/**
* Promise that resolves when this file has finished downloading from the remote server.
*/
get onLoadingComplete() {
return this._onLoadingComplete.promise;
}
/**
* Has the file finished downloading from the remote server.
*/
get isLoadingComplete() {
return this._isLoadingComplete;
}
/**
* This function must ba called in order to start downloading the file, if this function fail the file cannot be
* fetched from the server.
*/
startDownloading() {
return new Promise((resolve, reject) => {
// wait for libraries to be loaded
kLibPromise.then(() => {
const url = this.source instanceof gURL.URL ? this.source : gURL.parse(this.source);
const protocol = url.protocol === 'https' ? gHTTPS : gHTTP;
protocol.get(this.source, response => {
if (response.statusCode < 200 || response.statusCode >= 300) {
response.resume();
const notOK = new Error('Network response was not ok');
this._onLoadingComplete.reject(notOK);
reject(notOK);
return;
}
resolve();
// allow for the calling script to register events, etc
setTimeout(() => this.readFileStream(response));
});
});
});
}
/**
* Loads the file into an ArrayBuffer. Optionally a `start` and `end` can be specified to load a part of the file.
* @param start - The offset at which the data will start loading
* @param end - The offset at which the data will stop loading
*/
async loadData(start = 0, end = this._byteLength) {
if (this._isLoadingComplete && start >= this._byteLength) {
return new ArrayBuffer(0);
}
if (this._bytesLoaded >= end || this._isLoadingComplete) {
return this.buffer.slice(start, Math.min(end, this._bytesLoaded));
}
return new Promise(resolve => {
const handleEvent = (e, loaded) => {
if (loaded >= end || e === RemoteDataFile.LOADING_COMPLETE) {
this.off(RemoteDataFile.LOADING_PROGRESS, handleEvent);
this.off(RemoteDataFile.LOADING_COMPLETE, handleEvent);
resolve(this.buffer.slice(start, Math.min(end, loaded)));
}
};
this.on(RemoteDataFile.LOADING_PROGRESS, handleEvent);
this.on(RemoteDataFile.LOADING_COMPLETE, handleEvent);
});
}
/**
* Reads the data from a remote response using streams, this allows for data to be processed even if the file has
* not been completely loaded.
* @param response - A response object returned by `fetch`. This object will be used to retrieve the read stream.
*/
async readFileStream(response) {
const contentLength = response.headers['content-length'];
if (contentLength !== null && contentLength !== undefined) {
this._byteLength = parseInt(contentLength, 10);
this.buffer = new ArrayBuffer(this._byteLength);
}
else {
this._byteLength = -1;
this.buffer = new ArrayBuffer(kSizeOf4MB);
}
this._bytesLoaded = 0;
this.emit(RemoteDataFile.LOADING_START, this._byteLength);
if (this._byteLength === 0) {
this.emit(RemoteDataFile.LOADING_PROGRESS, this._bytesLoaded, this._byteLength);
this._isLoadingComplete = true;
this.emit(RemoteDataFile.LOADING_COMPLETE, this._byteLength);
this._onLoadingComplete.resolve(this._byteLength);
response.resume();
}
else {
let view = new Uint8Array(this.buffer);
response.on('error', error => {
this._onLoadingComplete.reject(error);
throw error;
});
response.on('data', chunk => {
if (this.buffer.byteLength < this._bytesLoaded + chunk.byteLength) {
const oldView = view;
this.buffer = new ArrayBuffer(this._bytesLoaded + Math.max(chunk.byteLength, kSizeOf4MB));
view = new Uint8Array(this.buffer);
view.set(oldView, 0);
}
view.set(chunk, this._bytesLoaded);
this.emit(RemoteDataFile.LOADING_PROGRESS, this._bytesLoaded, this._byteLength);
this._bytesLoaded += chunk.byteLength;
});
response.on('end', () => {
this._byteLength = this._bytesLoaded;
this._isLoadingComplete = true;
this.emit(RemoteDataFile.LOADING_COMPLETE, this._byteLength);
this._onLoadingComplete.resolve(this._byteLength);
});
}
}
}
/**
* Base class for data files on all platforms.
*/
class DataFile {
/**
* Utility function to wrap a local file as a [[DataFile]] for this platform.
* @param source - The file to wrap
*/
static async fromLocalSource(source) {
if (isNodeJS()) {
return LocalDataFileNode.fromSource(source);
}
else if (isDeno()) {
return LocalDataFileDeno.fromSource(source);
}
return LocalDataFileBrowser.fromSource(source);
}
/**
* Utility function to wrap a remote file as a [[DataFile]] for this platform.
* @param source - The file to wrap
*/
static async fromRemoteSource(source) {
if (isNodeJS()) {
return RemoteDataFileNode.fromSource(source);
}
else if (isDeno()) {
return RemoteDataFileBrowser.fromSource(source);
}
return RemoteDataFileBrowser.fromSource(source);
}
}
export { DataChunk, DataFile, LocalDataFile, LocalDataFileBrowser, LocalDataFileDeno, LocalDataFileNode, RemoteDataFile, RemoteDataFileBrowser, RemoteDataFileBrowser as RemoteDataFileDeno };
//# sourceMappingURL=mod.js.map