@openmeteo/file-reader
Version:
JavaScript reader for the om file format using WebAssembly
900 lines (892 loc) • 38.6 kB
JavaScript
'use strict';
var fs = require('node:fs/promises');
exports.CompressionType = void 0;
(function (CompressionType) {
/// Lossy compression using 2D delta coding and scale-factor.
/// Only supports float and scales to 16-bit signed integer.
CompressionType[CompressionType["PforDelta2dInt16"] = 0] = "PforDelta2dInt16";
/// Lossless float/double compression using 2D xor coding.
CompressionType[CompressionType["FpxXor2d"] = 1] = "FpxXor2d";
/// PFor integer compression.
/// f32 values are scaled to u32, f64 are scaled to u64.
CompressionType[CompressionType["PforDelta2d"] = 2] = "PforDelta2d";
/// Similar to `PforDelta2dInt16` but applies `log10(1+x)` before.
CompressionType[CompressionType["PforDelta2dInt16Logarithmic"] = 3] = "PforDelta2dInt16Logarithmic";
CompressionType[CompressionType["None"] = 4] = "None";
})(exports.CompressionType || (exports.CompressionType = {}));
exports.OmDataType = void 0;
(function (OmDataType) {
OmDataType[OmDataType["None"] = 0] = "None";
OmDataType[OmDataType["Int8"] = 1] = "Int8";
OmDataType[OmDataType["Uint8"] = 2] = "Uint8";
OmDataType[OmDataType["Int16"] = 3] = "Int16";
OmDataType[OmDataType["Uint16"] = 4] = "Uint16";
OmDataType[OmDataType["Int32"] = 5] = "Int32";
OmDataType[OmDataType["Uint32"] = 6] = "Uint32";
OmDataType[OmDataType["Int64"] = 7] = "Int64";
OmDataType[OmDataType["Uint64"] = 8] = "Uint64";
OmDataType[OmDataType["Float"] = 9] = "Float";
OmDataType[OmDataType["Double"] = 10] = "Double";
OmDataType[OmDataType["String"] = 11] = "String";
OmDataType[OmDataType["Int8Array"] = 12] = "Int8Array";
OmDataType[OmDataType["Uint8Array"] = 13] = "Uint8Array";
OmDataType[OmDataType["Int16Array"] = 14] = "Int16Array";
OmDataType[OmDataType["Uint16Array"] = 15] = "Uint16Array";
OmDataType[OmDataType["Int32Array"] = 16] = "Int32Array";
OmDataType[OmDataType["Uint32Array"] = 17] = "Uint32Array";
OmDataType[OmDataType["Int64Array"] = 18] = "Int64Array";
OmDataType[OmDataType["Uint64Array"] = 19] = "Uint64Array";
OmDataType[OmDataType["FloatArray"] = 20] = "FloatArray";
OmDataType[OmDataType["DoubleArray"] = 21] = "DoubleArray";
OmDataType[OmDataType["StringArray"] = 22] = "StringArray";
})(exports.OmDataType || (exports.OmDataType = {}));
// | BigInt64Array
// | BigUint64Array;
// Constants mapping
const DATA_TYPES = {
DATA_TYPE_NONE: 0,
DATA_TYPE_INT8: 1,
DATA_TYPE_UINT8: 2,
DATA_TYPE_INT16: 3,
DATA_TYPE_UINT16: 4,
DATA_TYPE_INT32: 5,
DATA_TYPE_UINT32: 6,
DATA_TYPE_INT64: 7,
DATA_TYPE_UINT64: 8,
DATA_TYPE_FLOAT: 9,
DATA_TYPE_DOUBLE: 10,
DATA_TYPE_STRING: 11,
DATA_TYPE_INT8_ARRAY: 12,
DATA_TYPE_UINT8_ARRAY: 13,
DATA_TYPE_INT16_ARRAY: 14,
DATA_TYPE_UINT16_ARRAY: 15,
DATA_TYPE_INT32_ARRAY: 16,
DATA_TYPE_UINT32_ARRAY: 17,
DATA_TYPE_INT64_ARRAY: 18,
DATA_TYPE_UINT64_ARRAY: 19,
DATA_TYPE_FLOAT_ARRAY: 20,
DATA_TYPE_DOUBLE_ARRAY: 21,
DATA_TYPE_STRING_ARRAY: 22,
};
const HEADER_TYPES = {
OM_HEADER_INVALID: 0,
OM_HEADER_LEGACY: 1,
OM_HEADER_READ_TRAILER: 2,
};
const ERROR_CODES = {
ERROR_OK: 0,
};
// Size of the decoder structure (estimate, will be updated during init)
const SIZEOF_DECODER = 256;
let wasmModuleWrapped = null;
async function initWasm() {
if (wasmModuleWrapped)
return wasmModuleWrapped;
try {
// Import the factory function that creates the module
// @ts-ignore
const OmFileFormat = await import('@openmeteo/file-format-wasm');
// Initialize the module by calling the factory function
const wasmModuleRaw = await OmFileFormat.default();
// Create our wrapped module with the expected interface
wasmModuleWrapped = createWrappedModule(wasmModuleRaw);
return wasmModuleWrapped;
}
catch (error) {
throw new Error(`Failed to initialize WASM module: ${error}`);
}
}
function createWrappedModule(rawModule) {
// Create a wrapper that maps the prefixed function names to our interface
return {
// Memory management functions
_malloc: rawModule._malloc,
_free: rawModule._free,
setValue: rawModule.setValue,
getValue: rawModule.getValue,
HEAPU8: rawModule.HEAPU8,
// Map all the C functions to their prefixed versions
om_header_size: rawModule._om_header_size,
om_header_type: rawModule._om_header_type,
om_trailer_size: rawModule._om_trailer_size,
om_trailer_read: rawModule._om_trailer_read,
om_variable_init: rawModule._om_variable_init,
om_variable_get_type: rawModule._om_variable_get_type,
om_variable_get_compression: rawModule._om_variable_get_compression,
om_variable_get_scale_factor: rawModule._om_variable_get_scale_factor,
om_variable_get_add_offset: rawModule._om_variable_get_add_offset,
om_variable_get_dimension_count: rawModule._om_variable_get_dimension_count,
om_variable_get_dimension_value: rawModule._om_variable_get_dimension_value,
om_variable_get_chunk_count: rawModule._om_variable_get_chunk_count,
om_variable_get_chunk_value: rawModule._om_variable_get_chunk_value,
om_variable_get_name_count: rawModule._om_variable_get_name_count,
om_variable_get_name_ptr: rawModule._om_variable_get_name_ptr,
om_variable_get_children_count: rawModule._om_variable_get_children_count,
om_variable_get_children: rawModule._om_variable_get_children,
om_variable_get_scalar: rawModule._om_variable_get_scalar,
om_decoder_init: rawModule._om_decoder_init,
om_decoder_init_index_read: rawModule._om_decoder_init_index_read,
om_decoder_init_data_read: rawModule._om_decoder_init_data_read,
om_decoder_read_buffer_size: rawModule._om_decoder_read_buffer_size,
om_decoder_next_index_read: rawModule._om_decoder_next_index_read,
om_decoder_next_data_read: rawModule._om_decoder_next_data_read,
om_decoder_decode_chunks: rawModule._om_decoder_decode_chunks,
// Constants
...HEADER_TYPES,
...ERROR_CODES,
...DATA_TYPES,
// Additional info
sizeof_decoder: SIZEOF_DECODER, // We could try to determine this dynamically if there's a C API for it
};
}
function getWasmModule() {
if (!wasmModuleWrapped) {
throw new Error("WASM module not initialized. Call initWasm() first.");
}
return wasmModuleWrapped;
}
class OmFileReader {
constructor(backend, wasm) {
this.backend = backend;
this.wasm = wasm || getWasmModule();
this.variable = null;
this.variableDataPtr = null;
this.verbose = false;
}
/**
* Static factory method to create and initialize an OmFileReader
*/
static async create(backend) {
// Make sure WASM is initialized
const wasm = await initWasm();
const reader = new OmFileReader(backend, wasm);
await reader.initialize();
return reader;
}
async initialize() {
// Similar to the 'new' method in Rust
const headerSize = this.wasm.om_header_size();
const headerData = await this.backend.getBytes(0, headerSize);
const headerPtr = this.wasm._malloc(headerData.length);
this.wasm.HEAPU8.set(headerData, headerPtr);
const headerType = this.wasm.om_header_type(headerPtr);
if (headerType === this.wasm.OM_HEADER_INVALID) {
this.wasm._free(headerPtr);
throw new Error("Not a valid OM file");
}
let variableData;
if (headerType === this.wasm.OM_HEADER_LEGACY) {
variableData = headerData;
}
else if (headerType === this.wasm.OM_HEADER_READ_TRAILER) {
const fileSize = await this.backend.count();
const trailerSize = this.wasm.om_trailer_size();
const trailerOffset = fileSize - trailerSize;
const trailerPtr = await this.readDataBlock(trailerOffset, trailerSize);
// Create pointers for offset and size (out parameters)
const offsetPtr = this.wasm._malloc(8); // 64-bit value = 8 bytes
const sizePtr = this.wasm._malloc(8);
const success = this.wasm.om_trailer_read(trailerPtr, offsetPtr, sizePtr);
if (!success) {
this.wasm._free(headerPtr);
this.wasm._free(trailerPtr);
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
throw new Error("Failed to read trailer");
}
// Read values from memory
const offset = Number(this.wasm.getValue(offsetPtr, "i64"));
const size = Number(this.wasm.getValue(sizePtr, "i64"));
// Free memory
this.wasm._free(trailerPtr);
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
// Get variable data
variableData = await this.backend.getBytes(offset, size);
}
else {
this.wasm._free(headerPtr);
throw new Error("Unknown header type");
}
// Initialize variable
const variableDataPtr = this.wasm._malloc(variableData.length);
this.wasm.HEAPU8.set(variableData, variableDataPtr);
this.variable = this.wasm.om_variable_init(variableDataPtr);
this.variableDataPtr = variableDataPtr;
this.wasm._free(headerPtr);
return this;
}
// Helper method to convert C strings to JS strings
_getString(strPtr, strLen) {
const bytes = this.wasm.HEAPU8.subarray(strPtr, strPtr + strLen);
return new TextDecoder("utf8").decode(bytes);
}
dataType() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_type(this.variable);
}
compression() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_compression(this.variable);
}
scaleFactor() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_scale_factor(this.variable);
}
addOffset() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_add_offset(this.variable);
}
getDimensions() {
if (this.variable === null)
throw new Error("Reader not initialized");
// Get count using the wrapper function
const count = Number(this.wasm.om_variable_get_dimension_count(this.variable));
// Get each dimension individually
const dimensions = [];
for (let i = 0; i < count; i++) {
dimensions.push(Number(this.wasm.om_variable_get_dimension_value(this.variable, BigInt(i))));
}
return dimensions;
}
getChunkDimensions() {
if (this.variable === null)
throw new Error("Reader not initialized");
// Get count using the wrapper function
const count = Number(this.wasm.om_variable_get_chunk_count(this.variable));
// Get each chunk dimension individually
const chunks = [];
for (let i = 0; i < count; i++) {
chunks.push(Number(this.wasm.om_variable_get_chunk_value(this.variable, BigInt(i))));
}
return chunks;
}
getName() {
if (this.variable === null)
throw new Error("Reader not initialized");
const size = this.wasm.om_variable_get_name_count(this.variable);
if (size === 0) {
return null;
}
const valuePtr = this.wasm.om_variable_get_name_ptr(this.variable);
if (valuePtr === 0) {
return null;
}
return this._getString(valuePtr, size);
}
numberOfChildren() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_children_count(this.variable);
}
async getChild(index) {
if (this.variable === null)
throw new Error("Reader not initialized");
// Allocate memory for the output parameters
const offsetPtr = this.wasm._malloc(8);
const sizePtr = this.wasm._malloc(8);
const success = this.wasm.om_variable_get_children(this.variable, index, 1, offsetPtr, sizePtr);
if (!success) {
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
return null;
}
const offset = Number(this.wasm.getValue(offsetPtr, "i64"));
const size = Number(this.wasm.getValue(sizePtr, "i64"));
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
return this.initChildFromOffsetSize({ offset, size });
}
async initChildFromOffsetSize(offsetSize) {
const childDataPtr = await this.readDataBlock(offsetSize.offset, offsetSize.size);
const childReader = new OmFileReader(this.backend, this.wasm);
childReader.variable = this.wasm.om_variable_init(childDataPtr);
childReader.variableDataPtr = childDataPtr;
return childReader;
}
// Method to read scalar values
readScalar(dataType) {
if (this.variable === null)
throw new Error("Reader not initialized");
if (this.dataType() !== dataType) {
return null;
}
// Allocate memory for output parameters
const ptrPtr = this.wasm._malloc(4); // pointer to pointer
const sizePtr = this.wasm._malloc(8); // u64
try {
const error = this.wasm.om_variable_get_scalar(this.variable, ptrPtr, sizePtr);
if (error !== this.wasm.ERROR_OK) {
return null;
}
const dataPtr = this.wasm.getValue(ptrPtr, "*");
const size = Number(this.wasm.getValue(sizePtr, "i64"));
if (dataPtr === 0) {
return null;
}
// Read data based on type
let result;
switch (dataType) {
case exports.OmDataType.Int8:
result = this.wasm.getValue(dataPtr, "i8");
break;
case exports.OmDataType.Uint8:
result = this.wasm.getValue(dataPtr, "i8") & 0xff;
break;
case exports.OmDataType.Int16:
result = this.wasm.getValue(dataPtr, "i16");
break;
case exports.OmDataType.Uint16:
result = this.wasm.getValue(dataPtr, "i16") & 0xffff;
break;
case exports.OmDataType.Int32:
result = this.wasm.getValue(dataPtr, "i32");
break;
case exports.OmDataType.Uint32:
result = this.wasm.getValue(dataPtr, "i32") >>> 0;
break;
case exports.OmDataType.Float:
result = this.wasm.getValue(dataPtr, "float");
break;
case exports.OmDataType.Double:
result = this.wasm.getValue(dataPtr, "double");
break;
default:
result = null;
}
return result;
}
finally {
this.wasm._free(ptrPtr);
this.wasm._free(sizePtr);
}
}
newIndexRead(decoderPtr) {
// Calculate proper size for OmDecoder_indexRead_t
const sizeOfRange = 16; // 8 bytes for lowerBound + 8 bytes for upperBound
const sizeOfIndexRead = 8 + 8 + sizeOfRange * 3; // offset + count + 3 range structs
// Allocate and zero the memory
const indexReadPtr = this.wasm._malloc(sizeOfIndexRead);
// Zero out the memory (equivalent to std::mem::zeroed())
const zeroBuffer = new Uint8Array(sizeOfIndexRead);
this.wasm.HEAPU8.set(zeroBuffer, indexReadPtr);
// Initialize the structure using C function
this.wasm.om_decoder_init_index_read(decoderPtr, indexReadPtr);
return indexReadPtr;
}
newDataRead(indexReadPtr) {
// Size of OmDecoder_dataRead_t
const sizeOfRange = 16; // 8 bytes for lowerBound + 8 bytes for upperBound
const sizeOfDataRead = 8 + 8 + sizeOfRange * 3; // offset + count + 3 range structs
// Allocate and zero the memory
const dataReadPtr = this.wasm._malloc(sizeOfDataRead);
// Zero out the memory (equivalent to std::mem::zeroed())
const zeroBuffer = new Uint8Array(sizeOfDataRead);
this.wasm.HEAPU8.set(zeroBuffer, dataReadPtr);
// Initialize the structure using C function
this.wasm.om_decoder_init_data_read(dataReadPtr, indexReadPtr);
return dataReadPtr;
}
async read(dataType, dimRanges, ioSizeMax = BigInt(65536), ioSizeMerge = BigInt(512)) {
if (this.variable === null)
throw new Error("Reader not initialized");
if (this.dataType() !== dataType) {
throw new Error(`Invalid data type: expected ${this.dataType()}, got ${dataType}`);
}
// Calculate output dimensions
const outDims = dimRanges.map((range) => Number(range.end - range.start));
const totalSize = outDims.reduce((a, b) => a * b, 1);
// Create output TypedArray based on data type
let output;
switch (dataType) {
case this.wasm.DATA_TYPE_INT8_ARRAY:
output = new Int8Array(totalSize);
break;
case this.wasm.DATA_TYPE_UINT8_ARRAY:
output = new Uint8Array(totalSize);
break;
case this.wasm.DATA_TYPE_INT16_ARRAY:
output = new Int16Array(totalSize);
break;
case this.wasm.DATA_TYPE_UINT16_ARRAY:
output = new Uint16Array(totalSize);
break;
case this.wasm.DATA_TYPE_INT32_ARRAY:
output = new Int32Array(totalSize);
break;
case this.wasm.DATA_TYPE_UINT32_ARRAY:
output = new Uint32Array(totalSize);
break;
case this.wasm.DATA_TYPE_FLOAT_ARRAY:
output = new Float32Array(totalSize);
break;
case this.wasm.DATA_TYPE_DOUBLE_ARRAY:
output = new Float64Array(totalSize);
break;
default:
throw new Error("Unsupported data type");
}
await this.readInto(dataType, output, dimRanges, ioSizeMax, ioSizeMerge);
return output;
}
/**
* Read data into an existing TypedArray with specified dimension ranges
* @param dataType The data type to read
* @param output The TypedArray to read data into
* @param dimRanges Ranges for each dimension to read
* @param ioSizeMax Maximum I/O size (default: 65536)
* @param ioSizeMerge Merge threshold for I/O operations (default: 512)
*/
async readInto(dataType, output, dimRanges, ioSizeMax = BigInt(65536), ioSizeMerge = BigInt(512)) {
if (this.variable === null)
throw new Error("Reader not initialized");
if (this.dataType() !== dataType) {
throw new Error(`Invalid data type: expected ${this.dataType()}, got ${dataType}`);
}
const nDims = dimRanges.length;
const fileDims = this.getDimensions();
// Validate dimension counts
if (fileDims.length !== nDims) {
throw new Error(`Mismatched dimensions: file has ${fileDims.length}, request has ${nDims}`);
}
// Calculate output dimensions and prepare arrays for WASM
const outDims = dimRanges.map((range) => range.end - range.start);
const intoCubeOffset = new Array(nDims).fill(0);
// Calculate total elements to ensure output array has correct size
const totalElements = outDims.reduce((a, b) => a * Number(b), 1);
if (output.length < totalElements) {
throw new Error(`Output array is too small: needs ${totalElements} elements, has ${output.length}`);
}
// Allocate memory for arrays
const readOffsetPtr = this.wasm._malloc(nDims * 8); // u64 array
const readCountPtr = this.wasm._malloc(nDims * 8);
const intoCubeOffsetPtr = this.wasm._malloc(nDims * 8);
const intoCubeDimensionPtr = this.wasm._malloc(nDims * 8);
try {
// Fill arrays
for (let i = 0; i < nDims; i++) {
// Validate ranges
if (dimRanges[i].start < 0 || dimRanges[i].end > fileDims[i] || dimRanges[i].start >= dimRanges[i].end) {
throw new Error(`Invalid range for dimension ${i}: ${JSON.stringify(dimRanges[i])}`);
}
this.wasm.setValue(readOffsetPtr + i * 8, BigInt(dimRanges[i].start), "i64");
this.wasm.setValue(readCountPtr + i * 8, BigInt(outDims[i]), "i64");
this.wasm.setValue(intoCubeOffsetPtr + i * 8, BigInt(intoCubeOffset[i]), "i64");
this.wasm.setValue(intoCubeDimensionPtr + i * 8, BigInt(outDims[i]), "i64");
}
// Create decoder
const decoderPtr = this.wasm._malloc(this.wasm.sizeof_decoder);
try {
// Initialize decoder
const error = this.wasm.om_decoder_init(decoderPtr, this.variable, BigInt(nDims), readOffsetPtr, readCountPtr, intoCubeOffsetPtr, intoCubeDimensionPtr, ioSizeMerge, ioSizeMax);
if (error !== this.wasm.ERROR_OK) {
throw new Error(`Decoder initialization failed: error code ${error}`);
}
await this.decode(decoderPtr, output);
}
finally {
this.wasm._free(decoderPtr);
}
}
finally {
// Clean up input arrays
this.wasm._free(readOffsetPtr);
this.wasm._free(readCountPtr);
this.wasm._free(intoCubeOffsetPtr);
this.wasm._free(intoCubeDimensionPtr);
}
}
async decode(decoderPtr, outputArray) {
if (this.verbose) {
console.log(`Starting decode with ${outputArray.constructor.name}, length=${outputArray.length}`);
}
const outputPtr = this.wasm._malloc(outputArray.byteLength);
const chunkBufferSize = Number(this.wasm.om_decoder_read_buffer_size(decoderPtr));
const chunkBufferPtr = this.wasm._malloc(chunkBufferSize);
// Create index_read struct
const indexReadPtr = this.newIndexRead(decoderPtr);
const errorPtr = this.wasm._malloc(4);
// Initialize error to OK
this.wasm.setValue(errorPtr, this.wasm.ERROR_OK, "i32");
try {
// Loop over index blocks
let indexBlockCount = 0;
while (this.wasm.om_decoder_next_index_read(decoderPtr, indexReadPtr)) {
indexBlockCount++;
// Get index_read parameters
const indexOffset = Number(this.wasm.getValue(indexReadPtr, "i64"));
const indexCount = Number(this.wasm.getValue(indexReadPtr + 8, "i64"));
if (this.verbose) {
console.log(`Index block #${indexBlockCount}: offset=${indexOffset}, count=${indexCount}`);
}
// Get bytes for index-read
const indexDataPtr = await this.readDataBlock(indexOffset, indexCount);
// Create data_read struct
let dataReadPtr = this.newDataRead(indexReadPtr);
try {
// Loop over data blocks and read compressed data chunks
// Loop over data blocks
let dataBlockCount = 0;
while (this.wasm.om_decoder_next_data_read(decoderPtr, dataReadPtr, indexDataPtr, BigInt(indexCount), errorPtr)) {
dataBlockCount++;
// Get data_read parameters
const dataOffset = Number(this.wasm.getValue(dataReadPtr, "i64"));
const dataCount = Number(this.wasm.getValue(dataReadPtr + 8, "i64"));
const chunkIndexPtr = dataReadPtr + 32; // offset(8), count(8), indexRange(16)
if (this.verbose) {
console.log(` Data block #${dataBlockCount}: offset=${dataOffset}, count=${dataCount}, chunkIndexPtr=${chunkIndexPtr}`);
}
// Get bytes for data-read
const dataBlockPtr = await this.readDataBlock(dataOffset, dataCount);
try {
// Decode chunks
const success = this.wasm.om_decoder_decode_chunks(decoderPtr, chunkIndexPtr, dataBlockPtr, BigInt(dataCount), outputPtr, chunkBufferPtr, errorPtr);
// Check for error
if (!success) {
const error = this.wasm.getValue(errorPtr, "i32");
throw new Error(`Decoder failed to decode chunks: error ${error}`);
}
}
finally {
this.wasm._free(dataBlockPtr);
}
}
// Check for errors after data_read loop
const error = this.wasm.getValue(errorPtr, "i32");
if (error !== this.wasm.ERROR_OK) {
throw new Error(`Data read error: ${error}`);
}
}
finally {
this.wasm._free(dataReadPtr);
this.wasm._free(indexDataPtr);
}
}
// Copy the data back to the output array with the correct type
this.copyToTypedArray(outputPtr, outputArray);
}
finally {
this.wasm._free(errorPtr);
this.wasm._free(indexReadPtr);
this.wasm._free(chunkBufferPtr);
this.wasm._free(outputPtr);
}
}
async readDataBlock(offset, size) {
const data = await this.backend.getBytes(offset, size);
const ptr = this.wasm._malloc(data.length);
this.wasm.HEAPU8.set(data, ptr);
return ptr;
}
/**
* Helper method to copy data from WASM memory to a TypedArray with the correct type
*/
copyToTypedArray(sourcePtr, targetArray) {
switch (targetArray.constructor) {
case Float32Array:
const f32Array = new Float32Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(f32Array);
break;
case Float64Array:
const f64Array = new Float64Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(f64Array);
break;
case Int8Array:
const i8Array = new Int8Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(i8Array);
break;
case Uint8Array:
const u8Array = new Uint8Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(u8Array);
break;
case Int16Array:
const i16Array = new Int16Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(i16Array);
break;
case Uint16Array:
const u16Array = new Uint16Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(u16Array);
break;
case Int32Array:
const i32Array = new Int32Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(i32Array);
break;
case Uint32Array:
const u32Array = new Uint32Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length);
targetArray.set(u32Array);
break;
default:
// Fallback to byte-by-byte copy
const byteArray = new Uint8Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.byteLength);
new Uint8Array(targetArray.buffer).set(byteArray);
}
}
// Clean up resources when done
dispose() {
if (this.variableDataPtr !== null) {
this.wasm._free(this.variableDataPtr);
this.variableDataPtr = null;
}
this.variable = null;
}
}
class FileBackendNode {
constructor(source) {
this.filePath = null;
this.memory = null;
this.fileSize = 0;
this.fileHandle = null;
if (typeof source === "string") {
this.filePath = source;
}
else if (source instanceof ArrayBuffer) {
this.memory = new Uint8Array(source);
this.fileSize = this.memory.length;
}
else if (source instanceof Uint8Array) {
this.memory = source;
this.fileSize = source.length;
}
else {
throw new Error("Unsupported file source type for Node.js FileBackendNode");
}
}
async count() {
if (this.memory) {
return this.fileSize;
}
if (this.filePath) {
if (this.fileSize > 0)
return this.fileSize;
const stats = await fs.stat(this.filePath);
this.fileSize = stats.size;
return this.fileSize;
}
throw new Error("Unable to determine file size");
}
async getBytes(offset, size) {
if (this.memory) {
return this.memory.slice(offset, offset + size);
}
if (this.filePath) {
if (!this.fileHandle) {
this.fileHandle = await fs.open(this.filePath, "r");
}
const buffer = new Uint8Array(size);
const { bytesRead } = await this.fileHandle.read(buffer, 0, size, offset);
if (bytesRead !== size) {
throw new Error(`Expected to read ${size} bytes but got ${bytesRead}`);
}
return buffer;
}
throw new Error("No file or memory buffer available");
}
async close() {
if (this.fileHandle) {
await this.fileHandle.close();
this.fileHandle = null;
}
}
}
class MemoryHttpBackend {
/**
* Create a new MemoryHttpBackend
* @param options Configuration options
*/
constructor(options) {
this.fileSize = null;
this.fileData = null;
this.loadPromise = null;
this.countPromise = null;
this.url = options.url;
this.maxFileSize = options.maxFileSize ?? 200 * 1024 * 1024; // 200 MB default
this.onProgress = options.onProgress;
this.debug = options.debug ?? false;
// Start loading the file in the background
this.loadFile().catch((err) => {
if (this.debug)
console.error("Background file load failed:", err);
});
}
/**
* Get the total size of the file
*/
async count() {
if (this.fileSize !== null) {
return this.fileSize;
}
if (!this.countPromise) {
if (this.debug)
console.log(`Making HEAD request to ${this.url}`);
this.countPromise = (async () => {
try {
const response = await fetch(this.url, {
method: "HEAD",
});
if (!response.ok) {
throw new Error(`HTTP error: ${response.status}`);
}
const contentLength = response.headers.get("content-length");
if (!contentLength) {
throw new Error("Content-Length header not available");
}
this.fileSize = parseInt(contentLength, 10);
if (this.fileSize > this.maxFileSize) {
throw new Error(`File size (${this.fileSize} bytes) exceeds maximum allowed size (${this.maxFileSize} bytes)`);
}
if (this.debug)
console.log(`File size: ${this.fileSize} bytes`);
return this.fileSize;
}
catch (error) {
this.countPromise = null;
throw new Error(`Failed to get file size: ${error instanceof Error ? error.message : String(error)}`);
}
})();
}
return this.countPromise;
}
/**
* Load the entire file into memory
*/
async loadFile() {
// If already loaded or loading, return that promise
if (this.fileData) {
return Promise.resolve();
}
if (this.loadPromise) {
return this.loadPromise;
}
this.loadPromise = (async () => {
try {
// First get the file size
const size = await this.count();
if (this.debug)
console.log(`Fetching entire file (${size} bytes) from ${this.url}`);
// Use fetch with streaming and progress tracking
const response = await fetch(this.url);
if (!response.ok) {
throw new Error(`HTTP error: ${response.status}`);
}
// Check if ReadableStream is supported and if progress tracking is needed
if (this.onProgress && response.body && "getReader" in response.body) {
// Stream the response with progress tracking
const contentLength = Number(response.headers.get("content-length") || size);
const reader = response.body.getReader();
const chunks = [];
let receivedLength = 0;
let lastProgressUpdate = 0;
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
chunks.push(value);
receivedLength += value.length;
// Don't update progress too frequently (throttle updates)
const now = Date.now();
if (now - lastProgressUpdate > 100) {
// update every 100ms
this.onProgress(receivedLength, contentLength);
lastProgressUpdate = now;
}
}
// Concatenate chunks into a single Uint8Array
this.fileData = new Uint8Array(receivedLength);
let position = 0;
for (const chunk of chunks) {
this.fileData.set(chunk, position);
position += chunk.length;
}
// Final progress update
this.onProgress(receivedLength, contentLength);
}
else {
// Simple approach without streaming
const buffer = await response.arrayBuffer();
this.fileData = new Uint8Array(buffer);
if (this.onProgress) {
this.onProgress(this.fileData.length, size);
}
}
if (this.debug)
console.log(`File loaded successfully (${this.fileData.length} bytes)`);
return;
}
catch (error) {
this.loadPromise = null;
throw new Error(`Failed to load file: ${error instanceof Error ? error.message : String(error)}`);
}
})();
return this.loadPromise;
}
/**
* Get bytes from the file
* @param offset The starting position in the file
* @param size The number of bytes to read
*/
async getBytes(offset, size) {
try {
// Make sure the file is loaded
if (!this.fileData) {
if (this.debug)
console.log(`getBytes(${offset}, ${size}): Waiting for file to load...`);
await this.loadFile();
if (this.debug)
console.log(`getBytes(${offset}, ${size}): File loaded`);
}
// At this point, fileData should be available
if (!this.fileData) {
throw new Error("File data is not available after load");
}
// Bounds check
if (offset < 0 || offset + size > this.fileData.length) {
throw new Error(`Requested range (${offset}:${offset + size}) is out of bounds (0:${this.fileData.length})`);
}
if (this.debug)
console.log(`Serving ${size} bytes from offset ${offset} from memory`);
// Return the requested slice of data
return this.fileData.slice(offset, offset + size);
}
catch (error) {
throw new Error(`Error in getBytes: ${error instanceof Error ? error.message : String(error)}`);
}
}
/**
* Check if the file is fully loaded
*/
isLoaded() {
return !!this.fileData;
}
/**
* Get the current loaded data or null if not loaded
*/
getFileData() {
return this.fileData;
}
/**
* Force a reload of the file
*/
async reload() {
this.fileData = null;
this.loadPromise = null;
return this.loadFile();
}
/**
* Close the backend and release any resources
*/
async close() {
this.fileData = null;
this.loadPromise = null;
}
}
exports.FileBackendNode = FileBackendNode;
exports.MemoryHttpBackend = MemoryHttpBackend;
exports.OmFileReader = OmFileReader;
exports.getWasmModule = getWasmModule;
exports.initWasm = initWasm;
//# sourceMappingURL=index.cjs.map