@openmeteo/file-reader
Version:
JavaScript reader for the om file format using WebAssembly
1,311 lines • 53.5 kB
JavaScript
var CompressionType;
(function (CompressionType) {
/// Lossy compression using 2D delta coding and scale-factor.
/// Only supports float and scales to 16-bit signed integer.
CompressionType[CompressionType["PforDelta2dInt16"] = 0] = "PforDelta2dInt16";
/// Lossless float/double compression using 2D xor coding.
CompressionType[CompressionType["FpxXor2d"] = 1] = "FpxXor2d";
/// PFor integer compression.
/// f32 values are scaled to u32, f64 are scaled to u64.
CompressionType[CompressionType["PforDelta2d"] = 2] = "PforDelta2d";
/// Similar to `PforDelta2dInt16` but applies `log10(1+x)` before.
CompressionType[CompressionType["PforDelta2dInt16Logarithmic"] = 3] = "PforDelta2dInt16Logarithmic";
CompressionType[CompressionType["None"] = 4] = "None";
})(CompressionType || (CompressionType = {}));
var OmDataType;
(function (OmDataType) {
OmDataType[OmDataType["None"] = 0] = "None";
OmDataType[OmDataType["Int8"] = 1] = "Int8";
OmDataType[OmDataType["Uint8"] = 2] = "Uint8";
OmDataType[OmDataType["Int16"] = 3] = "Int16";
OmDataType[OmDataType["Uint16"] = 4] = "Uint16";
OmDataType[OmDataType["Int32"] = 5] = "Int32";
OmDataType[OmDataType["Uint32"] = 6] = "Uint32";
OmDataType[OmDataType["Int64"] = 7] = "Int64";
OmDataType[OmDataType["Uint64"] = 8] = "Uint64";
OmDataType[OmDataType["Float"] = 9] = "Float";
OmDataType[OmDataType["Double"] = 10] = "Double";
OmDataType[OmDataType["String"] = 11] = "String";
OmDataType[OmDataType["Int8Array"] = 12] = "Int8Array";
OmDataType[OmDataType["Uint8Array"] = 13] = "Uint8Array";
OmDataType[OmDataType["Int16Array"] = 14] = "Int16Array";
OmDataType[OmDataType["Uint16Array"] = 15] = "Uint16Array";
OmDataType[OmDataType["Int32Array"] = 16] = "Int32Array";
OmDataType[OmDataType["Uint32Array"] = 17] = "Uint32Array";
OmDataType[OmDataType["Int64Array"] = 18] = "Int64Array";
OmDataType[OmDataType["Uint64Array"] = 19] = "Uint64Array";
OmDataType[OmDataType["FloatArray"] = 20] = "FloatArray";
OmDataType[OmDataType["DoubleArray"] = 21] = "DoubleArray";
OmDataType[OmDataType["StringArray"] = 22] = "StringArray";
})(OmDataType || (OmDataType = {}));
/**
* FNV-1a 64-bit hash implementation
*/
function fnv1aHash64(str) {
const FNV_OFFSET_BASIS = 0xcbf29ce484222325n;
const FNV_PRIME = 0x100000001b3n;
let hash = FNV_OFFSET_BASIS;
const bytes = new TextEncoder().encode(str);
for (const byte of bytes) {
hash ^= BigInt(byte);
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
}
return hash;
}
/**
* Fetch with exponential backoff retry on server-side errors (HTTP 5xx) and per-attempt timeout.
*/
async function fetchRetry(input, init, timeoutMs = 5000, retries = 3) {
let lastError;
function withTimeout(promise, ms) {
return Promise.race([
promise,
new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${ms}ms`)), ms)),
]);
}
for (let attempt = 0; attempt < retries; attempt++) {
try {
const response = await withTimeout(fetch(input, init), timeoutMs);
if (response.status >= 500 && response.status < 600) {
throw new Error(`Server error: ${response.status}`);
}
return response;
}
catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
if (attempt < retries - 1) {
const delay = Math.min(500 * Math.pow(2, attempt), 5000);
console.debug(`Attempt ${attempt + 1} failed, retrying in ${delay}ms: ${lastError.message}`);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
throw lastError;
}
async function runLimited(tasks, limit) {
const results = new Array(tasks.length);
for (let i = 0; i < tasks.length; i += limit) {
const batch = tasks.slice(i, i + limit);
const batchResults = await Promise.all(batch.map((task) => task()));
for (let j = 0; j < batchResults.length; j++) {
results[i + j] = batchResults[j];
}
}
return results;
}
// Constants mapping
const DATA_TYPES = {
DATA_TYPE_NONE: 0,
DATA_TYPE_INT8: 1,
DATA_TYPE_UINT8: 2,
DATA_TYPE_INT16: 3,
DATA_TYPE_UINT16: 4,
DATA_TYPE_INT32: 5,
DATA_TYPE_UINT32: 6,
DATA_TYPE_INT64: 7,
DATA_TYPE_UINT64: 8,
DATA_TYPE_FLOAT: 9,
DATA_TYPE_DOUBLE: 10,
DATA_TYPE_STRING: 11,
DATA_TYPE_INT8_ARRAY: 12,
DATA_TYPE_UINT8_ARRAY: 13,
DATA_TYPE_INT16_ARRAY: 14,
DATA_TYPE_UINT16_ARRAY: 15,
DATA_TYPE_INT32_ARRAY: 16,
DATA_TYPE_UINT32_ARRAY: 17,
DATA_TYPE_INT64_ARRAY: 18,
DATA_TYPE_UINT64_ARRAY: 19,
DATA_TYPE_FLOAT_ARRAY: 20,
DATA_TYPE_DOUBLE_ARRAY: 21,
DATA_TYPE_STRING_ARRAY: 22,
};
const HEADER_TYPES = {
OM_HEADER_INVALID: 0,
OM_HEADER_LEGACY: 1,
OM_HEADER_READ_TRAILER: 2,
};
const ERROR_CODES = {
ERROR_OK: 0,
};
// Size of the decoder structure
const SIZEOF_DECODER = 104;
let wasmModuleWrapped = null;
async function initWasm() {
if (wasmModuleWrapped)
return wasmModuleWrapped;
try {
// Import the factory function that creates the module
// @ts-expect-error module not found
const OmFileFormat = await import('@openmeteo/file-format-wasm');
// Initialize the module by calling the factory function
const wasmModuleRaw = await OmFileFormat.default();
// Create our wrapped module with the expected interface
wasmModuleWrapped = createWrappedModule(wasmModuleRaw);
return wasmModuleWrapped;
}
catch (error) {
throw new Error(`Failed to initialize WASM module: ${error}`);
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function createWrappedModule(rawModule) {
// Create a wrapper that maps the prefixed function names to our interface
return {
// Memory management functions
_malloc: rawModule._malloc,
_free: rawModule._free,
setValue: rawModule.setValue,
getValue: rawModule.getValue,
HEAPU8: rawModule.HEAPU8,
// Map all the C functions to their prefixed versions
om_header_size: rawModule._om_header_size,
om_header_type: rawModule._om_header_type,
om_trailer_size: rawModule._om_trailer_size,
om_trailer_read: rawModule._om_trailer_read,
om_variable_init: rawModule._om_variable_init,
om_variable_get_type: rawModule._om_variable_get_type,
om_variable_get_compression: rawModule._om_variable_get_compression,
om_variable_get_scale_factor: rawModule._om_variable_get_scale_factor,
om_variable_get_add_offset: rawModule._om_variable_get_add_offset,
om_variable_get_dimension_count: rawModule._om_variable_get_dimension_count,
om_variable_get_dimension_value: rawModule._om_variable_get_dimension_value,
om_variable_get_chunk_count: rawModule._om_variable_get_chunk_count,
om_variable_get_chunk_value: rawModule._om_variable_get_chunk_value,
om_variable_get_name_count: rawModule._om_variable_get_name_count,
om_variable_get_name_ptr: rawModule._om_variable_get_name_ptr,
om_variable_get_children_count: rawModule._om_variable_get_children_count,
om_variable_get_children: rawModule._om_variable_get_children,
om_variable_get_scalar: rawModule._om_variable_get_scalar,
om_decoder_init: rawModule._om_decoder_init,
om_decoder_init_index_read: rawModule._om_decoder_init_index_read,
om_decoder_init_data_read: rawModule._om_decoder_init_data_read,
om_decoder_read_buffer_size: rawModule._om_decoder_read_buffer_size,
om_decoder_next_index_read: rawModule._om_decoder_next_index_read,
om_decoder_next_data_read: rawModule._om_decoder_next_data_read,
om_decoder_decode_chunks: rawModule._om_decoder_decode_chunks,
// Constants
...HEADER_TYPES,
...ERROR_CODES,
...DATA_TYPES,
// Additional info
sizeof_decoder: SIZEOF_DECODER,
};
}
function getWasmModule() {
if (!wasmModuleWrapped) {
throw new Error("WASM module not initialized. Call initWasm() first.");
}
return wasmModuleWrapped;
}
class OmFileReader {
constructor(backend, wasm) {
this.backend = backend;
this.wasm = wasm || getWasmModule();
this.variable = null;
this.variableDataPtr = null;
this.metadataCache = new Map();
}
/**
* Static factory method to create and initialize an OmFileReader
*/
static async create(backend) {
// Make sure WASM is initialized
const wasm = await initWasm();
const reader = new OmFileReader(backend, wasm);
await reader.initialize();
return reader;
}
async initialize() {
// Similar to the 'new' method in Rust
const headerSize = this.wasm.om_header_size();
const headerData = await this.backend.getBytes(0, headerSize);
const headerPtr = this.wasm._malloc(headerData.length);
this.wasm.HEAPU8.set(headerData, headerPtr);
const headerType = this.wasm.om_header_type(headerPtr);
if (headerType === this.wasm.OM_HEADER_INVALID) {
this.wasm._free(headerPtr);
throw new Error("Not a valid OM file");
}
let variableData;
if (headerType === this.wasm.OM_HEADER_LEGACY) {
variableData = headerData;
}
else if (headerType === this.wasm.OM_HEADER_READ_TRAILER) {
const fileSize = await this.backend.count();
const trailerSize = this.wasm.om_trailer_size();
const trailerOffset = fileSize - trailerSize;
const trailerPtr = await this.readDataBlock(trailerOffset, trailerSize);
// Create pointers for offset and size (out parameters)
const offsetPtr = this.wasm._malloc(8); // 64-bit value = 8 bytes
const sizePtr = this.wasm._malloc(8);
const success = this.wasm.om_trailer_read(trailerPtr, offsetPtr, sizePtr);
if (!success) {
this.wasm._free(headerPtr);
this.wasm._free(trailerPtr);
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
throw new Error("Failed to read trailer");
}
// Read values from memory
const offset = Number(this.wasm.getValue(offsetPtr, "i64"));
const size = Number(this.wasm.getValue(sizePtr, "i64"));
// Free memory
this.wasm._free(trailerPtr);
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
// Get variable data
variableData = await this.backend.getBytes(offset, size);
}
else {
this.wasm._free(headerPtr);
throw new Error("Unknown header type");
}
// Initialize variable
const variableDataPtr = this.wasm._malloc(variableData.length);
this.wasm.HEAPU8.set(variableData, variableDataPtr);
this.variable = this.wasm.om_variable_init(variableDataPtr);
this.variableDataPtr = variableDataPtr;
this.wasm._free(headerPtr);
return this;
}
// Helper method to convert C strings to JS strings
_getString(strPtr, strLen) {
const bytes = this.wasm.HEAPU8.subarray(strPtr, strPtr + strLen);
return new TextDecoder("utf8").decode(bytes);
}
dataType() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_type(this.variable);
}
compression() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_compression(this.variable);
}
scaleFactor() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_scale_factor(this.variable);
}
addOffset() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_add_offset(this.variable);
}
getDimensions() {
if (this.variable === null)
throw new Error("Reader not initialized");
// Get count using the wrapper function
const count = Number(this.wasm.om_variable_get_dimension_count(this.variable));
// Get each dimension individually
const dimensions = [];
for (let i = 0; i < count; i++) {
dimensions.push(Number(this.wasm.om_variable_get_dimension_value(this.variable, BigInt(i))));
}
return dimensions;
}
getChunkDimensions() {
if (this.variable === null)
throw new Error("Reader not initialized");
// Get count using the wrapper function
const count = Number(this.wasm.om_variable_get_chunk_count(this.variable));
// Get each chunk dimension individually
const chunks = [];
for (let i = 0; i < count; i++) {
chunks.push(Number(this.wasm.om_variable_get_chunk_value(this.variable, BigInt(i))));
}
return chunks;
}
getName() {
if (this.variable === null)
throw new Error("Reader not initialized");
const size = this.wasm.om_variable_get_name_count(this.variable);
if (size === 0) {
return null;
}
const valuePtr = this.wasm.om_variable_get_name_ptr(this.variable);
if (valuePtr === 0) {
return null;
}
return this._getString(valuePtr, size);
}
numberOfChildren() {
if (this.variable === null)
throw new Error("Reader not initialized");
return this.wasm.om_variable_get_children_count(this.variable);
}
async getChild(index) {
if (this.variable === null)
throw new Error("Reader not initialized");
// Allocate memory for the output parameters
const offsetPtr = this.wasm._malloc(8);
const sizePtr = this.wasm._malloc(8);
const success = this.wasm.om_variable_get_children(this.variable, index, 1, offsetPtr, sizePtr);
if (!success) {
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
return null;
}
const offset = Number(this.wasm.getValue(offsetPtr, "i64"));
const size = Number(this.wasm.getValue(sizePtr, "i64"));
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
return this.initChildFromOffsetSize({ offset, size });
}
/**
* Searches direct children by name. Does not search recursively.
*/
async getChildByName(name) {
// Check cache first
const cachedMetadata = this.metadataCache.get(name);
if (cachedMetadata === null) {
return null;
}
if (cachedMetadata) {
return await this.initChildFromOffsetSize(cachedMetadata);
}
// Search through children and cache metadata
const numChildren = this.numberOfChildren();
for (let i = 0; i < numChildren; i++) {
const metadata = this._getChildMetadata(i);
if (metadata) {
const child = await this.initChildFromOffsetSize(metadata);
const childName = child.getName();
if (childName) {
// Cache the metadata
this.metadataCache.set(childName, metadata);
if (childName === name) {
return child;
}
}
}
}
// also remember invalid names
this.metadataCache.set(name, null);
return null;
}
async initChildFromOffsetSize(offsetSize) {
const childDataPtr = await this.readDataBlock(offsetSize.offset, offsetSize.size);
const childReader = new OmFileReader(this.backend, this.wasm);
childReader.variable = this.wasm.om_variable_init(childDataPtr);
childReader.variableDataPtr = childDataPtr;
return childReader;
}
/**
* Get child metadata by index.
*/
_getChildMetadata(index) {
if (this.variable === null)
throw new Error("Reader not initialized");
// Allocate memory for the output parameters
const offsetPtr = this.wasm._malloc(8);
const sizePtr = this.wasm._malloc(8);
const success = this.wasm.om_variable_get_children(this.variable, index, 1, offsetPtr, sizePtr);
if (!success) {
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
return null;
}
const offset = Number(this.wasm.getValue(offsetPtr, "i64"));
const size = Number(this.wasm.getValue(sizePtr, "i64"));
this.wasm._free(offsetPtr);
this.wasm._free(sizePtr);
return { offset, size };
}
/**
* Find a variable by its path (e.g., "parent/child/grandchild")
*/
async findByPath(path) {
const parts = path.split("/").filter((s) => s.length > 0);
return await this.navigatePath(parts);
}
/**
* Navigate through a path recursively
*/
async navigatePath(parts) {
if (parts.length === 0) {
return null;
}
const child = await this.getChildByName(parts[0]);
if (child) {
if (parts.length === 1) {
return child;
}
else {
return await child.navigatePath(parts.slice(1));
}
}
return null;
}
// Method to read scalar values
readScalar(dataType) {
if (this.variable === null)
throw new Error("Reader not initialized");
if (this.dataType() !== dataType) {
return null;
}
// Allocate memory for output parameters
const ptrPtr = this.wasm._malloc(4); // pointer to pointer
const sizePtr = this.wasm._malloc(8); // u64
try {
const error = this.wasm.om_variable_get_scalar(this.variable, ptrPtr, sizePtr);
if (error !== this.wasm.ERROR_OK) {
return null;
}
const dataPtr = this.wasm.getValue(ptrPtr, "*");
if (dataPtr === 0) {
return null;
}
// Read data based on type
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let result;
switch (dataType) {
case OmDataType.Int8:
result = this.wasm.getValue(dataPtr, "i8");
break;
case OmDataType.Uint8:
result = this.wasm.getValue(dataPtr, "i8") & 0xff;
break;
case OmDataType.Int16:
result = this.wasm.getValue(dataPtr, "i16");
break;
case OmDataType.Uint16:
result = this.wasm.getValue(dataPtr, "i16") & 0xffff;
break;
case OmDataType.Int32:
result = this.wasm.getValue(dataPtr, "i32");
break;
case OmDataType.Uint32:
result = this.wasm.getValue(dataPtr, "i32") >>> 0;
break;
case OmDataType.Float:
result = this.wasm.getValue(dataPtr, "float");
break;
case OmDataType.Double:
result = this.wasm.getValue(dataPtr, "double");
break;
default:
result = null;
}
return result;
}
finally {
this.wasm._free(ptrPtr);
this.wasm._free(sizePtr);
}
}
newIndexRead(decoderPtr) {
// Calculate proper size for OmDecoder_indexRead_t
const sizeOfRange = 16; // 8 bytes for lowerBound + 8 bytes for upperBound
const sizeOfIndexRead = 8 + 8 + sizeOfRange * 3; // offset + count + 3 range structs
// Allocate and zero the memory
const indexReadPtr = this.wasm._malloc(sizeOfIndexRead);
// Zero out the memory (equivalent to std::mem::zeroed())
const zeroBuffer = new Uint8Array(sizeOfIndexRead);
this.wasm.HEAPU8.set(zeroBuffer, indexReadPtr);
// Initialize the structure using C function
this.wasm.om_decoder_init_index_read(decoderPtr, indexReadPtr);
return indexReadPtr;
}
newDataRead(indexReadPtr) {
// Size of OmDecoder_dataRead_t
const sizeOfRange = 16; // 8 bytes for lowerBound + 8 bytes for upperBound
const sizeOfDataRead = 8 + 8 + sizeOfRange * 3; // offset + count + 3 range structs
// Allocate and zero the memory
const dataReadPtr = this.wasm._malloc(sizeOfDataRead);
// Zero out the memory (equivalent to std::mem::zeroed())
const zeroBuffer = new Uint8Array(sizeOfDataRead);
this.wasm.HEAPU8.set(zeroBuffer, dataReadPtr);
// Initialize the structure using C function
this.wasm.om_decoder_init_data_read(dataReadPtr, indexReadPtr);
return dataReadPtr;
}
async read(dataType, dimRanges, ioSizeMax = BigInt(65536), ioSizeMerge = BigInt(512), prefetch = true) {
if (this.variable === null)
throw new Error("Reader not initialized");
if (this.dataType() !== dataType) {
throw new Error(`Invalid data type: expected ${this.dataType()}, got ${dataType}`);
}
// Calculate output dimensions
const outDims = dimRanges.map((range) => Number(range.end - range.start));
const totalSize = outDims.reduce((a, b) => a * b, 1);
// Create output TypedArray based on data type
let output;
switch (dataType) {
case this.wasm.DATA_TYPE_INT8_ARRAY:
output = new Int8Array(totalSize);
break;
case this.wasm.DATA_TYPE_UINT8_ARRAY:
output = new Uint8Array(totalSize);
break;
case this.wasm.DATA_TYPE_INT16_ARRAY:
output = new Int16Array(totalSize);
break;
case this.wasm.DATA_TYPE_UINT16_ARRAY:
output = new Uint16Array(totalSize);
break;
case this.wasm.DATA_TYPE_INT32_ARRAY:
output = new Int32Array(totalSize);
break;
case this.wasm.DATA_TYPE_UINT32_ARRAY:
output = new Uint32Array(totalSize);
break;
case this.wasm.DATA_TYPE_FLOAT_ARRAY:
output = new Float32Array(totalSize);
break;
case this.wasm.DATA_TYPE_DOUBLE_ARRAY:
output = new Float64Array(totalSize);
break;
default:
throw new Error("Unsupported data type");
}
await this.readInto(dataType, output, dimRanges, ioSizeMax, ioSizeMerge, prefetch);
return output;
}
/**
* Read data into an existing TypedArray with specified dimension ranges
* @param dataType The data type to read
* @param output The TypedArray to read data into
* @param dimRanges Ranges for each dimension to read
* @param ioSizeMax Maximum I/O size (default: 65536)
* @param ioSizeMerge Merge threshold for I/O operations (default: 512)
*/
async readInto(dataType, output, dimRanges, ioSizeMax = BigInt(65536), ioSizeMerge = BigInt(512), prefetch = true) {
if (this.variable === null)
throw new Error("Reader not initialized");
if (this.dataType() !== dataType) {
throw new Error(`Invalid data type: expected ${this.dataType()}, got ${dataType}`);
}
const nDims = dimRanges.length;
const fileDims = this.getDimensions();
// Validate dimension counts
if (fileDims.length !== nDims) {
throw new Error(`Mismatched dimensions: file has ${fileDims.length}, request has ${nDims}`);
}
// Calculate output dimensions and prepare arrays for WASM
const outDims = dimRanges.map((range) => range.end - range.start);
// Calculate total elements to ensure output array has correct size
const totalElements = outDims.reduce((a, b) => a * Number(b), 1);
if (output.length < totalElements) {
throw new Error(`Output array is too small: needs ${totalElements} elements, has ${output.length}`);
}
// Allocate memory for arrays
const readOffsetPtr = this.wasm._malloc(nDims * 8); // u64 array
const readCountPtr = this.wasm._malloc(nDims * 8);
const intoCubeOffsetPtr = this.wasm._malloc(nDims * 8);
const intoCubeDimensionPtr = this.wasm._malloc(nDims * 8);
try {
// Fill arrays
for (let i = 0; i < nDims; i++) {
// Validate ranges
if (dimRanges[i].start < 0 || dimRanges[i].end > fileDims[i] || dimRanges[i].start >= dimRanges[i].end) {
throw new Error(`Invalid range for dimension ${i}: ${JSON.stringify(dimRanges[i])}`);
}
this.wasm.setValue(readOffsetPtr + i * 8, BigInt(dimRanges[i].start), "i64");
this.wasm.setValue(readCountPtr + i * 8, BigInt(outDims[i]), "i64");
this.wasm.setValue(intoCubeOffsetPtr + i * 8, BigInt(0), "i64");
this.wasm.setValue(intoCubeDimensionPtr + i * 8, BigInt(outDims[i]), "i64");
}
// Create decoder
const decoderPtr = this.wasm._malloc(this.wasm.sizeof_decoder);
try {
// Initialize decoder
const error = this.wasm.om_decoder_init(decoderPtr, this.variable, BigInt(nDims), readOffsetPtr, readCountPtr, intoCubeOffsetPtr, intoCubeDimensionPtr, ioSizeMerge, ioSizeMax);
if (error !== this.wasm.ERROR_OK) {
throw new Error(`Decoder initialization failed: error code ${error}`);
}
if (prefetch) {
await this.decodePrefetch(decoderPtr);
}
await this.decode(decoderPtr, output);
}
finally {
this.wasm._free(decoderPtr);
}
}
finally {
// Clean up input arrays
this.wasm._free(readOffsetPtr);
this.wasm._free(readCountPtr);
this.wasm._free(intoCubeOffsetPtr);
this.wasm._free(intoCubeDimensionPtr);
}
}
async decodePrefetch(decoderPtr) {
if (!this.backend.prefetchData) {
// Prefetch not supported by backend
return;
}
const indexReadPtr = this.newIndexRead(decoderPtr);
const errorPtr = this.wasm._malloc(4);
this.wasm.setValue(errorPtr, this.wasm.ERROR_OK, "i32");
try {
// Loop over index blocks
while (this.wasm.om_decoder_next_index_read(decoderPtr, indexReadPtr)) {
const indexOffset = Number(this.wasm.getValue(indexReadPtr, "i64"));
const indexCount = Number(this.wasm.getValue(indexReadPtr + 8, "i64"));
// Get bytes for index-read
const indexDataPtr = await this.readDataBlock(indexOffset, indexCount);
const dataReadPtr = this.newDataRead(indexReadPtr);
try {
// Collect prefetch tasks
const prefetchTasks = [];
while (this.wasm.om_decoder_next_data_read(decoderPtr, dataReadPtr, indexDataPtr, BigInt(indexCount), errorPtr)) {
const dataOffset = Number(this.wasm.getValue(dataReadPtr, "i64"));
const dataCount = Number(this.wasm.getValue(dataReadPtr + 8, "i64"));
prefetchTasks.push(() => this.backend.prefetchData(dataOffset, dataCount));
}
// Run prefetches in parallel
await runLimited(prefetchTasks, 5000);
// Check for errors after data_read loop
const error = this.wasm.getValue(errorPtr, "i32");
if (error !== this.wasm.ERROR_OK) {
throw new Error(`Data read error: ${error}`);
}
}
finally {
this.wasm._free(dataReadPtr);
this.wasm._free(indexDataPtr);
}
}
}
finally {
this.wasm._free(indexReadPtr);
this.wasm._free(errorPtr);
}
}
async decode(decoderPtr, outputArray) {
const outputPtr = this.wasm._malloc(outputArray.byteLength);
const chunkBufferSize = Number(this.wasm.om_decoder_read_buffer_size(decoderPtr));
const chunkBufferPtr = this.wasm._malloc(chunkBufferSize);
// Create index_read struct
const indexReadPtr = this.newIndexRead(decoderPtr);
const errorPtr = this.wasm._malloc(4);
// Initialize error to OK
this.wasm.setValue(errorPtr, this.wasm.ERROR_OK, "i32");
try {
// Loop over index blocks
while (this.wasm.om_decoder_next_index_read(decoderPtr, indexReadPtr)) {
// Get index_read parameters
const indexOffset = Number(this.wasm.getValue(indexReadPtr, "i64"));
const indexCount = Number(this.wasm.getValue(indexReadPtr + 8, "i64"));
// Get bytes for index-read
const indexDataPtr = await this.readDataBlock(indexOffset, indexCount);
const dataReadPtr = this.newDataRead(indexReadPtr);
try {
// Loop over data blocks and read compressed data chunks
while (this.wasm.om_decoder_next_data_read(decoderPtr, dataReadPtr, indexDataPtr, BigInt(indexCount), errorPtr)) {
// Get data_read parameters
const dataOffset = Number(this.wasm.getValue(dataReadPtr, "i64"));
const dataCount = Number(this.wasm.getValue(dataReadPtr + 8, "i64"));
const chunkIndexPtr = dataReadPtr + 32; // offset(8), count(8), indexRange(16)
// Get bytes for data-read
const dataBlockPtr = await this.readDataBlock(dataOffset, dataCount);
try {
// Decode chunks
const success = this.wasm.om_decoder_decode_chunks(decoderPtr, chunkIndexPtr, dataBlockPtr, BigInt(dataCount), outputPtr, chunkBufferPtr, errorPtr);
// Check for error
if (!success) {
const error = this.wasm.getValue(errorPtr, "i32");
throw new Error(`Decoder failed to decode chunks: error ${error}`);
}
}
finally {
this.wasm._free(dataBlockPtr);
}
}
// Check for errors after data_read loop
const error = this.wasm.getValue(errorPtr, "i32");
if (error !== this.wasm.ERROR_OK) {
throw new Error(`Data read error: ${error}`);
}
}
finally {
this.wasm._free(dataReadPtr);
this.wasm._free(indexDataPtr);
}
}
// Copy the data back to the output array with the correct type
this.copyToTypedArray(outputPtr, outputArray);
}
finally {
this.wasm._free(errorPtr);
this.wasm._free(indexReadPtr);
this.wasm._free(chunkBufferPtr);
this.wasm._free(outputPtr);
}
}
async readDataBlock(offset, size) {
const data = await this.backend.getBytes(offset, size);
const ptr = this.wasm._malloc(data.length);
this.wasm.HEAPU8.set(data, ptr);
return ptr;
}
/**
* Helper method to copy data from WASM memory to a TypedArray with the correct type
*/
copyToTypedArray(sourcePtr, targetArray) {
switch (targetArray.constructor) {
case Float32Array:
targetArray.set(new Float32Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case Float64Array:
targetArray.set(new Float64Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case Int8Array:
targetArray.set(new Int8Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case Uint8Array:
targetArray.set(new Uint8Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case Int16Array:
targetArray.set(new Int16Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case Uint16Array:
targetArray.set(new Uint16Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case Int32Array:
targetArray.set(new Int32Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case Uint32Array:
targetArray.set(new Uint32Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case BigInt64Array:
targetArray.set(new BigInt64Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
case BigUint64Array:
targetArray.set(new BigUint64Array(this.wasm.HEAPU8.buffer, sourcePtr, targetArray.length));
break;
default:
throw new Error("Unsupported TypedArray type in copyToTypedArray");
}
}
// Clean up resources when done
dispose() {
if (this.variableDataPtr !== null) {
this.wasm._free(this.variableDataPtr);
this.variableDataPtr = null;
}
this.variable = null;
}
}
class FileBackend {
constructor(source) {
this.fileObj = null;
this.memory = null;
this.fileSize = 0;
if (typeof File !== "undefined" && source instanceof File) {
this.fileObj = source;
this.fileSize = source.size;
}
else if (typeof Blob !== "undefined" && source instanceof Blob) {
this.fileObj = source;
this.fileSize = source.size;
}
else if (source instanceof ArrayBuffer) {
this.memory = new Uint8Array(source);
this.fileSize = this.memory.length;
}
else if (source instanceof Uint8Array) {
this.memory = source;
this.fileSize = source.length;
}
else {
throw new Error("Unsupported file source type for browser FileBackend");
}
}
async count() {
return this.fileSize;
}
async getBytes(offset, size) {
if (this.memory) {
return this.memory.slice(offset, offset + size);
}
if (this.fileObj) {
const blob = this.fileObj.slice(offset, offset + size);
const buffer = await blob.arrayBuffer();
return new Uint8Array(buffer);
}
throw new Error("No file or memory buffer available");
}
async prefetchData(_offset, _bytes) {
// No-op for now!
}
async close() {
// Nothing to clean up in browser
}
}
class MemoryHttpBackend {
/**
* Create a new MemoryHttpBackend
* @param options Configuration options
*/
constructor(options) {
this.fileSize = null;
this.fileData = null;
this.loadPromise = null;
this.countPromise = null;
this.url = options.url;
this.maxFileSize = options.maxFileSize ?? 200 * 1024 * 1024; // 200 MB default
this.onProgress = options.onProgress;
this.debug = options.debug ?? false;
// Start loading the file in the background
this.loadFile().catch((err) => {
if (this.debug)
console.error("Background file load failed:", err);
});
}
/**
* Get the total size of the file
*/
async count() {
if (this.fileSize !== null) {
return this.fileSize;
}
if (!this.countPromise) {
if (this.debug)
console.log(`Making HEAD request to ${this.url}`);
this.countPromise = (async () => {
try {
const response = await fetch(this.url, {
method: "HEAD",
});
if (!response.ok) {
throw new Error(`HTTP error: ${response.status}`);
}
const contentLength = response.headers.get("content-length");
if (!contentLength) {
throw new Error("Content-Length header not available");
}
this.fileSize = parseInt(contentLength, 10);
if (this.fileSize > this.maxFileSize) {
throw new Error(`File size (${this.fileSize} bytes) exceeds maximum allowed size (${this.maxFileSize} bytes)`);
}
if (this.debug)
console.log(`File size: ${this.fileSize} bytes`);
return this.fileSize;
}
catch (error) {
this.countPromise = null;
throw new Error(`Failed to get file size: ${error instanceof Error ? error.message : String(error)}`);
}
})();
}
return this.countPromise;
}
/**
* Load the entire file into memory
*/
async loadFile() {
// If already loaded or loading, return that promise
if (this.fileData) {
return Promise.resolve();
}
if (this.loadPromise) {
return this.loadPromise;
}
this.loadPromise = (async () => {
try {
// First get the file size
const size = await this.count();
if (this.debug)
console.log(`Fetching entire file (${size} bytes) from ${this.url}`);
// Use fetch with streaming and progress tracking
const response = await fetch(this.url);
if (!response.ok) {
throw new Error(`HTTP error: ${response.status}`);
}
// Check if ReadableStream is supported and if progress tracking is needed
if (this.onProgress && response.body && "getReader" in response.body) {
// Stream the response with progress tracking
const contentLength = Number(response.headers.get("content-length") || size);
const reader = response.body.getReader();
const chunks = [];
let receivedLength = 0;
let lastProgressUpdate = 0;
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
chunks.push(value);
receivedLength += value.length;
// Don't update progress too frequently (throttle updates)
const now = Date.now();
if (now - lastProgressUpdate > 100) {
// update every 100ms
this.onProgress(receivedLength, contentLength);
lastProgressUpdate = now;
}
}
// Concatenate chunks into a single Uint8Array
this.fileData = new Uint8Array(receivedLength);
let position = 0;
for (const chunk of chunks) {
this.fileData.set(chunk, position);
position += chunk.length;
}
// Final progress update
this.onProgress(receivedLength, contentLength);
}
else {
// Simple approach without streaming
const buffer = await response.arrayBuffer();
this.fileData = new Uint8Array(buffer);
if (this.onProgress) {
this.onProgress(this.fileData.length, size);
}
}
if (this.debug)
console.log(`File loaded successfully (${this.fileData.length} bytes)`);
return;
}
catch (error) {
this.loadPromise = null;
throw new Error(`Failed to load file: ${error instanceof Error ? error.message : String(error)}`);
}
})();
return this.loadPromise;
}
/**
* Get bytes from the file
* @param offset The starting position in the file
* @param size The number of bytes to read
*/
async getBytes(offset, size) {
try {
// Make sure the file is loaded
if (!this.fileData) {
if (this.debug)
console.log(`getBytes(${offset}, ${size}): Waiting for file to load...`);
await this.loadFile();
if (this.debug)
console.log(`getBytes(${offset}, ${size}): File loaded`);
}
// At this point, fileData should be available
if (!this.fileData) {
throw new Error("File data is not available after load");
}
// Bounds check
if (offset < 0 || offset + size > this.fileData.length) {
throw new Error(`Requested range (${offset}:${offset + size}) is out of bounds (0:${this.fileData.length})`);
}
if (this.debug)
console.log(`Serving ${size} bytes from offset ${offset} from memory`);
// Return the requested slice of data
return this.fileData.slice(offset, offset + size);
}
catch (error) {
throw new Error(`Error in getBytes: ${error instanceof Error ? error.message : String(error)}`);
}
}
/**
* Check if the file is fully loaded
*/
isLoaded() {
return !!this.fileData;
}
/**
* Get the current loaded data or null if not loaded
*/
getFileData() {
return this.fileData;
}
/**
* Force a reload of the file
*/
async reload() {
this.fileData = null;
this.loadPromise = null;
return this.loadFile();
}
async prefetchData(_offset, _bytes) {
// No-op for now!
}
/**
* Close the backend and release any resources
*/
async close() {
this.fileData = null;
this.loadPromise = null;
}
}
class BlockCacheCoordinator {
// constructor(cache: SharedBlockCache) {
// this.cache = cache;
// }
constructor(blockSize, maxBlocks) {
this.cache = new SharedBlockCache(blockSize, maxBlocks);
}
blockSize() {
return this.cache.blockSize;
}
maxBlocks() {
return this.cache.maxBlocks;
}
async get(key, fetchFn) {
const cached = this.cache.get(key);
if (cached)
return cached;
let inflight = this.cache.getInflight(key);
if (!inflight) {
inflight = fetchFn();
this.cache.setInflight(key, inflight);
inflight.then((data) => this.cache.set(key, data));
}
return inflight;
}
prefetch(key, fetchFn) {
if (!this.cache.get(key) && !this.cache.getInflight(key)) {
const inflight = fetchFn();
this.cache.setInflight(key, inflight);
inflight.then((data) => this.cache.set(key, data));
}
}
clear() {
this.cache.clear();
}
}
class SharedBlockCache {
constructor(blockSize, maxBlocks) {
this.blockSize = blockSize;
this.maxBlocks = maxBlocks;
this.cache = new Map();
this.lru = [];
this.inflight = new Map();
}
get(key) {
const entry = this.cache.get(key);
if (entry) {
entry.timestamp = Date.now();
// Move to end of LRU
this.lru = this.lru.filter((k) => k !== key);
this.lru.push(key);
return entry.data;
}
return undefined;
}
set(key, data) {
if (this.cache.size >= this.maxBlocks) {
// Evict LRU
const oldestKey = this.lru.shift();
if (oldestKey !== undefined)
this.cache.delete(oldestKey);
}
this.cache.set(key, { data, timestamp: Date.now() });
this.lru.push(key);
}
getInflight(key) {
return this.inflight.get(key);
}
setInflight(key, promise) {
this.inflight.set(key, promise);
promise.finally(() => this.inflight.delete(key));
}
clear() {
this.cache.clear();
this.lru = [];
this.inflight.clear();
}
}
/**
* Wraps a backend for caching blocks of data.
*/
class BlockCacheBackend {
constructor(backend, cacheCoordinator, cacheKey) {
this.backend = backend;
this.cacheKey = cacheKey;
this.cacheCoordinator = cacheCoordinator;
}
async count() {
return this.backend.count();
}
async prefetchData(offset, count) {
const blockSize = this.cacheCoordinator.blockSize();
const fileSize = await this.count();
const startBlock = Math.floor(offset / blockSize);
const endBlock = Math.floor((offset + count - 1) / blockSize);
for (let blockIdx = startBlock; blockIdx <= endBlock; blockIdx++) {
const blockKey = this.cacheKey + BigInt(blockIdx);
this.cacheCoordinator.prefetch(blockKey, () => this.backend.getBytes(blockIdx * blockSize, Math.min(blockSize, fileSize - blockIdx * blockSize)));
}
}
async getBytes(offset, size) {
const blockSize = this.cacheCoordinator.blockSize();
const fileSize = await this.count();
const startBlock = Math.floor(offset / blockSize);
const endBlock = Math.floor((offset + size - 1) / blockSize);
const output = new Uint8Array(size);
// Fetch all blocks in parallel
const tasks = [];
for (let blockIdx = startBlock; blockIdx <= endBlock; blockIdx++) {
const blockKey = this.cacheKey + BigInt(blockIdx);
tasks.push(async () => {
const blockStart = blockIdx * blockSize;
const blockEnd = Math.min(blockStart + blockSize, fileSize);
const block = await this.cacheCoordinator.get(blockKey, () => this.backend.getBytes(blockStart, blockEnd - blockStart));
return { blockIdx, block };
});
}
const fetchedBlocks = await Promise.all(tasks.map((task) => task()));
const blocks = new Map();
for (const { blockIdx, block } of fetchedBlocks) {
blocks.set(blockIdx, block);
}
// Copy relevant parts of each block to output
for (let blockIdx = startBlock; blockIdx <= endBlock; blockIdx++) {
const block = blocks.get(blockIdx);
const blockOffset = Math.max(offset, blockIdx * blockSize) - blockIdx * blockSize;
const outOffset = Math.max(blockIdx * blockSize, offset) - offset;
const copyLen = Math.min(blockSize - blockOffset, size - outOffset);
output.set(block.subarray(blockOffset, blockOffset + copyLen), outOffset);
}
return output;
}
async close() {
this.cacheCoordinator.clear();
await this.backend.close();
}
}
let globalCache = null;
function setupGlobalCache(blockSize = 64 * 1024, maxBlocks = 256) {
if (!globalCache) {
globalCache = new BlockCacheCoordinator(blockSize, maxBlocks);
}
else {
if (globalCache.blockSize() !== blockSize || globalCache.maxBlocks() !== maxBlocks) {
throw new Error("Global cache already set up with configuration " + blockSize + " " + maxBlocks);
}
}
}
class OmHttpBackendError extends Error {
constructor(message, statusCode) {
super(message);
this.statusCode = statusCode;
this.name = "OmHttpBackendError";
}
}
/**
* Backend for reading from HTTP servers with partial read support using Range requests.
* Checks last modified header and ETag.
*/
class OmHttpBackend {
constructor(options) {
this.fileSize = null;
this.lastModified = null;
this.eTag = null;
this.metadataPromise = null;
this.url = options.url;
this.debug = options.debug ?? false;
this.timeoutMs = options.timeoutMs ?? 30000;
this.retries = options.retries ?? 1;
}
/**
* Get cache key based on URL, ETag, and Last-Modified
*/
get cacheKey() {
const urlHash = fnv1aHash64(this.url);
const eTagHash = this.eTag ? fnv1aHash64(this.eTag) : 0n;
const lastModifiedHash = this.lastModified ? fnv1aHash64(this.lastModified) : 0n;
return urlHash ^ eTagHash ^ lastModifiedHash;
}
/**
* Fetch metadata using HEAD request
*/
async fetchMetadata() {
if (this.metadataPromise) {
return this.metadataPromise;
}
this.metadataPromise = (async () => {
const response = await fetchRetry(this.url, { method: "HEAD" }, this.timeoutMs ?? 5000, this.retries);
if (!response.ok) {
throw new OmHttpBackendError(response.status === 404 ? "File not found" : `HTTP error: ${response.status}`, response.status);
}
const contentLength = response.headers.get("content-length");
if (!contentLength)
throw new OmHttpBackendError("Content-Length header missing");
this.fileSize = parseInt(contentLength, 10);
this.lastModified = response.headers.get("last-modified");
this.eTag = response.headers.get("etag");
})();
return this.metadataPromise;
}
/**
* Get the total size of the file
*/
async count() {
if (this.fileSize !== null) {
return this.fileSize;
}
await this.fetchMetadata();
return this.fileSize;
}
/**
* Get bytes from the file using Range requests
*/
async getBytes(offset, size) {
if (offset < 0 || size <= 0) {
throw new OmHttpBackendError("Invalid offset or size");
}
// Ensure we have metadata
await this.fetchMetadata();
if (offset + size > this.fileSize) {
throw new OmHttpBackendError(`Requested range (${offset}:${offset + size}) exceeds file size (${this.fileSize})`);
}
// Prepare request
const headers = {
Range: `bytes=${offset}-${offset + size - 1}`,
};
// Add conditional headers for cache validation
if (this.lastModified) {
headers["If-Unmodified-Since"] = this.lastModified;
}
if (this.eTag) {
headers["If-Match"] = this.eTag;
}
if (this.debug) {
console.log(`Getting data range ${offset}-${offset + size - 1} from ${this.url}`);
}
const response = await fetchRetry(this.url, { headers }, this.timeoutMs, this.retries);
const buffer = await response.arrayBuffer();
const data = new Uint8Array(buffer);
if (data.length !== size) {
throw new OmHttpBackendError(`Received ${data.length} bytes, expected ${size}`);
}
return data;
}
async prefetchData(_offset, _bytes) {
// No-op for now!
}
async asCachedReader() {
if (globalCache) {
const cachedBackend = new BlockCacheBackend(this, globalCache, this.cacheKey);
return await OmFileReader.create(cachedBackend);
}
else {
throw new OmHttpBackendError("No global cache set up! Configure it with setupGlobalCache first!");
}
}
/**
* Close the backend and release resources
*/
async close() {
this.metadataPromise = null;
this.fileSize = null;
this.lastModified = null;
this.eTag = null;
}
}
export { BlockCacheBackend, CompressionType, FileBackend, MemoryHttpBackend, OmDataType, OmFileReader, OmHttpBackend, getWasmModule, initWasm, setupGlobalCache };
//# sourceMappingURL=index.browser.js.map