@openmeteo/file-reader/dist/index.d.ts

JavaScript reader for the om file format using WebAssembly

interface OmFileReaderBackend {
    /**
     * Get bytes from the backend at the specified offset and size
     * @param offset The offset in bytes from the start of the file
     * @param size The number of bytes to read
     * @param signal Optional AbortSignal to cancel the operation
     */
    getBytes(offset: number, size: number, signal?: AbortSignal): Promise<Uint8Array>;
    /**
     * Collects prefetch tasks for a given range without executing them.
     * Returns an array of functions that, when called, will fetch the data.
     * This allows the caller to control concurrency.
     * @param offset The offset in bytes from the start of the file
     * @param size The number of bytes to prefetch
     * @param signal Optional AbortSignal to cancel the operation
     */
    collectPrefetchTasks?(offset: number, size: number, signal?: AbortSignal): Promise<(() => Promise<void>)[]>;
    /**
     * Get the total size of the file in bytes
     * @param signal Optional AbortSignal to cancel the operation
     */
    count(signal?: AbortSignal): Promise<number>;
    /**
     * Close the backend and release any resources
     */
    close(): Promise<void>;
}

interface Range {
    start: number;
    end: number;
}
interface OffsetSize {
    offset: number;
    size: number;
}
interface OmFilePrefetchReadOptions {
    ranges: Range[];
    prefetchConcurrency?: number;
    ioSizeMax?: bigint;
    ioSizeMerge?: bigint;
    signal?: AbortSignal;
}
interface OmFileReadBaseOptions<T extends keyof OmDataTypeToTypedArray> extends OmFilePrefetchReadOptions {
    type: T;
    prefetch?: boolean;
}
interface OmFileReadOptions<T extends keyof OmDataTypeToTypedArray> extends OmFileReadBaseOptions<T> {
    intoSAB?: boolean;
}
interface OmFileReadIntoOptions<T extends keyof OmDataTypeToTypedArray> extends OmFileReadBaseOptions<T> {
    output: OmDataTypeToTypedArray[T];
}
declare enum CompressionType {
    PforDelta2dInt16 = 0,
    FpxXor2d = 1,
    PforDelta2d = 2,
    PforDelta2dInt16Logarithmic = 3,
    None = 4
}
declare enum OmDataType {
    None = 0,
    Int8 = 1,
    Uint8 = 2,
    Int16 = 3,
    Uint16 = 4,
    Int32 = 5,
    Uint32 = 6,
    Int64 = 7,
    Uint64 = 8,
    Float = 9,
    Double = 10,
    String = 11,
    Int8Array = 12,
    Uint8Array = 13,
    Int16Array = 14,
    Uint16Array = 15,
    Int32Array = 16,
    Uint32Array = 17,
    Int64Array = 18,
    Uint64Array = 19,
    FloatArray = 20,
    DoubleArray = 21,
    StringArray = 22
}
type TypedArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | BigInt64Array | BigUint64Array | Float32Array | Float64Array;
interface OmDataTypeToTypedArray {
    [OmDataType.Int8Array]: Int8Array;
    [OmDataType.Uint8Array]: Uint8Array;
    [OmDataType.Int16Array]: Int16Array;
    [OmDataType.Uint16Array]: Uint16Array;
    [OmDataType.Int32Array]: Int32Array;
    [OmDataType.Uint32Array]: Uint32Array;
    [OmDataType.Int64Array]: BigInt64Array;
    [OmDataType.Uint64Array]: BigUint64Array;
    [OmDataType.FloatArray]: Float32Array;
    [OmDataType.DoubleArray]: Float64Array;
}
type FileSource = string | File | Blob | Uint8Array | ArrayBuffer;

interface WasmModule {
    _malloc(size: number): number;
    _free(ptr: number): void;
    setValue(ptr: number, value: any, type: string): void;
    getValue(ptr: number, type: string): any;
    HEAPU8: Uint8Array;
    om_header_size(): number;
    om_header_type(ptr: number): number;
    om_trailer_size(): number;
    om_trailer_read(trailerPtr: number, offsetPtr: number, sizePtr: number): boolean;
    om_variable_init(dataPtr: number): number;
    om_variable_get_type(variable: number): number;
    om_variable_get_compression(variable: number): number;
    om_variable_get_scale_factor(variable: number): number;
    om_variable_get_add_offset(variable: number): number;
    om_variable_get_dimensions_count(variable: number): number;
    om_variable_get_dimensions_ptr(variable: number): number;
    om_variable_get_chunks_ptr(variable: number): number;
    om_variable_get_name_ptr(variable: number, length: number): number;
    om_variable_get_children_count(variable: number): number;
    om_variable_get_children(variable: number, index: number, count: number, offsetPtr: number, sizePtr: number): boolean;
    om_variable_get_scalar(variable: number, ptrPtr: number, sizePtr: number): number;
    om_decoder_init(decoderPtr: number, variable: number, nDims: bigint, readOffsetPtr: number, readCountPtr: number, intoCubeOffsetPtr: number, intoCubeDimensionPtr: number, ioSizeMerge: bigint, ioSizeMax: bigint): number;
    om_decoder_init_index_read(decoder: number, indexReadPtr: number): void;
    om_decoder_init_data_read(dataReadPtr: number, indexReadPtr: number): void;
    om_decoder_read_buffer_size(decoderPtr: number): number;
    om_decoder_next_index_read(decoder: number, indexRead: number): boolean;
    om_decoder_next_data_read(decoder: number, dataRead: number, indexData: number, indexCount: bigint, error: number): boolean;
    om_decoder_decode_chunks(decoder: number, chunkIndex: number, data: number, count: bigint, output: number, chunkBuffer: number, error: number): boolean;
    OM_HEADER_INVALID: number;
    OM_HEADER_LEGACY: number;
    OM_HEADER_READ_TRAILER: number;
    ERROR_OK: number;
    DATA_TYPE_INT8_ARRAY: number;
    DATA_TYPE_UINT8_ARRAY: number;
    DATA_TYPE_INT16_ARRAY: number;
    DATA_TYPE_UINT16_ARRAY: number;
    DATA_TYPE_INT32_ARRAY: number;
    DATA_TYPE_UINT32_ARRAY: number;
    DATA_TYPE_INT64_ARRAY: number;
    DATA_TYPE_UINT64_ARRAY: number;
    DATA_TYPE_FLOAT_ARRAY: number;
    DATA_TYPE_DOUBLE_ARRAY: number;
    sizeof_decoder: number;
}
declare function initWasm(): Promise<WasmModule>;
declare function getWasmModule(): WasmModule;

declare class OmFileReader {
    private backend;
    private wasm;
    private variable;
    private variableDataPtr;
    private metadataCache;
    constructor(backend: OmFileReaderBackend, wasm?: WasmModule);
    /**
     * Static factory method to create and initialize an OmFileReader
     */
    static create(backend: OmFileReaderBackend): Promise<OmFileReader>;
    initialize(): Promise<OmFileReader>;
    private _getString;
    dataType(): number;
    compression(): number;
    scaleFactor(): number;
    addOffset(): number;
    getDimensions(): number[];
    getChunkDimensions(): number[];
    getName(): string | null;
    numberOfChildren(): number;
    getChild(index: number): Promise<OmFileReader | null>;
    /**
     * Searches direct children by name. Does not search recursively.
     */
    getChildByName(name: string): Promise<OmFileReader | null>;
    initChildFromOffsetSize(offsetSize: OffsetSize): Promise<OmFileReader>;
    /**
     * Get child metadata by index.
     */
    private _getChildMetadata;
    /**
     * Find a variable by its path (e.g., "parent/child/grandchild")
     */
    findByPath(path: string): Promise<OmFileReader | null>;
    /**
     * Navigate through a path recursively
     */
    navigatePath(parts: string[]): Promise<OmFileReader | null>;
    readScalar<T>(dataType: OmDataType): T | null;
    private newIndexRead;
    private newDataRead;
    private allocateTypedArray;
    /**
     * Reads data from the file and returns a new TypedArray of the requested type.
     *
     * @param options Options for reading, including:
     *   - type: The data type to read.
     *   - ranges: Array of dimension ranges to read.
     *   - prefetch: Whether to prefetch data (default: true).
     *   - intoSAB: Use SharedArrayBuffer for output (default: false).
     *   - ioSizeMax: Maximum I/O size (default: 65536).
     *   - ioSizeMerge: Merge threshold for I/O operations (default: 2048).
     */
    read<T extends keyof OmDataTypeToTypedArray>(options: OmFileReadOptions<T>): Promise<OmDataTypeToTypedArray[T]>;
    /**
     * Reads data into an existing TypedArray with specified dimension ranges.
     *
     * @param options Options for reading, including:
     *   - type: The data type to read.
     *   - output: The TypedArray to read data into.
     *   - ranges: Array of dimension ranges to read.
     *   - prefetch: Whether to prefetch data (default: true).
     *   - ioSizeMax: Maximum I/O size (default: 65536).
     *   - ioSizeMerge: Merge threshold for I/O operations (default: 2048).
     */
    readInto<T extends keyof OmDataTypeToTypedArray>(options: OmFileReadIntoOptions<T>): Promise<void>;
    /**
     * Warms up the backend cache by requesting the necessary data blocks
     * without decoding them or copying them to a TypedArray.
     */
    readPrefetch(options: OmFilePrefetchReadOptions): Promise<void>;
    private decodePrefetch;
    private decode;
    /**
     * Internal helper to set up the decoder and execute a task.
     * Handles memory allocation and cleanup for ranges and the decoder.
     */
    private _runWithDecoder;
    private _iterateDataBlocks;
    private _readDataBlock;
    /**
     * Helper method to copy data from WASM memory to a TypedArray with the correct type
     */
    private copyToTypedArray;
    dispose(): void;
}

declare class FileBackendNode implements OmFileReaderBackend {
    private filePath;
    private memory;
    private fileSize;
    private fileHandle;
    constructor(source: FileSource);
    count(): Promise<number>;
    getBytes(offset: number, size: number): Promise<Uint8Array>;
    close(): Promise<void>;
}

declare class MemoryHttpBackend implements OmFileReaderBackend {
    private url;
    private fileSize;
    private fileData;
    private loadPromise;
    private countPromise;
    private maxFileSize;
    private onProgress?;
    private debug;
    /**
     * Create a new MemoryHttpBackend
     * @param options Configuration options
     */
    constructor(options: {
        url: string;
        maxFileSize?: number;
        onProgress?: (loaded: number, total: number) => void;
        debug?: boolean;
    });
    /**
     * Get the total size of the file
     */
    count(): Promise<number>;
    /**
     * Load the entire file into memory
     */
    loadFile(): Promise<void>;
    /**
     * Get bytes from the file
     * @param offset The starting position in the file
     * @param size The number of bytes to read
     */
    getBytes(offset: number, size: number): Promise<Uint8Array>;
    /**
     * Check if the file is fully loaded
     */
    isLoaded(): boolean;
    /**
     * Get the current loaded data or null if not loaded
     */
    getFileData(): Uint8Array | null;
    /**
     * Force a reload of the file
     */
    reload(): Promise<void>;
    /**
     * Close the backend and release any resources
     */
    close(): Promise<void>;
}

type KeyKind = "string" | "bigint";
/**
 * Interface for a block-level cache.
 * Implementations can be in-memory, persistent, or leverage browser APIs.
 */
interface BlockCache<K = bigint> {
    keyKind: KeyKind;
    /** Returns the block size used by the cache. */
    blockSize(): number;
    /** Retrieves a block from the cache or fetches it using the provided function. */
    get(key: K, fetchFn: () => Promise<Uint8Array>, fileSize?: number): Promise<Uint8Array>;
    /** Retrieves the total size of the cached file corresponding to key, if cached */
    size(key: K): Promise<number | undefined>;
    /** Optionally starts fetching a block into the cache without blocking. */
    prefetch(key: K, fetchFn: () => Promise<Uint8Array>, fileSize?: number): Promise<void>;
    /** Clears the cache contents. */
    clear(): void | Promise<void>;
}
declare class LruBlockCache implements BlockCache {
    readonly keyKind = "bigint";
    private readonly _blockSize;
    private readonly maxBlocks;
    private readonly cache;
    private readonly inflight;
    constructor(blockSize?: number, maxBlocks?: number);
    blockSize(): number;
    size(_key: bigint): Promise<number | undefined>;
    get(key: bigint, fetchFn: () => Promise<Uint8Array>): Promise<Uint8Array>;
    prefetch(key: bigint, fetchFn: () => Promise<Uint8Array>): Promise<void>;
    clear(): void;
}

interface OmHttpBackendOptions {
    url: string;
    eTagValidation?: boolean;
    debug?: boolean;
    timeoutMs?: number;
    retries?: number;
}
/**
 * Backend for reading from HTTP servers with partial read support using Range requests.
 * Checks last modified header and ETag.
 */
declare class OmHttpBackend implements OmFileReaderBackend {
    private readonly url;
    private readonly debug;
    private readonly timeoutMs;
    private readonly retries;
    private eTagValidation;
    private fileSize;
    private lastModified;
    private eTag;
    private metadataPromise;
    constructor(options: OmHttpBackendOptions);
    /**
     * Returns a bigint cache key for use with LruBlockCache.
     * Uniquely identifies the file based on its URL, ETag, and Last-Modified headers.
     * The ETag is only included if validation is enabled.
     */
    get cacheKeyBigInt(): bigint;
    /**
     * Returns a string cache key for use with BrowserBlockCache based on the underlying url.
     * If the upstream resource can change, this cache-key is not safe to use!
     * => Only use for static files!
     */
    get cacheKeyString(): string;
    /**
     * Fetch metadata using HEAD request
     */
    private fetchMetadata;
    /**
     * Get the total size of the file
     */
    count(signal?: AbortSignal): Promise<number>;
    /**
     * Get bytes from the file using Range requests
     */
    getBytes(offset: number, size: number, signal?: AbortSignal): Promise<Uint8Array>;
    asCachedReader(cache: BlockCache<string> | BlockCache<bigint>): Promise<OmFileReader>;
    /**
     * Close the backend and release resources
     */
    close(): Promise<void>;
}

/**
 * Wraps a backend for caching blocks of data.
 */
declare class BlockCacheBackend<K> implements OmFileReaderBackend {
    private readonly backend;
    private readonly cache;
    private readonly baseKey;
    private readonly keyBuilder;
    private cachedCount;
    constructor(backend: OmFileReaderBackend, cache: BlockCache<K>, baseKey: K, keyBuilder: (baseKey: K, blockIdx: number) => K);
    /**
     * Creates a BlockCacheBackend using bigint keys.
     */
    static withBigIntKeys(backend: OmFileReaderBackend, cache: BlockCache<bigint>, baseKey: bigint): BlockCacheBackend<bigint>;
    /**
     * Creates a BlockCacheBackend using string keys.
     */
    static withStringKeys(backend: OmFileReaderBackend, cache: BlockCache<string>, baseKey: string): BlockCacheBackend<string>;
    /**
     * Generates a unique key for a specific block.
     */
    private getBlockKey;
    /**
     * Get the byte range for a block indexed from the end.
     * Block 0 = last blockSize bytes, Block 1 = previous blockSize bytes, etc.
     */
    private getBlockRange;
    /**
     * Get the block index (from end) that contains a given offset.
     */
    private getBlockIdxFromEnd;
    count(signal?: AbortSignal): Promise<number>;
    getBytes(offset: number, size: number, signal?: AbortSignal): Promise<Uint8Array>;
    /**
     * Collects block fetch tasks for a given range without executing them.
     * Returns an array of functions that, when called, will fetch and cache the block.
     */
    collectPrefetchTasks(offset: number, size: number, signal?: AbortSignal): Promise<(() => Promise<void>)[]>;
    close(): Promise<void>;
}

/** Summary statistics for the cache */
interface CacheStats {
    /** Total entries in persistent cache */
    persistentEntries: number;
    /** Total bytes in persistent cache */
    persistentBytes: number;
    /** Entries currently in memory */
    memoryEntries: number;
    /** Bytes currently in memory */
    memoryBytes: number;
    /** Currently pending fetches */
    inflightCount: number;
    /** Maximum allowed bytes */
    maxBytes: number;
    /** Block size */
    blockSize: number;
}
interface BrowserBlockCacheOptions {
    blockSize?: number;
    cacheName?: string;
    /** Time in ms before an unused block is evicted from in-memory cache. Default: 1000 */
    memCacheTtlMs?: number;
    /** Maximum total size in bytes for persistent cache. Default: 1GB */
    maxBytes?: number;
    /** When evicting, remove this fraction of maxBytes to avoid frequent evictions. Default: 0.1 */
    evictionFraction?: number;
    maxConcurrentFetches?: number;
}
/**
 * A BlockCache implementation that uses the browser's Cache API.
 *
 * Features:
 * - Fast in-memory layer for recently accessed blocks
 * - Size-based eviction with configurable limits
 * - Metadata stored in response headers (no separate metadata cache)
 */
declare class BrowserBlockCache implements BlockCache<string> {
    readonly keyKind = "string";
    private readonly _blockSize;
    private readonly cacheName;
    private readonly inflight;
    /** In-memory cache for fast repeated access */
    private readonly memCache;
    /** Timers for automatic memory eviction */
    private readonly evictionTimers;
    /** Time in ms before an unused entry is evicted from memory */
    private readonly memCacheTtlMs;
    /** Maximum total bytes for persistent storage */
    private readonly maxBytes;
    /** Fraction of cache to clear during eviction */
    private readonly evictionFraction;
    /** Lock to prevent concurrent evictions */
    private evictionInProgress;
    /** Cached reference to the opened Cache object */
    private cachePromise;
    /** Maximum concurrent fetch operations */
    private readonly maxConcurrentFetches;
    /** Currently active fetch count */
    private activeFetches;
    /** Queue of pending fetch operations */
    private readonly fetchQueue;
    constructor(options?: BrowserBlockCacheOptions);
    blockSize(): number;
    size(key: string): Promise<number | undefined>;
    /**
     * Resolves a BlockKey into a URL string for the Cache API.
     */
    private resolveUrl;
    /**
     * Acquires a fetch slot. Resolves when a slot is available.
     */
    private acquireFetchSlot;
    /**
     * Releases a fetch slot and processes the next queued request.
     */
    private releaseFetchSlot;
    /**
     * Executes a fetch function with concurrency limiting.
     */
    private limitedFetch;
    /** Opens the cache once and reuses the reference */
    private getCache;
    /**
     * Scans the Cache API to get current total size and entry list.
     */
    private scanCache;
    /**
     * Evicts oldest entries until we're under the size limit.
     * Uses createdAt timestamp from headers for ordering.
     */
    private evictIfNeeded;
    /**
     * Refreshes the memory eviction timer for a URL.
     */
    private refreshEvictionTimer;
    /**
     * Stores data in the in-memory cache with automatic eviction.
     */
    private setMemCache;
    get(key: string, fetchFn: () => Promise<Uint8Array>, fileSize?: number): Promise<Uint8Array>;
    prefetch(key: string, fetchFn: () => Promise<Uint8Array>, fileSize?: number): Promise<void>;
    /**
     * Gets current cache statistics by scanning the Cache API.
     */
    getStats(): Promise<CacheStats>;
    clear(): Promise<void>;
}

export { BlockCacheBackend, BrowserBlockCache, CompressionType, FileBackendNode as FileBackend, LruBlockCache, MemoryHttpBackend, OmDataType, OmFileReader, OmHttpBackend, getWasmModule, initWasm };
export type { BlockCache, FileSource, OffsetSize, OmDataTypeToTypedArray, OmFilePrefetchReadOptions, OmFileReadBaseOptions, OmFileReadIntoOptions, OmFileReadOptions, OmFileReaderBackend, Range, TypedArray };