UNPKG

parquet-wasm

Version:

WebAssembly Parquet reader and writer.

1,539 lines (1,440 loc) 95.4 kB
let imports = {}; imports['__wbindgen_placeholder__'] = module.exports; let cachedUint8ArrayMemory0 = null; function getUint8ArrayMemory0() { if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); } return cachedUint8ArrayMemory0; } let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); cachedTextDecoder.decode(); function decodeText(ptr, len) { return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); } function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; return decodeText(ptr, len); } let WASM_VECTOR_LEN = 0; const cachedTextEncoder = new TextEncoder(); if (!('encodeInto' in cachedTextEncoder)) { cachedTextEncoder.encodeInto = function (arg, view) { const buf = cachedTextEncoder.encode(arg); view.set(buf); return { read: arg.length, written: buf.length }; } } function passStringToWasm0(arg, malloc, realloc) { if (realloc === undefined) { const buf = cachedTextEncoder.encode(arg); const ptr = malloc(buf.length, 1) >>> 0; getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); WASM_VECTOR_LEN = buf.length; return ptr; } let len = arg.length; let ptr = malloc(len, 1) >>> 0; const mem = getUint8ArrayMemory0(); let offset = 0; for (; offset < len; offset++) { const code = arg.charCodeAt(offset); if (code > 0x7F) break; mem[ptr + offset] = code; } if (offset !== len) { if (offset !== 0) { arg = arg.slice(offset); } ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); const ret = cachedTextEncoder.encodeInto(arg, view); offset += ret.written; ptr = realloc(ptr, len, offset, 1) >>> 0; } WASM_VECTOR_LEN = offset; return ptr; } let cachedDataViewMemory0 = null; function getDataViewMemory0() { if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { cachedDataViewMemory0 = new DataView(wasm.memory.buffer); } return cachedDataViewMemory0; } function addToExternrefTable0(obj) { const idx = wasm.__externref_table_alloc(); wasm.__wbindgen_export_4.set(idx, obj); return idx; } function handleError(f, args) { try { return f.apply(this, args); } catch (e) { const idx = addToExternrefTable0(e); wasm.__wbindgen_exn_store(idx); } } function isLikeNone(x) { return x === undefined || x === null; } function getArrayU8FromWasm0(ptr, len) { ptr = ptr >>> 0; return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); } function debugString(val) { // primitive types const type = typeof val; if (type == 'number' || type == 'boolean' || val == null) { return `${val}`; } if (type == 'string') { return `"${val}"`; } if (type == 'symbol') { const description = val.description; if (description == null) { return 'Symbol'; } else { return `Symbol(${description})`; } } if (type == 'function') { const name = val.name; if (typeof name == 'string' && name.length > 0) { return `Function(${name})`; } else { return 'Function'; } } // objects if (Array.isArray(val)) { const length = val.length; let debug = '['; if (length > 0) { debug += debugString(val[0]); } for(let i = 1; i < length; i++) { debug += ', ' + debugString(val[i]); } debug += ']'; return debug; } // Test for built-in const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val)); let className; if (builtInMatches && builtInMatches.length > 1) { className = builtInMatches[1]; } else { // Failed to match the standard '[object ClassName]' return toString.call(val); } if (className == 'Object') { // we're a user defined class or Object // JSON.stringify avoids problems with cycles, and is generally much // easier than looping through ownProperties of `val`. try { return 'Object(' + JSON.stringify(val) + ')'; } catch (_) { return 'Object'; } } // errors if (val instanceof Error) { return `${val.name}: ${val.message}\n${val.stack}`; } // TODO we could test for more things here, like `Set`s and `Map`s. return className; } const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry( state => { wasm.__wbindgen_export_5.get(state.dtor)(state.a, state.b); } ); function makeMutClosure(arg0, arg1, dtor, f) { const state = { a: arg0, b: arg1, cnt: 1, dtor }; const real = (...args) => { // First up with a closure we increment the internal reference // count. This ensures that the Rust closure environment won't // be deallocated while we're invoking it. state.cnt++; const a = state.a; state.a = 0; try { return f(a, state.b, ...args); } finally { if (--state.cnt === 0) { wasm.__wbindgen_export_5.get(state.dtor)(a, state.b); CLOSURE_DTORS.unregister(state); } else { state.a = a; } } }; real.original = state; CLOSURE_DTORS.register(real, state, state); return real; } function takeFromExternrefTable0(idx) { const value = wasm.__wbindgen_export_4.get(idx); wasm.__externref_table_dealloc(idx); return value; } function getArrayJsValueFromWasm0(ptr, len) { ptr = ptr >>> 0; const mem = getDataViewMemory0(); const result = []; for (let i = ptr; i < ptr + 4 * len; i += 4) { result.push(wasm.__wbindgen_export_4.get(mem.getUint32(i, true))); } wasm.__externref_drop_slice(ptr, len); return result; } /** * Read a Parquet file into a stream of Arrow `RecordBatch`es. * * This returns a ReadableStream containing RecordBatches in WebAssembly memory. To transfer the * Arrow table to JavaScript memory you have two options: * * - (Easier): Call {@linkcode RecordBatch.intoIPCStream} to construct a buffer that can be parsed * with Arrow JS's `tableFromIPC` function. (The table will have a single internal record * batch). * - (More performant but bleeding edge): Call {@linkcode RecordBatch.intoFFI} to construct a data * representation that can be parsed zero-copy from WebAssembly with * [arrow-js-ffi](https://github.com/kylebarron/arrow-js-ffi) using `parseRecordBatch`. * * Example with IPC stream: * * ```js * import { tableFromIPC, Table } from "apache-arrow"; * import initWasm, {readParquetStream} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * const stream = await readParquetStream(url); * * const batches = []; * for await (const wasmRecordBatch of stream) { * const arrowTable = tableFromIPC(wasmRecordBatch.intoIPCStream()); * batches.push(...arrowTable.batches); * } * const table = new Table(batches); * ``` * * Example with `arrow-js-ffi`: * * ```js * import { Table } from "apache-arrow"; * import { parseRecordBatch } from "arrow-js-ffi"; * import initWasm, {readParquetStream, wasmMemory} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * const WASM_MEMORY = wasmMemory(); * * const stream = await readParquetStream(url); * * const batches = []; * for await (const wasmRecordBatch of stream) { * const ffiRecordBatch = wasmRecordBatch.intoFFI(); * const recordBatch = parseRecordBatch( * WASM_MEMORY.buffer, * ffiRecordBatch.arrayAddr(), * ffiRecordBatch.schemaAddr(), * true * ); * batches.push(recordBatch); * } * const table = new Table(batches); * ``` * * @param url URL to Parquet file * @param {string} url * @param {number | null} [content_length] * @returns {Promise<ReadableStream>} */ exports.readParquetStream = function(url, content_length) { const ptr0 = passStringToWasm0(url, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; const ret = wasm.readParquetStream(ptr0, len0, isLikeNone(content_length) ? 0x100000001 : (content_length) >>> 0); return ret; }; function _assertClass(instance, klass) { if (!(instance instanceof klass)) { throw new Error(`expected instance of ${klass.name}`); } } /** * Transform a ReadableStream of RecordBatches to a ReadableStream of bytes * * Browser example with piping to a file via the File System API: * * ```js * import initWasm, {ParquetFile, transformParquetStream} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * const fileInstance = await ParquetFile.fromUrl("https://example.com/file.parquet"); * const recordBatchStream = await fileInstance.stream(); * const serializedParquetStream = await transformParquetStream(recordBatchStream); * // NB: requires transient user activation - you would typically do this before ☝️ * const handle = await window.showSaveFilePicker(); * const writable = await handle.createWritable(); * await serializedParquetStream.pipeTo(writable); * ``` * * NodeJS (ESM) example with piping to a file: * ```js * import { open } from "node:fs/promises"; * import { Writable } from "node:stream"; * import initWasm, {ParquetFile, transformParquetStream} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * const fileInstance = await ParquetFile.fromUrl("https://example.com/file.parquet"); * const recordBatchStream = await fileInstance.stream(); * const serializedParquetStream = await transformParquetStream(recordBatchStream); * * // grab a file handle via fsPromises * const handle = await open("file.parquet"); * const destinationStream = Writable.toWeb(handle.createWriteStream()); * await serializedParquetStream.pipeTo(destinationStream); * * ``` * NB: the above is a little contrived - `await writeFile("file.parquet", serializedParquetStream)` * is enough for most use cases. * * Browser kitchen sink example - teeing to the Cache API, using as a streaming post body, transferring * to a Web Worker: * ```js * // prelude elided - see above * const serializedParquetStream = await transformParquetStream(recordBatchStream); * const [cacheStream, bodyStream] = serializedParquetStream.tee(); * const postProm = fetch(targetUrl, { * method: "POST", * duplex: "half", * body: bodyStream * }); * const targetCache = await caches.open("foobar"); * await targetCache.put("https://example.com/file.parquet", new Response(cacheStream)); * // this could have been done with another tee, but beware of buffering * const workerStream = await targetCache.get("https://example.com/file.parquet").body; * const worker = new Worker("worker.js"); * worker.postMessage(workerStream, [workerStream]); * await postProm; * ``` * * @param stream A {@linkcode ReadableStream} of {@linkcode RecordBatch} instances * @param writer_properties (optional) Configuration for writing to Parquet. Use the {@linkcode * WriterPropertiesBuilder} to build a writing configuration, then call `.build()` to create an * immutable writer properties to pass in here. * @returns ReadableStream containing serialized Parquet data. * @param {ReadableStream} stream * @param {WriterProperties | null} [writer_properties] * @returns {Promise<ReadableStream>} */ exports.transformParquetStream = function(stream, writer_properties) { let ptr0 = 0; if (!isLikeNone(writer_properties)) { _assertClass(writer_properties, WriterProperties); ptr0 = writer_properties.__destroy_into_raw(); } const ret = wasm.transformParquetStream(stream, ptr0); return ret; }; function passArray8ToWasm0(arg, malloc) { const ptr = malloc(arg.length * 1, 1) >>> 0; getUint8ArrayMemory0().set(arg, ptr / 1); WASM_VECTOR_LEN = arg.length; return ptr; } /** * Read a Parquet file into Arrow data. * * This returns an Arrow table in WebAssembly memory. To transfer the Arrow table to JavaScript * memory you have two options: * * - (Easier): Call {@linkcode Table.intoIPCStream} to construct a buffer that can be parsed with * Arrow JS's `tableFromIPC` function. * - (More performant but bleeding edge): Call {@linkcode Table.intoFFI} to construct a data * representation that can be parsed zero-copy from WebAssembly with * [arrow-js-ffi](https://github.com/kylebarron/arrow-js-ffi) using `parseTable`. * * Example with IPC stream: * * ```js * import { tableFromIPC } from "apache-arrow"; * import initWasm, {readParquet} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmTable = readParquet(parquetUint8Array); * const arrowTable = tableFromIPC(arrowWasmTable.intoIPCStream()); * ``` * * Example with `arrow-js-ffi`: * * ```js * import { parseTable } from "arrow-js-ffi"; * import initWasm, {readParquet, wasmMemory} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * const WASM_MEMORY = wasmMemory(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmTable = readParquet(parquetUint8Array); * const ffiTable = arrowWasmTable.intoFFI(); * const arrowTable = parseTable( * WASM_MEMORY.buffer, * ffiTable.arrayAddrs(), * ffiTable.schemaAddr() * ); * ``` * * @param parquet_file Uint8Array containing Parquet data * @param options * * Options for reading Parquet data. Optional keys include: * * - `batchSize`: The number of rows in each batch. If not provided, the upstream parquet * default is 1024. * - `rowGroups`: Only read data from the provided row group indexes. * - `limit`: Provide a limit to the number of rows to be read. * - `offset`: Provide an offset to skip over the given number of rows. * - `columns`: The column names from the file to read. * @param {Uint8Array} parquet_file * @param {ReaderOptions | null} [options] * @returns {Table} */ exports.readParquet = function(parquet_file, options) { const ptr0 = passArray8ToWasm0(parquet_file, wasm.__wbindgen_malloc); const len0 = WASM_VECTOR_LEN; const ret = wasm.readParquet(ptr0, len0, isLikeNone(options) ? 0 : addToExternrefTable0(options)); if (ret[2]) { throw takeFromExternrefTable0(ret[1]); } return Table.__wrap(ret[0]); }; /** * Read an Arrow schema from a Parquet file in memory. * * This returns an Arrow schema in WebAssembly memory. To transfer the Arrow schema to JavaScript * memory you have two options: * * - (Easier): Call {@linkcode Schema.intoIPCStream} to construct a buffer that can be parsed with * Arrow JS's `tableFromIPC` function. This results in an Arrow JS Table with zero rows but a * valid schema. * - (More performant but bleeding edge): Call {@linkcode Schema.intoFFI} to construct a data * representation that can be parsed zero-copy from WebAssembly with * [arrow-js-ffi](https://github.com/kylebarron/arrow-js-ffi) using `parseSchema`. * * Example with IPC Stream: * * ```js * import { tableFromIPC } from "apache-arrow"; * import initWasm, {readSchema} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmSchema = readSchema(parquetUint8Array); * const arrowTable = tableFromIPC(arrowWasmSchema.intoIPCStream()); * const arrowSchema = arrowTable.schema; * ``` * * Example with `arrow-js-ffi`: * * ```js * import { parseSchema } from "arrow-js-ffi"; * import initWasm, {readSchema, wasmMemory} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * const WASM_MEMORY = wasmMemory(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmSchema = readSchema(parquetUint8Array); * const ffiSchema = arrowWasmSchema.intoFFI(); * const arrowTable = parseSchema(WASM_MEMORY.buffer, ffiSchema.addr()); * const arrowSchema = arrowTable.schema; * ``` * * @param parquet_file Uint8Array containing Parquet data * @param {Uint8Array} parquet_file * @returns {Schema} */ exports.readSchema = function(parquet_file) { const ptr0 = passArray8ToWasm0(parquet_file, wasm.__wbindgen_malloc); const len0 = WASM_VECTOR_LEN; const ret = wasm.readSchema(ptr0, len0); if (ret[2]) { throw takeFromExternrefTable0(ret[1]); } return Schema.__wrap(ret[0]); }; /** * Write Arrow data to a Parquet file. * * For example, to create a Parquet file with Snappy compression: * * ```js * import { tableToIPC } from "apache-arrow"; * // Edit the `parquet-wasm` import as necessary * import initWasm, { * Table, * WriterPropertiesBuilder, * Compression, * writeParquet, * } from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * // Given an existing arrow JS table under `table` * const wasmTable = Table.fromIPCStream(tableToIPC(table, "stream")); * const writerProperties = new WriterPropertiesBuilder() * .setCompression(Compression.SNAPPY) * .build(); * const parquetUint8Array = writeParquet(wasmTable, writerProperties); * ``` * * If `writerProperties` is not provided or is `null`, the default writer properties will be used. * This is equivalent to `new WriterPropertiesBuilder().build()`. * * @param table A {@linkcode Table} representation in WebAssembly memory. * @param writer_properties (optional) Configuration for writing to Parquet. Use the {@linkcode * WriterPropertiesBuilder} to build a writing configuration, then call `.build()` to create an * immutable writer properties to pass in here. * @returns Uint8Array containing written Parquet data. * @param {Table} table * @param {WriterProperties | null} [writer_properties] * @returns {Uint8Array} */ exports.writeParquet = function(table, writer_properties) { _assertClass(table, Table); var ptr0 = table.__destroy_into_raw(); let ptr1 = 0; if (!isLikeNone(writer_properties)) { _assertClass(writer_properties, WriterProperties); ptr1 = writer_properties.__destroy_into_raw(); } const ret = wasm.writeParquet(ptr0, ptr1); if (ret[3]) { throw takeFromExternrefTable0(ret[2]); } var v3 = getArrayU8FromWasm0(ret[0], ret[1]).slice(); wasm.__wbindgen_free(ret[0], ret[1] * 1, 1); return v3; }; /** * Returns a handle to this wasm instance's `WebAssembly.Table` which is the indirect function * table used by Rust * @returns {FunctionTable} */ exports._functionTable = function() { const ret = wasm._functionTable(); return ret; }; /** * Returns a handle to this wasm instance's `WebAssembly.Memory` * @returns {Memory} */ exports.wasmMemory = function() { const ret = wasm.wasmMemory(); return ret; }; let cachedUint32ArrayMemory0 = null; function getUint32ArrayMemory0() { if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) { cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer); } return cachedUint32ArrayMemory0; } function getArrayU32FromWasm0(ptr, len) { ptr = ptr >>> 0; return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len); } function __wbg_adapter_8(arg0, arg1, arg2) { wasm.closure3806_externref_shim(arg0, arg1, arg2); } function __wbg_adapter_15(arg0, arg1) { wasm.wasm_bindgen__convert__closures_____invoke__h7767827484ad8911(arg0, arg1); } function __wbg_adapter_266(arg0, arg1, arg2, arg3) { wasm.closure3830_externref_shim(arg0, arg1, arg2, arg3); } /** * Supported compression algorithms. * * Codecs added in format version X.Y can be read by readers based on X.Y and later. * Codec support may vary between readers based on the format version and * libraries available at runtime. * @enum {0 | 1 | 2 | 3 | 4 | 5 | 6 | 7} */ exports.Compression = Object.freeze({ UNCOMPRESSED: 0, "0": "UNCOMPRESSED", SNAPPY: 1, "1": "SNAPPY", GZIP: 2, "2": "GZIP", BROTLI: 3, "3": "BROTLI", /** * @deprecated as of Parquet 2.9.0. * Switch to LZ4_RAW */ LZ4: 4, "4": "LZ4", ZSTD: 5, "5": "ZSTD", LZ4_RAW: 6, "6": "LZ4_RAW", LZO: 7, "7": "LZO", }); /** * Controls the level of statistics to be computed by the writer * @enum {0 | 1 | 2} */ exports.EnabledStatistics = Object.freeze({ /** * Compute no statistics */ None: 0, "0": "None", /** * Compute chunk-level statistics but not page-level */ Chunk: 1, "1": "Chunk", /** * Compute page-level and chunk-level statistics */ Page: 2, "2": "Page", }); /** * Encodings supported by Parquet. * Not all encodings are valid for all types. These enums are also used to specify the * encoding of definition and repetition levels. * @enum {0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8} */ exports.Encoding = Object.freeze({ /** * Default byte encoding. * - BOOLEAN - 1 bit per value, 0 is false; 1 is true. * - INT32 - 4 bytes per value, stored as little-endian. * - INT64 - 8 bytes per value, stored as little-endian. * - FLOAT - 4 bytes per value, stored as little-endian. * - DOUBLE - 8 bytes per value, stored as little-endian. * - BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes. * - FIXED_LEN_BYTE_ARRAY - just the bytes are stored. */ PLAIN: 0, "0": "PLAIN", /** * **Deprecated** dictionary encoding. * * The values in the dictionary are encoded using PLAIN encoding. * Since it is deprecated, RLE_DICTIONARY encoding is used for a data page, and * PLAIN encoding is used for dictionary page. */ PLAIN_DICTIONARY: 1, "1": "PLAIN_DICTIONARY", /** * Group packed run length encoding. * * Usable for definition/repetition levels encoding and boolean values. */ RLE: 2, "2": "RLE", /** * Bit packed encoding. * * This can only be used if the data has a known max width. * Usable for definition/repetition levels encoding. */ BIT_PACKED: 3, "3": "BIT_PACKED", /** * Delta encoding for integers, either INT32 or INT64. * * Works best on sorted data. */ DELTA_BINARY_PACKED: 4, "4": "DELTA_BINARY_PACKED", /** * Encoding for byte arrays to separate the length values and the data. * * The lengths are encoded using DELTA_BINARY_PACKED encoding. */ DELTA_LENGTH_BYTE_ARRAY: 5, "5": "DELTA_LENGTH_BYTE_ARRAY", /** * Incremental encoding for byte arrays. * * Prefix lengths are encoded using DELTA_BINARY_PACKED encoding. * Suffixes are stored using DELTA_LENGTH_BYTE_ARRAY encoding. */ DELTA_BYTE_ARRAY: 6, "6": "DELTA_BYTE_ARRAY", /** * Dictionary encoding. * * The ids are encoded using the RLE encoding. */ RLE_DICTIONARY: 7, "7": "RLE_DICTIONARY", /** * Encoding for floating-point data. * * K byte-streams are created where K is the size in bytes of the data type. * The individual bytes of an FP value are scattered to the corresponding stream and * the streams are concatenated. * This itself does not reduce the size of the data but can lead to better compression * afterwards. */ BYTE_STREAM_SPLIT: 8, "8": "BYTE_STREAM_SPLIT", }); /** * The Parquet version to use when writing * @enum {0 | 1} */ exports.WriterVersion = Object.freeze({ V1: 0, "0": "V1", V2: 1, "1": "V2", }); const __wbindgen_enum_ReadableStreamType = ["bytes"]; const __wbindgen_enum_RequestCache = ["default", "no-store", "reload", "no-cache", "force-cache", "only-if-cached"]; const __wbindgen_enum_RequestCredentials = ["omit", "same-origin", "include"]; const __wbindgen_enum_RequestMode = ["same-origin", "no-cors", "cors", "navigate"]; const ColumnChunkMetaDataFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_columnchunkmetadata_free(ptr >>> 0, 1)); /** * Metadata for a Parquet column chunk. */ class ColumnChunkMetaData { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(ColumnChunkMetaData.prototype); obj.__wbg_ptr = ptr; ColumnChunkMetaDataFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; ColumnChunkMetaDataFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_columnchunkmetadata_free(ptr, 0); } /** * Total number of values in this column chunk. * @returns {number} */ numValues() { const ret = wasm.columnchunkmetadata_numValues(this.__wbg_ptr); return ret; } /** * Path (or identifier) of this column. * @returns {string[]} */ columnPath() { const ret = wasm.columnchunkmetadata_columnPath(this.__wbg_ptr); var v1 = getArrayJsValueFromWasm0(ret[0], ret[1]).slice(); wasm.__wbindgen_free(ret[0], ret[1] * 4, 4); return v1; } /** * Compression for this column. * @returns {Compression} */ compression() { const ret = wasm.columnchunkmetadata_compression(this.__wbg_ptr); return ret; } /** * Byte offset in `file_path()`. * @returns {bigint} */ fileOffset() { const ret = wasm.columnchunkmetadata_fileOffset(this.__wbg_ptr); return ret; } /** * Returns the total compressed data size of this column chunk. * @returns {number} */ compressedSize() { const ret = wasm.columnchunkmetadata_compressedSize(this.__wbg_ptr); return ret; } /** * Returns the total uncompressed data size of this column chunk. * @returns {number} */ uncompressedSize() { const ret = wasm.columnchunkmetadata_uncompressedSize(this.__wbg_ptr); return ret; } /** * All encodings used for this column. * @returns {any[]} */ encodings() { const ret = wasm.columnchunkmetadata_encodings(this.__wbg_ptr); var v1 = getArrayJsValueFromWasm0(ret[0], ret[1]).slice(); wasm.__wbindgen_free(ret[0], ret[1] * 4, 4); return v1; } /** * File where the column chunk is stored. * * If not set, assumed to belong to the same file as the metadata. * This path is relative to the current file. * @returns {string | undefined} */ filePath() { const ret = wasm.columnchunkmetadata_filePath(this.__wbg_ptr); let v1; if (ret[0] !== 0) { v1 = getStringFromWasm0(ret[0], ret[1]).slice(); wasm.__wbindgen_free(ret[0], ret[1] * 1, 1); } return v1; } } if (Symbol.dispose) ColumnChunkMetaData.prototype[Symbol.dispose] = ColumnChunkMetaData.prototype.free; exports.ColumnChunkMetaData = ColumnChunkMetaData; const FFIDataFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_ffidata_free(ptr >>> 0, 1)); /** * An Arrow array exported to FFI. * * Using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi), you can view or copy Arrow * these objects to JavaScript. * * Note that this also includes an ArrowSchema C struct as well, so that extension type * information can be maintained. * ## Memory management * * Note that this array will not be released automatically. You need to manually call `.free()` to * release memory. */ class FFIData { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(FFIData.prototype); obj.__wbg_ptr = ptr; FFIDataFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; FFIDataFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_ffidata_free(ptr, 0); } /** * Access the pointer to the * [`ArrowArray`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct. This can be viewed or copied (without serialization) to an Arrow JS `RecordBatch` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmRecordBatch: FFIRecordBatch = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmRecordBatch.arrayAddr(), * wasmRecordBatch.schemaAddr(), * true * ); * ``` * @returns {number} */ arrayAddr() { const ret = wasm.ffidata_arrayAddr(this.__wbg_ptr); return ret >>> 0; } /** * Access the pointer to the * [`ArrowSchema`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct. This can be viewed or copied (without serialization) to an Arrow JS `Field` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmRecordBatch: FFIRecordBatch = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmRecordBatch.arrayAddr(), * wasmRecordBatch.schemaAddr(), * true * ); * ``` * @returns {number} */ schemaAddr() { const ret = wasm.ffidata_schemaAddr(this.__wbg_ptr); return ret >>> 0; } } if (Symbol.dispose) FFIData.prototype[Symbol.dispose] = FFIData.prototype.free; exports.FFIData = FFIData; const FFISchemaFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_ffischema_free(ptr >>> 0, 1)); class FFISchema { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(FFISchema.prototype); obj.__wbg_ptr = ptr; FFISchemaFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; FFISchemaFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_ffischema_free(ptr, 0); } /** * Access the pointer to the * [`ArrowSchema`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct. This can be viewed or copied (without serialization) to an Arrow JS `Field` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmRecordBatch: FFIRecordBatch = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmRecordBatch.arrayAddr(), * wasmRecordBatch.schemaAddr(), * true * ); * ``` * @returns {number} */ addr() { const ret = wasm.ffischema_addr(this.__wbg_ptr); return ret >>> 0; } } if (Symbol.dispose) FFISchema.prototype[Symbol.dispose] = FFISchema.prototype.free; exports.FFISchema = FFISchema; const FFIStreamFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_ffistream_free(ptr >>> 0, 1)); /** * A representation of an Arrow C Stream in WebAssembly memory exposed as FFI-compatible * structs through the Arrow C Data Interface. * * Unlike other Arrow implementations outside of JS, this always stores the "stream" fully * materialized as a sequence of Arrow chunks. */ class FFIStream { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(FFIStream.prototype); obj.__wbg_ptr = ptr; FFIStreamFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; FFIStreamFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_ffistream_free(ptr, 0); } /** * Get the pointer to one ArrowArray FFI struct for a given chunk index and column index * * Access the pointer to one * [`ArrowArray`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct representing one of the internal `RecordBatch`es. This can be viewed or copied (without serialization) to an Arrow JS `RecordBatch` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import * as arrow from "apache-arrow"; * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmTable: FFITable = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * const jsBatches: arrow.RecordBatch[] = [] * for (let i = 0; i < wasmTable.numBatches(); i++) { * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmTable.arrayAddr(i), * wasmTable.schemaAddr(), * true * ); * jsBatches.push(jsRecordBatch); * } * const jsTable = new arrow.Table(jsBatches); * ``` * * @param chunk number The chunk index to use * @returns number pointer to an ArrowArray FFI struct in Wasm memory * @param {number} chunk * @returns {number} */ arrayAddr(chunk) { const ret = wasm.ffistream_arrayAddr(this.__wbg_ptr, chunk); return ret >>> 0; } /** * Get the total number of elements in this stream * @returns {number} */ numArrays() { const ret = wasm.ffistream_numArrays(this.__wbg_ptr); return ret >>> 0; } /** * @returns {Uint32Array} */ arrayAddrs() { const ret = wasm.ffistream_arrayAddrs(this.__wbg_ptr); var v1 = getArrayU32FromWasm0(ret[0], ret[1]).slice(); wasm.__wbindgen_free(ret[0], ret[1] * 4, 4); return v1; } /** * Get the pointer to the ArrowSchema FFI struct * @returns {number} */ schemaAddr() { const ret = wasm.ffistream_schemaAddr(this.__wbg_ptr); return ret >>> 0; } drop() { const ptr = this.__destroy_into_raw(); wasm.ffistream_drop(ptr); } } if (Symbol.dispose) FFIStream.prototype[Symbol.dispose] = FFIStream.prototype.free; exports.FFIStream = FFIStream; const FileMetaDataFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_filemetadata_free(ptr >>> 0, 1)); /** * Metadata for a Parquet file. */ class FileMetaData { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(FileMetaData.prototype); obj.__wbg_ptr = ptr; FileMetaDataFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; FileMetaDataFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_filemetadata_free(ptr, 0); } /** * String message for application that wrote this file. * * This should have the following format: * `<application> version <application version> (build <application build hash>)`. * * ```shell * parquet-mr version 1.8.0 (build 0fda28af84b9746396014ad6a415b90592a98b3b) * ``` * @returns {string | undefined} */ createdBy() { const ret = wasm.filemetadata_createdBy(this.__wbg_ptr); let v1; if (ret[0] !== 0) { v1 = getStringFromWasm0(ret[0], ret[1]).slice(); wasm.__wbindgen_free(ret[0], ret[1] * 1, 1); } return v1; } /** * Returns key_value_metadata of this file. * @returns {Map<any, any>} */ keyValueMetadata() { const ret = wasm.filemetadata_keyValueMetadata(this.__wbg_ptr); if (ret[2]) { throw takeFromExternrefTable0(ret[1]); } return takeFromExternrefTable0(ret[0]); } /** * Returns version of this file. * @returns {number} */ version() { const ret = wasm.filemetadata_version(this.__wbg_ptr); return ret; } /** * Returns number of rows in the file. * @returns {number} */ numRows() { const ret = wasm.filemetadata_numRows(this.__wbg_ptr); return ret; } } if (Symbol.dispose) FileMetaData.prototype[Symbol.dispose] = FileMetaData.prototype.free; exports.FileMetaData = FileMetaData; const IntoUnderlyingByteSourceFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_intounderlyingbytesource_free(ptr >>> 0, 1)); class IntoUnderlyingByteSource { __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; IntoUnderlyingByteSourceFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_intounderlyingbytesource_free(ptr, 0); } /** * @returns {number} */ get autoAllocateChunkSize() { const ret = wasm.intounderlyingbytesource_autoAllocateChunkSize(this.__wbg_ptr); return ret >>> 0; } /** * @param {ReadableByteStreamController} controller * @returns {Promise<any>} */ pull(controller) { const ret = wasm.intounderlyingbytesource_pull(this.__wbg_ptr, controller); return ret; } /** * @param {ReadableByteStreamController} controller */ start(controller) { wasm.intounderlyingbytesource_start(this.__wbg_ptr, controller); } /** * @returns {ReadableStreamType} */ get type() { const ret = wasm.intounderlyingbytesource_type(this.__wbg_ptr); return __wbindgen_enum_ReadableStreamType[ret]; } cancel() { const ptr = this.__destroy_into_raw(); wasm.intounderlyingbytesource_cancel(ptr); } } if (Symbol.dispose) IntoUnderlyingByteSource.prototype[Symbol.dispose] = IntoUnderlyingByteSource.prototype.free; exports.IntoUnderlyingByteSource = IntoUnderlyingByteSource; const IntoUnderlyingSinkFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_intounderlyingsink_free(ptr >>> 0, 1)); class IntoUnderlyingSink { __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; IntoUnderlyingSinkFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_intounderlyingsink_free(ptr, 0); } /** * @param {any} reason * @returns {Promise<any>} */ abort(reason) { const ptr = this.__destroy_into_raw(); const ret = wasm.intounderlyingsink_abort(ptr, reason); return ret; } /** * @returns {Promise<any>} */ close() { const ptr = this.__destroy_into_raw(); const ret = wasm.intounderlyingsink_close(ptr); return ret; } /** * @param {any} chunk * @returns {Promise<any>} */ write(chunk) { const ret = wasm.intounderlyingsink_write(this.__wbg_ptr, chunk); return ret; } } if (Symbol.dispose) IntoUnderlyingSink.prototype[Symbol.dispose] = IntoUnderlyingSink.prototype.free; exports.IntoUnderlyingSink = IntoUnderlyingSink; const IntoUnderlyingSourceFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_intounderlyingsource_free(ptr >>> 0, 1)); class IntoUnderlyingSource { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(IntoUnderlyingSource.prototype); obj.__wbg_ptr = ptr; IntoUnderlyingSourceFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; IntoUnderlyingSourceFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_intounderlyingsource_free(ptr, 0); } /** * @param {ReadableStreamDefaultController} controller * @returns {Promise<any>} */ pull(controller) { const ret = wasm.intounderlyingsource_pull(this.__wbg_ptr, controller); return ret; } cancel() { const ptr = this.__destroy_into_raw(); wasm.intounderlyingsource_cancel(ptr); } } if (Symbol.dispose) IntoUnderlyingSource.prototype[Symbol.dispose] = IntoUnderlyingSource.prototype.free; exports.IntoUnderlyingSource = IntoUnderlyingSource; const ParquetFileFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_parquetfile_free(ptr >>> 0, 1)); class ParquetFile { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(ParquetFile.prototype); obj.__wbg_ptr = ptr; ParquetFileFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; ParquetFileFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_parquetfile_free(ptr, 0); } /** * Read from the Parquet file in an async fashion. * * @param options * * Options for reading Parquet data. Optional keys include: * * - `batchSize`: The number of rows in each batch. If not provided, the upstream parquet * default is 1024. * - `rowGroups`: Only read data from the provided row group indexes. * - `limit`: Provide a limit to the number of rows to be read. * - `offset`: Provide an offset to skip over the given number of rows. * - `columns`: The column names from the file to read. * @param {ReaderOptions | null} [options] * @returns {Promise<Table>} */ read(options) { const ret = wasm.parquetfile_read(this.__wbg_ptr, isLikeNone(options) ? 0 : addToExternrefTable0(options)); return ret; } /** * @returns {Schema} */ schema() { const ret = wasm.parquetfile_schema(this.__wbg_ptr); if (ret[2]) { throw takeFromExternrefTable0(ret[1]); } return Schema.__wrap(ret[0]); } /** * Create a readable stream of record batches. * * Each item in the stream will be a {@linkcode RecordBatch}. * * @param options * * Options for reading Parquet data. Optional keys include: * * - `batchSize`: The number of rows in each batch. If not provided, the upstream parquet * default is 1024. * - `rowGroups`: Only read data from the provided row group indexes. * - `limit`: Provide a limit to the number of rows to be read. * - `offset`: Provide an offset to skip over the given number of rows. * - `columns`: The column names from the file to read. * - `concurrency`: The number of concurrent requests to make * @param {ReaderOptions | null} [options] * @returns {Promise<ReadableStream>} */ stream(options) { const ret = wasm.parquetfile_stream(this.__wbg_ptr, isLikeNone(options) ? 0 : addToExternrefTable0(options)); return ret; } /** * Construct a ParquetFile from a new URL. * @param {string} url * @returns {Promise<ParquetFile>} */ static fromUrl(url) { const ptr0 = passStringToWasm0(url, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; const ret = wasm.parquetfile_fromUrl(ptr0, len0); return ret; } /** * @returns {ParquetMetaData} */ metadata() { const ret = wasm.parquetfile_metadata(this.__wbg_ptr); if (ret[2]) { throw takeFromExternrefTable0(ret[1]); } return ParquetMetaData.__wrap(ret[0]); } /** * Construct a ParquetFile from a new [Blob] or [File] handle. * * [Blob]: https://developer.mozilla.org/en-US/docs/Web/API/Blob * [File]: https://developer.mozilla.org/en-US/docs/Web/API/File * * Safety: Do not use this in a multi-threaded environment, * (transitively depends on `!Send` `web_sys::Blob`) * @param {Blob} handle * @returns {Promise<ParquetFile>} */ static fromFile(handle) { const ret = wasm.parquetfile_fromFile(handle); return ret; } } if (Symbol.dispose) ParquetFile.prototype[Symbol.dispose] = ParquetFile.prototype.free; exports.ParquetFile = ParquetFile; const ParquetMetaDataFinalization = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(ptr => wasm.__wbg_parquetmetadata_free(ptr >>> 0, 1)); /** * Global Parquet metadata. */ class ParquetMetaData { static __wrap(ptr) { ptr = ptr >>> 0; const obj = Object.create(ParquetMetaData.prototype); obj.__wbg_ptr = ptr; ParquetMetaDataFinalization.register(obj, obj.__wbg_ptr, obj); return obj; } __destroy_into_raw() { const ptr = this.__wbg_ptr; this.__wbg_ptr = 0; ParquetMetaDataFinalization.unregister(this); return ptr; } free() { const ptr = this.__destroy_into_raw(); wasm.__wbg_parquetmetadata_free(ptr, 0); } /** * Returns row group metadata for all row groups * @returns {RowGroupMetaData[]} */ rowGroups() { const ret = wasm.parquetmetadata_rowGroups(this.__wbg_ptr); var v1 = getArrayJsValueFromWasm0(ret[0], ret[1]).slice(); wasm.__wbindgen_free(ret[0], ret[1] * 4, 4); return v1; } /** * Returns file metadata as reference. * @returns {FileMetaData} */ fileMetadata() { const ret = wasm.parquetmetadata_fileMetadata(this.__wbg_ptr); return FileMetaData.__wrap(ret); } /** * Returns number of row groups in this file. * @returns {number} */ numRowGroups() { const ret = wasm.parquetmetadata_numRowGroups(this.__wbg_ptr); return ret >>> 0; } /** * Returns row group metadata for `i`th position. * Position should be less than number of row groups `num_row_groups`. * @param {number} i * @returns {RowGroupMetaData} */ rowGroup(i) { const ret = wasm.parquetmetadata_rowGroup(this.__wbg_ptr, i); return RowGroupMetaData.__wrap(ret); } } if (Symbol.dispose) ParquetMetaData.prototype[Symbol.dis