@dobesv/parquets
Version:
TypeScript implementation of the Parquet file format, based on parquet.js
776 lines • 31.4 kB
JavaScript
"use strict";
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
var __asyncDelegator = (this && this.__asyncDelegator) || function (o) {
var i, p;
return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i;
function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === "return" } : f ? f(v) : v; } : f; }
};
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var g = generator.apply(thisArg, _arguments || []), i, q = [];
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
function fulfill(value) { resume("next", value); }
function reject(value) { resume("throw", value); }
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ParquetEnvelopeBufferReader = exports.ParquetBufferReader = exports.ParquetBufferCursor = exports.ParquetEnvelopeReader = exports.ParquetReader = exports.ParquetCursor = void 0;
const codec_1 = require("./codec");
const Compression = require("./compression");
const schema_1 = require("./schema");
const Shred = require("./shred");
// tslint:disable-next-line:max-line-length
const thrift_1 = require("./thrift");
const Util = require("./util");
const concatValueArrays_1 = require("./concatValueArrays");
const util_1 = require("./util");
const shred_1 = require("./shred");
/**
* Parquet File Magic String
*/
const PARQUET_MAGIC = 'PAR1';
/**
* Parquet File Format Version
*/
const PARQUET_VERSION = 1;
/**
* Internal type used for repetition/definition levels
*/
const PARQUET_RDLVL_TYPE = 'INT32';
const PARQUET_RDLVL_ENCODING = 'RLE';
/**
* A parquet cursor is used to retrieve rows from a parquet file in order
*/
class ParquetCursor {
/**
* Create a new parquet reader from the file metadata and an envelope reader.
* It is usually not recommended to call this constructor directly except for
* advanced and internal use cases. Consider using getCursor() on the
* ParquetReader instead
*/
constructor(metadata, envelopeReader, schema, columnList) {
this.metadata = metadata;
this.envelopeReader = envelopeReader;
this.schema = schema;
this.columnList = columnList;
this.rowGroup = [];
this.rowGroupIndex = 0;
this.cursorIndex = 0;
}
/**
* Retrieve the next row from the cursor. Returns a row or NULL if the end
* of the file was reached
*/
async next() {
if (this.cursorIndex >= this.rowGroup.length) {
if (this.rowGroupIndex >= this.metadata.row_groups.length) {
return null;
}
const rowBuffer = await this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
this.rowGroup = Shred.materializeRecords(this.schema, rowBuffer);
this.rowGroupIndex++;
this.cursorIndex = 0;
}
return this.rowGroup[this.cursorIndex++];
}
/**
* Rewind the cursor the the beginning of the file
*/
rewind() {
this.rowGroup = [];
this.rowGroupIndex = 0;
}
/**
* Implement AsyncIterable
*/
// tslint:disable-next-line:function-name
[Symbol.asyncIterator]() {
let done = false;
return {
next: async () => {
if (done) {
return { done, value: null };
}
const value = await this.next();
if (value === null) {
return { done: true, value };
}
return { done: false, value };
},
return: async () => {
done = true;
return { done, value: null };
},
throw: async () => {
done = true;
return { done: true, value: null };
},
};
}
}
exports.ParquetCursor = ParquetCursor;
/**
* A parquet reader allows retrieving the rows from a parquet file in order.
* The basic usage is to create a reader and then retrieve a cursor/iterator
* which allows you to consume row after row until all rows have been read. It is
* important that you call close() after you are finished reading the file to
* avoid leaking file descriptors.
*/
class ParquetReader {
/**
* Open the parquet file pointed to by the specified path and return a new
* parquet reader
*/
static async openFile(filePath) {
const envelopeReader = await ParquetEnvelopeReader.openFile(filePath);
try {
await envelopeReader.readHeader();
const metadata = await envelopeReader.readFooter();
return new ParquetReader(metadata, envelopeReader);
}
catch (err) {
await envelopeReader.close();
throw err;
}
}
static async openBuffer(buffer) {
const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
try {
await envelopeReader.readHeader();
const metadata = await envelopeReader.readFooter();
return new ParquetReader(metadata, envelopeReader);
}
catch (err) {
await envelopeReader.close();
throw err;
}
}
/**
* Create a new parquet reader from the file metadata and an envelope reader.
* It is not recommended to call this constructor directly except for advanced
* and internal use cases. Consider using one of the open{File,Buffer} methods
* instead
*/
constructor(metadata, envelopeReader) {
if (metadata.version !== PARQUET_VERSION) {
throw new Error('invalid parquet version');
}
this.metadata = metadata;
this.envelopeReader = envelopeReader;
const root = this.metadata.schema[0];
const { schema } = decodeSchema(this.metadata.schema, 1, root.num_children);
this.schema = new schema_1.ParquetSchema(schema);
}
getCursor(columnList) {
if (!columnList) {
// tslint:disable-next-line:no-parameter-reassignment
columnList = [];
}
// tslint:disable-next-line:no-parameter-reassignment
columnList = columnList.map(x => (Array.isArray(x) ? x : [x]));
return new ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
}
/**
* Get an iterable over a single column. The column is specified as an array of
* strings in order to support nested records.
*
* The path should not reference a nested record column.
*
* When a column is repeated the iterable will an array for each row.
*
* When a column is optional the iterable will produce null for any row missing
* the value.
*
* If a column is repeated and also nested inside another repeated object, then an array of arrays
* is returned for each row in the dataset.
*
* If a column is optional and also nested inside a repeated nested object, then it will be in an array
* where the array elements may be null.
*
* This means you can iterate multiple of these in parallel to walk multiple
* columns at once and they will stay in sync as long as the calls to next()
* are made in sync.
*
* @param columnPath
*/
getColumnValues(columnPath) {
return __asyncGenerator(this, arguments, function* getColumnValues_1() {
for (const rowGroup of this.metadata.row_groups) {
const colChunk = (0, util_1.findColumnChunk)(rowGroup, columnPath);
const data = yield __await(this.envelopeReader.readColumnChunk(this.schema, colChunk));
yield __await(yield* __asyncDelegator(__asyncValues((0, shred_1.materializeColumn)(this.schema, data, columnPath))));
}
});
}
/**
* Return the number of rows in this file. Note that the number of rows is
* not neccessarily equal to the number of rows in each column.
*/
getRowCount() {
return +this.metadata.num_rows;
}
/**
* Returns the ParquetSchema for this file
*/
getSchema() {
return this.schema;
}
/**
* Returns the user (key/value) metadata for this file
*/
getMetadata() {
const md = {};
for (const kv of this.metadata.key_value_metadata) {
md[kv.key] = kv.value;
}
return md;
}
/**
* Returns the column metadata for all columns.
*/
getColumnMetadata() {
const columnMetadata = {};
for (const rowGroup of this.metadata.row_groups) {
for (const columnChunk of rowGroup.columns) {
const columnPath = columnChunk.meta_data.path_in_schema.join('.');
if (!(columnPath in columnMetadata)) {
columnMetadata[columnPath] = [];
}
columnMetadata[columnPath].push(columnChunk.meta_data);
}
}
return columnMetadata;
}
/**
* Close this parquet reader. You MUST call this method once you're finished
* reading rows
*/
async close() {
await this.envelopeReader.close();
this.envelopeReader = null;
this.metadata = null;
}
/**
* Implement AsyncIterable
*/
// tslint:disable-next-line:function-name
[Symbol.asyncIterator]() {
return this.getCursor()[Symbol.asyncIterator]();
}
}
exports.ParquetReader = ParquetReader;
/**
* The parquet envelope reader allows direct, unbuffered access to the individual
* sections of the parquet file, namely the header, footer and the row groups.
* This class is intended for advanced/internal users; if you just want to retrieve
* rows from a parquet file use the ParquetReader instead
*/
class ParquetEnvelopeReader {
static async openFile(filePath) {
const fileStat = await Util.fstat(filePath);
const fileDescriptor = await Util.fopen(filePath);
const readFn = Util.fread.bind(undefined, fileDescriptor);
const closeFn = Util.fclose.bind(undefined, fileDescriptor);
return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);
}
/**
* Read parquet data from an in-memory buffer. This provides an asynchronous
* interface compatible with reading from a file.
*
* Note that you can also use ParquetEnvelopeBufferReader if you don't need your code to be able
* to handle files and buffers both. It may offer some performance benefit because it does not yield
* to the event loop in between operations.
*/
static async openBuffer(buffer) {
const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
const closeFn = () => Promise.resolve();
return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
}
constructor(read, close, fileSize) {
this.read = read;
this.close = close;
this.fileSize = fileSize;
}
async readHeader() {
const buf = await this.read(0, PARQUET_MAGIC.length);
if (buf.toString() !== PARQUET_MAGIC) {
throw new Error('not valid parquet file');
}
}
async readRowGroup(schema, rowGroup, columnList) {
const buffer = {
rowCount: +rowGroup.num_rows,
columnData: {},
};
for (const colChunk of rowGroup.columns) {
const colMetadata = colChunk.meta_data;
const colKey = colMetadata.path_in_schema;
if (columnList.length > 0 && Util.fieldIndexOf(columnList, colKey) < 0) {
continue;
}
buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
}
return buffer;
}
async readColumnChunk(schema, colChunk) {
if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
throw new Error('external references are not supported');
}
const pagesOffset = +colChunk.meta_data.data_page_offset;
const pagesSize = +colChunk.meta_data.total_compressed_size;
const pagesBuf = await this.read(pagesOffset, pagesSize);
return decodeColumnChunk(schema, colChunk, pagesBuf);
}
async readFooter() {
const trailerLen = PARQUET_MAGIC.length + 4;
const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
if (trailerBuf.slice(4).toString() !== PARQUET_MAGIC) {
throw new Error('not a valid parquet file');
}
const metadataSize = trailerBuf.readUInt32LE(0);
const metadataOffset = this.fileSize - metadataSize - trailerLen;
if (metadataOffset < PARQUET_MAGIC.length) {
throw new Error('invalid metadata size');
}
const metadataBuf = await this.read(metadataOffset, metadataSize);
// let metadata = new parquet_thrift.FileMetaData();
// parquet_util.decodeThrift(metadata, metadataBuf);
const { metadata } = Util.decodeFileMetadata(metadataBuf);
return metadata;
}
}
exports.ParquetEnvelopeReader = ParquetEnvelopeReader;
/**
* A parquet cursor is used to retrieve rows from a parquet file in order
*/
class ParquetBufferCursor {
/**
* Create a new parquet reader from the file metadata and an envelope reader.
* It is usually not recommended to call this constructor directly except for
* advanced and internal use cases. Consider using getCursor() on the
* ParquetReader instead
*/
constructor(metadata, envelopeReader, schema, columnList) {
this.metadata = metadata;
this.envelopeReader = envelopeReader;
this.schema = schema;
this.columnList = columnList;
this.rows = [];
this.rowsIndex = 0;
this.cursorIndex = 0;
}
/**
* Retrieve the next row from the cursor. Returns a row or NULL if the end
* of the file was reached
*/
next() {
if (this.cursorIndex >= this.rows.length) {
if (this.rowsIndex >= this.metadata.row_groups.length) {
return null;
}
const rowBuffer = this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowsIndex], this.columnList);
this.rows = Shred.materializeRecords(this.schema, rowBuffer);
this.rowsIndex++;
this.cursorIndex = 0;
}
return this.rows[this.cursorIndex++];
}
/**
* Rewind the cursor the the beginning of the file
*/
rewind() {
this.rows = [];
this.rowsIndex = 0;
}
/**
* Implement Iterable
*/
// tslint:disable-next-line:function-name
[Symbol.iterator]() {
let done = false;
return {
next: () => {
if (done) {
return { done, value: null };
}
const value = this.next();
if (value === null) {
return { done: true, value };
}
return { done: false, value };
},
return: () => {
done = true;
return { done, value: null };
},
throw: () => {
done = true;
return { done: true, value: null };
},
};
}
}
exports.ParquetBufferCursor = ParquetBufferCursor;
/**
* A parquet reader allows retrieving the rows from a parquet file in order.
* The basic usage is to create a reader and then retrieve a cursor/iterator
* which allows you to consume row after row until all rows have been read. It is
* important that you call close() after you are finished reading the file to
* avoid leaking file descriptors.
*/
class ParquetBufferReader {
static openBuffer(buffer) {
return new ParquetBufferReader(buffer);
}
/**
* Create a new parquet reader from a buffer. This version of ParquetReader
* runs synchronously so it may be more efficient when reading from a Buffer.
*
* However, it doesn't have a compatible API with ParquetReader.
*/
constructor(buffer) {
this.buffer = buffer;
this.envelopeReader = new ParquetEnvelopeBufferReader(buffer);
this.metadata = this.envelopeReader.readFooter();
if (this.metadata.version !== PARQUET_VERSION) {
throw new Error('invalid parquet version');
}
const root = this.metadata.schema[0];
const { schema } = decodeSchema(this.metadata.schema, 1, root.num_children);
this.schema = new schema_1.ParquetSchema(schema);
}
getCursor(columnList) {
const normalizedColumnList = (columnList || []).map(x => Array.isArray(x) ? x : [x]);
return new ParquetBufferCursor(this.metadata, this.envelopeReader, this.schema, normalizedColumnList);
}
/**
* Get an iterable over a single column. The column is specified as an array of
* strings in order to support nested records.
*
* The path should not reference a nested record column.
*
* When a column is repeated the iterable will an array for each row.
*
* When a column is optional the iterable will produce null for any row missing
* the value.
*
* If a column is repeated and also nested inside another repeated object, then an array of arrays
* is returned for each row in the dataset.
*
* If a column is optional and also nested inside a repeated nested object, then it will be in an array
* where the array elements may be null.
*
* This means you can iterate multiple of these in parallel to walk multiple
* columns at once and they will stay in sync as long as the calls to next()
* are made in sync.
*
* @param columnPath
*/
*getColumnValues(columnPath) {
for (const rowGroup of this.metadata.row_groups) {
const colChunk = (0, util_1.findColumnChunk)(rowGroup, columnPath);
const data = this.envelopeReader.readColumnChunk(this.schema, colChunk);
yield* (0, shred_1.materializeColumn)(this.schema, data, columnPath);
}
}
/**
* Return the number of rows in this file. Note that the number of rows is
* not necessarily equal to the number of rows in each column.
*/
getRowCount() {
return +this.metadata.num_rows;
}
/**
* Returns the ParquetSchema for this file
*/
getSchema() {
return this.schema;
}
/**
* Returns the user (key/value) metadata for this file
*/
getMetadata() {
const md = {};
for (const kv of this.metadata.key_value_metadata) {
md[kv.key] = kv.value;
}
return md;
}
/**
* Implement Iterable
*/
// tslint:disable-next-line:function-name
[Symbol.iterator]() {
return this.getCursor()[Symbol.iterator]();
}
}
exports.ParquetBufferReader = ParquetBufferReader;
class ParquetEnvelopeBufferReader {
constructor(buffer) {
this.buffer = buffer;
}
read(offset, length) {
return this.buffer.slice(offset, offset + length);
}
readHeader() {
const buf = this.read(0, PARQUET_MAGIC.length);
if (buf.toString() !== PARQUET_MAGIC) {
throw new Error('not valid parquet file');
}
}
readRowGroup(schema, rowGroup, columnList) {
const buffer = {
rowCount: +rowGroup.num_rows,
columnData: {},
};
for (const colChunk of rowGroup.columns) {
const colMetadata = colChunk.meta_data;
const colKey = colMetadata.path_in_schema;
if (columnList.length > 0 && Util.fieldIndexOf(columnList, colKey) < 0) {
continue;
}
buffer.columnData[colKey.join()] = this.readColumnChunk(schema, colChunk);
}
return buffer;
}
readColumnChunk(schema, colChunk) {
if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
throw new Error('external references are not supported');
}
const pagesOffset = +colChunk.meta_data.data_page_offset;
const pagesSize = +colChunk.meta_data.total_compressed_size;
const pagesBuf = this.read(pagesOffset, pagesSize);
return decodeColumnChunk(schema, colChunk, pagesBuf);
}
readFooter() {
const trailerLen = PARQUET_MAGIC.length + 4;
const trailerBuf = this.read(this.buffer.length - trailerLen, trailerLen);
if (trailerBuf.slice(4).toString() !== PARQUET_MAGIC) {
throw new Error('not a valid parquet file');
}
const metadataSize = trailerBuf.readUInt32LE(0);
const metadataOffset = this.buffer.length - metadataSize - trailerLen;
if (metadataOffset < PARQUET_MAGIC.length) {
throw new Error('invalid metadata size');
}
const metadataBuf = this.read(metadataOffset, metadataSize);
// let metadata = new parquet_thrift.FileMetaData();
// parquet_util.decodeThrift(metadata, metadataBuf);
const { metadata } = Util.decodeFileMetadata(metadataBuf);
return metadata;
}
}
exports.ParquetEnvelopeBufferReader = ParquetEnvelopeBufferReader;
/**
* Decode column chunk
*
* This calculates the field type and compression setting using
* the schema and column chunk metadata and calls decodeDataPages
* to do the heavy lifting.
*/
function decodeColumnChunk(schema, colChunk, pagesBuf) {
const field = schema.findField(colChunk.meta_data.path_in_schema);
const type = Util.getThriftEnum(thrift_1.Type, colChunk.meta_data.type);
if (type !== field.primitiveType) {
throw new Error('chunk type not matching schema: ' + type);
}
const compression = Util.getThriftEnum(thrift_1.CompressionCodec, colChunk.meta_data.codec);
const numValues = +colChunk.meta_data.num_values;
return decodeDataPages(pagesBuf, field, compression, numValues);
}
/**
* Decode a consecutive array of data using one of the parquet encodings
*/
function decodeValues(type, encoding, cursor, count, opts) {
if (!(encoding in codec_1.PARQUET_CODEC)) {
throw new Error(`invalid encoding: ${encoding}`);
}
return codec_1.PARQUET_CODEC[encoding].decodeValues(type, cursor, count, opts);
}
function decodeDataPages(buffer, column, compression, numValues) {
const cursor = {
buffer,
offset: 0,
size: buffer.length,
};
const rLevelPages = [];
const dLevelPages = [];
const valuePages = [];
let count = 0;
while (cursor.offset < cursor.size) {
// Stop once we have decoded all expected values (guards against trailing
// bytes from dictionary pages being included in total_compressed_size).
if (numValues !== undefined && count >= numValues) {
break;
}
// const pageHeader = new parquet_thrift.PageHeader();
// cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer);
const { pageHeader, length } = Util.decodePageHeader(cursor.buffer, cursor.offset);
cursor.offset += length;
const pageType = Util.getThriftEnum(thrift_1.PageType, pageHeader.type);
if (pageType !== 'DATA_PAGE_V2' && pageType !== 'DATA_PAGE') {
throw new Error(`Unsupported data page type ${pageType}`);
}
// Record cursor position before decoding so we can advance past the full
// page body regardless of how the decoders advance the cursor internally.
const pageEnd = cursor.offset + pageHeader.compressed_page_size;
const pageData = pageType === 'DATA_PAGE_V2'
? decodeDataPageV2(cursor, pageHeader, column, compression)
: decodeDataPage(cursor, pageHeader, column, compression);
// For UNCOMPRESSED data, decoders advance cursor.offset as they read;
// for COMPRESSED data, decodeDataPage/V2 already sets cursor.offset = cursorEnd.
// In either case, ensure we are positioned at the start of the next page.
cursor.offset = pageEnd;
rLevelPages.push(pageData.rLevels);
dLevelPages.push(pageData.dLevels);
valuePages.push(pageData.values);
count += pageData.count;
}
return {
rLevels: (0, concatValueArrays_1.default)(rLevelPages),
dLevels: (0, concatValueArrays_1.default)(dLevelPages),
values: (0, concatValueArrays_1.default)(valuePages),
count,
};
}
function decodeDataPage(cursor, header, column, compression) {
const cursorEnd = cursor.offset + header.compressed_page_size;
const valueCount = header.data_page_header.num_values;
// uncompress page
let dataCursor = cursor;
if (compression !== 'UNCOMPRESSED') {
const valuesBuf = Compression.inflate(compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
dataCursor = {
buffer: valuesBuf,
offset: 0,
size: valuesBuf.length,
};
cursor.offset = cursorEnd;
}
// read repetition levels
const rLevelEncoding = Util.getThriftEnum(thrift_1.Encoding, header.data_page_header.repetition_level_encoding);
// tslint:disable-next-line:prefer-array-literal
const rLevels = column.rLevelMax > 0
? decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
bitWidth: Util.getBitWidth(column.rLevelMax),
disableEnvelope: false,
// column: opts.column
})
: new Int32Array(valueCount);
// read definition levels
const dLevelEncoding = Util.getThriftEnum(thrift_1.Encoding, header.data_page_header.definition_level_encoding);
// tslint:disable-next-line:prefer-array-literal
const dLevels = column.dLevelMax > 0
? decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
bitWidth: Util.getBitWidth(column.dLevelMax),
disableEnvelope: false,
// column: opts.column
})
: new Int32Array(valueCount);
let valueCountNonNull = 0;
for (const dlvl of dLevels) {
if (dlvl === column.dLevelMax) {
valueCountNonNull++;
}
}
/* read values */
const valueEncoding = Util.getThriftEnum(thrift_1.Encoding, header.data_page_header.encoding);
const values = decodeValues(column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, {
typeLength: column.typeLength,
bitWidth: column.typeLength,
});
return {
dLevels,
rLevels,
values,
count: valueCount,
};
}
function decodeDataPageV2(cursor, header, column, compression) {
const cursorEnd = cursor.offset + header.compressed_page_size;
const valueCount = header.data_page_header_v2.num_values;
const valueCountNonNull = valueCount - header.data_page_header_v2.num_nulls;
const valueEncoding = Util.getThriftEnum(thrift_1.Encoding, header.data_page_header_v2.encoding);
// read repetition levels
const rLevels = column.rLevelMax > 0
? decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
bitWidth: Util.getBitWidth(column.rLevelMax),
disableEnvelope: true,
})
: new Int32Array(valueCount);
// read definition levels
const dLevels = column.dLevelMax > 0
? decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
bitWidth: Util.getBitWidth(column.dLevelMax),
disableEnvelope: true,
})
: new Int32Array(valueCount);
/* read values */
let valuesBufCursor = cursor;
if (header.data_page_header_v2.is_compressed) {
const valuesBuf = Compression.inflate(compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
valuesBufCursor = {
buffer: valuesBuf,
offset: 0,
size: valuesBuf.length,
};
cursor.offset = cursorEnd;
}
const values = decodeValues(column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, {
typeLength: column.typeLength,
bitWidth: column.typeLength,
});
return {
dLevels,
rLevels,
values,
count: valueCount,
};
}
function decodeSchema(schemaElements, offset, len) {
const schema = {};
let next = offset;
for (let i = 0; i < len; i++) {
const schemaElement = schemaElements[next];
const repetitionType = next > 0
? Util.getThriftEnum(thrift_1.FieldRepetitionType, schemaElement.repetition_type)
: 'ROOT';
const optional = repetitionType === 'OPTIONAL';
const repeated = repetitionType === 'REPEATED';
if (schemaElement.num_children > 0) {
const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
next = res.next;
schema[schemaElement.name] = {
// type: undefined,
optional,
repeated,
fields: res.schema,
};
}
else {
let logicalType = Util.getThriftEnum(thrift_1.Type, schemaElement.type);
if (schemaElement.converted_type != null) {
logicalType = Util.getThriftEnum(thrift_1.ConvertedType, schemaElement.converted_type);
}
schema[schemaElement.name] = {
type: logicalType,
typeLength: schemaElement.type_length,
optional,
repeated,
};
next++;
}
}
return { schema, offset, next };
}
//# sourceMappingURL=reader.js.map