@dobesv/parquets
Version:
TypeScript implementation of the Parquet file format, based on parquet.js
280 lines • 10.3 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.decodeValues = exports.encodeValues = void 0;
const INT53 = require("int53");
const systemIsLittleEndian = new DataView(new Int32Array([1]).buffer).getInt32(0, true) === 1;
function encodeValues(type, values, opts) {
switch (type) {
case 'BOOLEAN':
return encodeValues_BOOLEAN(values);
case 'INT32':
return encodeValues_INT32(values);
case 'INT64':
return encodeValues_INT64(values);
case 'INT96':
return encodeValues_INT96(values);
case 'FLOAT':
return encodeValues_FLOAT(values);
case 'DOUBLE':
return encodeValues_DOUBLE(values);
case 'BYTE_ARRAY':
return encodeValues_BYTE_ARRAY(values);
case 'FIXED_LEN_BYTE_ARRAY':
return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
default:
throw new Error(`unsupported type: ${type}`);
}
}
exports.encodeValues = encodeValues;
function decodeValues(type, cursor, count, opts) {
switch (type) {
case 'BOOLEAN':
return decodeValues_BOOLEAN(cursor, count);
case 'INT32':
return decodeValues_INT32(cursor, count);
case 'INT64':
return decodeValues_INT64(cursor, count);
case 'INT96':
return decodeValues_INT96(cursor, count);
case 'FLOAT':
return decodeValues_FLOAT(cursor, count);
case 'DOUBLE':
return decodeValues_DOUBLE(cursor, count);
case 'BYTE_ARRAY':
return decodeValues_BYTE_ARRAY(cursor, count);
case 'FIXED_LEN_BYTE_ARRAY':
return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
default:
throw new Error(`unsupported type: ${type}`);
}
}
exports.decodeValues = decodeValues;
/**
* Encode an array of booleans as a bit sequence.
*
* The resulting buffer will be rounded up in size to the nearest whole byte.
*
* If the parameter is not actually an array of booleans, "truthy" values will
* be written with 1, other values will be written as 0.
*/
function encodeValues_BOOLEAN(values) {
const buf = Buffer.alloc(Math.ceil(values.length / 8));
buf.fill(0);
for (let i = 0; i < values.length; i++) {
if (values[i]) {
buf[Math.floor(i / 8)] |= 1 << i % 8;
}
}
return buf;
}
/**
* Read a bit sequence from a buffer to an array of booleans.
*/
function decodeValues_BOOLEAN(cursor, count) {
const values = [];
for (let i = 0; i < count; i++) {
const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
values.push((b & (1 << i % 8)) > 0);
}
cursor.offset += Math.ceil(count / 8);
return values;
}
/**
* Encode INT32 values to binary.
*
* Note that if the input is not an array of number or an Int32Array
* this may throw an exception.
*/
function encodeValues_INT32(values) {
// On little-endian systems we can use typed array to avoid data copying
if (systemIsLittleEndian) {
const tab = values instanceof Int32Array
? values
: Int32Array.from(values);
return Buffer.from(tab.buffer.slice(tab.byteOffset, tab.byteLength));
}
const buf = Buffer.alloc(4 * values.length);
for (let i = 0; i < values.length; i++) {
buf.writeInt32LE(values[i], i * 4);
}
return buf;
}
/**
* Decode values into an Int32Array.
*/
function decodeValues_INT32(cursor, count) {
const values = systemIsLittleEndian && (cursor.buffer.byteOffset + cursor.offset) % 4 === 0
? // On little-endian systems we can just use the data as-is
new Int32Array(cursor.buffer.buffer, cursor.buffer.byteOffset + cursor.offset, count)
: // Otherwise we have to copy and convert the data
// tslint:disable-next-line:prefer-array-literal
Int32Array.from(new Array(count), (_, i) => cursor.buffer.readInt32LE(cursor.offset + i * Int32Array.BYTES_PER_ELEMENT));
cursor.offset += count * Int32Array.BYTES_PER_ELEMENT;
return values;
}
/**
* Encode INT64 values to a buffer.
*/
function encodeValues_INT64(values) {
const buf = Buffer.alloc(8 * values.length);
for (let i = 0; i < values.length; i++) {
INT53.writeInt64LE(values[i], buf, i * 8);
}
return buf;
}
/**
* Decode INT64 values from a buffer to an array of numbers
*/
function decodeValues_INT64(cursor, count) {
const values = [];
for (let i = 0; i < count; i++) {
values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));
cursor.offset += 8;
}
return values;
}
/**
* Encode INT96 values to a buffer
*/
function encodeValues_INT96(values) {
const buf = Buffer.alloc(12 * values.length);
for (let i = 0; i < values.length; i++) {
if (values[i] >= 0) {
INT53.writeInt64LE(values[i], buf, i * 12);
buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision
}
else {
INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);
buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision
}
}
return buf;
}
/**
* Decode INT96 values to an array of numbers
*/
function decodeValues_INT96(cursor, count) {
const values = [];
for (let i = 0; i < count; i++) {
const low = INT53.readInt64LE(cursor.buffer, cursor.offset);
const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
if (high === 0xffffffff) {
values.push(~-low + 1); // truncate to 64 actual precision
}
else {
values.push(low); // truncate to 64 actual precision
}
cursor.offset += 12;
}
return values;
}
/**
* Encode FLOAT values from an array of numbers or a Float32Array
*/
function encodeValues_FLOAT(values) {
// On little-endian systems we can use typed array
if (systemIsLittleEndian) {
const tab = values instanceof Float32Array
? values
: Float32Array.from(values);
return Buffer.from(tab.buffer.slice(tab.byteOffset, tab.byteLength));
}
const buf = Buffer.alloc(4 * values.length);
for (let i = 0; i < values.length; i++) {
buf.writeFloatLE(values[i], i * 4);
}
return buf;
}
/**
* Decode FLOAT values to a Float32Array
*/
function decodeValues_FLOAT(cursor, count) {
const values = systemIsLittleEndian && (cursor.buffer.byteOffset + cursor.offset) % 4 === 0
? // On little-endian systems with 4-byte aligned data we can avoid data copying
new Float32Array(cursor.buffer.buffer, cursor.buffer.byteOffset + cursor.offset, count)
: // Otherwise we have to copy and convert the data
// tslint:disable-next-line:prefer-array-literal
Float32Array.from(new Array(count), (_, i) => cursor.buffer.readFloatLE(cursor.offset + i * Float32Array.BYTES_PER_ELEMENT));
cursor.offset += count * Float32Array.BYTES_PER_ELEMENT;
return values;
}
/**
* Encode DOUBLE values from an array of numbers or a Float64Array.
*/
function encodeValues_DOUBLE(values) {
// On little-endian systems with 8-byte aligned data we can avoid data copying
if (systemIsLittleEndian) {
const tab = values instanceof Float64Array
? values
: Float64Array.from(values);
return Buffer.from(tab.buffer.slice(tab.byteOffset, tab.byteLength));
}
const buf = Buffer.alloc(8 * values.length);
for (let i = 0; i < values.length; i++) {
buf.writeDoubleLE(values[i], i * 8);
}
return buf;
}
function decodeValues_DOUBLE(cursor, count) {
const values = systemIsLittleEndian && (cursor.buffer.byteOffset + cursor.offset) % 8 === 0
? // On little-endian systems with 8-byte aligned data we can avoid data copying
new Float64Array(cursor.buffer.buffer, cursor.buffer.byteOffset + cursor.offset, count)
: // Otherwise we have to copy and convert the data
// tslint:disable-next-line:prefer-array-literal
Float64Array.from(new Array(count), (_, i) => cursor.buffer.readDoubleLE(cursor.offset + i * Float64Array.BYTES_PER_ELEMENT));
cursor.offset += count * Float64Array.BYTES_PER_ELEMENT;
return values;
}
function encodeValues_BYTE_ARRAY(values) {
// tslint:disable-next-line:variable-name
let buf_len = 0;
for (let i = 0; i < values.length; i++) {
const value = values[i];
const buf = (values[i] = Buffer.from(value));
buf_len += 4 + buf.length;
}
const buf = Buffer.alloc(buf_len);
// tslint:disable-next-line:variable-name
let buf_pos = 0;
for (let i = 0; i < values.length; i++) {
const value = values[i];
buf.writeUInt32LE(value.length, buf_pos);
value.copy(buf, buf_pos + 4);
buf_pos += 4 + value.length;
}
return buf;
}
function decodeValues_BYTE_ARRAY(cursor, count) {
const values = [];
for (let i = 0; i < count; i++) {
const len = cursor.buffer.readUInt32LE(cursor.offset);
cursor.offset += 4;
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
cursor.offset += len;
}
return values;
}
function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
if (!opts.typeLength) {
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
}
if (!values.every(val => val.length === opts.typeLength)) {
throw new Error('not all values for FIXED_LEN_BYTE_ARRAY have the correct length');
}
if (values.every(val => Buffer.isBuffer(val))) {
return Buffer.concat(values);
}
return Buffer.concat(values.map(val => (Buffer.isBuffer(val) ? val : Buffer.from(val))));
}
function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
const values = [];
if (!opts.typeLength) {
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
}
for (let i = 0; i < count; i++) {
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
cursor.offset += opts.typeLength;
}
return values;
}
//# sourceMappingURL=plain.js.map