@dobesv/parquets
Version:
TypeScript implementation of the Parquet file format, based on parquet.js
215 lines • 6.42 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.load = exports.findColumnChunk = exports.stringArraysEqual = exports.fieldIndexOf = exports.osopen = exports.osclose = exports.oswrite = exports.fclose = exports.fread = exports.fstat = exports.fopen = exports.getThriftEnum = exports.getBitWidth = exports.decodePageHeader = exports.decodeFileMetadata = exports.decodeThrift = exports.serializeThrift = void 0;
const fs = require("fs");
const thrift_1 = require("thrift");
const thrift_2 = require("./thrift");
class UFramedTransport extends thrift_1.TFramedTransport {
}
/**
* Helper function that serializes a thrift object into a buffer
*/
function serializeThrift(obj) {
const output = [];
const transport = new thrift_1.TBufferedTransport(null, buf => {
output.push(buf);
});
const protocol = new thrift_1.TCompactProtocol(transport);
obj.write(protocol);
transport.flush();
return Buffer.concat(output);
}
exports.serializeThrift = serializeThrift;
function decodeThrift(obj, buf, offset) {
if (!offset) {
// tslint:disable-next-line:no-parameter-reassignment
offset = 0;
}
const transport = new UFramedTransport(buf);
transport.readPos = offset;
const protocol = new thrift_1.TCompactProtocol(transport);
obj.read(protocol);
return transport.readPos - offset;
}
exports.decodeThrift = decodeThrift;
function decodeFileMetadata(buf, offset) {
if (!offset) {
// tslint:disable-next-line:no-parameter-reassignment
offset = 0;
}
const transport = new UFramedTransport(buf);
transport.readPos = offset;
const protocol = new thrift_1.TCompactProtocol(transport);
const metadata = thrift_2.FileMetaData.read(protocol);
return { length: transport.readPos - offset, metadata };
}
exports.decodeFileMetadata = decodeFileMetadata;
function decodePageHeader(buf, offset) {
if (!offset) {
// tslint:disable-next-line:no-parameter-reassignment
offset = 0;
}
const transport = new UFramedTransport(buf);
transport.readPos = offset;
const protocol = new thrift_1.TCompactProtocol(transport);
const pageHeader = thrift_2.PageHeader.read(protocol);
return { length: transport.readPos - offset, pageHeader };
}
exports.decodePageHeader = decodePageHeader;
/**
* Get the number of bits required to store a given value
*/
function getBitWidth(val) {
if (val === 0) {
return 0;
// tslint:disable-next-line:no-else-after-return
}
else {
return Math.ceil(Math.log2(val + 1));
}
}
exports.getBitWidth = getBitWidth;
/**
* FIXME not ideal that this is linear
*/
function getThriftEnum(klass, value) {
for (const k in klass) {
if (klass[k] === value) {
return k;
}
}
throw new Error('Invalid ENUM value');
}
exports.getThriftEnum = getThriftEnum;
function fopen(filePath) {
return new Promise((resolve, reject) => {
fs.open(filePath, 'r', (err, fd) => {
if (err) {
reject(err);
}
else {
resolve(fd);
}
});
});
}
exports.fopen = fopen;
function fstat(filePath) {
return new Promise((resolve, reject) => {
fs.stat(filePath, (err, stat) => {
if (err) {
reject(err);
}
else {
resolve(stat);
}
});
});
}
exports.fstat = fstat;
function fread(fd, position, length) {
const buffer = Buffer.alloc(length);
return new Promise((resolve, reject) => {
fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
if (err || bytesRead !== length) {
reject(err || Error('read failed'));
}
else {
resolve(buf);
}
});
});
}
exports.fread = fread;
function fclose(fd) {
return new Promise((resolve, reject) => {
fs.close(fd, err => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
}
exports.fclose = fclose;
function oswrite(os, buf) {
return new Promise((resolve, reject) => {
os.write(buf, err => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
}
exports.oswrite = oswrite;
function osclose(os) {
if ('close' in os) {
return new Promise((resolve, reject) => {
os.close((err) => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
}
return Promise.resolve();
}
exports.osclose = osclose;
function osopen(path, opts) {
return new Promise((resolve, reject) => {
const outputStream = fs.createWriteStream(path, opts);
outputStream.once('open', fd => resolve(outputStream));
outputStream.once('error', err => reject(err));
});
}
exports.osopen = osopen;
// Supports MQTT path wildcards
// + all immediate children
// # all descendents
function fieldIndexOf(arr, elem) {
for (let j = 0; j < arr.length; j++) {
if (arr[j].length > elem.length)
continue;
let m = true;
for (let i = 0; i < elem.length; i++) {
if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#')
continue;
if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#')
continue;
m = false;
break;
}
if (m)
return j;
}
return -1;
}
exports.fieldIndexOf = fieldIndexOf;
/**
* Compare two arrays of strings. Used to compare field paths.
*/
function stringArraysEqual(a, b) {
return a.length === b.length && a.every((aa, idx) => aa === b[idx]);
}
exports.stringArraysEqual = stringArraysEqual;
/**
* Find a column chunk in a row group matching the given field path
* @param name
*/
function findColumnChunk(rowGroup, name) {
return rowGroup.columns.find(cc => stringArraysEqual(cc.meta_data.path_in_schema, name));
}
exports.findColumnChunk = findColumnChunk;
function load(name) {
return (module || global)['require'](name);
}
exports.load = load;
//# sourceMappingURL=util.js.map