s3-query-json/lib/index.js

Query (ND)JSON files on S3

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const aws_sdk_1 = require("aws-sdk");
const entities_1 = require("./entities");
const asserts_1 = require("./asserts");
exports.query = async (bucket, key, expression, opts = { documentType: entities_1.DocumentType.NDJSON, compressionType: 'NONE', delimiter: '\n' }) => {
    var _a, _b;
    asserts_1.assertIsString(bucket, `Bucket \`${bucket}\` should be a string`);
    asserts_1.assertIsString(key, `Key \`${key}\` should be a string`);
    asserts_1.assertIsString(expression, `Expression \`${expression}\` should be a string`);
    asserts_1.assertIsString(opts.delimiter, `Delimiter \`${opts.delimiter}\` should be a string`);
    asserts_1.assert(opts.delimiter.length === 1, `Delimiter must have length \`1\`, found ${opts.delimiter.length}`);
    asserts_1.optionalIn(opts.documentType, Object.keys(entities_1.DocumentType), `Unknown documentType \`${opts.documentType}\``);
    asserts_1.optionalIn(opts.compressionType, Object.keys(entities_1.CompressionType), `Unknown compressionType \`${opts.compressionType}\``);
    asserts_1.optionalIn(Object.keys(opts.scanRange || {}), ['start', 'end'], `In ScanRange only \`start\` and \`end\` are allowed, got ${JSON.stringify(opts.scanRange)}`);
    const s3 = new aws_sdk_1.S3();
    const request = {
        Bucket: bucket,
        Key: key,
        Expression: expression,
        ExpressionType: 'SQL',
        InputSerialization: {
            JSON: {
                Type: opts.documentType === entities_1.DocumentType.JSON ? 'DOCUMENT' : 'LINES'
            },
            CompressionType: opts.compressionType || entities_1.CompressionType.NONE
        },
        OutputSerialization: {
            JSON: {
                RecordDelimiter: opts.delimiter
            }
        },
        ScanRange: {
            Start: (_a = opts.scanRange) === null || _a === void 0 ? void 0 : _a.start,
            End: (_b = opts.scanRange) === null || _b === void 0 ? void 0 : _b.end
        }
    };
    let result;
    try {
        const { Payload } = await s3.selectObjectContent(request).promise();
        result = Payload;
    }
    catch (err) {
        return Promise.reject(err);
    }
    if (opts.stream === true) {
        return result;
    }
    return new Promise((resolve, reject) => {
        let data = '';
        result.on('data', ({ Records }) => {
            if (!Records) {
                return;
            }
            data += Records.Payload.toString();
        });
        result.on('error', err => reject(err));
        result.on('end', () => {
            const results = [];
            for (const record of data.split(opts.delimiter || '\n')) {
                if (!record.trim()) {
                    continue;
                }
                results.push(JSON.parse(record));
            }
            resolve(results);
        });
    });
};