@forzalabs/remora/engines/file/FileExporter.js

A powerful CLI tool for seamless data translation.

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Algo_1 = __importDefault(require("../../core/Algo"));
const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
const Environment_1 = __importDefault(require("../Environment"));
const FileContentBuilder_1 = __importDefault(require("./FileContentBuilder"));
class FileExporterClass {
    constructor() {
        this.export = (consumer, output, data) => __awaiter(this, void 0, void 0, function* () {
            (0, Affirm_1.default)(consumer, `Invalid consumer`);
            (0, Affirm_1.default)(output, `Invalid output`);
            (0, Affirm_1.default)(data, `Invalid export data`);
            const preparedData = this._prepareData(consumer, output, data);
            const writeRes = yield this._writeData(consumer, output, preparedData);
            return writeRes;
        });
        this._prepareData = (consumer, output, data) => {
            (0, Affirm_1.default)(consumer, `Invalid consumer`);
            (0, Affirm_1.default)(output, `Invalid output`);
            (0, Affirm_1.default)(data, `Invalid export data`);
            let exportDataChunks = null;
            let extension = null;
            // build the actual file in the requested format
            switch (output.format.toUpperCase()) {
                case 'CSV': {
                    const lines = [];
                    const keys = Object.keys(data[0]);
                    lines.push(keys.map(x => `"${x}"`).join(','));
                    for (let i = 0; i < data.length; i++) {
                        const row = data[i];
                        const columns = keys.map(key => row[key]);
                        const rowValues = columns.map(c => { var _a; return Algo_1.default.replaceAll((_a = c === null || c === void 0 ? void 0 : c.toString()) !== null && _a !== void 0 ? _a : '', '"', '""'); }).map(c => `"${c}"`).join(',');
                        lines.push(rowValues);
                    }
                    exportDataChunks = FileContentBuilder_1.default.compose(lines, '\n');
                    extension = 'csv';
                    break;
                }
                case 'JSON': {
                    const lines = [];
                    for (let i = 0; i < data.length; i++) {
                        const row = data[i];
                        lines.push(JSON.stringify(row));
                    }
                    exportDataChunks = FileContentBuilder_1.default.compose(lines, '\n');
                    extension = 'jsonl';
                    break;
                }
                case 'PARQUET': {
                    throw new Error(`Consumer output "${output.format}" not implemented yet`);
                }
            }
            return { textChunks: exportDataChunks, extension };
        };
        this._writeData = (consumer, output, preparedData) => __awaiter(this, void 0, void 0, function* () {
            (0, Affirm_1.default)(consumer, `Invalid consumer`);
            (0, Affirm_1.default)(output, `Invalid output`);
            (0, Affirm_1.default)(preparedData, `Invalid prepared data`);
            // export it where it needs to go
            const source = Environment_1.default.getSource(output.exportDestination);
            (0, Affirm_1.default)(source, `Invalid consumer "${consumer.name}" export location source. Make sure that the export location is an available source.`);
            const { extension, textChunks } = preparedData;
            const name = this._composeFileName(consumer, extension);
            switch (source.engine) {
                case 'local': {
                    const driver = yield DriverFactory_1.default.instantiateDestination(source);
                    const res = yield driver.multipartUpload({ contents: textChunks, name });
                    return res.key;
                }
                case 'aws-s3': {
                    const driver = yield DriverFactory_1.default.instantiateDestination(source);
                    if (textChunks.length === 1) {
                        const res = yield driver.uploadFile({ content: textChunks[0], name });
                        return res.key;
                    }
                    else {
                        const res = yield driver.multipartUpload({ contents: textChunks, name });
                        return res.key;
                    }
                }
            }
        });
        this._composeFileName = (consumer, extension) => `${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-')}.${extension}`;
    }
}
const FileExporter = new FileExporterClass();
exports.default = FileExporter;