@forzalabs/remora/executors/OutputExecutor.js

A powerful CLI tool for seamless data translation.

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Algo_1 = __importDefault(require("../core/Algo"));
const DSTE_1 = __importDefault(require("../core/dste/DSTE"));
const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
const Environment_1 = __importDefault(require("../engines/Environment"));
const CSVParser_1 = __importDefault(require("../engines/parsing/CSVParser"));
const ExecutorScope_1 = __importDefault(require("./ExecutorScope"));
class OutputExecutorClass {
    constructor() {
        this._getInternalRecordFormat = (consumer) => {
            const output = consumer.outputs[0];
            const format = output.format === 'API'
                ? 'JSON'
                : output.format === 'PARQUET'
                    ? 'CSV'
                    : output.format;
            return format;
        };
        this.outputRecord = (record, consumer, fields) => {
            const format = this._getInternalRecordFormat(consumer);
            switch (format) {
                case 'CSV':
                    return this.toCSV(record, fields, ',');
                case 'JSON':
                    return this.toJSON(record, fields);
                default:
                    throw new Error(`Export format ${format} not implemented yet.`);
            }
        };
        this.toCSV = (record, fields, delimiter) => {
            const myDelimtier = delimiter !== null && delimiter !== void 0 ? delimiter : ',';
            // remove the not wanted dimension
            const line = fields
                .filter(x => !x.cField.hidden)
                .map(x => { var _a, _b; return `"${Algo_1.default.replaceAll((_b = (_a = record[x.finalKey]) === null || _a === void 0 ? void 0 : _a.toString()) !== null && _b !== void 0 ? _b : '', '"', '""')}"`; })
                .join(myDelimtier);
            return line;
        };
        this.toJSON = (record, fields) => {
            if (fields.some(x => x.cField.hidden)) {
                // remove the un-wanted dimensions
                for (const dim of fields) {
                    if (dim.cField.hidden)
                        delete record[dim.finalKey];
                }
                return JSON.stringify(record);
            }
            else {
                return JSON.stringify(record);
            }
        };
        this.exportResult = (consumer, fields, scope) => __awaiter(this, void 0, void 0, function* () {
            const internalFormat = this._getInternalRecordFormat(consumer);
            for (const output of consumer.outputs) {
                const destination = Environment_1.default.getSource(output.exportDestination);
                const driver = yield DriverFactory_1.default.instantiateDestination(destination);
                const currentPath = ExecutorScope_1.default.getMainPath(scope);
                const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
                if (output.format === internalFormat) {
                    return yield driver.move(currentPath, destinationName);
                }
                else {
                    switch (output.format) {
                        case 'CSV':
                            return yield driver.transformAndMove(currentPath, line => {
                                const parsed = JSON.parse(line);
                                return Object.keys(parsed).map(x => `"${parsed[x]}"`).join(',');
                            }, destinationName);
                        case 'API':
                        case 'JSON':
                            return yield driver.transformAndMove(currentPath, line => {
                                const parts = CSVParser_1.default.parseRow(line, ',');
                                const value = {};
                                for (const [index, field] of fields.entries())
                                    value[field.finalKey] = parts[index];
                                return JSON.stringify(value);
                            }, destinationName);
                        case 'PARQUET':
                        default:
                            throw new Error(`Export result to format ${output.format} not implemented yet.`);
                    }
                }
            }
        });
        this._getExtension = (output) => {
            return output.format === 'CSV'
                ? 'csv'
                : output.format === 'JSON'
                    ? 'jsonl'
                    : 'txt';
        };
        this._composeFileName = (consumer, output, extension, executionId) => {
            if (output.exportName && output.exportName.trim().length > 0) {
                // Ensure no extension duplication
                const sanitized = output.exportName.replace(/\.[^.]+$/, '');
                return `${sanitized}.${extension}`;
            }
            const baseTs = Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-');
            const suffix = executionId ? `_${executionId}` : '';
            return `${consumer.name}_${baseTs}${suffix}.${extension}`;
        };
    }
}
const OutputExecutor = new OutputExecutorClass();
exports.default = OutputExecutor;