ffc-pay-etl-framework/dist/cjs/destinations/postgresDestination.js

A framework for creating ETL pipelines in node

const EventEmitter = require('node:events');
const { Transform } = require('node:stream');
const debug = require('debug')('destination');
const DEFAULT_PORT = 5432;
function isKeyWord(column) {
    return ['USER'].includes(column.toUpperCase());
}
function getMappingForColumn(mapping, column) {
    if (mapping.length === 0) {
        return {};
    }
    const [map] = mapping.filter(m => m.column === column);
    return map;
}
function hasReturningColumns(mapping) {
    return mapping.filter(m => m.returning === true).length > 0;
}
function getReturningColumns(mapping) {
    return mapping.filter(m => m.returning === true).map(m => m === null || m === void 0 ? void 0 : m.targetColumn);
}
function writeInsertStatement(columnMapping, table, chunk, schema, ignoredColumns = []) {
    const filteredColumns = chunk._columns.filter(column => {
        const mapping = getMappingForColumn(columnMapping, column);
        const targetColumn = (mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn) ? mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn : mapping === null || mapping === void 0 ? void 0 : mapping.column;
        return !ignoredColumns.includes(targetColumn);
    });
    let statement = `INSERT INTO ${schema !== null && schema !== void 0 ? schema : 'public'}."${table}" (${filteredColumns.map(column => {
        const mapping = getMappingForColumn(columnMapping, column);
        return (mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn) ? `"${mapping.targetColumn}"` : `"${mapping === null || mapping === void 0 ? void 0 : mapping.column}"`;
    })
        .join(",")}) VALUES (${filteredColumns.map(column => {
        const index = chunk._columns.indexOf(column);
        const mapping = getMappingForColumn(columnMapping, column);
        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
            debug('Source data is not a number.');
            return 0;
        }
        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "varchar" || (mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "char") {
            return `'${chunk[index]}'`;
        }
        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "date") {
            if (!chunk[index]) {
                return `''`;
            }
            return `to_timestamp('${chunk[index]}','${mapping === null || mapping === void 0 ? void 0 : mapping.format}')`;
        }
        return chunk[index] ? chunk[index] : 'null';
    })})`;
    if (hasReturningColumns(columnMapping)) {
        statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`;
    }
    return statement;
}
/**
 *
 * @param {Object} options
 * @param {Object} options.table
 * @param {Object} options.connectionname
 * @param {Object} options.mapping
 * @param {Object} options.includeErrors
 * @param {String} [options.schema]
 * @param {Array<String>} [options.ignoredColumns]
 * @returns Transform
 */
function PostgresDestination(options) {
    var _a;
    EventEmitter.call(this);
    const table = options.table;
    const connectionname = options.connectionname;
    const mapping = options.mapping;
    const includeErrors = options.includeErrors;
    const schema = options.schema;
    const ignoredColumns = (_a = options.ignoredColumns) !== null && _a !== void 0 ? _a : [];
    let lastChunk;
    const transform = new Transform({
        objectMode: true,
        emitClose: true,
        construct(callback) {
            // @ts-ignore
            this.connectionname = connectionname;
            callback();
        },
        write(chunk, _, callback) {
            let insertStatement;
            // @ts-ignore
            if (chunk.errors.length === 0 || options.includeErrors) {
                insertStatement = writeInsertStatement(mapping, table, chunk, schema, ignoredColumns);
                debug('Insert statement: [%s]', insertStatement);
                // @ts-ignore
                this.connection.db.query(insertStatement)
                    .then(result => {
                    var _a;
                    debug('result %o', result);
                    chunk._result = result;
                    lastChunk = chunk;
                    // @ts-ignore
                    (_a = this.tasks) === null || _a === void 0 ? void 0 : _a.forEach(task => task.write(chunk));
                    // @ts-ignore
                    callback(null, chunk);
                }).catch(error => {
                    debug('error %o', error);
                    chunk.errors.push(error);
                    lastChunk = chunk;
                    // @ts-ignore
                    callback(error, chunk);
                });
            }
            else {
                debug('Chunk has errors %o', chunk);
            }
        },
        final(callback) {
            this.emit('result', lastChunk);
            callback();
        }
    });
    Object.assign(Transform.prototype, {
        type: 'PostgresDestination',
        setConnection: function (connection) {
            this.connection = connection;
        }.bind(transform),
        getConnectionName: function () {
            var _a;
            return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
        }.bind(transform),
        setTasks: function (tasks) {
            this.tasks = tasks;
        }.bind(transform)
    });
    return transform;
}
module.exports = {
    PostgresDestination,
    writeInsertStatement,
    isKeyWord,
    getMappingForColumn,
    hasReturningColumns,
    getReturningColumns
};