@forzalabs/remora/engines/consumer/ConsumerOnFinishManager.js

A powerful CLI tool for seamless data translation.

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Environment_1 = __importDefault(require("../Environment"));
const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
class ConsumerOnFinishManagerClass {
    constructor() {
        this.onConsumerSuccess = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
            void executionId;
            for (const output of consumer.outputs) {
                if (output.onSuccess)
                    yield this.performOnSuccessActions(consumer, output);
            }
        });
        this.onConsumerError = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
            void executionId;
            for (const output of consumer.outputs) {
                if (output.onError)
                    yield this.performOnSuccessActions(consumer, output);
            }
        });
        this.performOnSuccessActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
            (0, Affirm_1.default)(consumer, 'Invalid consumer');
            (0, Affirm_1.default)(output, 'Invalid output');
            if (!output.onSuccess || output.onSuccess.length === 0)
                return;
            for (const onSuccess of output.onSuccess) {
                switch (onSuccess.action) {
                    case 'move-file': {
                        yield this.moveSourceFiles(consumer, onSuccess.moveToDestination);
                        break;
                    }
                    default:
                        throw new Error(`On success action "${onSuccess.action}" is not implemented yet.`);
                }
            }
        });
        this.moveSourceFiles = (consumer, moveDestination) => __awaiter(this, void 0, void 0, function* () {
            (0, Affirm_1.default)(moveDestination, 'Move destination is required for move-file action');
            const destinationSource = Environment_1.default.getSource(moveDestination);
            (0, Affirm_1.default)(destinationSource, `Destination source "${moveDestination}" not found`);
            // Get all unique source files from all producers used by this consumer
            const sourceFilesToMove = [];
            for (const consumerProducer of consumer.producers) {
                const producer = Environment_1.default.getProducer(consumerProducer.name);
                (0, Affirm_1.default)(producer, `Producer "${consumerProducer.name}" not found`);
                const source = Environment_1.default.getSource(producer.source);
                (0, Affirm_1.default)(source, `Source "${producer.source}" not found`);
                // Only handle file-based sources that have fileKey
                if (producer.settings.fileKey) {
                    // Handle wildcard patterns (files with %)
                    if (producer.settings.fileKey.includes('%')) {
                        // Get all files matching the pattern
                        const matchingFiles = yield this.getMatchingFiles(source, producer.settings.fileKey);
                        for (const fileKey of matchingFiles) {
                            sourceFilesToMove.push({
                                sourceName: producer.source,
                                filePath: this.getSourceFilePath(source, fileKey),
                                fileKey
                            });
                        }
                    }
                    else {
                        // Single file
                        sourceFilesToMove.push({
                            sourceName: producer.source,
                            filePath: this.getSourceFilePath(source, producer.settings.fileKey),
                            fileKey: producer.settings.fileKey
                        });
                    }
                }
            }
            // Move all files to destination
            yield this.moveFiles(sourceFilesToMove, destinationSource);
        });
        this.getMatchingFiles = (source, fileKeyPattern) => __awaiter(this, void 0, void 0, function* () {
            const sourceDriver = yield DriverFactory_1.default.instantiateSource(source);
            if (source.engine === 'aws-s3') {
                // S3 driver has a public listFiles method that handles patterns
                const s3Driver = sourceDriver;
                return yield s3Driver.listFiles(fileKeyPattern);
            }
            else if (source.engine === 'local') {
                // Local driver now has a public listFiles method consistent with S3
                const localDriver = sourceDriver;
                return localDriver.listFiles(fileKeyPattern);
            }
            throw new Error(`Unsupported source engine for file listing: ${source.engine}`);
        });
        this.getSourceFilePath = (source, fileKey) => {
            if (source.engine === 'local') {
                return `${source.authentication['path']}/${fileKey}`;
            }
            else if (source.engine === 'aws-s3') {
                // For S3, we return the key as the path since S3 uses keys instead of file paths
                return fileKey;
            }
            throw new Error(`Unsupported source engine for file move: ${source.engine}`);
        };
        this.moveFiles = (files, destinationSource) => __awaiter(this, void 0, void 0, function* () {
            for (const file of files) {
                const sourceSource = Environment_1.default.getSource(file.sourceName);
                if (sourceSource.engine === 'local' && destinationSource.engine === 'local') {
                    // Local to Local move
                    yield this.moveLocalToLocal(file.filePath, destinationSource, file.fileKey);
                }
                else if (sourceSource.engine === 'local' && destinationSource.engine === 'aws-s3') {
                    // Local to S3 move
                    yield this.moveLocalToS3(file.filePath, destinationSource, file.fileKey);
                }
                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'local') {
                    // S3 to Local move
                    yield this.moveS3ToLocal(sourceSource, file.fileKey, destinationSource);
                }
                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'aws-s3') {
                    // S3 to S3 move
                    yield this.moveS3ToS3(sourceSource, file.fileKey, destinationSource);
                }
                else {
                    throw new Error(`Unsupported move operation from ${sourceSource.engine} to ${destinationSource.engine}`);
                }
            }
        });
        this.moveLocalToLocal = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
            // Read file from source
            const fileContent = sourceDriver.readFile(fileKey);
            // Save to destination
            yield destinationDriver.saveFile(fileKey, fileContent);
            // Delete from source
            sourceDriver.deleteFile(fileKey);
        });
        this.findSourceForPath = (filePath) => {
            // Extract directory from file path for source creation
            const directory = filePath.substring(0, filePath.lastIndexOf('/'));
            return {
                name: 'temp-source',
                engine: 'local',
                authentication: { path: directory }
            };
        };
        this.moveLocalToS3 = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
            // Read file from local source
            const fileContent = sourceDriver.readFile(fileKey);
            // Upload to S3 destination
            yield destinationDriver.saveFile(fileKey, fileContent);
            // Remove source file after successful upload
            sourceDriver.deleteFile(fileKey);
        });
        this.moveS3ToLocal = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
            // Download from S3
            const content = yield sourceDriver.downloadFile(fileKey);
            // Save to local destination
            yield destinationDriver.saveFile(fileKey, content);
            // Delete from S3 source
            yield sourceDriver.deleteFile(fileKey);
        });
        this.moveS3ToS3 = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
            // Copy from source S3 to destination S3
            yield destinationDriver.copyFromS3(sourceSource.authentication['bucket'], fileKey, fileKey);
            // Delete from source S3
            yield sourceDriver.deleteFile(fileKey);
        });
        this.performOnErrorActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
            (0, Affirm_1.default)(consumer, 'Invalid consumer');
            (0, Affirm_1.default)(output, 'Invalid output');
            if (!output.onError || output.onError.length === 0)
                return;
            for (const onError of output.onError) {
                switch (onError.action) {
                    case 'move-file': {
                        yield this.moveSourceFiles(consumer, onError.moveToDestination);
                        break;
                    }
                    default:
                        throw new Error(`On success action "${onError.action}" is not implemented yet.`);
                }
            }
        });
    }
}
const ConsumerOnFinishManager = new ConsumerOnFinishManagerClass();
exports.default = ConsumerOnFinishManager;