@flowlab/all/packages/data/src/loaders/mongoLoader.ts

A cool library focusing on handling various flows

// src/loaders/mongoLoader.ts
import { Collection as MongoCollection, Document, AnyBulkWriteOperation } from 'mongodb';
import { ILoader, PipelineContext, DatabaseTargetConfig, MongoConnection } from '../core/interfaces';
import { ComponentError } from '../core/errors';

export class MongoLoader<TInput extends Document> implements ILoader<TInput> {
    private config: DatabaseTargetConfig;
    private collection: MongoCollection<TInput>;
    private operation: 'insertMany' | 'bulkWrite';
    private generateBulkOps?: (item: TInput) => AnyBulkWriteOperation<TInput>;

    constructor(config: DatabaseTargetConfig) {
        if (config.type !== 'mongodb' || !(config.connection as MongoConnection)?.db || !config.collection) {
             throw new ComponentError('MongoLoader requires config type "mongodb", "collection" name, and "connection" object with "db" (MongoDB Db instance).');
        }
        this.config = config;
        const mongoConn = config.connection as MongoConnection;
        this.collection = mongoConn.db.collection<TInput>(config.collection);
        this.operation = config.mongoOperation || 'insertMany';

        if (this.operation === 'bulkWrite') {
             if (typeof config.bulkWriteOperations !== 'function') {
                 throw new ComponentError('MongoLoader with operation "bulkWrite" requires a "bulkWriteOperations" function in config.');
             }
             this.generateBulkOps = config.bulkWriteOperations;
        }
    }

    async loadBatch(batch: TInput[], context: PipelineContext): Promise<void> {
        if (batch.length === 0) return;

        context.logger.debug(`Loading batch of ${batch.length} items to MongoDB collection ${this.collection.collectionName} using ${this.operation}`);

        try {
            if (this.operation === 'insertMany') {
                // Ordered: false allows inserts to continue even if some documents fail (e.g., duplicate key)
                const result = await this.collection.insertMany(batch, { ordered: false });
                context.logger.info(`MongoDB insertMany inserted ${result.insertedCount} documents (requested: ${batch.length}).`);
                 if (result.insertedCount !== batch.length) {
                    context.logger.warn(`MongoDB insertMany: Not all documents were inserted (<span class="math-inline">\{result\.insertedCount\}/</span>{batch.length}). Check MongoDB logs for potential duplicate key errors.`);
                    // Note: result.insertedIds only contains IDs of successfully inserted docs.
                    // Identifying specific failures requires more complex error handling or trying inserts one-by-one.
                }
            } else if (this.operation === 'bulkWrite' && this.generateBulkOps) {
                const operations: AnyBulkWriteOperation<TInput>[] = batch.map(this.generateBulkOps);
                if (operations.length > 0) {
                     // Ordered: false allows operations to continue on error
                    const result = await this.collection.bulkWrite(operations, { ordered: false });
                    context.logger.info(
                        `MongoDB bulkWrite result: inserted=<span class="math-inline">\{result\.insertedCount\}, matched\=</span>{result.matchedCount}, modified=<span class="math-inline">\{result\.modifiedCount\}, upserted\=</span>{result.upsertedCount}`
                     );
                     if (result.hasWriteErrors()) {
                         context.logger.warn({ errors: result.getWriteErrors() }, `MongoDB bulkWrite encountered errors.`);
                         // Handle or log errors as needed
                     }
                } else {
                     context.logger.info('MongoDB bulkWrite: No operations generated for the batch.');
                }
            }
        } catch (error: any) {
             context.logger.error({ err: error }, `Error loading batch to MongoDB collection ${this.collection.collectionName}`);
             // Rethrow error to be caught by pipeline retry logic
             throw new ComponentError(`Failed to load batch to MongoDB collection ${this.collection.collectionName}`, 'MongoLoader', error);
        }
    }
}