@flowlab/all/packages/data/src/loaders/fileLoader.ts

A cool library focusing on handling various flows

// src/loaders/fileLoader.ts
import * as fs from 'fs/promises';
import { ILoader, PipelineContext, FileTargetConfig } from '../core/interfaces';
import { ComponentError } from '../core/errors';
import { EOL } from 'os'; // End Of Line constant

// MARK: - FileLoader
export class FileLoader<TInput extends (string | object)> implements ILoader<TInput> {
    private config: FileTargetConfig;
    private writeStream: fs.FileHandle | null = null; // Keep handle open for append mode

    constructor(config: FileTargetConfig) {
         if (!config.path || !config.format) {
            throw new ComponentError('FileLoader requires "path" and "format" in config.');
        }
        this.config = config;
        this.config.mode = config.mode || 'overwrite';
    }

    // MARK: - initStream
    private async initializeStream(context: PipelineContext): Promise<void> {
        if (!this.writeStream) {
            const flags = this.config.mode === 'append' ? 'a' : 'w';
            context.logger.debug(`Opening file <span class="math-inline">\{this\.config\.path\} in mode '</span>{flags}'`);
            try {
                this.writeStream = await fs.open(this.config.path, flags);
            } catch (error: any) {
                throw new ComponentError(`Failed to open file ${this.config.path} for writing`, 'FileLoader', error);
            }
        }
    }

    // MARK: - loadBatch    
    async loadBatch(batch: TInput[], context: PipelineContext): Promise<void> {
        if (batch.length === 0) return;
        await this.initializeStream(context); // Ensure stream is open

        let contentToWrite = '';
        for (const item of batch) {
             if (this.config.format === 'json') {
                // Write JSON Lines format
                contentToWrite += JSON.stringify(item) + EOL;
             } else if (this.config.format === 'csv') {
                // Basic CSV - join array elements, or stringify object values
                // Consider a proper CSV library ('csv-stringify') for complex objects/quoting
                const line = Array.isArray(item) ? item.join(',') : typeof item === 'object' ? Object.values(item).join(',') : String(item);
                contentToWrite += line + EOL;
             } else { // text format
                contentToWrite += String(item) + EOL;
             }
        }

        try {
             await this.writeStream!.writeFile(contentToWrite, { encoding: this.config.encoding || 'utf-8' });
             context.logger.trace(`Wrote batch of ${batch.length} items to ${this.config.path}`);
        } catch (error: any) {
            context.logger.error({ err: error, file: this.config.path }, `Error writing batch to file`);
             throw new ComponentError(`Error writing to file ${this.config.path}`, 'FileLoader', error);
        }
        // Note: We keep the stream open in append mode until explicitly closed or pipeline ends.
        // Need a mechanism to close it (e.g., a `shutdown` method on the loader called by the pipeline).
    }

    // MARK: - shutdown
    // Optional: Add a shutdown method to close the file handle
    async shutdown(context: PipelineContext): Promise<void> {
        if (this.writeStream) {
            context.logger.debug(`Closing file handle for ${this.config.path}`);
            try {
                 await this.writeStream.close();
                 this.writeStream = null;
            } catch (error: any) {
                 context.logger.error({ err: error, file: this.config.path }, `Error closing file handle`);
            }
        }
    }
}