@flowlab/all/packages/data/src/loaders/fileDlqLoader.ts

A cool library focusing on handling various flows

// src/loaders/fileDlqLoader.ts
import * as fs from 'fs/promises';
import { ILoader, PipelineContext, FileTargetConfig, FailedItemInfo } from '../core/interfaces';
import { ComponentError } from '../core/errors';
import { EOL } from 'os';

// This FileTargetConfig is used specifically for the DLQ file path
export class FileDlqLoader implements ILoader<FailedItemInfo> {
    private config: FileTargetConfig;
    private fileHandle: fs.FileHandle | null = null;

    constructor(config: FileTargetConfig) {
        if (!config || !config.path) {
            throw new ComponentError('FileDlqLoader requires "path" in config.');
        }
        this.config = { ...config, format: 'json', mode: 'append' }; // Force JSON Lines append mode
    }

    private async initialize(context: PipelineContext): Promise<void> {
        if (!this.fileHandle) {
            context.logger.debug(`Initializing DLQ file writer for ${this.config.path}`);
            try {
                // Ensure directory exists
                await fs.mkdir(path.dirname(this.config.path), { recursive: true });
                this.fileHandle = await fs.open(this.config.path, 'a'); // Always append
            } catch (error: any) {
                 throw new ComponentError(`Failed to open DLQ file ${this.config.path} for writing`, 'FileDlqLoader', error);
            }
        }
    }

    async loadBatch(batch: FailedItemInfo[], context: PipelineContext): Promise<void> {
        if (batch.length === 0) return;
        await this.initialize(context); // Ensure file is open

        let contentToWrite = '';
        for (const item of batch) {
            contentToWrite += JSON.stringify(item) + EOL;
        }

        try {
             await this.fileHandle!.writeFile(contentToWrite, { encoding: this.config.encoding || 'utf-8' });
             context.logger.warn(`Wrote ${batch.length} failed item(s) to DLQ file: ${this.config.path}`);
        } catch (error: any) {
            context.logger.error({ err: error, file: this.config.path }, `Error writing batch to DLQ file`);
             throw new ComponentError(`Error writing to DLQ file ${this.config.path}`, 'FileDlqLoader', error);
        }
    }

    async shutdown(context: PipelineContext): Promise<void> {
         if (this.fileHandle) {
             context.logger.debug(`Closing DLQ file handle for ${this.config.path}`);
             try {
                 await this.fileHandle.close();
                 this.fileHandle = null;
             } catch (error: any) {
                  context.logger.error({ err: error, file: this.config.path }, `Error closing DLQ file handle`);
             }
         }
    }
}