@flowlab/all/packages/data/src/config/loader.ts

A cool library focusing on handling various flows

// src/config/loader.ts
import * as fs from 'fs/promises';
import * as path from 'path';
import * as yaml from 'js-yaml';
import { DataPipeline } from '../core/pipeline';
import { PipelineConfig, PipelineStepConfig } from '../core/interfaces';
import { createExtractor, createLoader, createPipelineStep } from '../core/registry';
import { ConfigurationError } from '../core/errors';
import { Logger } from 'pino';
import { createLogger } from '../utils/logger';

/**
 * Loads pipeline configuration from a JSON or YAML file.
 * @param configPath Path to the configuration file.
 * @param logger Optional logger instance.
 * @returns A configured DataPipeline instance.
 */
export async function loadPipelineFromConfig(
    configPath: string,
    logger: Logger = createLogger()
): Promise<DataPipeline<any, any>> { // Returns pipeline with 'any' types, specific types are internal
    logger.info(`Loading pipeline configuration from: ${configPath}`);

    let rawConfig: string;
    try {
        rawConfig = await fs.readFile(configPath, 'utf-8');
    } catch (error: any) {
        throw new ConfigurationError(`Failed to read configuration file: ${configPath}`, configPath, error.message);
    }

    let config: PipelineConfig;
    try {
        const ext = path.extname(configPath).toLowerCase();
        if (ext === '.yaml' || ext === '.yml') {
            config = yaml.load(rawConfig) as PipelineConfig;
        } else if (ext === '.json') {
            config = JSON.parse(rawConfig);
        } else {
            throw new ConfigurationError(`Unsupported configuration file format: ${ext}. Use .json or .yaml/.yml.`);
        }
        logger.debug({ config }, 'Configuration loaded successfully.');
        // TODO: Add validation using Zod or similar schema validator here
    } catch (error: any) {
        throw new ConfigurationError(`Failed to parse configuration file: ${configPath}`, configPath, error.message);
    }

    if (!config || !config.id || !config.source || !config.steps || !config.target) {
         throw new ConfigurationError(`Invalid configuration structure in ${configPath}. Missing required fields (id, source, steps, target).`);
    }

    const pipeline = new DataPipeline<any, any>(config.id, { logger });

    // Configure pipeline options
    if (config.options) {
        pipeline.configure({
            batchSize: config.options.batchSize,
            retries: config.options.errorHandling?.retries,
            retryDelay: config.options.errorHandling?.delay,
            // Pass other options if DataPipeline supports them
        });
        logger.info({ options: config.options }, 'Pipeline options configured.');
    }

    // Instantiate and set extractor
    try {
        const extractor = createExtractor(config.source);
        pipeline.extract(extractor);
        logger.info(`Extractor '${config.source.type}' configured.`);
    } catch (error) {
         if (error instanceof ConfigurationError || error instanceof ComponentError) throw error;
         throw new ComponentError(`Error configuring extractor from config`, 'Extractor', error as Error);
    }


    // Instantiate and add steps
    try {
        config.steps.forEach((stepConfig, index) => {
            const stepComponent = createPipelineStep(stepConfig);
            // The pipeline's chain methods don't retain specific types well here,
            // so we cast internally. The actual execution relies on the component interfaces.
            if ('clean' in stepComponent) {
                pipeline.clean(stepComponent as any);
                 logger.info(`Step ${index + 1}: Cleaner '${stepConfig.cleaner || stepConfig.type}' configured.`);
            } else if ('transform' in stepComponent) {
                pipeline.transform(stepComponent as any);
                 logger.info(`Step ${index + 1}: Transformer '${stepConfig.transformer || stepConfig.type}' configured.`);
            } else {
                 // Should not happen if createPipelineStep is correct
                 throw new ConfigurationError(`Invalid component returned for step ${index + 1}`);
            }
        });
    } catch (error) {
         if (error instanceof ConfigurationError || error instanceof ComponentError) throw error;
         throw new ComponentError(`Error configuring pipeline steps from config`, 'Steps', error as Error);
    }

    // Instantiate and set loader
    try {
        const loader = createLoader(config.target);
        pipeline.load(loader);
        logger.info(`Loader '${config.target.type}' configured.`);
    } catch (error) {
         if (error instanceof ConfigurationError || error instanceof ComponentError) throw error;
         throw new ComponentError(`Error configuring loader from config`, 'Loader', error as Error);
    }


    logger.info(`Pipeline '${config.id}' successfully created from configuration.`);
    return pipeline;
}