@flowlab/all/packages/data/src/core/interfaces.ts

A cool library focusing on handling various flows

// src/core/interfaces.ts
import { Logger } from 'pino';
import { Stream } from 'stream'; // Node.js stream type
import { Db as MongoDb, Collection as MongoCollection, Filter as MongoFilter } from 'mongodb';
import { Redis as IORedisClient } from 'ioredis';
import { PrismaClient } from '@prisma/client'; // Already there
import {
    IFlowNode as IFlowNodeCore,
    FlowExecutionContext,
    FlowRegistry, // Assuming registry instance is imported elsewhere or passed
    InputPayload,
    OutputResult,
    NodeConfigSchema
} from '@flowlab/core'; // <--- ASSUMED IMPORT PATH AND TYPES

// --- Configuration Types ---
export interface BaseSourceConfig {
    type: string; // e.g., 'postgresql', 'mongodb', 'redis', 'mysql', 'file', 'api'
    connection?: any; // Connection details (varies by type)
    [key: string]: any; // Allow other options
}

export interface DatabaseSourceConfig extends BaseSourceConfig {
    type: 'postgresql' | 'mongodb' | 'redis' | 'mysql';
    connection: any; // Specific connection object (e.g., PrismaClient instance, connection string)
    query?: string | object | MongoFilter<any>; // SQL query or MongoDB query object
    batchSize?: number; // For batch extraction
}

export interface FileSourceConfig extends BaseSourceConfig {
    type: 'file';
    path: string;
    format: 'json' | 'csv' | 'text'; // Add more formats as needed
    encoding?: BufferEncoding;
}

export interface ApiSourceConfig extends BaseSourceConfig {
    type: 'api';
    url: string;
    method?: 'GET' | 'POST'; // etc.
    headers?: Record<string, string>;
    body?: any;
}

export type ProcessingErrorStrategy = 'fail' | 'skip' | 'log' | 'dlq';

export type SourceConfig = DatabaseSourceConfig | FileSourceConfig | ApiSourceConfig | BaseSourceConfig; // Union type

export type IFlowLabNode = IFlowNodeCore;


export interface BaseTargetConfig {
    type: string; // e.g., 'postgresql', 'mongodb', 'redis', 'mysql', 'file', 'api'
    connection?: any;
    [key: string]: any;
}

// MARK: - DatabaseTarget
export interface DatabaseTargetConfig extends BaseTargetConfig {
     type: 'postgresql' | 'mongodb' | 'redis' | 'mysql';
     connection: any; // e.g., PrismaClient
     table?: string; // Or collection name
     // Add options like 'upsert', 'update', etc.
}

// MARK: - FileTarget
export interface FileTargetConfig extends BaseTargetConfig {
    type: 'file';
    path: string;
    format: 'json' | 'csv' | 'text';
    encoding?: BufferEncoding;
    mode?: 'append' | 'overwrite'; // Default to overwrite?
}

// MARK: - ApiTarget
export interface ApiTargetConfig extends BaseTargetConfig {
    type: 'api';
    url: string;
    method?: 'POST' | 'PUT'; // etc.
    headers?: Record<string, string>;
}

// MARK: - Pipeline
export interface PipelineConfig {
    // ... id, source, steps, target ...
    options?: {
        batchSize?: number;
        concurrency?: number; // Added for parallel processing later
        errorHandling?: {
            retries?: number; // Retries for Extract/Load operations
            delay?: number; // ms
            itemProcessingErrorStrategy?: ProcessingErrorStrategy; // Strategy for step errors
            dlqTarget?: TargetConfig; // Configuration for the Dead Letter Queue loader
        };
    };
}

// MARK: - Connections
export interface PrismaConnection { client: PrismaClient, model: string };
export interface MongoConnection { db: MongoDb, collection: string };
export interface RedisConnection { client: IORedisClient }; 

// MARK: - DatabaseSource
export interface DatabaseSourceConfig extends BaseSourceConfig {
    type: 'postgresql' | 'mongodb' | 'redis' | 'mysql';
    // Use a union for more specific connection types based on 'type'
    connection: PrismaConnection | MongoConnection | RedisConnection | any; // Keep 'any' for flexibility if needed
    query?: string | MongoFilter<any> | object | undefined; // SQL query, MongoDB filter, or other object
    batchSize?: number;
    // Add specific options per DB type if necessary
    redisScanMatch?: string; // For Redis extractor SCAN pattern
    redisKeyType?: 'string' | 'hash' | 'list' // For Redis extractor
}

export interface DatabaseTargetConfig extends BaseTargetConfig {
    type: 'postgresql' | 'mongodb' | 'redis' | 'mysql';
    connection: PrismaConnection | MongoConnection | RedisConnection | any;
    table?: string; // SQL table, Redis key prefix?
    collection?: string; // Mongo collection
    // Add specific options per DB type
    mongoOperation?: 'insertMany' | 'bulkWrite'; // Default insertMany
    bulkWriteOperations?: (item: any) => any; // Function to generate bulkWrite ops array
    redisOperation?: 'set' | 'hmset' | 'lpush' | 'rpush' // etc.
    redisKeyField?: string; // Field in item to use as Redis key
}



export type TargetConfig = DatabaseTargetConfig | FileTargetConfig | ApiTargetConfig | BaseTargetConfig; // Union type


// MARK: - FailedItemInfo
export interface FailedItemInfo {
    pipelineId: string;
    runId: string;
    timestamp: string;
    step: string; // Name/type of step that failed
    error: {
        message: string;
        stack?: string;
        name?: string;
    };
    originalItem: any; // The item that failed processing
}


// --- Component Interfaces ---

/** Represents an object or stream that can yield data items */
export type DataSource<T> = T[] | AsyncIterable<T>;


// MARK: - PipelineContext
export interface PipelineContext {
    logger: Logger;
    runId: string; // Unique identifier for this pipeline run
    [key: string]: any; // Allow extensions
    errorStrategy?: ProcessingErrorStrategy;
    dlqLoader?: ILoader<FailedItemInfo>; // DLQ loader instance
}

/** Extracts data from a source */
export interface IExtractor<TOutput> {
    extract(context: PipelineContext): Promise<DataSource<TOutput>>;
}

/** Cleans a single data item */
export interface ICleaner<TInput> {
    clean(data: TInput, context: PipelineContext): Promise<TInput | null | undefined>; // Return null/undefined to filter out item
}

/** Transforms a single data item */
export interface ITransformer<TInput, TOutput> {
    transform(data: TInput, context: PipelineContext): Promise<TOutput>;
}

/** Loads a batch of data items to a target */
export interface ILoader<TInput> {
    loadBatch(batch: TInput[], context: PipelineContext): Promise<void>;
}

/** Processes a real-time stream */
// TInput is the raw message type from the stream, TOutput is the processed type
export interface IStreamProcessor<TInput, TOutput> {
    initialize?(context: PipelineContext): Promise<void>; // Optional setup
    process(message: TInput, context: PipelineContext): Promise<TOutput | null | undefined>; // Process single message
    handleBatch?(batch: TOutput[], context: PipelineContext): Promise<void>; // Optional: Load/handle processed batch
    onError?(error: Error, message: TInput | null, context: PipelineContext): Promise<void>; // Handle processing errors
    shutdown?(context: PipelineContext): Promise<void>; // Optional cleanup
}

/** Configuration for stream processing */
export interface StreamProcessingConfig {
    source: {
        type: 'kafka' | 'redis-streams' | string; // Extensible
        config: any; // e.g., KafkaJS client config, Redis connection options
        topic?: string; // Kafka topic
        group?: string; // Consumer group
        streamKey?: string; // Redis stream key
        // ... other stream-specific options
    };
    processor: IStreamProcessor<any, any>;
    batchSize?: number; // Process messages in batches?
    concurrency?: number;
    errorHandling?: {
        retries?: number;
        delay?: number; // ms
        deadLetterTopic?: string; // Topic/queue for failed messages
    };
}


// --- FlowLab Node Registration ---
// Assume a core @flowlab/core package defines these
export interface IFlowLabNode {
    readonly id: string; // Unique ID for this node instance
    readonly type: string; // Type of node (e.g., 'extractor', 'loader', '@flowlab/data:pipeline')
    execute(payload: any, context: any): Promise<any>; // Execution logic
    // potentially methods for configuration schema, validation, etc.
}

export interface IFlowLabRegistry {
    register(node: IFlowLabNode): void;
}

// --- Pipeline Configuration (for JSON/YAML) ---
export interface PipelineStepConfig {
    type: 'clean' | 'transform' | 'map' | 'extract' | 'load' | 'custom';
    // Options depend on type:
    cleaner?: string | object; // reference to registered cleaner or inline config
    transformer?: string | object; // reference to registered transformer or inline config
    rules?: Record<string, string | object>; // For mapping transformer
    customFunctionPath?: string; // Path to a JS file exporting a transform function
}

export interface PipelineConfig {
    id: string; // Unique ID for this pipeline definition
    source: SourceConfig;
    steps: PipelineStepConfig[];
    target: TargetConfig;
    options?: {
        batchSize?: number;
        errorHandling?: {
            retries?: number;
            delay?: number; // ms
        };
        // concurrency options?
    };
}