@flowlab/all/packages/data/src/loaders/redisLoader.ts

A cool library focusing on handling various flows

// src/loaders/redisLoader.ts
import { Redis as IORedisClient } from 'ioredis';
import { ILoader, PipelineContext, DatabaseTargetConfig, RedisConnection } from '../core/interfaces';
import { ComponentError } from '../core/errors';

export class RedisLoader<TInput> implements ILoader<TInput> {
    private config: DatabaseTargetConfig;
    private client: IORedisClient;
    private operation: string;
    private keyField?: string; // Field in the input object to use as the key

    constructor(config: DatabaseTargetConfig) {
        if (config.type !== 'redis' || !(config.connection as RedisConnection)?.client) {
             throw new ComponentError('RedisLoader requires config type "redis" and "connection" object with "client" (ioredis Redis instance).');
        }
         if (!config.redisOperation) {
             throw new ComponentError('RedisLoader requires "redisOperation" in config (e.g., "set", "hmset", "lpush").');
         }
        this.config = config;
        const redisConn = config.connection as RedisConnection;
        this.client = redisConn.client;
        this.operation = config.redisOperation;
        this.keyField = config.redisKeyField; // Optional: key derived from data item

         if (!this.keyField && ['set', 'hmset' /* other key-specific ops */].includes(this.operation)) {
              // LPush/RPush might use a fixed key (config.table?) or derive it. Clarify requirements.
              // For now, require keyField for set/hmset.
             throw new ComponentError(`RedisLoader operation "${this.operation}" requires "redisKeyField" to be specified in config.`);
         }
    }

    async loadBatch(batch: TInput[], context: PipelineContext): Promise<void> {
        if (batch.length === 0) return;

        context.logger.debug(`Loading batch of ${batch.length} items to Redis using operation ${this.operation}`);

        // Use Redis Pipeline for batching commands
        const pipeline = this.client.pipeline();
        let commandsAdded = 0;

        for (const item of batch) {
            let key: string | undefined;

             // Determine the key
            if (this.keyField) {
                 key = (item as any)?.[this.keyField];
                 if (!key) {
                     context.logger.warn({ item, keyField: this.keyField }, `Skipping item: Key field "${this.keyField}" not found or is empty.`);
                     continue;
                 }
            } else if (this.config.table) {
                // Use fixed key from config.table (e.g., for LPUSH to a specific list)
                key = this.config.table;
            } else if (['lpush', 'rpush', 'sadd' /* ops without required keyField */].includes(this.operation)) {
                // Allow operations like LPUSH to potentially use a hardcoded key from config.table
                // This logic might need refinement based on exact use cases. For now assume config.table is the list/set key.
                 key = this.config.table;
                 if (!key) {
                    context.logger.warn({ item, operation: this.operation }, `Skipping item: Operation "${this.operation}" requires a target key (use "table" or "redisKeyField" in config).`);
                    continue;
                 }
            }
             else {
                 context.logger.warn({ item, operation: this.operation }, `Skipping item: Could not determine Redis key for operation "${this.operation}". Specify "redisKeyField" or "table".`);
                 continue;
             }


            // Add command to pipeline based on operation
            try {
                switch (this.operation) {
                    case 'set':
                         const valueToSet = typeof item === 'string' ? item : JSON.stringify(item);
                         pipeline.set(key, valueToSet);
                         commandsAdded++;
                         break;
                    case 'hmset':
                         if (typeof item !== 'object' || item === null) {
                            context.logger.warn({ item, key }, `Skipping item for HMSET: Input must be an object.`);
                            continue;
                         }
                         // ioredis HMSET args can be object or key, field, value, field, value...
                         // Ensure all values are strings for HMSET
                         const hashData: { [key: string]: string } = {};
                         for (const prop in item) {
                             if (Object.prototype.hasOwnProperty.call(item, prop)) {
                                hashData[prop] = String((item as any)[prop]);
                             }
                         }
                         pipeline.hmset(key, hashData);
                         commandsAdded++;
                         break;
                     case 'lpush':
                     case 'rpush':
                         const valuesToPush = Array.isArray(item) ? item.map(String) : [String(item)];
                         if (valuesToPush.length > 0) {
                             pipeline[this.operation](key, ...valuesToPush);
                             commandsAdded++;
                         }
                         break;
                    // Add cases for 'sadd', 'zadd', etc. if needed
                    default:
                         context.logger.warn(`Unsupported Redis operation: ${this.operation}. Skipping item.`);
                         continue; // Skip this item
                }
            } catch (error: any) {
                 context.logger.error({ err: error, item, key, operation: this.operation }, `Error preparing Redis command for item.`);
                 // Optionally skip item or fail batch? For now, skip.
            }
        }

        if (commandsAdded > 0) {
            context.logger.debug(`Executing Redis pipeline with ${commandsAdded} commands.`);
            try {
                const results = await pipeline.exec();
                // Check results for errors (each result is [error, result])
                let errors = 0;
                results?.forEach(([err, res], index) => {
                     if (err) {
                         errors++;
                         // Log detailed error, potentially linking back to the item if possible (hard with pipeline)
                         context.logger.error({ pipelineError: err, commandIndex: index }, `Error in Redis pipeline execution.`);
                     }
                });
                if (errors > 0) {
                    context.logger.warn(`Encountered ${errors} errors during Redis pipeline execution for batch.`);
                    // Decide if this constitutes a batch failure for retry logic
                    // throw new ComponentError(`${errors} errors occurred during Redis pipeline execution.`, 'RedisLoader');
                } else {
                     context.logger.info(`Redis pipeline executed successfully (${commandsAdded} commands).`);
                }
            } catch (error: any) {
                 context.logger.error({ err: error }, `Error executing Redis pipeline.`);
                 throw new ComponentError(`Failed to execute Redis pipeline`, 'RedisLoader', error);
            }
        } else {
             context.logger.debug('No Redis commands generated for this batch.');
        }
    }
}