@flowlab/all/packages/data/src/loaders/apiLoader.ts

A cool library focusing on handling various flows

// src/loaders/apiLoader.ts
import { ILoader, PipelineContext, ApiTargetConfig } from '../core/interfaces';
import { ComponentError } from '../core/errors';
// import fetch from 'node-fetch'; // Uncomment if using node-fetch

export class ApiLoader<TInput> implements ILoader<TInput> {
    private config: ApiTargetConfig;

    constructor(config: ApiTargetConfig) {
         if (!config.url) {
            throw new ComponentError('ApiLoader requires "url" in config.');
        }
        this.config = config;
        this.config.method = config.method || 'POST';
    }

    async loadBatch(batch: TInput[], context: PipelineContext): Promise<void> {
        if (batch.length === 0) return;
        context.logger.debug(`Loading batch of ${batch.length} items to API: ${this.config.method} ${this.config.url}`);

        // Option 1: Send items one by one (simple, but potentially slow)
        // Option 2: Send the whole batch if the API supports it (often preferred)

        // --- Option 2: Send whole batch ---
        try {
             const response = await fetch(this.config.url, {
                 method: this.config.method,
                 headers: { 'Content-Type': 'application/json', ...this.config.headers },
                 body: JSON.stringify(batch), // Send the array as JSON
             });

             if (!response.ok) {
                 const errorBody = await response.text();
                  // Log details about the failed batch?
                 throw new ComponentError(`API load request failed with status ${response.status}: ${response.statusText}. Body: ${errorBody}`, 'ApiLoader');
             }
              context.logger.trace(`Successfully loaded batch of ${batch.length} items via API.`);
              // Handle response body if needed (e.g., check for partial success)
             await response.text(); // Consume response body

        } catch (error: any) {
            context.logger.error({ err: error, url: this.config.url }, `Error loading batch to API`);
             if (error instanceof ComponentError) throw error;
            throw new ComponentError(`Failed to load data to API ${this.config.url}`, 'ApiLoader', error);
        }

        // --- Option 1: Send one by one (example) ---
        /*
        for (const item of batch) {
            try {
                const response = await fetch(this.config.url, {
                    method: this.config.method,
                    headers: { 'Content-Type': 'application/json', ...this.config.headers },
                    body: JSON.stringify(item),
                });
                if (!response.ok) {
                     const errorBody = await response.text();
                     context.logger.error({ item, status: response.status, errorBody }, `Failed to load item to API ${this.config.url}`);
                     // Decide how to handle single item failure (log, collect errors, stop?)
                     continue; // Continue with next item for now
                }
                 await response.text(); // Consume body
            } catch (error: any) {
                 context.logger.error({ err: error, item, url: this.config.url }, `Network error loading item to API`);
                 // Handle network error for single item
            }
        }
        */
    }
}