@flowlab/all/packages/data/src/extractors/apiExtractor.ts

A cool library focusing on handling various flows

// src/extractors/apiExtractor.ts
import { IExtractor, PipelineContext, DataSource, ApiSourceConfig } from '../core/interfaces';
import { ComponentError } from '../core/errors';
// import fetch from 'node-fetch'; // Uncomment if using node-fetch

// MARK: - ApiExtractor
export class ApiExtractor<TOutput> implements IExtractor<TOutput> {
    private config: ApiSourceConfig;

    constructor(config: ApiSourceConfig) {
         if (!config.url) {
            throw new ComponentError('ApiExtractor requires "url" in config.');
        }
        this.config = config;
        this.config.method = config.method || 'GET';
    }

    // MARK: - extract
    async extract(context: PipelineContext): Promise<DataSource<TOutput>> {
        context.logger.info(`Extracting data from API: ${this.config.method} ${this.config.url}`);

        try {
            const response = await fetch(this.config.url, {
                method: this.config.method,
                headers: this.config.headers,
                body: this.config.body ? JSON.stringify(this.config.body) : undefined,
                // Add timeout, agent, etc. if needed
            });

            if (!response.ok) {
                const errorBody = await response.text();
                throw new ComponentError(`API request failed with status ${response.status}: ${response.statusText}. Body: ${errorBody}`, 'ApiExtractor');
            }

            // Assume JSON response for now, add handling for other types
            const data = await response.json();

            // Check if the response is an array (common for list endpoints)
            // or needs further unwrapping (e.g., data contained in response.data)
            if (Array.isArray(data)) {
                return data as TOutput[];
            } else if (typeof data === 'object' && data !== null && 'results' in data && Array.isArray((data as any).results)) {
                 // Example: Handle paginated responses where items are in 'results' array
                 context.logger.warn(`API response is not an array, attempting to extract 'results' property.`);
                 return (data as any).results as TOutput[];
                 // TODO: Implement proper pagination handling if API supports it
            }
             else {
                // Return single object as an array
                context.logger.warn(`API response is not an array. Returning as single item array.`);
                return [data as TOutput];
             }

        } catch (error: any) {
             context.logger.error({ err: error, url: this.config.url }, `Error during API extraction`);
            if (error instanceof ComponentError) throw error;
            throw new ComponentError(`Failed to fetch data from API ${this.config.url}`, 'ApiExtractor', error);
        }
    }
}