@rudderstack/integrations-lib/build/utils/batch-processing.d.ts

A comprehensive TypeScript library providing shared utilities, SDKs, and tools for RudderStack integrations and destinations.

/**
 * Options for batch processing operations
 */
export interface BatchProcessingOptions {
    /**
     * Number of items to process in each batch (default: 10)
     * Must be a positive integer.
     */
    batchSize?: number;
    /**
     * Time threshold in milliseconds (default: 10) before yielding control back to the event loop.
     * Set to 0 to yield after every batch. Must be a non-negative integer.
     */
    yieldThreshold?: number;
    /**
     * Whether to process items sequentially within each batch. When true (default), each item in a batch
     * will be processed one at a time. When false, all items in a batch will be processed concurrently.
     * Consider the implications of concurrency on your processing logic before setting this to false, e.g. race conditions, rate limits, memory pressure, etc.
     */
    sequentialProcessing?: boolean;
}
/**
 * Configure global defaults for batch processing operations
 *
 * @param config - Configuration options for batch processing
 * @returns The current configuration after applying changes
 *
 * @example
 * ```typescript
 * // Set both defaults
 * configureBatchProcessingDefaults({ batchSize: 20, yieldThreshold: 5 });
 *
 * // Set only batch size
 * configureBatchProcessingDefaults({ batchSize: 50 });
 *
 * // Enable concurrent processing within batches
 * configureBatchProcessingDefaults({ sequentialProcessing: false });
 *
 * // Get current configuration
 * const currentConfig = configureBatchProcessingDefaults();
 * ```
 */
export declare function configureBatchProcessingDefaults(config?: BatchProcessingOptions): BatchProcessingOptions;
/**
 * Maps over an array in batches to avoid blocking the event loop.
 * Processes items in chunks and yields control back to the event loop between batches
 * when the processing time exceeds the threshold.
 *
 * @template T - The type of items in the input array
 * @template R - The type of items in the result array
 * @param items - The array to map over
 * @param mapFn - The mapping function to apply to each item. Receives the item and its index.
 * @param options - Batch processing options
 * @returns A promise that resolves to the mapped array
 * @throws {Error} When batchSize is not a positive integer
 *
 * @example
 * ```typescript
 * // Synchronous mapping with default options (sequential processing)
 * const doubled = await mapInBatches([1, 2, 3, 4], (x) => x * 2);
 *
 * // With concurrent processing within batches
 * const doubled = await mapInBatches([1, 2, 3, 4], (x) => x * 2, { sequentialProcessing: false });
 * ```
 */
export declare function mapInBatches<T, R>(items: T[], mapFn: (item: T, index: number) => R | Promise<R>, options?: BatchProcessingOptions): Promise<R[]>;
/**
 * Filters an array in batches to avoid blocking the event loop.
 * Processes items in chunks and yields control back to the event loop between batches
 * when the processing time exceeds the threshold.
 *
 * @template T - The type of items in the array
 * @param items - The array to filter
 * @param predicate - The predicate function to test each item. Receives the item and its index.
 * @param options - Batch processing options
 * @returns A promise that resolves to the filtered array
 * @throws {Error} When batchSize is not a positive integer
 *
 * @example
 * ```typescript
 * // Synchronous filtering with default options
 * const evens = await filterInBatches([1, 2, 3, 4, 5], (x) => x % 2 === 0);
 * // Result: [2, 4]
 *
 * // With custom batch size
 * const evens = await filterInBatches([1, 2, 3, 4, 5], (x) => x % 2 === 0, { batchSize: 2 });
 * ```
 */
export declare function filterInBatches<T>(items: T[], predicate: (item: T, index: number) => boolean | Promise<boolean>, options?: BatchProcessingOptions): Promise<T[]>;
/**
 * Groups an array by a key function in batches to avoid blocking the event loop.
 * Processes items in chunks and yields control back to the event loop between batches
 * when the processing time exceeds the threshold.
 *
 * @template T - The type of items in the array
 * @template K - The type of the grouping key (must extend PropertyKey)
 * @param items - The array to group
 * @param keyFn - The function to extract the grouping key from each item. Receives the item and its index.
 * @param options - Batch processing options
 * @returns A promise that resolves to an object with grouped items
 * @throws {Error} When batchSize is not a positive integer
 *
 * @example
 * ```typescript
 * // Group by property with default options
 * const byType = await groupByInBatches(
 *   [{type: 'A', value: 1}, {type: 'B', value: 2}, {type: 'A', value: 3}],
 *   (item) => item.type
 * );
 * // Result: {A: [{type: 'A', value: 1}, {type: 'A', value: 3}], B: [{type: 'B', value: 2}]}
 *
 * // With custom batch size
 * const byType = await groupByInBatches(
 *   [{type: 'A', value: 1}, {type: 'B', value: 2}, {type: 'A', value: 3}],
 *   (item) => item.type,
 *   { batchSize: 2 }
 * );
 * ```
 */
export declare function groupByInBatches<T, K extends PropertyKey>(items: T[], keyFn: (item: T, index: number) => K | Promise<K>, options?: BatchProcessingOptions): Promise<Record<K, T[]>>;
/**
 * Reduces an array in batches to avoid blocking the event loop.
 * Processes items in chunks and yields control back to the event loop between batches
 * when the processing time exceeds the threshold. Sequential processing is always used for the reducer function,
 * irrespective of the `sequentialProcessing` option.
 *
 * @template T - The type of items in the array
 * @template R - The type of the accumulator/result
 * @param items - The array to reduce
 * @param reducer - The reducer function. Receives the accumulator, current item, and index.
 * @param initialValue - The initial value for the accumulator
 * @param options - Batch processing options
 * @returns A promise that resolves to the reduced value
 * @throws {Error} When batchSize is not a positive integer
 *
 * @example
 * ```typescript
 * // Sum numbers with default options
 * const sum = await reduceInBatches([1, 2, 3, 4], (acc, x) => acc + x, 0);
 * // Result: 10
 *
 * // With custom batch size
 * const sum = await reduceInBatches([1, 2, 3, 4], (acc, x) => acc + x, 0, { batchSize: 2 });
 * ```
 */
export declare function reduceInBatches<T, R>(items: T[], reducer: (acc: R, item: T, index: number) => R | Promise<R>, initialValue: R, options?: BatchProcessingOptions): Promise<R>;
/**
 * FlatMaps an array in batches to avoid blocking the event loop.
 * Processes items in chunks, flattens the results, and yields control back to the event loop between batches
 * when the processing time exceeds the threshold.
 *
 * @template T - The type of items in the input array
 * @template R - The type of items in the flattened result array
 * @param items - The array to flatMap over
 * @param mapFn - The mapping function that returns an array for each item. Receives the item and its index.
 * @param options - Batch processing options
 * @returns A promise that resolves to the flattened mapped array
 * @throws {Error} When batchSize is not a positive integer
 *
 * @example
 * ```typescript
 * // Duplicate each item with default options
 * const duplicated = await flatMapInBatches([1, 2, 3], (x) => [x, x]);
 * // Result: [1, 1, 2, 2, 3, 3]
 *
 * // With custom batch size
 * const duplicated = await flatMapInBatches([1, 2, 3], (x) => [x, x], { batchSize: 2 });
 * ```
 */
export declare function flatMapInBatches<T, R>(items: T[], mapFn: (item: T, index: number) => R[] | Promise<R[]>, options?: BatchProcessingOptions): Promise<R[]>;
/**
 * forEach over an array in batches to avoid blocking the event loop.
 * Processes items in chunks and yields control back to the event loop between batches
 * when the processing time exceeds the threshold.
 *
 * @template T - The type of items in the input array
 * @param items - The array to iterate over
 * @param fn - The function to apply to each item. Receives the item and its index. Can be async.
 * @param options - Batch processing options
 * @returns A promise that resolves when all items have been processed
 * @throws {Error} When batchSize is not a positive integer
 *
 * @example
 * ```typescript
 * // Process items in batches with default options
 * await forEachInBatches([1, 2, 3, 4], async (x) => {
 *   await doSomething(x);
 * });
 *
 * // With custom batch size
 * await forEachInBatches([1, 2, 3, 4], async (x) => {
 *   await doSomething(x);
 * }, { batchSize: 2 });
 * ```
 */
export declare function forEachInBatches<T>(items: T[], fn: (item: T, index: number) => void | Promise<void>, options?: BatchProcessingOptions): Promise<void>;
//# sourceMappingURL=batch-processing.d.ts.map