simple-in-memory-queue/dist/logic/withConsumer/createQueueWithResilientRemoteConsumer.d.ts

A simple in-memory queue, for nodejs and the browser, with consumers for common usecases.

/**
 * creates a queue with a consumer which consumes each item one at a time with a remote calls, resiliently
 *
 * features
 * - is event driven: processes items one at a time as soon as they are queued, w/ a `maxConcurrency`
 * - is resilient: retries the items up to `retryThreshold` times in a row, removing it from queue after threshold exceeded
 *   - calls "onFailureAttempt" method when an item's failure count is below the `retryThreshold`
 *   - calls "onFailurePermanent" method when an item's failure count exceeds the `retryThreshold`
 * - is intelligent: pauses consuming items if more than `pauseThreshold` errors in a row on different items
 *   - calls "onPause" method when the `pauseThreshold` is exceeded
 * - is efficient: supports concurrency and non-blocking processing of items while others are delayed
 *   - allows consuming items in parallel
 *   - allows consumption of items while other items that have had a failure are delayed in the background
 *
 * usecases
 * - make calls against a remote api for each item
 *   - automatically retry items to recover from intermittent networking errors
 *   - intelligently pause processing if all items are failing
 */
export declare const createQueueWithResilientRemoteConsumer: <T>({ consumer, threshold, delay, on, }: {
    /**
     * the consumer to invoke with each item, which may throw an error
     */
    consumer: ({ item }: {
        item: T;
    }) => Promise<void>;
    /**
     * the thresholds which affect consumption
     */
    threshold: {
        /**
         * the max concurrency with which to consume items
         */
        concurrency: 1;
        /**
         * how many times to retry an item before considering it as permanently failing, calling the onFailurePermanent hook, and removing it from the queue
         *
         * note
         * - this ensures that intermittent errors are resolved automatically while not blocking other items if only one item has an issue
         */
        retry: number;
        /**
         * how many items in a row can fail before we pause consumption of the items and call the onPause hook
         *
         * note
         * - this ensures that if there is a systematic issue causing all items to fail, the system wont keep wasting resources pointlessly
         */
        pause: number;
    };
    /**
     * the delays which affect consumption
     */
    delay: {
        /**
         * the number of milliseconds to delay retrying after experiencing an error
         */
        retry: number;
        /**
         * the number of milliseconds to wait before an item in the queue should be visible to the consume
         *
         * usecases
         * - e.g., wait a second for some other stores to be updated before attempting to process the event
         */
        visibility?: number;
    };
    /**
     * the hooks available to subscribe to
     */
    on?: {
        /**
         * a hook that is called when an item fails but will be retried
         */
        failureAttempt?: (({ item, attempt, error, }: {
            item: T;
            attempt: number;
            error: unknown;
        }) => void) | undefined;
        /**
         * a hook that is called when an item failed beyond the retry threshold and will be removed from the queue
         */
        failurePermanent?: (({ item, error }: {
            item: T;
            error: unknown;
        }) => void) | undefined;
        /**
         * a hook that is called when the pause threshold is exceeded and consumption has been paused
         */
        pause?: (({ failures }: {
            failures: {
                item: T;
                error: unknown;
            }[];
        }) => void) | undefined;
    } | undefined;
}) => import("../queue/createQueue").Queue<T>;