@shutterstock/chunker/dist/src/chunker.js

Calls a callback before size would be exceeded or when count limit is reached

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Chunker = void 0;
const p_map_iterable_1 = require("@shutterstock/p-map-iterable");
/**
 * Collects items up to `countLimit`, calling `writer` before `sizeLimit` would be exceeded.
 *
 * @remarks
 *
 * Always call {@link onIdle} when done to ensure that the last `writer` call is made.
 *
 * @template T Type of items to be chunked
 */
class Chunker {
    /**
     * Collects items up to `countLimit`, calling `writer` before `sizeLimit` would be exceeded.
     *
     * @remarks
     *
     * Always call {@link onIdle} when done to ensure that the last `writer` call is made.
     *
     * @template T Type of items to be chunked
     * @param options Chunker options
     * @param options.sizeLimit User-defined size before (not to exceed) which `writer` should be called
     * @param options.countLimit User-defined size at which `writer` should be called
     * @param options.sizer Function that returns user-defined size of an item
     * @param options.writer Function that writes the pending items when `sizeLimit` or `countLimit` would be exceeded.
     *    This is not a `mapper` as it does not return a result at all.
     *    For example, this may write the items to a file or send them to a service.
     *    If the results need to be passed along, add them to an `IterableQueueMapper` for example.
     */
    constructor(options) {
        this._iterableQueue = new p_map_iterable_1.IterableQueue({ maxUnread: 0 });
        this._errors = [];
        this._pendingSize = 0;
        this._pendingItems = [];
        const { sizeLimit, countLimit, sizer, writer } = options;
        this._writer = writer;
        // Start the background flushing process
        this._backgroundWriter = (async () => {
            for await (const item of this._iterableQueue) {
                // Get size of new item
                const size = await sizer(item);
                // Check if we need to flush before adding
                if (this._pendingItems.length === countLimit ||
                    (this._pendingSize > 0 && this._pendingSize + size > sizeLimit)) {
                    await this.flush();
                }
                // Add the new item to the queue
                this._pendingItems.push(item);
                this._pendingSize += size;
                // Check if we need to flush after adding
                if (this._pendingItems.length === countLimit) {
                    await this.flush();
                }
            }
            // Final flush if any items are left
            if (this._pendingItems.length > 0) {
                await this.flush();
            }
        })();
    }
    /**
     * Sum of the user-defined size of all pending items
     */
    get pendingSize() {
        return this._pendingSize;
    }
    /**
     * Number of items pending
     */
    get pendingCount() {
        return this._pendingItems.length;
    }
    /**
     * Accumulated errors from background flushes
     */
    get errors() {
        return this._errors;
    }
    /**
     * Wait for all background writes to finish.
     * MUST be called before exit to ensure no lost writes.
     */
    async onIdle() {
        // Indicate that we're done writing requests
        this._iterableQueue.done();
        // Wait for the background writer to exit
        await this._backgroundWriter;
    }
    /**
     * Calls `writer` for any pending items and clears pending items queue.
     * @returns Result from `writer` function or `undefined` if no items pending
     */
    async flush() {
        const itemsToWrite = this._pendingItems;
        this._pendingItems = [];
        this._pendingSize = 0;
        if (itemsToWrite.length === 0) {
            return undefined;
        }
        else {
            try {
                await this._writer(itemsToWrite);
                // eslint-disable-next-line @typescript-eslint/no-explicit-any
            }
            catch (error) {
                this._errors.push(error);
            }
        }
    }
    /**
     * Adds an item to the pending queue, flushing the queue before
     * adding the item if the new item would cause the item limit
     * or size limit to be exceeded.
     *
     * @param item Item to be added to the queue
     */
    async enqueue(item) {
        await this._iterableQueue.enqueue(item);
    }
}
exports.Chunker = Chunker;
//# sourceMappingURL=chunker.js.map