@speckle/shared/dist/commonjs/core/helpers/batch.js

Shared code between various Speckle JS packages

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.batchAsyncOperations = batchAsyncOperations;
/* eslint-disable @typescript-eslint/no-explicit-any */
const _lodash_1 = require("#lodash");
/**
 * Utility for batching async operations. Useful when you have thousands of async operations and you can't
 * just invoke them all at once because of resource constraints (e.g. network bandwidth).
 *
 * 'operationParams' should be an array of parameters for each async operation. The size of this array coresponds
 * to the amount of async operations that will be invoked.
 * 'operationPromiseGenerator' will be invoked sequentially with each params of 'operationParams' and with it
 * you can specify what the actual async operation should be
 *
 * TODO: Some tests would be nice, although it does work when tested through `yarn cli download commit` in speckle-server
 */
async function batchAsyncOperations(name, operationParams, operationPromiseGenerator, options) {
    const { retryCount = 3, batchSize = 100, dropReturns = false, logger = (...args) => console.log(...args) } = options || {};
    const finalLogger = (message, ...args) => logger(`[${name}] ${message}`, ...args);
    finalLogger('Starting batched operation...');
    const operationCount = operationParams.length;
    let allResults = [];
    let executedOperationCount = 0;
    const batchCount = Math.ceil(operationCount / batchSize);
    for (let i = 0; i < batchCount; i++) {
        finalLogger(`Processing batch ${i + 1} out of ${batchCount}...`);
        // Figure out how many operations we can execute in this batch
        const newExecutedOperationCount = Math.min(executedOperationCount + batchSize, operationCount);
        const operationsToExecuteCount = newExecutedOperationCount - executedOperationCount;
        if (operationsToExecuteCount <= 0)
            return;
        // Invoke operation generator
        const batchParams = operationParams.slice(executedOperationCount, newExecutedOperationCount);
        const batchRequests = [];
        let currentOperationIdx = executedOperationCount;
        for (const params of batchParams) {
            const currentOperationNumber = currentOperationIdx + 1;
            const label = `${currentOperationNumber}/${operationCount}`;
            const operationPromise = (async () => {
                finalLogger(`Queuing operation ${label}...`);
                const execute = () => operationPromiseGenerator(params);
                let promise = execute().then((res) => {
                    finalLogger(`...finished operation ${label}`);
                    return res;
                });
                // Attach retries
                (0, _lodash_1.range)(retryCount).forEach((retry) => {
                    promise = promise.catch((e) => {
                        finalLogger(`...failure in operation ${label}: "${e}". Triggering retry ${retry + 1}...`);
                        return execute();
                    });
                });
                promise = promise.catch((e) => {
                    finalLogger(`...final failure in operation ${label}!`);
                    throw e;
                });
                return promise;
            })();
            batchRequests.push(operationPromise);
            currentOperationIdx++;
        }
        const batchResults = await Promise.all(batchRequests);
        if (!dropReturns) {
            allResults = allResults.concat(batchResults);
        }
        executedOperationCount = newExecutedOperationCount;
    }
    return dropReturns ? true : allResults;
}
//# sourceMappingURL=batch.js.map