caminho/dist/esm/operators/generator.js

Tool for creating efficient data pipelines in a JavaScript environment

import { sleep } from '../utils/sleep';
import { getNewValueBag } from '../utils/valueBag';
const SLEEP_FOR_BACKPRESSURE_MS = 10;
export function wrapGenerator(generatorParams, loggers) {
    return async function* wrappedGenerator(initialBag) {
        const bagArrayForLogger = [initialBag];
        loggers.onStepStarted(bagArrayForLogger);
        let isStart = true;
        let startTime = new Date();
        try {
            for await (const value of generatorParams.fn(initialBag)) {
                if (!isStart) {
                    loggers.onStepStarted(bagArrayForLogger);
                }
                isStart = false;
                const newValueBag = getNewValueBag(initialBag, generatorParams.provides, value);
                loggers.onStepFinished([newValueBag], startTime);
                yield newValueBag;
                startTime = new Date();
            }
        }
        catch (err) {
            loggers.onStepFinished([initialBag], startTime, err);
            throw err;
        }
    };
}
export function wrapGeneratorWithBackPressure(generatorParams, maxItemsFlowing, pendingDataControl, loggers) {
    const wrappedGenerator = wrapGenerator(generatorParams, loggers);
    return async function* wrappedGeneratorWithBackPressure(initialBag, runId) {
        for await (const value of wrappedGenerator({ ...initialBag })) {
            pendingDataControl.increment(runId);
            yield value;
            if (needsToWaitForBackpressure(pendingDataControl, maxItemsFlowing)) {
                await waitOnBackpressure(maxItemsFlowing, pendingDataControl);
            }
        }
    };
}
function needsToWaitForBackpressure(pendingDataControl, maxItemsFlowing) {
    return pendingDataControl.size >= maxItemsFlowing;
}
async function waitOnBackpressure(maxItemsFlowing, pendingDataControl) {
    while (pendingDataControl.size >= maxItemsFlowing) {
        await sleep(SLEEP_FOR_BACKPRESSURE_MS);
    }
}
//# sourceMappingURL=generator.js.map