caminho
Version:
Tool for creating efficient data pipelines in a JavaScript environment
52 lines • 2.19 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.wrapGenerator = wrapGenerator;
exports.wrapGeneratorWithBackPressure = wrapGeneratorWithBackPressure;
const sleep_1 = require("../utils/sleep");
const valueBag_1 = require("../utils/valueBag");
const SLEEP_FOR_BACKPRESSURE_MS = 10;
function wrapGenerator(generatorParams, loggers) {
return async function* wrappedGenerator(initialBag) {
const bagArrayForLogger = [initialBag];
loggers.onStepStarted(bagArrayForLogger);
let isStart = true;
let startTime = new Date();
try {
for await (const value of generatorParams.fn(initialBag)) {
if (!isStart) {
loggers.onStepStarted(bagArrayForLogger);
}
isStart = false;
const newValueBag = (0, valueBag_1.getNewValueBag)(initialBag, generatorParams.provides, value);
loggers.onStepFinished([newValueBag], startTime);
yield newValueBag;
startTime = new Date();
}
}
catch (err) {
loggers.onStepFinished([initialBag], startTime, err);
throw err;
}
};
}
function wrapGeneratorWithBackPressure(generatorParams, maxItemsFlowing, pendingDataControl, loggers) {
const wrappedGenerator = wrapGenerator(generatorParams, loggers);
return async function* wrappedGeneratorWithBackPressure(initialBag, runId) {
for await (const value of wrappedGenerator({ ...initialBag })) {
pendingDataControl.increment(runId);
yield value;
if (needsToWaitForBackpressure(pendingDataControl, maxItemsFlowing)) {
await waitOnBackpressure(maxItemsFlowing, pendingDataControl);
}
}
};
}
function needsToWaitForBackpressure(pendingDataControl, maxItemsFlowing) {
return pendingDataControl.size >= maxItemsFlowing;
}
async function waitOnBackpressure(maxItemsFlowing, pendingDataControl) {
while (pendingDataControl.size >= maxItemsFlowing) {
await (0, sleep_1.sleep)(SLEEP_FOR_BACKPRESSURE_MS);
}
}
//# sourceMappingURL=generator.js.map