@rudderstack/workflow-engine
Version:
A generic workflow execution engine
73 lines • 3.33 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BatchUtils = void 0;
const object_sizeof_1 = __importDefault(require("object-sizeof"));
const batch_1 = require("../../errors/batch");
class BatchUtils {
static chunkArrayBySizeAndLength(array, options = {}) {
const items = [];
const indices = [];
if (!array?.length) {
return { items, indices };
}
const { maxSizeInBytes, maxItems } = options;
let currentChunk = [array[0]];
let currentIndices = [0];
let currentSize = maxSizeInBytes ? (0, object_sizeof_1.default)(array[0]) : 0;
for (let idx = 1; idx < array.length; idx += 1) {
const item = array[idx];
const itemSizeInBytes = maxSizeInBytes ? (0, object_sizeof_1.default)(item) : 0;
if (this.isSizeLimitReached(itemSizeInBytes, currentSize, maxSizeInBytes) ||
this.isLengthLimitReached(currentChunk.length, maxItems)) {
items.push(currentChunk);
indices.push(currentIndices);
currentChunk = [];
currentIndices = [];
currentSize = 0;
}
currentChunk.push(item);
currentIndices.push(idx);
currentSize += itemSizeInBytes;
}
if (currentChunk.length > 0) {
items.push(currentChunk);
indices.push(currentIndices);
}
return { items, indices };
}
static validateBatchResults(input, batchResults) {
if (!Array.isArray(batchResults)) {
throw new batch_1.BatchError('batch step requires array output from batch executor');
}
const batchIndices = batchResults.reduce((acc, batchResult) => acc.concat(batchResult.indices), []);
batchIndices.sort((a, b) => a - b);
batchIndices.forEach((index, idx) => {
if (index !== idx) {
throw new batch_1.BatchError('batch step requires return all indices');
}
});
const batchItems = batchResults.reduce((acc, batchResult) => acc.concat(batchResult.items), []);
if (batchIndices.length !== input.length || batchItems.length !== input.length) {
throw new batch_1.BatchError('batch step requires batch executor to return same number of items as input');
}
batchResults.forEach((batchResult) => {
if (!batchResult.key) {
throw new batch_1.BatchError('batch step requires batch executor to return key for each batch result');
}
if (batchResult.indices.length !== batchResult.items.length) {
throw new batch_1.BatchError('batch step requires batch executor to return same number of items and indices');
}
});
}
static isSizeLimitReached(itemSizeInBytes, currentSize, maxSizeInBytes) {
return maxSizeInBytes && currentSize + itemSizeInBytes > maxSizeInBytes;
}
static isLengthLimitReached(currentLength, maxLength) {
return maxLength && currentLength + 1 > maxLength;
}
}
exports.BatchUtils = BatchUtils;
//# sourceMappingURL=batch.js.map