frolyk
Version:
Stream processing library for Kafka in Node
82 lines • 4.06 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createPipeline = void 0;
const highland_1 = __importDefault(require("highland"));
const long_1 = __importDefault(require("long"));
const abandon = Symbol('abandon');
function createPipeline(assignmentContext, processors) {
return __awaiter(this, void 0, void 0, function* () {
const messageProcessors = yield processors.reduce((p, setupProcessor) => __awaiter(this, void 0, void 0, function* () {
const processors = yield p;
const setupResult = yield setupProcessor(assignmentContext);
const newProcessors = Array.isArray(setupResult) ? setupResult : [setupResult];
return [...processors, ...newProcessors];
}), Promise.resolve([]));
const processedOffsets = highland_1.default();
const pipeline = (controlledStream) => {
const processedStream = controlledStream.consume(function (err, x, push, next) {
if (err) {
// forward errors
push(err);
next();
return;
}
else if (highland_1.default.isNil(x)) {
// forward end of stream
processedOffsets.end();
push(null, x);
return;
}
const message = x;
const { highWaterOffset, offset, partition, topic, timestamp } = message;
const context = {
abandon,
toString: () => `processor context (o=${offset} p=${partition} t=${topic}, ho=${highWaterOffset})`,
commit: (metadata) => assignmentContext.commitOffset(long_1.default.fromValue(offset).add(1), metadata),
/* istanbul ignore next */
log(tags, payload) {
return assignmentContext.log(tags, payload);
},
group: () => assignmentContext.group,
offset: () => offset,
partition: () => partition,
topic: () => topic,
timestamp: () => timestamp
};
const processingMessage = messageProcessors.reduce((r, messageProcessor) => __awaiter(this, void 0, void 0, function* () {
const prevResult = yield r;
if (prevResult === abandon)
return prevResult;
const result = yield messageProcessor(prevResult, context);
return result;
}), Promise.resolve(message));
processingMessage.then((result) => {
processedOffsets.write(offset);
if (result !== abandon) {
push(null, result);
}
next();
}, (err) => {
push(err);
next();
});
});
return processedStream;
};
return [pipeline, processedOffsets];
});
}
exports.createPipeline = createPipeline;
//# sourceMappingURL=processors.js.map