frolyk
Version:
Stream processing library for Kafka in Node
160 lines • 7.03 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const stream_1 = require("stream");
const events_1 = require("events");
const long_1 = __importDefault(require("long"));
const invariant_1 = __importDefault(require("invariant"));
class SeekOp {
constructor({ id, offset }) {
this.id = id;
this.offset = offset;
}
}
class TPStream extends stream_1.Transform {
constructor({ topic, partition, consumer }, streamOptions = {}) {
super(Object.assign(Object.assign({}, streamOptions), { objectMode: true, readableHighWaterMark: 0, writableHighWaterMark: 16, emitClose: true, autoDestroy: true }));
this.topic = topic;
this.partition = partition;
this.consumer = consumer;
this.seekOpIdSeq = -1;
this.observedOpId = -1;
}
_transform(messageOrOp, encoding, callback) {
if (this.seekOpIdSeq !== this.observedOpId) {
// there's a pending seek operation to be completed
if (messageOrOp instanceof SeekOp) {
let op = messageOrOp;
this.observedOpId = op.id;
}
return callback();
}
else {
let message = messageOrOp;
callback(null, message);
}
}
seek(offset) {
try {
offset = long_1.default.fromValue(offset);
}
catch (parseError) {
invariant_1.default(!parseError, 'Valid offset (parseable as Long) is required to seek stream to offset');
}
const { topic, partition } = this;
const operation = new SeekOp({
id: ++this.seekOpIdSeq,
offset: offset.toString()
});
this.consumer.seek({
topic,
partition,
offset: offset.toString()
});
this.write(operation);
}
}
class TaskStreams {
constructor(consumer) {
this.consumer = consumer;
this.streams = [];
this.consumerEvents = new events_1.EventEmitter();
consumer.on(consumer.events.STOP, (...args) => {
this.consumerEvents.emit(consumer.events.STOP, ...args);
});
}
stream({ topic, partition }) {
const { consumer, consumerEvents } = this;
const stream = this.streams
.find((topparStream) => topparStream.topic === topic && topparStream.partition === partition);
if (!stream) {
let newStream = new TPStream({ topic, partition, consumer });
let onConsumerStop = () => newStream.end();
let onStreamClose = () => {
this.streams = this.streams.filter((stream) => stream !== newStream);
};
let onStreamFinish = () => cleanup();
let cleanup = () => {
consumerEvents.removeListener(consumer.events.STOP, onConsumerStop);
newStream.destroy(); // no more writing to this stream
};
newStream.once('close', onStreamClose);
newStream.once('finish', onStreamFinish);
consumerEvents.once(consumer.events.STOP, onConsumerStop);
this.streams.push(newStream);
return newStream;
}
else {
return stream;
}
}
start() {
return __awaiter(this, void 0, void 0, function* () {
const { consumer } = this;
const pauseConsumption = ({ topic, partition }) => __awaiter(this, void 0, void 0, function* () {
const stream = this.stream({ topic, partition });
// let isDrained = Symbol('drained')
let onDrain;
let draining = new Promise((resolve, reject) => {
onDrain = () => resolve();
stream.once('drain', onDrain);
});
let onEnd;
let ending = new Promise((resolve, reject) => {
onEnd = () => resolve();
stream.once('end', onEnd);
});
consumer.pause([{ topic, partitions: [partition] }]);
try {
yield Promise.race([draining, ending]);
}
finally {
stream.removeListener('drain', onDrain);
stream.removeListener('end', onEnd);
}
});
yield consumer.run({
autoCommit: false,
eachBatchAutoResolve: false,
partitionsConsumedConcurrently: 4,
eachBatch: ({ batch, resolveOffset: checkpoint, commitOffsetsIfNecessary: commitOffset, heartbeat, isRunning, isStale }) => __awaiter(this, void 0, void 0, function* () {
const stream = this.stream({ topic: batch.topic, partition: batch.partition });
for (let message of batch.messages) {
if (!isRunning() || stream.writableEnded || isStale())
break;
const more = stream.write(Object.assign(Object.assign({ topic: batch.topic, partition: batch.partition, highWaterOffset: batch.highWatermark, offset: message.offset }, message), { timestamp: message.timestamp }));
checkpoint(message.offset);
if (!more && isRunning()) {
yield pauseConsumption({ topic: batch.topic, partition: batch.partition });
if (!isRunning() || stream.writableEnded)
break;
}
yield heartbeat();
}
yield heartbeat();
// whether we drained or stopped consuming this partition, we'll
// want it to be okay for consumer to fetch for this partition again.
if (isRunning()) {
consumer.resume([{ topic: batch.topic, partitions: [batch.partition] }]);
}
})
});
});
}
}
function create(consumer) {
return new TaskStreams(consumer);
}
exports.default = create;
//# sourceMappingURL=streams.js.map