frolyk
Version:
Stream processing library for Kafka in Node
152 lines • 7.44 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const highland_1 = __importDefault(require("highland"));
const kafkajs_1 = require("kafkajs");
const long_1 = __importDefault(require("long"));
const invariant_1 = __importDefault(require("invariant"));
const lodash_groupby_1 = __importDefault(require("lodash.groupby"));
const offsets_1 = require("../offsets");
const processors_1 = require("../processors");
function createContext({ assignment, processors, stream: rawStream, admin, createProducer, consumer }) {
return __awaiter(this, void 0, void 0, function* () {
const controlledStream = highland_1.default(rawStream);
const producer = createProducer();
/* istanbul ignore next */
const fetchWatermarks = () => __awaiter(this, void 0, void 0, function* () {
const offsets = yield admin.fetchTopicOffsets(assignment.topic);
const partitionOffset = offsets.find((offset) => assignment.partition === offset.partition);
return {
high: partitionOffset.high && long_1.default.fromString(partitionOffset.high),
low: partitionOffset.low && long_1.default.fromString(partitionOffset.low)
};
});
const assignmentContext = {
caughtUp(offset) {
return __awaiter(this, void 0, void 0, function* () {
var offsetLong;
try {
offsetLong = long_1.default.fromValue(offset);
}
catch (parseError) {
invariant_1.default(false, 'Valid offset (parseable as Long) is required to verify the assignment is caught up at it');
}
const watermarks = yield fetchWatermarks();
return watermarks.high.lt(1) || offsetLong.gte(watermarks.high);
});
},
commitOffset(newOffset, metadata = null) {
return __awaiter(this, void 0, void 0, function* () {
try {
newOffset = long_1.default.fromValue(newOffset);
}
catch (parseError) {
invariant_1.default(false, 'Valid offset (parseable as Long) is required to commit offset for assignment');
}
yield consumer.commitOffsets([{
topic: assignment.topic,
partition: assignment.partition,
offset: newOffset.toString(),
metadata
}]);
});
},
committed() {
return __awaiter(this, void 0, void 0, function* () {
const payload = { groupId: assignment.group, topic: assignment.topic };
const partitionOffsets = yield admin.fetchOffsets(payload);
const { offset, metadata } = partitionOffsets.find(({ partition }) => partition === assignment.partition);
return { offset, metadata };
});
},
isEmpty() {
return __awaiter(this, void 0, void 0, function* () {
const watermarks = yield fetchWatermarks();
return watermarks.high.subtract(watermarks.low).lte(0);
});
},
/* istanbul ignore next */
log(tags, payload) {
return __awaiter(this, void 0, void 0, function* () { });
},
seek(offset) {
return __awaiter(this, void 0, void 0, function* () {
if (offsets_1.isEarliest(offset))
offset = (yield fetchWatermarks()).low;
if (offsets_1.isLatest(offset))
offset = (yield fetchWatermarks()).high;
return rawStream.seek(offset);
});
},
send(messages) {
return __awaiter(this, void 0, void 0, function* () {
if (!Array.isArray(messages))
messages = [messages];
const messagesByTopic = lodash_groupby_1.default(messages, (message) => message.topic);
const topics = Object.keys(messagesByTopic);
const topicMessages = topics.reduce((topicMessages, topic) => {
topicMessages.push({
topic,
messages: messagesByTopic[topic]
});
return topicMessages;
}, []);
// TODO: incorporate acks, timeouts and compression into API somehow
return producer.sendBatch({
topicMessages,
acks: -1,
timeout: 30 * 1000,
compression: kafkajs_1.CompressionTypes.None
});
});
},
/* istanbul ignore next */
watermarks() {
return __awaiter(this, void 0, void 0, function* () {
const { high, low } = yield fetchWatermarks();
return {
highOffset: high.toString(),
lowOffset: low.toString()
};
});
},
topic: assignment.topic,
partition: assignment.partition,
group: assignment.group
};
const [processingPipeline, processedOffsets] = yield processors_1.createPipeline(assignmentContext, processors);
const processedStream = controlledStream.through(processingPipeline);
processedOffsets.each((offset) => {
// this stream is mostly used for testing, so here we'll just want to make sure it doesn't
// become a memory leak by instantly relieving all back-pressure
});
return {
topic: assignment.topic,
partition: assignment.partition,
stream: processedStream,
start() {
return __awaiter(this, void 0, void 0, function* () {
yield producer.connect();
});
},
stop() {
return __awaiter(this, void 0, void 0, function* () {
yield producer.disconnect();
});
}
};
});
}
exports.default = createContext;
//# sourceMappingURL=kafka.js.map