roach-storm
Version:
Apache Kafka to Google Pub/Sub Gateway, API controlled
161 lines (160 loc) • 7.18 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const Debug = require("debug");
const R = require("ramda");
const moment = require("moment");
const debug = Debug("roach:handler");
class MessageHandler {
constructor(roachStorm) {
this.mongoPoller = roachStorm.mongoPoller;
this.metrics = roachStorm.metrics;
this.mongoWrapper = roachStorm.mongoWrapper;
this.pubSubHandler = roachStorm.pubSubHandler;
}
findConfigForTopic(topic) {
const topicConfigs = this.mongoPoller.getCollected().topicConfigs;
for (let i = topicConfigs.length - 1; i >= 0; i--) {
if (topicConfigs[i].sourceTopic === topic) {
return topicConfigs[i];
}
}
return null;
}
handleSortedMessageBatch(sortedBatch) {
// parallel processing on topic level
const topicPromises = Object.keys(sortedBatch).map((topic) => {
const topicConfig = this.findConfigForTopic(topic);
if (!topicConfig) {
this.metrics.inc("processed_messages_failed_no_config");
throw new Error("topic configuration missing for topic " + topic);
}
// parallel processing on partition level
const partitionPromises = Object.keys(sortedBatch[topic]).map((partition) => {
// sequential processing on message level (to respect ORDER)
const messages = sortedBatch[topic][partition]
.map((message) => this.handleMessage(message, topicConfig.parseAsJson))
.filter((message) => !!message);
return Promise.all(topicConfig.pipes.map((pipe) => {
return this.processMessagesOnPipe(pipe, messages);
}));
});
// wait until all partitions of this topic are processed and commit its offset
return Promise.all(partitionPromises);
});
return Promise.all(topicPromises);
}
async processMessagesOnPipe(pipe, messages) {
if (!pipe.publishTombstones) {
messages = messages.filter((message) => {
return !(message.value === null || message.value === "null" || typeof message.value === "undefined");
});
}
if (!pipe.filter || Object.keys(pipe.filter).length === 0) {
return this.publishMessagesChunkified(messages, pipe.targetTopic, pipe.chunkSize);
}
const messageFilter = R.allPass(Object.keys(pipe.filter).map((key) => {
if (key.indexOf("[") !== -1 || key.indexOf("]") !== -1) {
throw new Error("Character not allowed in filter key [], only dot strings as path allowed.");
}
if (Array.isArray(pipe.filter[key]) || (typeof pipe.filter[key] === "object"
&& pipe.filter[key] !== null)) {
throw new Error("Filter field values, must not be arrays or objects, please resolve via flat string paths. " + key);
}
return R.pathEq(key.split("."), pipe.filter[key]);
}));
messages = messages.filter((message) => {
return messageFilter(message);
});
return this.publishMessagesChunkified(messages, pipe.targetTopic, pipe.chunkSize);
}
publishMessagesChunkified(messages, targetTopic, chunkSize = 1) {
if (!messages.length) {
return Promise.resolve([]);
}
let index = 0;
let chunk = [];
const chunkPromises = [];
for (const message of messages) {
index++;
chunk.push(message);
if (chunk.length && (chunk.length >= chunkSize || index >= messages.length - 1)) {
chunkPromises.push(this.pubSubHandler.publish(targetTopic, JSON.stringify(chunk)));
chunk = [];
}
}
return Promise.all(chunkPromises);
}
handleMessage(message, parseAsJson = false) {
this.metrics.inc("processed_messages");
if (!message || !message.topic || typeof message.topic !== "string" || typeof message.partition !== "number") {
this.metrics.inc(`message_dropped`, 1, { topic: message.topic });
debug("Dropping message because of bad format, not an object or no topic", message);
return null;
}
if (!this.mongoWrapper.isConnected()) {
throw new Error("MongoDB connection is not established.");
}
const startTime = Date.now();
let keyAsBuffer = null;
let keyAsString = null;
if (message.key) {
if (Buffer.isBuffer(message.key)) {
keyAsBuffer = message.key;
keyAsString = message.key.toString("utf8");
}
else {
keyAsBuffer = Buffer.from(message.key);
keyAsString = message.key + "";
}
}
const messageHasTimestamp = message.timestamp && typeof message.timestamp === "number";
const timeOfStoring = moment().valueOf();
// try to strip the value as raw, yet parsed as possible before storing
// happy path here is to turn a message buffer into its JSON object and store as such
// in case the value does not contain a JSON payload, it should be stored as RAW (message.value) representative
// NOTE: Also check if topic shold be queryable, otherwise message value should be stored as buffer
let alteredMessageValue = null;
if (message.value && parseAsJson) {
if (Buffer.isBuffer(message.value)) {
alteredMessageValue = message.value.toString("utf8");
}
else {
alteredMessageValue = message.value;
}
try {
alteredMessageValue = JSON.parse(alteredMessageValue);
// no way to validate the output here
}
catch (_) {
alteredMessageValue = message.value;
}
}
// elif - always ensure we store value as buffer
if (message.value && !parseAsJson) {
if (Buffer.isBuffer(message.value)) {
alteredMessageValue = message.value;
}
else {
if (typeof message.value !== "string") {
alteredMessageValue = Buffer.from(JSON.stringify(message.value));
}
else {
alteredMessageValue = Buffer.from(message.value);
}
}
}
const parsedMessage = {
key: parseAsJson && keyAsString ? keyAsString : keyAsBuffer,
timestamp: messageHasTimestamp ? message.timestamp : timeOfStoring,
partition: message.partition,
offset: message.offset,
value: alteredMessageValue,
processedAt: timeOfStoring,
};
const duration = Date.now() - startTime;
this.metrics.set("processed_message_ms", duration);
this.metrics.inc("processed_messages_success");
return parsedMessage;
}
}
exports.default = MessageHandler;