node-red-contrib-kafka-consumer/kafkaConsumer.js

An Apache Kafka connector for node-red

module.exports = function (RED) {
	"use strict";
	
	var kafka = require('kafka-node');
	var util = require('util');
	
	/* 
	Kafka Producer Parameters:
		- topics
		- zkquorum
			- example:
				- zkquorum = "local:2181"
		- groupId
			- example:
				- groupId = "test-consumer-group" (this is the default setting in the kafka quickstart guide)
	*/
	
	function kafkaConsumer(config) {
		RED.nodes.createNode(this, config);
		
		var node = this;
		
		/* Config params */
		var topic = config.topic;
		var topics = config.topics;
		var clusterZookeeper = config.zkquorum;
		var groupId = config.groupId;
		var options = {
			groudId: groupId,
			autoCommit: config.autoCommit,
			autoCommitMsgCount: 10
		};
		
		var topicsJSONArray = [];
		
		var Client = kafka.Client;
		var client = new Client(clusterZookeeper);
		var HighLevelConsumer = kafka.HighLevelConsumer;
		var consumer = new HighLevelConsumer(client, [{topic: 'Temperature'}, {topic: 'management'}], {groupId: 'clique'});
		
		try {
            
            consumer.on('message', function (msg) {
				// check if multiple topics
				var payloads = [];
				if (msg.topic.indexOf(",") > -1) {
					var topicArray = msg.topic.split(',');
					console.log(topicArray);
					console.log(topicArray.length);


					for (i = 0; i < topicArray.length; i++) {
						console.log(topicArray[i]);
						topicsJSONArray.push({topic: topicArray[i]});
						payloads.push({topic: topicsJSONArray, messages: msg.value});
					}
					topics = topicsJSONArray;
				} else {
					
					//node.log("recieved payload " + util.inspect(msg, {showHidden: false, depth: null}));
					payloads = [{topic: msg.topic, messages: msg.value}];
				}
                node.send(msg.value, function(err, data) {
					if (err) {
						node.error(err);
					}
					node.log("Message recieved: " + data)
				});
            });

            
            this.on('error', function (err) {
				console.error(err);
            });
        } catch (e) {
			node.error(e);
			return;
		}
            this.log("Consumer created...");

		
	}
	RED.nodes.registerType("Kafka Consumer", kafkaConsumer);
};