kafka-node-reply
Version:
Kafka node reply is a function support service can send message as request and receive response from other consumer to complete request. Write base package kafka-node https://www.npmjs.com/package/kafka-node
366 lines (332 loc) • 12.8 kB
JavaScript
"use strict";
/**
*
* KafkaNodeReply v.1.0.0
*
* Kafka Reply is package base on kafka-node https://www.npmjs.com/package/kafka-node
*
* This package resolve problem which developer want to using kafka as request-response like http protocol
*
* Author QuyenCH v.quyench@cloudhms.net
*/
/**
* Module dependencies
*/
var events = require("events");
var kafka = require("kafka-node");
/**
* Function dependencies
*/
let RandomKey = (length) => {
var text = "";
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
for (var i = 0; i < length; i++)
text += possible.charAt(Math.floor(Math.random() * possible.length));
return text;
}
/**
*
* Class KafkaNodeReply
*
* @param {KafkaNode.client instancies} client The kafka-node client. See more: https://www.npmjs.com/package/kafka-node#kafkaclient
* @param {KafkaNode.admin instancies} admin The kafka-node admin. See more: https://www.npmjs.com/package/kafka-node#admin
* @param {Object} topicRequest The topic request config
* @param {String} topicRequest.topic The topic request name that producer will send message base on `topicRequest.options`
* @param {Integer} topicRequest.requestTimeout The request timeout in milisecond. Default 30000 ms
* @param {Object} topicRequest.options The options for kafka high level producer. See more: https://www.npmjs.com/package/kafka-node#producerkafkaclient-options-custompartitioner
*
* @param {Object} topicReply The topic request config
* @param {String} topicReply.topic The topic request name that producer will send message base on `topicReply.options`
* @param {Integer} topicReply.requestTimeout The request timeout in milisecond. Default 30000 ms
* @param {Object} topicReply.options The options for kafka high level producer. See more: https://www.npmjs.com/package/kafka-node#consumergroupoptions-topics
*
* Returns:
* kafka.HighLevelProducer with property function requestSync
*/
class KafkaNodeReply {
constructor(client, admin, topicRequest={}, topicReply={}, options={}) {
if (!client) {
throw new Error("[KafkaNodeReply] Require kafka-node client. See more: https://www.npmjs.com/package/kafka-node#kafkaclient")
}
if (!admin) {
throw new Error("[KafkaNodeReply] Require kafka-node admin. See more: https://www.npmjs.com/package/kafka-node#admin")
}
this.client = client
this._admin = admin
this.topicRequest = topicRequest;
this.topicReply = topicReply;
this.defaultTopicRequestOptions = {
requireAcks: 0,
ackTimeoutMs: 100,
partitionerType: 2
}
this.options = options;
this.connected = false
this.ready = false
this.producer = null
this.consumer = null
this.replyEmitter = new events.EventEmitter()
this.consumer = this.newConsumerReply(this.topicReply)
this.producer = this.newHighLevelProducer(this.topicRequest)
this.producer.sendMessage = this.sendMessage
this.sendMessage = this.sendMessage
// Start consumer
this.listenReplyEvent()
return this
}
/**
* Create new kafka client base on module kafka-node.
* See more at: https://www.npmjs.com/package/kafka-node
*
* @param {Object} `config` base on https://www.npmjs.com/package/kafka-node#kafkaclient
*/
newClient(config=this.configTopic) {
if (config.kafkaHost === undefined) {
throw("[KafkaNodeReply] Config require field `kafkaHost`")
}
/**
* Create and return client with status connected
*/
return new Promise((resolve, reject) => {
this.client = new kafka.KafkaClient(config);
// kafka on connected ready
this.client.on('ready', function (){
this.connected = true
resolve(this.client)
})
// kafka on connected error
this.client.on('error', function (err){
reject(err)
})
this._admin = new kafka.Admin(this.client);
})
}
/**
* Create consumer group, config and options same to initialize a https://www.npmjs.com/package/kafka-node#consumer
*
* @param {Object} `topicReply`
* @param {String} `topicReply.topic` Name of topic that reply consumer receive message
* @param {Object} `topicReply.options`
* @param {String} `topicReply.options.groupId` Unique groupId to identified group consume topic
* Example:
* {
* topicRequest: "PmsPropertyRoomCreate_Request",
* topicReply: "PmsPropertyRoomCreate_Request",
* options: {
* groupId: "hms.cts.organization"
* }
* }
*/
newConsumerReply(topicReply) {
let ConsumerGroup = kafka.ConsumerGroup
let options = topicReply.options
// Set consumer host if not define
if (options == undefined) {
options = this.options
} else {
options = Object.assign({},this.options,options)
}
let topicPartition = {}
topicPartition[topicReply.topic] = undefined
/**
* Event on consumer being rebalance.
* A message transition to wrong destitional partition while consumer group being rebalance
* So producer has attribute `ready` to control this
*
* Args:
* - isAlreadyMember: True if consumer is first connect else false
* - callback: Callback function to continue consume
*/
let KafkaNodeReply = this
options.onRebalance = async (isAlreadyMember, callback) => {
this.ready = false
function refresh() {
return new Promise((r,j)=> {
setTimeout(() => {
KafkaNodeReply._admin.describeGroups([options.groupId], async (err, result)=>{
let consumerClientId = KafkaNodeReply.consumer.memberId
if (err || result[options.groupId].state != "Stable" ) {
await refresh()
} else {
for (const consumerClient of result[options.groupId].members) {
if (consumerClient.memberId == consumerClientId) {
KafkaNodeReply.partitions = consumerClient.memberAssignment.partitions[topicReply.topic]
if(KafkaNodeReply.partitions == undefined) {
await refresh()
} else {
KafkaNodeReply.ready = true
return r()
}
break
}
}
console.debug(`[KafkaNodeReply] Loaded new partitions assigned [${options.groupId}][${topicReply.topic}] ${consumerClientId}: ${KafkaNodeReply.partitions}`)
}
})
},2000)
})
}
await refresh()
callback()
}
let consumerGroup = new ConsumerGroup(
options,
topicReply.topic
)
consumerGroup.on("error", (err)=>{
let consumerConnect = false
let t = setTimeout(()=>{
if (consumerConnect == false){
this.newConsumerReply(this.topicReply)
} else {
consumerConnect = true
clearTimeout(t)
}
}, 1000)
})
return consumerGroup
}
/**
* Create new Producer base on KafkaNode.HighLevelProducer
*
* Returns:
* Kafka producer
*/
newHighLevelProducer(options={}) {
if (options == undefined) {
options = this.defaultTopicRequestOptions
} else {
options = Object.assign({},this.defaultTopicRequestOptions,options)
}
let Producer = kafka.HighLevelProducer
let producer = new Producer(this.client)
producer.on('ready', function(data){
console.debug("[KafkaNodeReply] New Producer already")
})
producer.on('error', function (err) {
console.error("[KafkaNodeReply] ERROR when new Producer ", err)
// throw err
})
return producer
}
/**
* Create request synchronous.
* An message will be publish to topic `config.topicRequest` which format json below
* {
* "action": "getBatch",
* "body": {
* "limit": 20
* },
* "contentType": "application/json",
* "headers": {
* "Authorization": "Bearer AuthenicationToken"
* },
* "callback": {
* "type": "sync",
* "kafka": {
* "topic": "PmsPropertyRoomCreate_Request",
* "partition": 8,
* "key": "Tg9PHN5tlSoI42X0VmoLqhyMsPpSqf"
* }
* },
* "date": "2019-04-11T04:59:22.006Z"
* }
*
* @param {Object} message The message
* @param {String} message.action The action that consumer process
* @param {Object} message.body The request body message
* @param {String} message.contentType The message content type
* @param {Object} message.headers The request headers, like http request headers
* @param {Obejct} message.callback The callback info. Include type of request and method callback. Default callback through kafka service
* @param {String} message.callback.type Type of request. Value `sync` it mean request require response from consumer to complete request
* @param {Object} message.callback.kafka Callback method, current support only kafka
* @param {String} message.callback.kafka.topic The topic name which consumer will send message
* @param {String} message.callback.kafka.partition The partition specify that consumer will send message to topic above
* @param {String} message.callback.kafka.key The request key that consumer must send message with key of message for requestor determine it's response
* @param {Integer} timeout Request timeout in milisecond
* @param {Boolean} force When kafka consumer is rebalancing, the producer will have property `ready` state is false.
* In this case, producer cannot send request. But this value `force` = true will force producer send request.
* NOTE: While consumer is rebalancing, message may be send to wrong partition and request cannot get response
*
* Other consumer consume topic above and process base on business logic.
* After process, if request has callback info. Consumer must send response to callback info.
*
* Returns:
* Promise response from consumer
*
* TODO: Support other message type such as: binary, byte...
*/
requestSync(message, timeout, force=false) {
if (!this.ready && force == false) {
throw("ERROR Consumer group being rebalance.")
}
if (this.partitions == undefined) {
throw("ERROR This consumer is not assigned partition. Cannot send and consume message.")
}
var timeout = timeout?timeout:this.topicRequest.requestTimeout?this.topicRequest.requestTimeout:30000
var partition = this.ready?this.partitions[Math.floor(Math.random() * this.partitions.length)]:undefined
// Random key
let key = RandomKey(30)
if (!message.callback) {
let callback = {
type: "sync",
kafka: {
topic: this.topicReply.topic,
partition: partition,
key: key
}
}
message.callback = callback
}
// Stringify message
message = JSON.stringify(message)
// Prepare && Wrap message
let topic = {
messages: message,
key: key,
// partition: partition,
topic: this.topicRequest.topic
}
let payloads = [
topic
]
return new Promise((resolve, reject)=>{
let settimeout = setTimeout(() => {
let error = new Error(`[KafkaNodeReply] Request ${key} TIMEOUT after ${timeout}ms`)
return reject(error)
}, timeout);
this.replyEmitter.once(topic.key, (response)=> {
clearTimeout(settimeout)
return resolve(response)
})
this.producer.send(payloads, function(err, data){
if (err) {
console.log("DEBUG", data)
return reject(err)
}
})
})
}
/**
* Start consumer and wait for message receive
* When receive a message. The events emit will publish this message value with key in message
*/
listenReplyEvent() {
let consumer = this.consumer
this.consumer.on("message", (message)=>{
let response
try {
response = JSON.parse(message.value)
} catch (err) {
response = message.value
}
this.replyEmitter.emit(message.key, response)
consumer.commit(function(err, data) {
});
})
this.consumer.on("error", (error)=>{
console.error("[KafkaNodeReply] ERROR when consume: ", error)
})
}
}
module.exports = KafkaNodeReply