@itentialopensource/adapter-kafka
Version:
[Deprecated] Itential adapter to connect to kafka
1,271 lines (1,171 loc) • 65.3 kB
JavaScript
/* @copyright Itential, LLC 2019 (pre-modifications) */
// Set globals
/* global eventSystem log */
/* eslint no-underscore-dangle: warn */
/* eslint no-loop-func: warn */
/* eslint no-cond-assign: warn */
/* eslint no-unused-vars: warn */
/* eslint consistent-return: warn */
/* eslint import/no-dynamic-require: warn */
/* eslint global-require: warn */
/* eslint no-await-in-loop: warn */
/* eslint no-promise-executor-return: warn */
/* Required libraries. */
const fs = require('fs-extra');
const path = require('path');
const util = require('util');
/* Fetch in the other needed components for the this Adaptor */
const EventEmitterCl = require('events').EventEmitter;
const http = require('http');
const kafkaLogging = require('kafka-node/logging');
function consoleLoggerProvider(name) {
return {
debug: log.debug,
info: log.info,
warn: log.warn,
error: log.error
};
}
kafkaLogging.setLoggerProvider(consoleLoggerProvider);
const kafka = require('kafka-node');
const pjson = require(path.resolve(__dirname, 'pronghorn.json'));
let myid = null;
let errors = [];
let firstRun = false;
let firstDone = false;
const mytopics = Object.keys(pjson.topics);
const pronghornFile = path.join(__dirname, '/pronghorn.json');
async function fetchSchema(registryUrl, topic, version) {
const reqOptions = {
hostname: registryUrl,
path: `/subjects/${topic}/versions/${version}`,
method: 'GET'
};
return new Promise((resolve, reject) => {
log.debug(`${registryUrl}/subjects/${topic}/versions/${version}`);
let respStr = '';
try {
const req = http.request(reqOptions, (res) => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error(`Schema registry error: Did not receive a valid status code: ${res.statusCode}`));
}
// process data from response
res.on('data', (replyData) => {
respStr += replyData;
});
res.on('end', () => {
// this is the schema response
log.debug('returning schema response');
return resolve(JSON.parse(respStr));
});
});
req.on('error', (error) => reject(new Error(
`Schema registry error: ${error.code} - ${error.message}`
)));
req.end();
} catch (e) {
log.error(`error sending request: ${e} \n${e.stack}`);
return reject(e);
}
});
}
/**
* @summary Build a standard error object from the data provided
*
* @function formatErrorObject
* @param {String} origin - the originator of the error (optional).
* @param {String} type - the internal error type (optional).
* @param {String} variables - the variables to put into the error message (optional).
* @param {Integer} sysCode - the error code from the other system (optional).
* @param {Object} sysRes - the raw response from the other system (optional).
* @param {Exception} stack - any available stack trace from the issue (optional).
*
* @return {Object} - the error object, null if missing pertinent information
*/
function formatErrorObject(origin, type, variables, sysCode, sysRes, stack) {
log.trace(`${myid}-adapter-formatErrorObject`);
// add the required fields
const errorObject = {
icode: 'AD.999',
IAPerror: {
origin: `${myid}-unidentified`,
displayString: 'error not provided',
recommendation: 'report this issue to the adapter team!'
}
};
if (origin) {
errorObject.IAPerror.origin = origin;
}
if (type) {
errorObject.IAPerror.displayString = type;
}
// add the messages from the error.json
for (let e = 0; e < errors.length; e += 1) {
if (errors[e].key === type) {
errorObject.icode = errors[e].icode;
errorObject.IAPerror.displayString = errors[e].displayString;
errorObject.IAPerror.recommendation = errors[e].recommendation;
} else if (errors[e].icode === type) {
errorObject.icode = errors[e].icode;
errorObject.IAPerror.displayString = errors[e].displayString;
errorObject.IAPerror.recommendation = errors[e].recommendation;
}
}
// replace the variables
let varCnt = 0;
while (errorObject.IAPerror.displayString.indexOf('$VARIABLE$') >= 0) {
let curVar = '';
// get the current variable
if (variables && Array.isArray(variables) && variables.length >= varCnt + 1) {
curVar = variables[varCnt];
}
varCnt += 1;
errorObject.IAPerror.displayString = errorObject.IAPerror.displayString.replace('$VARIABLE$', curVar);
}
// add all of the optional fields
if (sysCode) {
errorObject.IAPerror.code = sysCode;
}
if (sysRes) {
errorObject.IAPerror.raw_response = sysRes;
}
if (stack) {
errorObject.IAPerror.stack = stack;
}
// return the object
return errorObject;
}
/**
* This is the adapter/interface into Kafka
*/
class Kafka extends EventEmitterCl {
/**
* Kafka Adapter
* @constructor
*/
constructor(prongid, properties) {
super();
this.props = properties;
this.alive = false;
this.healthy = false;
this.id = prongid;
myid = prongid;
// put topics from file into memory
this.topicsEvents = [];
let topicsFile = path.join(__dirname, `/.topics-${this.id}.json`);
// if the file does not exist - error
if (fs.existsSync(topicsFile)) {
this.topicsEvents = JSON.parse(fs.readFileSync(topicsFile, 'utf-8'));
} else if (fs.existsSync(path.join(__dirname, '/.topics.json'))) {
log.debug('Found old .topics.json file.');
topicsFile = path.join(__dirname, '/.topics.json');
this.topicsEvents = JSON.parse(fs.readFileSync(topicsFile, 'utf-8'));
}
// get the topics into the right format --- fix older .topics.json files so they have the right format
if (!Array.isArray(this.topicsEvents)) {
// original format
const keys = Object.keys(this.topicsEvents);
const tempTops = [];
keys.forEach((item) => {
tempTops.push({
topic: item,
partition: this.topicsEvents[item].partitions || this.topicsEvents[item].partition || 0,
offset: this.topicsEvents[item].offset || 0,
processed: this.topicsEvents[item].offset || 0,
subscribers: this.topicsEvents[item].subscribers || 0,
avro: this.topicsEvents[item].avro || 'NO',
subInfo: [
{
subname: 'default',
filters: [],
rabbit: 'kafka',
throttle: {}
}
]
});
});
this.topicsEvents = tempTops;
} else {
for (let t = 0; t < this.topicsEvents.length; t += 1) {
if (!Object.hasOwnProperty.call(this.topicsEvents[t], 'processed')) {
this.topicsEvents[t].processed = this.topicsEvents[t].offset || 0;
}
if (!Object.hasOwnProperty.call(this.topicsEvents[t], 'subInfo')) {
this.topicsEvents[t].subInfo = [
{
subname: 'default',
filters: [],
rabbit: 'kafka',
throttle: {}
}
];
}
}
}
// If we have a rabbit queue - need to add that to topics.json
// if there are topics in the properties
if (this.props && this.props.topics) {
let needRestart = false;
for (let t = 0; t < this.props.topics.length; t += 1) {
if (typeof this.props.topics[t] === 'string') {
const topName = this.props.topics[t];
if (!mytopics.includes(topName)) {
pjson.topics[this.props.topics[t]] = {};
needRestart = true;
}
for (let te = 0; te < this.topicsEvents.length; te += 1) {
// only set the rabbit in the topic if it is the default (kafka)
if (this.topicsEvents[te].topic === topName && this.topicsEvents[te].partition === 0
&& this.topicsEvents[te].subInfo.rabbit === 'kafka') {
this.topicsEvents[te].subInfo.rabbit = topName;
break;
}
}
} else if (typeof this.props.topics[t] === 'object') {
const topName = this.props.topics[t].name;
// if the topic is not in the pronghorn.json
if (!mytopics.includes(topName)) {
pjson.topics[topName] = {};
needRestart = true;
}
const topPart = this.props.topics[t].partitions || this.props.topics[t].partition || 0;
let useAvro = 'NO';
if (this.props.topics[t].avro !== undefined && this.props.topics[t].avro !== null && this.props.topics[t].avro === true) {
useAvro = 'YES';
}
let subInfo = [
{
subname: 'default',
filters: [],
rabbit: 'kafka',
throttle: {}
}
];
if (this.props.topics[t].subscriberInfo) {
subInfo = this.props.topics[t].subscriberInfo;
for (let r = 0; r < subInfo.length; r += 1) {
// if the rabbit topic is not in the pronghorn.json
if (!mytopics.includes(subInfo[r].rabbit)) {
pjson.topics[subInfo[r].rabbit] = {};
needRestart = true;
}
}
}
let found = false;
for (let te = 0; te < this.topicsEvents.length; te += 1) {
// see if we find the topic and partition
let updatedPartitionArray = false;
if (Array.isArray(this.topicsEvents[te].partition)) {
if (Array.isArray(topPart)) {
this.topicsEvents[te].partition = topPart;
this.topicsEvents[te].offset = Array(topPart.length).fill(0);
updatedPartitionArray = true;
} else if (this.topicsEvents[te].partition.includes(topPart)) {
updatedPartitionArray = true;
} else {
this.topicsEvents[te].partition.push(topPart);
this.topicsEvents[te].offset.push(0);
updatedPartitionArray = true;
}
}
if (this.topicsEvents[te].topic === topName && (updatedPartitionArray === true || this.topicsEvents[te].partition === topPart)) {
found = true;
// only set the rabbit in the topic if it is the default (kafka)
if (this.topicsEvents[te].subInfo.rabbit === 'kafka') {
this.topicsEvents[te].subInfo.rabbit = topName;
}
// set the subscribers if always
if (this.props.topics[t].always) {
this.topicsEvents[te].subscribers = 99999;
}
// set the avro accordingly
this.topicsEvents[te].avro = useAvro;
// check to see if we need to add a subscriber
for (let sub = 0; sub < subInfo.length; sub += 1) {
let subFound = false;
for (let s = 0; s < this.topicsEvents[te].subInfo.length; s += 1) {
// if the subscriber is found, no need to add anything
if (subInfo[sub].subname === this.topicsEvents[te].subInfo[s].subname) {
subFound = true;
// set the filters
if (subInfo[sub].filters) {
this.topicsEvents[te].subInfo[s].filters = subInfo[sub].filters;
}
// set the rabbit
if (subInfo[sub].rabbit) {
this.topicsEvents[te].subInfo[s].rabbit = subInfo[sub].rabbit;
}
// set the throttle
if (subInfo[sub].throttle) {
this.topicsEvents[te].subInfo[s].throttle = subInfo[sub].throttle;
}
break;
}
}
if (!subFound) {
// add the subscriber if it was not found
this.topicsEvents[te].subInfo.push(subInfo[sub]);
}
}
break;
}
}
// if the topic was not found but should always be subscribed to, add it
if (!found && this.props.topics[t].always) {
if (Array.isArray(topPart)) {
const partOffset = Array(topPart.length).fill(0);
this.topicsEvents.push({
topic: topName,
partition: topPart,
offset: partOffset,
processed: partOffset,
subscribers: 99999,
avro: useAvro,
subInfo
});
} else {
this.topicsEvents.push({
topic: topName,
partition: topPart,
offset: 0,
processed: 0,
subscribers: 99999,
avro: useAvro,
subInfo
});
}
}
}
}
// if we need to restart the adapter due to changes to pronghorn.json
if (needRestart) {
fs.writeFileSync(pronghornFile, JSON.stringify(pjson, null, 2));
log.error('NEED TO RESTART ADAPTER - EXITING');
const errorObj = {
origin: `${this.id}-adapter-constructor`,
type: 'Restarting to add new topics',
vars: []
};
// log and throw the error
log.error(`${errorObj.origin}: ${errorObj.displayString}`);
setTimeout(() => {
throw new Error(JSON.stringify(errorObj));
}, 1000);
}
}
this.KafkaClient = null;
this.producer = null;
this.KafkaProducerClient = null;
this.consumer = null;
this.KafkaConsumerClient = null;
this.registryUrl = null;
this.registry = null;
if (this.props && this.props.registry_url) {
this.registryUrl = this.props.registry_url;
this.registry = require('avro-schema-registry')(this.props.registry_url);
}
// get the path for the specific error file
const errorFile = path.join(__dirname, '/error.json');
// if the file does not exist - error
if (!fs.existsSync(errorFile)) {
const origin = `${this.id}-adapter-constructor`;
log.warn(`${origin}: Could not locate ${errorFile} - errors will be missing details`);
}
// Read the action from the file system
const errorData = JSON.parse(fs.readFileSync(errorFile, 'utf-8'));
({ errors } = errorData);
// rewrite the topics file for persistence
if (this.props && this.props.stub === false) {
const intTime = this.props.interval_time || 30000;
setInterval(() => {
try {
const split = topicsFile.split('/');
log.debug(`Update ${split[split.length - 1]} file.`);
fs.writeFileSync(topicsFile, JSON.stringify(this.topicsEvents, null, 2));
} catch (err) {
log.error(err);
}
}, intTime);
}
}
/**
* @summary Connect function is used during Pronghorn startup to provide instantiation feedback.
*
* @function connect
*/
connect() {
const meth = 'adapter-connect';
const origin = `${this.id}-${meth}`;
log.trace(origin);
// initially set as off
this.emit('OFFLINE', { id: this.id });
this.alive = true;
if (this.props && this.props.stub === true) {
this.emit('ONLINE', {
id: this.id
});
log.info('EMITTED ONLINE ON STUB MODE');
return;
}
try {
// Create a kafka client
const combinedProps = this.props.client || {};
combinedProps.kafkaHost = this.props.hostList || this.props.host.concat(':', this.props.port);
const consumerClientProps = combinedProps;
const producerClientProps = combinedProps;
if (this.props.client && this.props.client.consumersasl) {
consumerClientProps.sasl = this.props.client.consumersasl;
}
if (this.props.client && this.props.client.producersasl) {
producerClientProps.sasl = this.props.client.producersasl;
}
this.KafkaClient = new kafka.KafkaClient(combinedProps);
this.KafkaClient.on('ready', () => {
this.emit('ONLINE', {
id: this.id
});
log.info('EMITTED ONLINE');
this.KafkaProducerClient = new kafka.KafkaClient(producerClientProps);
this.producer = new kafka.Producer(this.KafkaProducerClient, this.props.producer || {});
this.offsetMgr = new kafka.Offset(this.KafkaClient);
// get the topics
const consumerPayloads = [];
// Need to handle the topics that are already defined
for (let t = 0; t < this.topicsEvents.length; t += 1) {
if (Array.isArray(this.topicsEvents[t].partition)) {
for (let p = 0; p < this.topicsEvents[t].partition.length; p += 1) {
consumerPayloads.push({
topic: this.topicsEvents[t].topic,
partition: this.topicsEvents[t].partition[p] || 0,
offset: this.topicsEvents[t].offset[p] || 0
});
}
} else {
consumerPayloads.push({
topic: this.topicsEvents[t].topic,
partition: this.topicsEvents[t].partition || 0,
offset: this.topicsEvents[t].offset || 0
});
}
}
// create the consumer
log.debug(origin, 'consumerPayloads', consumerPayloads, this.props.consumer);
this.KafkaConsumerClient = new kafka.KafkaClient(consumerClientProps);
this.consumer = new kafka.Consumer(this.KafkaConsumerClient, consumerPayloads, this.props.consumer || {});
// consume the messages
this.consumer.on('error', (serr) => {
log.error(`Consumer Error: ${serr}`);
});
const setCurrentOffset = (topicDetails, topic, partition, offset) => {
log.debug(origin, 'setOffset', topicDetails, topic, partition, offset);
this.consumer.pause();
this.consumer.setOffset(topic, partition, offset);
this.consumer.resume();
topicDetails.offset = offset; // eslint-disable-line no-param-reassign
};
this.consumer.on('offsetOutOfRange', (err) => {
const { topic } = err;
log.error(`OffsetOutOfRange, topic: ${topic}`);
const topicEvent = this.topicsEvents.find(((e) => e.topic === topic));
if (!topicEvent) {
log.error(origin, 'Topic', topic, 'not found in', this.topicsEvents);
return;
}
/*
Read current offset range from kafka server (earliest, latest),
if current consumer offset is less than the earliest offset, set consumer offset to earliest
if current consumer offset is greater than the latest offset, set consumer offset to latest
*/
this.offsetMgr.fetchEarliestOffsets([topic], (earliestError, earliestOffsets) => {
if (earliestError) {
log.error(origin, `Could not get earliest offset for topic: ${topic}`, earliestError);
} else {
log.debug(origin, topic, 'earliestOffsets', earliestOffsets, 'current offset', topicEvent.offset, 'earlistOffset', earliestOffsets[topic][topicEvent.partition]);
if (topicEvent.offset < earliestOffsets[topic][topicEvent.partition]) {
setCurrentOffset(topicEvent, topic, topicEvent.partition, earliestOffsets[topic][topicEvent.partition]);
} else {
this.offsetMgr.fetchLatestOffsets([topic], (lastestError, lastestOffsets) => {
if (lastestError) {
log.error(origin, `Could not get latest offset for topic: ${topic}`, lastestError);
} else {
log.debug(origin, topic, 'lastestOffsets', lastestOffsets, 'current offset', topicEvent.offset, 'latestOffset', lastestOffsets[topic][topicEvent.partition]);
if (topicEvent.offset > lastestOffsets[topic][topicEvent.partition]) {
setCurrentOffset(topicEvent, topic, topicEvent.partition, lastestOffsets[topic][topicEvent.partition]);
}
}
});
}
}
});
});
const isPassingFiltering = (message, filters) => {
const topicConfig = this.topicsEvents.find((e) => e.topic === message.topic);
if (!topicConfig) return true;
if (filters.length === 0) return true;
const matched = filters.find((filter) => {
const re = new RegExp(filter);
if (message.value.match(re)) {
log.debug('Matched filter: ', re);
return true;
}
return false;
});
return matched;
};
this.consumer.on('message', (message) => {
(async () => {
log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${message.topic} PARTITION: ${message.partition} WITH OFFSET: ${message.offset}`);
const newMsg = message;
let avroUsed = false;
const desiredTopic = [];
let passesFiltering = false;
let goodMsg = false;
/*
To handle adapter restart when adapter is configured with prop ("fromOffset": true).
If the message's offset is less or equal to currently stored offset for given (topic:partition),
then the mesage has already been emmited before adapter restart.
Note that the current offset setting after adapter restart depends on .topic.json content
and outOfRange resolution.
*/
let duplicate = false;
const topicConfig = this.topicsEvents.find((e) => e.topic === message.topic);
if (!topicConfig) {
goodMsg = true;
}
// should always find the topic here to change the offset
for (let t = 0; t < this.topicsEvents.length; t += 1) {
if (this.topicsEvents[t].topic === message.topic && (this.topicsEvents[t].partition === message.partition || Object.values(this.topicsEvents[t].partition).includes(message.partition))) {
if (this.topicsEvents[t].partition === message.partition) {
if (this.topicsEvents[t].offset < message.offset) {
this.topicsEvents[t].offset = message.offset;
} else {
duplicate = true;
}
} else if (Array.isArray(this.topicsEvents[t].partition) && Object.values(this.topicsEvents[t].partition).includes(message.partition)) {
const offsetPosition = this.topicsEvents[t].partition.indexOf(message.partition);
if (this.topicsEvents[t].offset[offsetPosition] < message.offset) {
this.topicsEvents[t].offset[offsetPosition] = message.offset;
} else {
duplicate = true;
}
}
// determine if we are using AVRO and set flag
if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
avroUsed = true;
}
// go through the filters to see if this is a message we want and set desired topic for messages passing filtering
if (this.topicsEvents[t].subInfo) {
for (let k = 0; k < this.topicsEvents[t].subInfo.length; k += 1) {
if (this.topicsEvents[t].subInfo[k].filters) {
passesFiltering = isPassingFiltering(newMsg, this.topicsEvents[t].subInfo[k].filters);
}
if (this.topicsEvents[t].subInfo[k].rabbit && passesFiltering) {
desiredTopic.push(this.topicsEvents[t].subInfo[k].rabbit);
goodMsg = true;
} else if (passesFiltering) {
desiredTopic.push(this.topicsEvents[t].topic);
goodMsg = true;
}
}
} else if (!this.topicsEvents[t].subInfo || !Array.isArray(this.topicsEvents[t].subInfo)) {
goodMsg = true;
}
break;
}
}
// if using avro
if (this.registry && avroUsed) {
// will give us an interval between 5001 and 60000 milliseconds (5 seconds to 1 minute)
const fastInt = Math.floor(Math.random() * 55000) + 5001;
// This interval is to help prevent the adapter from bombarding the registry by sending requests at random intervals
const intervalObject = setInterval(async () => {
// Prevents running the request while the first one is running
if (!firstRun) {
if (!firstDone) {
firstRun = true;
}
try {
log.debug('GET AVRO KEY');
newMsg.key = await this.registry.decode(newMsg.key);
log.debug(`AVRO KEY: ${newMsg.key}`);
newMsg.value = await this.registry.decode(newMsg.value);
log.debug(`AVRO MSG: ${newMsg.value}`);
firstDone = true;
firstRun = false;
clearInterval(intervalObject);
} catch (ex) {
log.warn(`Had issue getting registry will try again in ${fastInt} milliseconds`);
firstRun = false;
}
}
}, fastInt);
}
if (duplicate) {
return log.warn(`Emitting: ${newMsg.value}/${newMsg.offset} SKIPPED, duplicated message`);
}
if (goodMsg) {
log.info(`Message: '${newMsg.value}' being CONSUMED as it does meet the filter condition`);
// see if the topic/partition has its own rabbitMQ topic
for (let j = 0; j < desiredTopic.length; j += 1) {
// yes - publish on specific topic
if (mytopics.includes(desiredTopic[j])) {
log.info(`Emitting: ${newMsg.value} TO ${desiredTopic[j]}`);
} else {
// no - publish on general topic
log.info(`The desired topic ${desiredTopic[j]} does not exist. If it should, make sure it is in pronghorn.json`);
log.info(`Emitting: ${newMsg.value} TO kafka`);
desiredTopic[j] = 'kafka';
}
}
// NOTE: ADD THROTTLING
for (let j = 0; j < desiredTopic.length; j += 1) {
eventSystem.publish(desiredTopic[j], newMsg);
}
} else {
log.info(`Message: '${newMsg.value}' being DROPPED as it fails to meet any filter condition`);
}
})();
});
});
} catch (ex) {
const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
log.error(JSON.stringify(ex));
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return (null, errorObj);
}
}
/**
* @summary HealthCheck function is used to provide Pronghorn the status of this adapter.
*
* @function healthCheck
* @param callback - a callback function to return the result id and status
*/
healthCheck(callback) {
// need to work on this later
const retstatus = { id: this.id, status: 'success' };
return callback(retstatus);
}
/**
* getAllFunctions is used to get all of the exposed function in the adapter
*
* @function getAllFunctions
*/
getAllFunctions() {
let myfunctions = [];
let obj = this;
// find the functions in this class
do {
const l = Object.getOwnPropertyNames(obj)
.concat(Object.getOwnPropertySymbols(obj).map((s) => s.toString()))
.sort()
.filter((p, i, arr) => typeof obj[p] === 'function' && p !== 'constructor' && (i === 0 || p !== arr[i - 1]) && myfunctions.indexOf(p) === -1);
myfunctions = myfunctions.concat(l);
}
while (
(obj = Object.getPrototypeOf(obj)) && Object.getPrototypeOf(obj)
);
return myfunctions;
}
/**
* getWorkflowFunctions is used to get all of the workflow function in the adapter
*
* @function getWorkflowFunctions
*/
getWorkflowFunctions() {
const myIgnore = [
'subscribeAvroWithSubscriber',
'subscribeWithSubscriber'
];
const myfunctions = this.getAllFunctions();
const wffunctions = [];
// remove the functions that should not be in a Workflow
for (let m = 0; m < myfunctions.length; m += 1) {
if (myfunctions[m] === 'addListener') {
// got to the second tier (adapterBase)
break;
}
if (myfunctions[m] !== 'connect' && myfunctions[m] !== 'healthCheck'
&& myfunctions[m] !== 'getAllFunctions' && myfunctions[m] !== 'getWorkflowFunctions') {
let found = false;
if (myIgnore && Array.isArray(myIgnore)) {
for (let i = 0; i < myIgnore.length; i += 1) {
if (myfunctions[m].toUpperCase() === myIgnore[i].toUpperCase()) {
found = true;
}
}
}
if (!found) {
wffunctions.push(myfunctions[m]);
}
}
}
return wffunctions;
}
/**
* Call to create topics on the Kafka server.
* @function createTopics
* @param topics - an array of topics (required)
* @param callback - a callback function to return a result
*/
createTopics(topics, callback) {
const meth = 'adapter-createTopics';
const origin = `${this.id}-${meth}`;
log.trace(origin);
try {
// verify the required data has been provided
if (!topics) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!Array.isArray(topics)) {
const errorObj = formatErrorObject(origin, 'Invalid topics format - topics must be an array', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!this.KafkaClient) {
const errorObj = formatErrorObject(origin, 'KafkaClient not created', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
// create the topic
this.KafkaClient.createTopics(topics, (error, result) => {
if (error) {
const errorObj = formatErrorObject(origin, 'Topic creation failed', null, null, null, error);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
return callback({
status: 'success',
code: 200,
response: topics
});
});
} catch (ex) {
const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
}
/**
* Call to send message.
* @function send
* @param payloads - array of ProduceRequest, ProduceRequest is a JSON object (required)
* @param callback - a callback function to return a result
*/
sendMessage(payloads, callback) {
const meth = 'adapter-sendMessage';
const origin = `${this.id}-${meth}`;
log.trace(origin);
try {
if (!payloads) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['payloads'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!Array.isArray(payloads)) {
const errorObj = formatErrorObject(origin, 'Invalid payloads format - payloads must be an array', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!this.producer) {
const errorObj = formatErrorObject(origin, 'Producer not created', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
// if using avro
const newPay = payloads;
// need to go through the payloads that were provided
return (async () => {
for (let p = 0; p < payloads.length; p += 1) {
if (!payloads[p].topic) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['payloads[p].topic'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!payloads[p].messages) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['payloads[p].messages'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (typeof payloads[p].messages !== 'string') {
newPay[p].messages = JSON.stringify(newPay[p].messages);
}
if (this.registry && newPay[p].simple && newPay[p].simple.toUpperCase() === 'NO') {
log.debug(`Handling AVRO Message: ${JSON.stringify(newPay[p])}`);
let keyVer = 1;
let valVer = 1;
if (newPay[p].key_version) {
keyVer = newPay[p].key_version;
}
if (newPay[p].value_version) {
valVer = newPay[p].value_version;
}
// will give us an interval between 2501 and 10000 milliseconds (2.5 to 10 seconds)
const fastInt = Math.floor(Math.random() * 7500) + 2501;
let currValue = '';
let encodeKey = '';
// This interval is to help prevent the adapter from bombarding the registry by sending requests at random intervals
const intervalObject = setInterval(async () => {
try {
// get the key and value for the topic of this message
log.spam('fetching schema values');
currValue = await fetchSchema(this.registryUrl, `${newPay[p].topic}-value`, valVer);
if (newPay[p].key) {
let schKey = '';
log.spam('fetching schema keys');
schKey = await fetchSchema(this.registryUrl, `${newPay[p].topic}-key`, keyVer);
// encode the key
encodeKey = await this.registry.encodeKey(newPay[p].topic, schKey.schema, newPay[p].key);
}
clearInterval(intervalObject);
} catch (ex) {
log.warn(`Had issue getting registry will try again in ${fastInt} milliseconds`);
firstRun = false;
}
}, fastInt);
if (Array.isArray(newPay[p].messages)) {
log.debug('encoding message array');
const newMsgs = [];
// need to deal with an array of messages - encoding each
for (let m = 0; m < newPay[p].messages.length; m += 1) {
let msg = '';
log.debug(`attempting to encode message ${m}`);
try {
msg = await this.registry.encodeMessage(newPay[p].topic, currValue.schema, newPay[p].messages[m]);
} catch (encodeErr) {
log.error(encodeErr.stack);
return callback(null, encodeErr);
}
if (newPay[p].key) {
log.debug('creating and pushing keyed message');
newMsgs.push(new kafka.KeyedMessage(encodeKey, msg));
} else {
log.debug('pushing encoded message');
newMsgs.push(msg);
}
}
newPay[p].messages = newMsgs;
} else {
log.debug('encoding single message');
let msg = '';
log.debug('attempting to encode message');
try {
msg = await this.registry.encodeMessage(newPay[p].topic, currValue.schema, newPay[p].messages);
} catch (encodeErr) {
log.error(encodeErr.stack);
return callback(null, encodeErr);
}
if (newPay[p].key) {
log.debug('creating and pushing keyed message');
newPay[p].messages = [new kafka.KeyedMessage(encodeKey, msg)];
} else {
log.debug('pushing encoded message');
newPay[p].messages = [msg];
}
}
log.debug(`Finished Keying Message: ${JSON.stringify(newPay[p].messages)}`);
} else if (newPay[p].key) {
// handle any keyed messages
log.debug('Keying Message');
if (Array.isArray(newPay[p].messages)) {
const newMsgs = [];
// need to deal with an array of messages - encoding each
for (let m = 0; m < newPay[p].messages.length; m += 1) {
newMsgs.push(new kafka.KeyedMessage(newPay[p].key, newPay[p].messages[m]));
}
newPay[p].messages = newMsgs;
} else {
newPay[p].messages = [new kafka.KeyedMessage(newPay[p].key, newPay[p].messages)];
}
log.debug(`Finished Keying Message: ${JSON.stringify(newPay[p].messages)}`);
}
}
// send message
return this.producer.send(newPay, (err, data) => {
if (err) {
const errorObj = formatErrorObject(origin, 'Producer sending message failed', null, null, null, err);
log.error('error: '.concat(JSON.stringify(err)));
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
return callback({
status: 'success',
code: 200,
response: data
});
});
})();
} catch (ex) {
const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
}
/**
* Call to add topics to current consumer.
* @function subscribe
* @param {Array} topics - array of topics to add (required)
* @param partition - partition to set offset (optional)
* @param offset - offset to set (optional)
* @param {Callback} callback - a callback function to return a result
*/
subscribe(topics, partition, offset, callback) {
const meth = 'adapter-subscribe';
const origin = `${this.id}-${meth}`;
log.trace(origin);
// verify the required data has been provided
if (!topics) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!Array.isArray(topics)) {
const errorObj = formatErrorObject(origin, 'Invalid topics format - topics must be an array', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (topics.length === 0) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
// add the partition and offset into the topics
const topicArray = [];
topics.forEach((item) => {
topicArray.push({
topic: item,
offset: offset || 0,
partition: partition || 0
});
});
this.subscribeWithSubscriber(topicArray, null, callback);
}
/**
* Call to add topics to current consumer with Subscriber Info
* @function subscribeWithSubscriber
* @param {Array} topics - array of topics to remove - can be array ['topic1', { topic: 'topic2', partition: 0, offset: 0 }] (required)
* @param {Array} subscriberInfo - array of subscriber info to add (required)
* @param {Callback} callback - a callback function to return a result
*/
subscribeWithSubscriber(topics, subscriberInfo, callback) {
const meth = 'adapter-subscribeWithSubscriber';
const origin = `${this.id}-${meth}`;
log.trace(origin);
try {
// verify the required data has been provided
if (!topics) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!Array.isArray(topics)) {
const errorObj = formatErrorObject(origin, 'Invalid topics format - topics must be an array', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (topics.length === 0) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!this.consumer) {
const errorObj = formatErrorObject(origin, 'Consumer not created', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
// scan each topic to see if we are already listening for it
const topicAdd = [];
topics.forEach((item) => {
// see if we already are subscribed to the topic/partition
let found = false;
for (let t = 0; t < this.topicsEvents.length; t += 1) {
if ((typeof item === 'string' && this.topicsEvents[t].topic === item) || (typeof item === 'object'
&& this.topicsEvents[t].topic === item.topic && this.topicsEvents[t].partition === item.partition)) {
found = true;
// increase the number of subscribers
if (this.topicsEvents[t].subscribers >= 999999) {
this.topicsEvents[t].subscribers += 1;
}
// can only set the offset if no one else listening
if (item.offset && item.offset !== this.topicsEvents[t].offset) {
log.warn('Can not reset the offset of a shared topic!');
}
}
}
if (!found) {
// create the topic info to be added to consumer
topicAdd.push({
topic: item.topic || item,
offset: item.offset || 0,
partition: item.partition || 0
});
}
});
// then add topics to current consumer
if (topicAdd.length > 0) {
log.debug(`topicsToAdd: ${JSON.stringify(topicAdd)}`);
this.consumer.pause();
this.consumer.addTopics(topicAdd, (error, result) => {
this.consumer.resume();
if (error) {
const errorObj = formatErrorObject(origin, 'Adding consumer topics failed', null, null, null, error);
log.error('error: '.concat(JSON.stringify(error)));
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
// create tne topic information in adapter memory
for (let ta = 0; ta < topicAdd.length; ta += 1) {
let subInfo = [
{
subname: 'default',
filters: [],
rabbit: 'kafka',
throttle: {}
}
];
if (subscriberInfo && Array.isArray(subscriberInfo)) {
subInfo = subscriberInfo;
}
// need to ad subscribed topic/partition to adapter topics
this.topicsEvents.push({
topic: topicAdd[ta].topic,
offset: topicAdd[ta].offset,
partition: topicAdd[ta].partition,
subscribers: 1,
avro: 'YES',
processed: topicAdd[ta].offset,
subInfo
});
}
log.debug('Topics subscribed succesfully.');
return callback({
status: 'success',
code: 200,
response: result
});
});
} else {
return callback({
status: 'success',
code: 200,
response: topics
});
}
} catch (ex) {
const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
log.error(ex);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
}
/**
* Call to add topics to current consumer.
* @function subscribeAvro
* @param {Array} topics - array of topics to add (required)
* @param partition - partition to set offset (optional)
* @param offset - offset to set (optional)
* @param {Callback} callback - a callback function to return a result
*/
subscribeAvro(topics, partition, offset, callback) {
const meth = 'adapter-subscribeAvro';
const origin = `${this.id}-${meth}`;
log.trace(origin);
// verify the required data has been provided
if (!topics) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!Array.isArray(topics)) {
const errorObj = formatErrorObject(origin, 'Invalid topics format - topics must be an array', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (topics.length === 0) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
// add the partition and offset into the topics
const topicArray = [];
topicArray.forEach((item) => {
topicArray.push({
topic: item,
offset: offset || 0,
partition: partition || 0
});
});
this.subscribeAvroWithSubscriber(topicArray, null, callback);
}
/**
* Call to add topics to current consumer with Avro and Subscriber Info
* @function subscribeAvroWithSubscriber
* @param {Array} topics - array of topics to remove - can be array ['topic1', { topic: 'topic2', partition: 0, offset: 0 }] (required)
* @param {Array} subscriberInfo - array of subscriber info to add (required)
* @param {Callback} callback - a callback function to return a result
*/
subscribeAvroWithSubscriber(topics, subscriberInfo, callback) {
const meth = 'adapter-subscribeAvroWithSubscriber';
const origin = `${this.id}-${meth}`;
log.trace(origin);
try {
// verify the required data has been provided
if (!topics) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!Array.isArray(topics)) {
const errorObj = formatErrorObject(origin, 'Invalid topics format - topics must be an array', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (topics.length === 0) {
const errorObj = formatErrorObject(origin, 'Missing Data', ['topics'], null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
if (!this.consumer) {
const errorObj = formatErrorObject(origin, 'Consumer not created', null, null, null, null);
log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
return callback(null, errorObj);
}
// scan each topic to see if we are already listening for it
const topicAdd = [];
topics.forEach((item) => {
// see if we already are subscribed to the topic/partition
let found = false;
for (let t = 0; t < this.topicsEvents.length; t += 1) {
if ((typeof item === 'string' && this.topicsEvents[t].topic === item) || (typeof item === 'object'
&& this.topicsEvents[t].topic === item.topic && this.topicsEvents[t].partition === item.partition)) {
found = true;
// increase the number of subscribers
if (this.topicsEvents[t].subscribers >= 999999) {
this.topicsEvents[t].subscribers += 1;
}
// can only set the offset if no one else listening
if (item.offset && item.offset !== this.topicsEvents[t].offset) {
log.warn('Can not reset the offset of a shared topic!');
}
}
}
if (!found) {
// create the topic info to be added to consumer
topicAdd.push({
topic: item.topic || item,
offset: item.offset || 0,
partition: item.partition || 0
});
}
});
// then add topics to current consumer
if (topicAdd.length > 0) {
log.debug(`topicsToAdd: ${JSON.stringify(topicAdd)}`);
this.consumer.pause();
this.consumer.addTopics(topicAdd, (error, result) => {
this.consumer.resume();
if (error) {
const errorObj = formatErrorObject(origin, 'Adding consumer topics failed', null, null, null, error);
log.error('error: '.concat(JSON.stringify(error)));
log.error(`${orig