ihub-framework-js/kafka/index.js

Legacy version of iHub Framework written in Javascript

/* eslint-disable no-unused-vars */
const {
  Kafka,
  ProducerRecord,
  RecordMetadata,
  ConsumerSubscribeTopic,
  ConsumerRunConfig,
} = require('kafkajs');
const log = require('winston');
const { v4: uuidv4 } = require('uuid');
const { IsJsonString } = require('../plugins/JSONUtils');
const { acksEnums } = require('../plugins/kafkaEnums');

const {
  KAFKA_USERNAME: username,
  KAFKA_PASSWORD: password,
  PROJECT_NAMESPACE: projectNamespace,
  KAFKA_BROKERS: brokers,
} = process.env;
const sasl = username && password ? { username, password, mechanism: 'plain' } : null;
const ssl = !!sasl;
const brokersArray = brokers ? brokers.split(',').map(b => b.trim()) : [];

const kafka = new Kafka({
  clientId: projectNamespace,
  brokers: brokersArray,
  ssl,
  sasl,
  retry: 1,
});

const producer = kafka.producer({
  retry: 1,
  idempotent: false,
});

const consumer = kafka.consumer({
  groupId: projectNamespace,
  retry: 1,
});

const admin = kafka.admin();

/**
 * Send messages in topic
 * @param {ProducerRecord} record
 * @returns {ProducerRecord}
 */
function validateRecordMessages(record) {
  if (record.messages && Array.isArray(record.messages)) {
    record.messages.forEach((message) => {
      if (typeof message.value === 'object') {
        /* eslint-disable-next-line no-param-reassign */
        message.value = JSON.stringify(message.value);
      }
    });
  }

  return record;
}

/**
 * Send messages in topic
 * @param {ProducerRecord} record
 * @returns {Promise<RecordMetadata[]>}
 */
async function send(record) {
  try {
    validateRecordMessages(record);

    const sentMessage = await producer.send({
      acks: acksEnums.WAIT_FOR_LEADER,
      ...record,
    });

    return sentMessage;
  } catch (error) {
    log.error(`Error while trying to send message in topic ${record.topic}, message: ${typeof error === 'object' ? JSON.stringify(error) : error}`);
    throw error;
  }
}

/**
 * Send transational messages in topic
 * @param {ProducerRecord} record
 * @returns {Promise<RecordMetadata[]>}
 */
async function sendTransational(record) {
  const transationalProducer = kafka.producer({
    maxInFlightRequests: 1,
    idempotent: true,
    transactionalId: uuidv4(),
  });

  await transationalProducer.connect();

  const transaction = await transationalProducer.transaction();

  try {
    validateRecordMessages(record);

    const sentMessage = await transaction.send({
      ...record,
      acks: acksEnums.WAIT_FOR_LEADER_AND_REPLICAS,
    });

    await transaction.commit();

    return sentMessage;
  } catch (error) {
    await transaction.abort();
    log.error(`Error while trying to send message in topic ${record.topic}, message: ${typeof error === 'object' ? JSON.stringify(error) : error}`);
    throw error;
  } finally {
    await transationalProducer.disconnect();
  }
}

/**
 * Kafka topics subscriber
 * @param {ConsumerSubscribeTopic} topic
 * @returns {Promise<void>}
 */
async function subscribe(topic) {
  return consumer.subscribe(topic);
}

/**
 * Kafka topics subscriptions listener
 * @param {Array<any>} topics
 * @param {ConsumerRunConfig} options
 * @returns {Promise<void>}
 */
async function run(topics, options) {
  try {
    await consumer.run({
      ...options,
      eachMessage: async (payload) => {
        try {
          const topicFound = topics.find(task => task.topic === payload.topic);

          if (topicFound) {
            /* eslint-disable-next-line no-param-reassign */
            payload.message.key = payload.message.key && payload.message.key != null ? payload.message.key.toString() : undefined;
            /* eslint-disable-next-line no-param-reassign */
            payload.message.value = IsJsonString(payload.message.value.toString())
              ? JSON.parse(payload.message.value.toString())
              : payload.message.value.toString();

            const message = {
              offset: payload.message.offset,
              partition: payload.partition,
              topic: payload.topic,
              key: payload.message.key,
              value: payload.message.value,
              timestamp: payload.message.timestamp,
            };
            topicFound.handler(message);
          } else {
            log.error(`Topic with name ${payload.topic} not found`);
          }
        } catch (e) {
          log.error(e);
        }
      },
    });
  } catch (error) {
    log.info('Error while trying to listen subscribed topics');
    throw error;
  }
}

/**
 * Verifies the topic existence and create it if doesn't exists or update if the number of the partitions increased.
 * @param {object} topicObj
 *  @returns {Promise<void>}
 */
async function createOrUpdateTopic(topicObj) {
  try {
    // Start admin
    await admin.connect();

    const topicCreated = await admin.createTopics({
      validateOnly: false,
      waitForLeaders: true,
      topics: [
        {
          topic: topicObj.topic,
          numPartitions: topicObj.partitions,
          replicationFactor: brokersArray.length,
        },
      ],
    });

    if (topicCreated) {
      log.info(`Topic ${topicObj.topic} successfully created.`);
    } else {
      log.info(`Topic ${topicObj.topic} already exists.`);
    }
  } catch (error) {
    log.info(`Error while trying to create topic: ${topicObj.topic}`);
    throw error;
  }
}

async function exitHandler(options, exitCode) {
  await producer.disconnect();
  log.debug(`Kafka producer for groupId: ${projectNamespace}  successfully disconected`);

  await consumer.disconnect();
  log.debug(`Kafka consumer for groupId: ${projectNamespace}  successfully disconected`);

  log.debug(`Process exited with code ${exitCode}`);

  if (options.exit) process.exit();
}

function disconnectOnProcessExit() {
  // do something when app is closing
  process.on('exit', exitHandler.bind(null, { cleanup: true }));

  // catches ctrl+c event
  process.on('SIGINT', exitHandler.bind(null, { exit: true }));

  // catches "kill pid" (for example: nodemon restart)
  process.on('SIGUSR1', exitHandler.bind(null, { exit: true }));
  process.on('SIGUSR2', exitHandler.bind(null, { exit: true }));

  // catches uncaught exceptions
  process.on('uncaughtException', exitHandler.bind(null, { exit: true }));
}

/**
 * Initiate the customer connection and return all methods that can be used.
 * @returns
 */
module.exports = async function init() {
  try {
    await consumer.connect();
    await producer.connect();

    disconnectOnProcessExit();

    return {
      send,
      sendTransational,
      subscribe,
      run,
      kafka,
      createOrUpdateTopic,
      consumer,
    };
  } catch (error) {
    log.info('Error while to do the initial connection as consumer');
    throw error;
  }
};