egg-yanxin-kafkajs/lib/kafka.js

egg kafka plugin base kafkajs module

const assert = require('assert');
const { Kafka, Partitioners } = require('kafkajs');
const KAFKA_CLIENT = Symbol('Plugin#kafkaClient');
const PRODUCER = Symbol('Plugin#kafkaProducer');
const CONSUMER = Symbol('Plugin#kafkaConsumer');
const PRODUCER_CON_STATE = Symbol('Plugin#kafkaProducerCon');
const CONSUMER_CON_STATE = Symbol('Plugin#kafkaConsumerCon');
const APP_LOG = Symbol('Plugin#kafkaAppLog');
const kafka = {};
module.exports = (app) => {
    // app.addSingleton('kafka', (config, app)=>{})

    app.kafka = createKafka(app.config.kafkajs, app);
};

function createKafka(config, app) {
    assert(config.client && config.client.brokers && config.client.brokers.length && config.client.clientId, '[egg-yanxin-kafkajs] configuration-(client = { clientId="",brokers=[] } ) is required when use kafka');
    assert(config.consumer && config.consumer.groupId, '[egg-yanxin-kafkajs] configuration-(consumer = { groupId="" } ) is required when use kafka');

    const { client = {}, consumer = {}, producer = {} } = config;
    const defaultClient = {
        clientId: 'broker-data-sync',
        brokers: ['127.0.0.1:9092'],
        connectionTimeout: 30000,
        retry: {
            initialRetryTime: 100,
            retries: Number.MAX_SAFE_INTEGER,
        }
    };
    const defaultConsumer = {
        groupId: 'group-default',
    }
    const defaultProducer = {
        allowAutoTopicCreation: true,
        createPartitioner: Partitioners.LegacyPartitioner
    }
    Object.assign(defaultClient, client);
    Object.assign(defaultConsumer, consumer);
    Object.assign(defaultProducer, producer);
    if (!kafka[KAFKA_CLIENT]) kafka[KAFKA_CLIENT] = new Kafka(defaultClient);
    if (!kafka[PRODUCER]) kafka[PRODUCER] = kafka[KAFKA_CLIENT].producer(defaultProducer);
    if (!kafka[CONSUMER]) kafka[CONSUMER] = kafka[KAFKA_CLIENT].consumer(defaultConsumer);
    kafka[PRODUCER_CON_STATE] = false;
    kafka[CONSUMER_CON_STATE] = false;
    kafka[PRODUCER].sendMessage = sendMessage;
    kafka[CONSUMER].initSubscribe = initSubscribe;
    kafka[CONSUMER].fetchLogger = consumer.fetchLogger;
    kafka[APP_LOG] = app.logger;
    return {
        consumer: kafka[CONSUMER],
        producer: kafka[PRODUCER],
        client: kafka[KAFKA_CLIENT],
    }
}
async function initSubscribe(options) {
    kafka[CONSUMER].on('consumer.connect', r => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.connect at ${r.timestamp}`);
        kafka[CONSUMER_CON_STATE] = true;
    });
    if (kafka[CONSUMER_CON_STATE]) return;
    await kafka[CONSUMER].connect();
    await kafka[CONSUMER].subscribe(options);
    kafka[CONSUMER].on('consumer.disconnect', r => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.disconnect at ${r.timestamp}`);
        kafka[CONSUMER_CON_STATE] = false;
    });
    kafka[CONSUMER].on('consumer.crash', r => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.crash at ${r.timestamp}`);
        kafka[CONSUMER_CON_STATE] = false;
    });
    kafka[CONSUMER].on('consumer.network.request_timeout', r => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.network.request_timeout at ${r.timestamp}`);
        kafka[CONSUMER_CON_STATE] = false;
    });
    kafka[CONSUMER].on('consumer.group_join', r => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.group_join at ${r.timestamp}`);
    });

    kafka[CONSUMER].on('consumer.start_batch_process', r => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.start_batch_process at ${r.timestamp}`);
    });
    kafka[CONSUMER].on('consumer.end_batch_process', r => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.end_batch_process at ${r.timestamp}`);
    });

    if (kafka[CONSUMER].fetchLogger) {
        kafka[CONSUMER].on('consumer.fetch', r => {
            kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.fetch at ${r.timestamp}`);
        });
        kafka[CONSUMER].on('consumer.fetch_start', r => {
            kafka[APP_LOG].info(`[egg-yanxin-kafkajs] consumer.fetch_start at ${r.timestamp}`);
        });
    }
}

async function sendMessage(topic, messages = []) {
    kafka[PRODUCER].on('producer.connect', (r) => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] producer.connect at ${r.timestamp}`);
        kafka[PRODUCER_CON_STATE] = true;
    });

    kafka[PRODUCER].on('producer.disconnect', (r) => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] producer.disconnect at ${r.timestamp}`);
        kafka[PRODUCER_CON_STATE] = false;
    });
    kafka[PRODUCER].on('producer.network.request_timeout', (r) => {
        kafka[APP_LOG].info(`[egg-yanxin-kafkajs] producer.network.request_timeout at ${r.timestamp}`);
        kafka[PRODUCER_CON_STATE] = false;
    });
    if (!kafka[PRODUCER_CON_STATE]) {
        await kafka[PRODUCER].connect();
    }
    const res = await kafka[PRODUCER].send({
        topic,
        messages,
    });
    return res
}