egg-kafka-cluster/lib/kafka.js

provides egg bindings for the kafka.

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Kafka = exports.Consumer = void 0;
const node_rdkafka_1 = require("node-rdkafka");
const node_path_1 = __importDefault(require("node:path"));
const lodash_1 = require("lodash");
class Consumer {
    app;
    constructor(app) {
        this.app = app;
    }
}
exports.Consumer = Consumer;
class Kafka {
    app;
    consumer;
    producers;
    constructor(app) {
        this.app = app;
        this.producers = new Map();
    }
    initConsumer() {
        if ((0, lodash_1.isArray)(this.app.config.kafka.readStreamOptions.topics)) {
            if (this.app.config.kafka.readStreamOptions.topics.length === 0) {
                return;
            }
        }
        // metadata.broker.list
        const consumerGlobalConfig = {};
        if (process.env.BROKER_LIST)
            consumerGlobalConfig['metadata.broker.list'] = process.env.BROKER_LIST;
        this.consumer = node_rdkafka_1.KafkaConsumer.createReadStream({ ...this.app.config.kafka.consumerGlobalConfig, ...consumerGlobalConfig }, this.app.config.kafka.consumerTopicConfig, this.app.config.kafka.readStreamOptions);
        this.consumer.on('data', (message) => {
            try {
                const data = JSON.parse(message.value.toString());
                this.app.messenger.sendRandom('kafka:read', { topic: message.topic, data });
            }
            catch (e) {
                this.app.logger.error('[mq-consume] MQ消费数据解析失败: %s, %s, %s', message.topic, message.value.toString(), e);
            }
        });
        this.consumer.consumer.once('subscribed', (topics) => {
            this.app.logger.info('[mq-consume] MQ消费订阅成功: %s', topics.join(','));
            this.app.emit('kafka:consumer-subscribed');
        });
    }
    initProducer(topic, opts) {
        const producerGlobalConfig = {};
        if (process.env.BROKER_LIST)
            producerGlobalConfig['metadata.broker.list'] = process.env.BROKER_LIST;
        const stream = node_rdkafka_1.Producer.createWriteStream({ ...this.app.config.kafka.producerGlobalConfig, ...producerGlobalConfig, ...opts?.producerGlobalConfig }, { ...this.app.config.kafka.producerTopicConfig, ...opts?.producerTopicConfig }, { topic, ...this.app.config.kafka.writeStreamOptions, ...opts?.writeStreamOptions });
        this.producers.set(topic, stream);
        stream.on('error', err => {
            this.app.logger.error('[mq-produce] MQ生产错误: %s, %s', topic, err);
        });
    }
    appDelegate() {
        const dir = node_path_1.default.join(this.app.config.baseDir, 'app/mq');
        this.app.loader.loadToApp(dir, 'MQ', {
            call: true,
            caseStyle: 'lower',
            initializer: (model) => {
                return new model(this.app);
            },
        });
        this.app.messenger.on('kafka:read', ({ topic, data }) => {
            const file = topic.split('-')[0];
            const cla = this.app.MQ[file];
            if (!cla) {
                this.app.logger.warn('[mq-consume] MQ执行类不存在: %s, %s', topic, JSON.stringify(data));
                return;
            }
            let method = (0, lodash_1.lowerFirst)(topic.split('-')[1] || '');
            if (!method) {
                method = 'run';
            }
            if (!(0, lodash_1.isFunction)(cla[method])) {
                this.app.logger.warn('[mq-consume] MQ执行类没有对应处理tag方法: %s, %s', topic, JSON.stringify(data));
                return;
            }
            cla[method](data);
        });
    }
    agentDelegate() {
        this.app.messenger.on('kafka:write', ({ topic, data }) => {
            this.write(topic, data);
        });
    }
    write(topic, data) {
        // @ts-ignore
        if (this.app.options.type !== 'agent') {
            this.app.messenger.sendToAgent('kafka:write', { topic, data });
            return;
        }
        if (!this.producers.has(topic)) {
            this.producers.set(topic, node_rdkafka_1.Producer.createWriteStream(this.app.config.kafka.producerGlobalConfig, this.app.config.kafka.producerTopicConfig, { topic, ...this.app.config.kafka.writeStreamOptions }));
        }
        this.producers.get(topic)?.write(Buffer.from(JSON.stringify(data)), err => {
            if (err)
                this.app.logger.error('[mq-produce] MQ生产数据失败: %s, %s, %s', topic, JSON.stringify(data), err);
        });
    }
}
exports.Kafka = Kafka;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoia2Fma2EuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJrYWZrYS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7QUFDQSwrQ0FBZ007QUFDaE0sMERBQTZCO0FBQzdCLG1DQUF5RDtBQVl6RCxNQUFhLFFBQVE7SUFDQSxHQUFHLENBQWM7SUFDcEMsWUFBWSxHQUFnQjtRQUMxQixJQUFJLENBQUMsR0FBRyxHQUFHLEdBQUcsQ0FBQztJQUNqQixDQUFDO0NBQ0Y7QUFMRCw0QkFLQztBQUdELE1BQWEsS0FBSztJQUVoQixHQUFHLENBQWlCO0lBQ3BCLFFBQVEsQ0FBaUI7SUFDekIsU0FBUyxDQUE4QjtJQUV2QyxZQUFZLEdBQW1CO1FBQzdCLElBQUksQ0FBQyxHQUFHLEdBQUcsR0FBRyxDQUFDO1FBQ2YsSUFBSSxDQUFDLFNBQVMsR0FBRyxJQUFJLEdBQUcsRUFBRSxDQUFDO0lBQzdCLENBQUM7SUFFRCxZQUFZO1FBRVYsSUFBSSxJQUFBLGdCQUFPLEVBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLGlCQUFpQixDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUM7WUFDNUQsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsaUJBQWlCLENBQUMsTUFBTSxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUUsQ0FBQztnQkFDaEUsT0FBTztZQUNULENBQUM7UUFDSCxDQUFDO1FBRUQsdUJBQXVCO1FBQ3ZCLE1BQU0sb0JBQW9CLEdBQXdCLEVBQUUsQ0FBQztRQUNyRCxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsV0FBVztZQUFFLG9CQUFvQixDQUFDLHNCQUFzQixDQUFDLEdBQUcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxXQUFXLENBQUM7UUFFcEcsSUFBSSxDQUFDLFFBQVEsR0FBRyw0QkFBYSxDQUFDLGdCQUFnQixDQUM1QyxFQUFFLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLG9CQUFvQixFQUFFLEdBQUcsb0JBQW9CLEVBQUUsRUFDMUUsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLG1CQUFtQixFQUN6QyxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsaUJBQWlCLENBQ3hDLENBQUM7UUFFRixJQUFJLENBQUMsUUFBUSxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxPQUFxQixFQUFFLEVBQUU7WUFDakQsSUFBSSxDQUFDO2dCQUNILE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxRQUFRLEVBQUUsQ0FBNEIsQ0FBQztnQkFDN0UsSUFBSSxDQUFDLEdBQUcsQ0FBQyxTQUFTLENBQUMsVUFBVSxDQUFDLFlBQVksRUFBRSxFQUFFLEtBQUssRUFBRSxPQUFPLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7WUFDOUUsQ0FBQztZQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUM7Z0JBQ1gsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLHFDQUFxQyxFQUFFLE9BQU8sQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLEtBQUssQ0FBQyxRQUFRLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQztZQUMzRyxDQUFDO1FBQ0gsQ0FBQyxDQUFDLENBQUM7UUFFSCxJQUFJLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsWUFBWSxFQUFFLENBQUMsTUFBMEIsRUFBRSxFQUFFO1lBQ3ZFLElBQUksQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQywyQkFBMkIsRUFBRSxNQUFNLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUM7WUFDcEUsSUFBSSxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsMkJBQTJCLENBQUMsQ0FBQztRQUM3QyxDQUFDLENBQUMsQ0FBQztJQUVMLENBQUM7SUFFRCxZQUFZLENBQUMsS0FBYSxFQUFFLElBQTBJO1FBRXBLLE1BQU0sb0JBQW9CLEdBQXlCLEVBQUUsQ0FBQztRQUN0RCxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsV0FBVztZQUFFLG9CQUFvQixDQUFDLHNCQUFzQixDQUFDLEdBQUcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxXQUFXLENBQUM7UUFFcEcsTUFBTSxNQUFNLEdBQUcsdUJBQVEsQ0FBQyxpQkFBaUIsQ0FDdkMsRUFBRSxHQUFHLElBQUksQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxvQkFBb0IsRUFBRSxHQUFHLG9CQUFvQixFQUFFLEdBQUcsSUFBSSxFQUFFLG9CQUFvQixFQUFFLEVBQ3pHLEVBQUUsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsbUJBQW1CLEVBQUUsR0FBRyxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsRUFDOUUsRUFBRSxLQUFLLEVBQUUsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsa0JBQWtCLEVBQUUsR0FBRyxJQUFJLEVBQUUsa0JBQWtCLEVBQUUsQ0FDcEYsQ0FBQztRQUNGLElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEtBQUssRUFBRSxNQUFNLENBQUMsQ0FBQztRQUVsQyxNQUFNLENBQUMsRUFBRSxDQUFDLE9BQU8sRUFBRSxHQUFHLENBQUMsRUFBRTtZQUN2QixJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsNkJBQTZCLEVBQUUsS0FBSyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1FBQ25FLENBQUMsQ0FBQyxDQUFDO0lBRUwsQ0FBQztJQUVELFdBQVc7UUFDVCxNQUFNLEdBQUcsR0FBRyxtQkFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxPQUFPLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFDekQsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLEdBQUcsRUFBRSxJQUFJLEVBQUU7WUFDbkMsSUFBSSxFQUFFLElBQUk7WUFDVixTQUFTLEVBQUUsT0FBTztZQUNsQixXQUFXLEVBQUUsQ0FBQyxLQUFVLEVBQUUsRUFBRTtnQkFDMUIsT0FBTyxJQUFJLEtBQUssQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7WUFDN0IsQ0FBQztTQUNGLENBQUMsQ0FBQztRQUVILElBQUksQ0FBQyxHQUFHLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQyxZQUFZLEVBQUUsQ0FBQyxFQUFFLEtBQUssRUFBRSxJQUFJLEVBQW9ELEVBQUUsRUFBRTtZQUV4RyxNQUFNLElBQUksR0FBRyxLQUFLLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO1lBQ2pDLE1BQU0sR0FBRyxHQUFJLElBQUksQ0FBQyxHQUFtQixDQUFDLEVBQUUsQ0FBQyxJQUFJLENBQUMsQ0FBQztZQUMvQyxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUM7Z0JBQ1QsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLCtCQUErQixFQUFFLEtBQUssRUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7Z0JBQ25GLE9BQU87WUFDVCxDQUFDO1lBRUQsSUFBSSxNQUFNLEdBQUcsSUFBQSxtQkFBVSxFQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUM7WUFDbkQsSUFBSSxDQUFDLE1BQU0sRUFBRSxDQUFDO2dCQUNaLE1BQU0sR0FBRyxLQUFLLENBQUM7WUFDakIsQ0FBQztZQUVELElBQUksQ0FBQyxJQUFBLG1CQUFVLEVBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDLEVBQUUsQ0FBQztnQkFDN0IsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLHVDQUF1QyxFQUFFLEtBQUssRUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7Z0JBQzNGLE9BQU87WUFDVCxDQUFDO1lBRUQsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQUksQ0FBQyxDQUFDO1FBQ3BCLENBQUMsQ0FBQyxDQUFDO0lBQ0wsQ0FBQztJQUVELGFBQWE7UUFDWCxJQUFJLENBQUMsR0FBRyxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUMsYUFBYSxFQUFFLENBQUMsRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFvRCxFQUFFLEVBQUU7WUFDekcsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLEVBQUUsSUFBSSxDQUFDLENBQUM7UUFDMUIsQ0FBQyxDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQsS0FBSyxDQUFDLEtBQWEsRUFBRSxJQUE2QjtRQUVoRCxhQUFhO1FBQ2IsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLE9BQU8sQ0FBQyxJQUFJLEtBQUssT0FBTyxFQUFFLENBQUM7WUFDdEMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxTQUFTLENBQUMsV0FBVyxDQUFDLGFBQWEsRUFBRSxFQUFFLEtBQUssRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO1lBQy9ELE9BQU87UUFDVCxDQUFDO1FBRUQsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7WUFDL0IsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsS0FBSyxFQUFFLHVCQUFRLENBQUMsaUJBQWlCLENBQ2xELElBQUksQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxvQkFBb0IsRUFDMUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLG1CQUFtQixFQUN6QyxFQUFFLEtBQUssRUFBRSxHQUFHLElBQUksQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxrQkFBa0IsRUFBRSxDQUN2RCxDQUFDLENBQUM7UUFDTCxDQUFDO1FBRUQsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsS0FBSyxDQUFDLEVBQUUsS0FBSyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxFQUFFO1lBQ3hFLElBQUksR0FBRztnQkFBRSxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsbUNBQW1DLEVBQUUsS0FBSyxFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLEVBQUUsR0FBRyxDQUFDLENBQUM7UUFDeEcsQ0FBQyxDQUFDLENBQUM7SUFFTCxDQUFDO0NBR0Y7QUE3SEQsc0JBNkhDIn0=