@lskjs/kafka
Version:
LSK module for kafka.
387 lines (385 loc) • 17 kB
JavaScript
"use strict";
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
Object.defineProperty(exports, "__esModule", {
value: true
});
exports["default"] = exports.KafkaModule = void 0;
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
var _objectWithoutProperties2 = _interopRequireDefault(require("@babel/runtime/helpers/objectWithoutProperties"));
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
var _get2 = _interopRequireDefault(require("@babel/runtime/helpers/get"));
var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
var _err = _interopRequireDefault(require("@lskjs/err"));
var _module = _interopRequireDefault(require("@lskjs/module"));
var _bluebird = _interopRequireDefault(require("bluebird"));
var _kafkajs = require("kafkajs");
var _chunk = _interopRequireDefault(require("lodash/chunk"));
var _get3 = _interopRequireDefault(require("lodash/get"));
var _last = _interopRequireDefault(require("lodash/last"));
var _excluded = ["message"],
_excluded2 = ["concurrency"];
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2["default"])(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2["default"])(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2["default"])(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2["default"])(this, result); }; }
function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
var toLskLogLevel = function toLskLogLevel(level) {
switch (level) {
case _kafkajs.logLevel.ERROR:
return 'error';
case _kafkajs.logLevel.WARN:
return 'warn';
case _kafkajs.logLevel.INFO:
return 'info';
case _kafkajs.logLevel.DEBUG:
return 'debug';
case _kafkajs.logLevel.NOTHING:
return 'trace';
default:
return 'trace';
}
};
var KafkaModule = /*#__PURE__*/function (_Module) {
(0, _inherits2["default"])(KafkaModule, _Module);
var _super = _createSuper(KafkaModule);
function KafkaModule() {
var _this;
(0, _classCallCheck2["default"])(this, KafkaModule);
for (var _len = arguments.length, args = new Array(_len), _key = 0; _key < _len; _key++) {
args[_key] = arguments[_key];
}
_this = _super.call.apply(_super, [this].concat(args));
_this.Kafka = _kafkajs.Kafka;
_this.defaultGroupId = 'kafka-module';
return _this;
}
(0, _createClass2["default"])(KafkaModule, [{
key: "getOptions",
value: function getOptions() {
var logLevel = (0, _get3["default"])(this.config, 'log.level') || _kafkajs.logLevel.ERROR;
var logger = this.log.createChild({
ns: "".concat(this.log.ns, ":client"),
name: null
});
var options = _objectSpread(_objectSpread({
logLevel: logLevel
}, this.config), {}, {
logCreator: function logCreator() {
return function (_ref) {
var namespace = _ref.namespace,
level = _ref.level,
_ref$log = _ref.log,
message = _ref$log.message,
info = (0, _objectWithoutProperties2["default"])(_ref$log, _excluded);
//eslint-disable-line
var lskLevel = toLskLogLevel(level);
var args = [message];
if (level === _kafkajs.logLevel.WARN || level === _kafkajs.logLevel.ERROR) args.push(info);
var name = logger.name;
logger.name = namespace; // TODO: bad practrice
logger[lskLevel].apply(logger, args);
logger.name = name;
// this.log[lskLevel](`[${namespace}]`, message, info);
};
}
});
if (!options.clientId) options.clientId = (0, _get3["default"])(process, 'env.WORKER', "nodejs_".concat((0, _get3["default"])(process, 'env.USER')));
return options;
}
}, {
key: "init",
value: function () {
var _init = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee() {
var options;
return _regenerator["default"].wrap(function _callee$(_context) {
while (1) switch (_context.prev = _context.next) {
case 0:
_context.next = 2;
return (0, _get2["default"])((0, _getPrototypeOf2["default"])(KafkaModule.prototype), "init", this).call(this);
case 2:
if (this.config.brokers) {
_context.next = 5;
break;
}
this.log.warn('!config.brokers');
return _context.abrupt("return");
case 5:
options = this.getOptions();
this.log.debug('[options]', options);
this.client = new this.Kafka(options);
case 8:
case "end":
return _context.stop();
}
}, _callee, this);
}));
function init() {
return _init.apply(this, arguments);
}
return init;
}()
}, {
key: "insert",
value: function () {
var _insert = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2() {
var _this$producer;
var _args2 = arguments;
return _regenerator["default"].wrap(function _callee2$(_context2) {
while (1) switch (_context2.prev = _context2.next) {
case 0:
if (this.producer) {
_context2.next = 2;
break;
}
throw new _err["default"]('!producer');
case 2:
return _context2.abrupt("return", (_this$producer = this.producer).send.apply(_this$producer, _args2));
case 3:
case "end":
return _context2.stop();
}
}, _callee2, this);
}));
function insert() {
return _insert.apply(this, arguments);
}
return insert;
}()
}, {
key: "createConsumer",
value: function () {
var _createConsumer = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee3(topic) {
var _ref2,
groupId,
topics,
consumer,
props,
_args3 = arguments;
return _regenerator["default"].wrap(function _callee3$(_context3) {
while (1) switch (_context3.prev = _context3.next) {
case 0:
_ref2 = _args3.length > 1 && _args3[1] !== undefined ? _args3[1] : {}, groupId = _ref2.groupId;
if (Array.isArray(topic)) {
topics = topic;
} else if (typeof topic === 'string') {
if (topic.includes(',')) {
topics = topic.split(',');
} else {
topics = [topic];
}
}
if (topics) {
_context3.next = 4;
break;
}
throw new _err["default"]('!topic');
case 4:
consumer = this.client.consumer({
groupId: groupId || this.defaultGroupId,
maxBytesPerPartition: 4096
});
_context3.next = 7;
return consumer.connect();
case 7:
props = {
fromBeginning: true
};
this.log.trace('[consumer.subscribe]', topics, props);
_context3.next = 11;
return consumer.subscribe(_objectSpread({
topics: topics
}, props));
case 11:
return _context3.abrupt("return", consumer);
case 12:
case "end":
return _context3.stop();
}
}, _callee3, this);
}));
function createConsumer(_x) {
return _createConsumer.apply(this, arguments);
}
return createConsumer;
}()
}, {
key: "consume",
value: function () {
var _consume = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee4(topic, onConsume, _ref3) {
var _this2 = this;
var concurrency, options, consumer;
return _regenerator["default"].wrap(function _callee4$(_context4) {
while (1) switch (_context4.prev = _context4.next) {
case 0:
concurrency = _ref3.concurrency, options = (0, _objectWithoutProperties2["default"])(_ref3, _excluded2);
_context4.next = 3;
return this.createConsumer(topic, options);
case 3:
consumer = _context4.sent;
_context4.next = 6;
return consumer.run({
partitionsConsumedConcurrently: 1,
// сколько partiotions может сразу слушать consumer
eachBatch: function eachBatch(props) {
return _this2.onEachBatchMessage(onConsume, props, {
concurrency: concurrency
});
}
});
case 6:
case "end":
return _context4.stop();
}
}, _callee4, this);
}));
function consume(_x2, _x3, _x4) {
return _consume.apply(this, arguments);
}
return consume;
}()
}, {
key: "onEachBatchMessage",
value: function () {
var _onEachBatchMessage = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee7(onConsume) {
var _this3 = this;
var options,
_ref4,
_ref4$concurrency,
concurrency,
batch,
resolveOffset,
heartbeat,
_args7 = arguments;
return _regenerator["default"].wrap(function _callee7$(_context7) {
while (1) switch (_context7.prev = _context7.next) {
case 0:
options = _args7.length > 1 && _args7[1] !== undefined ? _args7[1] : {};
_ref4 = _args7.length > 2 && _args7[2] !== undefined ? _args7[2] : {}, _ref4$concurrency = _ref4.concurrency, concurrency = _ref4$concurrency === void 0 ? 1 : _ref4$concurrency;
if (onConsume) {
_context7.next = 4;
break;
}
throw new _err["default"]('!onConsume');
case 4:
batch = options.batch, resolveOffset = options.resolveOffset, heartbeat = options.heartbeat;
return _context7.abrupt("return", _bluebird["default"].mapSeries((0, _chunk["default"])(batch.messages, concurrency), /*#__PURE__*/function () {
var _ref5 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee6(messages) {
var res;
return _regenerator["default"].wrap(function _callee6$(_context6) {
while (1) switch (_context6.prev = _context6.next) {
case 0:
_context6.next = 2;
return _bluebird["default"].map(messages, /*#__PURE__*/function () {
var _ref6 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee5(msg) {
var res2;
return _regenerator["default"].wrap(function _callee5$(_context5) {
while (1) switch (_context5.prev = _context5.next) {
case 0:
_this3.log.trace('[onConsume] start', msg);
_context5.next = 3;
return onConsume(msg, options);
case 3:
res2 = _context5.sent;
_this3.log.trace('[onConsume] res', res2);
return _context5.abrupt("return", res2);
case 6:
case "end":
return _context5.stop();
}
}, _callee5);
}));
return function (_x7) {
return _ref6.apply(this, arguments);
};
}());
case 2:
res = _context6.sent;
_context6.next = 5;
return resolveOffset((0, _last["default"])(messages).offset);
case 5:
_context6.next = 7;
return heartbeat();
case 7:
return _context6.abrupt("return", res);
case 8:
case "end":
return _context6.stop();
}
}, _callee6);
}));
return function (_x6) {
return _ref5.apply(this, arguments);
};
}()));
case 6:
case "end":
return _context7.stop();
}
}, _callee7);
}));
function onEachBatchMessage(_x5) {
return _onEachBatchMessage.apply(this, arguments);
}
return onEachBatchMessage;
}()
}, {
key: "nack",
value: function nack(message, options) {
// this.log.warn('nack', message, options);
}
}, {
key: "ack",
value: function ack(message, options) {
// this.log.info('ack', message, options);
}
}, {
key: "nackError",
value: function nackError(message, options) {
// this.log.error('nackError', message, options);
}
}, {
key: "ackError",
value: function ackError(message, options) {
// this.log.error('ackError', message, options);
}
}, {
key: "run",
value: function () {
var _run = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee8() {
return _regenerator["default"].wrap(function _callee8$(_context8) {
while (1) switch (_context8.prev = _context8.next) {
case 0:
if (this.client) {
_context8.next = 2;
break;
}
return _context8.abrupt("return");
case 2:
_context8.next = 4;
return (0, _get2["default"])((0, _getPrototypeOf2["default"])(KafkaModule.prototype), "run", this).call(this);
case 4:
this.producer = this.client.producer();
_context8.next = 7;
return this.producer.connect();
case 7:
case "end":
return _context8.stop();
}
}, _callee8, this);
}));
function run() {
return _run.apply(this, arguments);
}
return run;
}()
}]);
return KafkaModule;
}(_module["default"]);
exports.KafkaModule = KafkaModule;
var _default = KafkaModule;
exports["default"] = _default;
//# sourceMappingURL=KafkaModule.js.map