kafka-sagas
Version:
Build sagas that consume from a kafka topic
971 lines (948 loc) • 39.4 kB
JavaScript
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; }
var Bluebird = _interopDefault(require('bluebird'));
var EventEmitter = _interopDefault(require('events'));
var uuid = _interopDefault(require('uuid'));
var kafkajs = require('kafkajs');
var pino = _interopDefault(require('pino'));
class ActionChannelBuffer {
constructor() {
this.actions = [];
this.observers = [];
}
put(action) {
this.actions = [...this.actions, action];
this.notifyObservers();
}
take() {
const promise = new Promise(resolve => this.observers.push(resolve));
this.notifyObservers();
return promise;
}
// tslint:disable-next-line: cyclomatic-complexity
notifyObservers() {
const countObservers = this.observers.length;
const countActions = this.actions.length;
// if there are no observers and no actions, do nothing
if (!countObservers || !countActions) {
return;
}
let numberOfItemsToShift = Math.min(countObservers, countActions);
while (numberOfItemsToShift) {
const observer = this.observers.shift();
const action = this.actions.shift();
if (!observer || !action) {
throw new Error('Possible mutation of observers or actions during notification loop');
}
observer(action);
numberOfItemsToShift -= 1;
}
}
}
class EphemeralBuffer {
constructor() {
this.observers = [];
}
put(action) {
for (const observerResolve of this.observers) {
observerResolve(action);
}
delete this.observers;
this.observers = [];
}
take() {
const promise = new Promise(resolve => this.observers.push(resolve));
return promise;
}
}
(function (EffectDescriptionKind) {
EffectDescriptionKind["PUT"] = "PUT";
EffectDescriptionKind["TAKE"] = "TAKE";
EffectDescriptionKind["CALL"] = "CALL";
EffectDescriptionKind["DELAY"] = "DELAY";
EffectDescriptionKind["COMBINATOR"] = "COMBINATOR";
EffectDescriptionKind["ACTION_CHANNEL"] = "ACTION_CHANNEL";
EffectDescriptionKind["TAKE_ACTION_CHANNEL"] = "TAKE_ACTION_CHANNEL";
})(exports.EffectDescriptionKind || (exports.EffectDescriptionKind = {}));
function isTransactionMessage(messageValue) {
return messageValue && messageValue.transaction_id;
}
function isTakeEffectDescription(effectDescription) {
return effectDescription.kind === exports.EffectDescriptionKind.TAKE;
}
function isPutEffectDescription(effectDescription) {
return effectDescription.kind === exports.EffectDescriptionKind.PUT;
}
function isCallEffectDescription(effectDescription) {
return effectDescription.kind === exports.EffectDescriptionKind.CALL;
}
function isEffectCombinatorDescription(effectDescription) {
return !!(effectDescription.combinator &&
effectDescription.effects);
}
function isActionChannelEffectDescription(effectDescription) {
return effectDescription.kind === exports.EffectDescriptionKind.ACTION_CHANNEL;
}
function isTakeActionChannelEffectDescription(effectDescription) {
return effectDescription.kind === exports.EffectDescriptionKind.TAKE_ACTION_CHANNEL;
}
function isDelayEffectDescription(effectDescription) {
return effectDescription.kind === exports.EffectDescriptionKind.DELAY;
}
function actionPatternIsPredicateRecord(pattern) {
return !!(typeof pattern !== 'string' &&
!Array.isArray(pattern) &&
pattern.pattern &&
pattern.predicate);
}
function isTakePatternActuallyActionChannelEffectDescription(effectDescription) {
return (effectDescription.kind ===
exports.EffectDescriptionKind.ACTION_CHANNEL);
}
function takeInputIsActionPattern(takeInput) {
return (typeof takeInput === 'string' ||
Array.isArray(takeInput) ||
actionPatternIsPredicateRecord(takeInput.pattern));
}
function takeInputIsActionChannelEffectDescription(input) {
return (typeof input !== 'string' &&
!Array.isArray(input) &&
input.kind ===
exports.EffectDescriptionKind.ACTION_CHANNEL);
}
function isGenerator(possibleGenerator) {
return (!!possibleGenerator &&
!!possibleGenerator.next &&
!!possibleGenerator.throw &&
!!possibleGenerator.return);
}
function isKafkaJSProtocolError(error) {
return error && typeof error === 'object' && error.name === 'KafkaJSProtocolError';
}
var type_guard = /*#__PURE__*/Object.freeze({
__proto__: null,
isTransactionMessage: isTransactionMessage,
isTakeEffectDescription: isTakeEffectDescription,
isPutEffectDescription: isPutEffectDescription,
isCallEffectDescription: isCallEffectDescription,
isEffectCombinatorDescription: isEffectCombinatorDescription,
isActionChannelEffectDescription: isActionChannelEffectDescription,
isTakeActionChannelEffectDescription: isTakeActionChannelEffectDescription,
isDelayEffectDescription: isDelayEffectDescription,
actionPatternIsPredicateRecord: actionPatternIsPredicateRecord,
isTakePatternActuallyActionChannelEffectDescription: isTakePatternActuallyActionChannelEffectDescription,
takeInputIsActionPattern: takeInputIsActionPattern,
takeInputIsActionChannelEffectDescription: takeInputIsActionChannelEffectDescription,
isGenerator: isGenerator,
isKafkaJSProtocolError: isKafkaJSProtocolError
});
async function racePromiseRecord(raceable) {
const [resolvedKey, resolvedValue] = await new Promise((resolve, reject) => {
Object.entries(raceable).forEach(([key, promiseValue]) => {
promiseValue.then(value => resolve([key, value]), err => reject(err));
});
});
const withNullifiedValues = Object.keys(raceable).reduce((record, keyName) => {
return {
...record,
[keyName]: null
};
}, {});
return {
...withNullifiedValues,
[resolvedKey]: resolvedValue
};
}
async function racePromise(raceable) {
if (Array.isArray(raceable)) {
return Bluebird.race(raceable);
}
else {
return racePromiseRecord(raceable);
}
}
function allPromise(promises) {
if (Array.isArray(promises)) {
return Bluebird.all(promises);
}
else if (
// tslint:disable-next-line: triple-equals
promises != null &&
typeof promises === 'object') {
return Bluebird.props(promises);
}
throw new Error(`allPromise cannot handle the type provided: ${promises}`);
}
class EffectBuilder {
constructor(transactionId) {
this.transactionId = transactionId;
this.put = (...args) => {
const pattern = args[0];
const payload = args[1];
return {
pattern,
topic: pattern,
payload,
transactionId: this.transactionId,
kind: exports.EffectDescriptionKind.PUT
};
};
this.delay = (...[delayInMilliseconds, payload]) => {
return {
transactionId: this.transactionId,
kind: exports.EffectDescriptionKind.DELAY,
delayInMilliseconds,
payload
};
};
this.put = this.put.bind(this);
this.take = this.take.bind(this);
this.callFn = this.callFn.bind(this);
this.actionChannel = this.actionChannel.bind(this);
this.all = this.all.bind(this);
this.race = this.race.bind(this);
}
take(patterns) {
if (takeInputIsActionChannelEffectDescription(patterns)) {
return {
transactionId: this.transactionId,
patterns: patterns.pattern,
kind: exports.EffectDescriptionKind.TAKE_ACTION_CHANNEL,
buffer: patterns.buffer,
topics: this.generateTopics(patterns),
observer: this.generateTopicStreamObserver(patterns.pattern, patterns.buffer)
};
}
if (typeof patterns === 'string' ||
Array.isArray(patterns) ||
actionPatternIsPredicateRecord(patterns)) {
const buffer = new EphemeralBuffer();
const takeEffectDescription = {
transactionId: this.transactionId,
patterns,
kind: exports.EffectDescriptionKind.TAKE,
buffer,
topics: this.generateTopics(patterns),
observer: this.generateTopicStreamObserver(patterns, buffer)
};
return takeEffectDescription;
}
throw new Error('Invalid input provided for take: expected string | string[] | IPredicateRecord | ActionChannelEffectDescription');
}
callFn(effect, args) {
return {
transactionId: this.transactionId,
kind: exports.EffectDescriptionKind.CALL,
effect,
args
};
}
actionChannel(input, actionBuffer) {
const defaultActionBuffer = new ActionChannelBuffer();
const buffer = actionBuffer || defaultActionBuffer;
return {
transactionId: this.transactionId,
pattern: input,
buffer,
kind: exports.EffectDescriptionKind.ACTION_CHANNEL,
topics: this.generateTopics(input),
observer: this.generateTopicStreamObserver(input, buffer)
};
}
all(effects) {
return {
transactionId: this.transactionId,
kind: exports.EffectDescriptionKind.COMBINATOR,
combinator: allPromise,
effects
};
}
race(effects) {
return {
transactionId: this.transactionId,
kind: exports.EffectDescriptionKind.COMBINATOR,
combinator: racePromise,
effects
};
}
// tslint:disable-next-line: cyclomatic-complexity
generateTopics(input) {
if (takeInputIsActionChannelEffectDescription(input)) {
return input.topics;
}
if (actionPatternIsPredicateRecord(input)) {
return this.generateTopics(input.pattern);
}
if (Array.isArray(input)) {
return input;
}
if (typeof input === 'string') {
return [input];
}
throw new Error('Cannot handle patterns of type ' + typeof input);
}
generateTopicStreamObserver(input, buffer) {
return (action) => {
if (actionPatternIsPredicateRecord(input)) {
if (input.predicate(action)) {
buffer.put(action);
}
}
else {
buffer.put(action);
}
};
}
}
function parseHeaders(headers) {
if (!headers) {
return {};
}
const keys = Object.keys(headers);
return keys.reduce((parsed, key) => {
const header = headers[key];
return {
...parsed,
[key.toString()]: header ? header.toString() : undefined
};
}, {});
}
function transformKafkaMessageToAction(topic, message, headerParser = parseHeaders, valueParser = JSON.parse) {
const { headers, value } = message;
const parsedValue = value ? valueParser(value.toString()) : value;
if (!isTransactionMessage(parsedValue)) {
throw new Error('Received a misshapen payload');
}
const action = {
topic,
transaction_id: parsedValue.transaction_id,
payload: parsedValue.payload,
headers: headerParser(headers)
};
return action;
}
class TopicAdministrator {
constructor(kafka, topicConfig = {}, adminConfig = TopicAdministrator.adminClientConfiguration) {
this.kafka = kafka;
this.topicConfig = topicConfig;
this.adminConfig = adminConfig;
this.createTopic = this.createTopic.bind(this);
this.deleteTopic = this.deleteTopic.bind(this);
}
async createTopic(topic) {
const adminClient = this.kafka.admin(this.adminConfig);
await adminClient.connect();
await adminClient.createTopics({
topics: [
{
topic,
...this.topicConfig
}
],
waitForLeaders: true
});
await adminClient.disconnect();
}
async deleteTopic(topic) {
const adminClient = this.kafka.admin(this.adminConfig);
await adminClient.connect();
await adminClient.deleteTopics({
topics: [topic]
});
await adminClient.disconnect();
}
}
TopicAdministrator.adminClientConfiguration = {
retry: {
retries: 1,
initialRetryTime: 300,
maxRetryTime: 500
}
};
class ConsumerPool {
constructor(kafka, rootTopic, consumerConfig = {}, topicAdministrator) {
this.kafka = kafka;
this.rootTopic = rootTopic;
this.consumerConfig = consumerConfig;
this.consumers = new Map();
this.observersByTransaction = new Map();
this.topicAdministrator = topicAdministrator || new TopicAdministrator(kafka);
}
async streamActionsFromTopic(topic) {
if (this.consumers.has(topic)) {
return;
}
const consumer = this.kafka.consumer({
groupId: `${this.rootTopic}-${uuid.v4()}`,
allowAutoTopicCreation: false,
heartbeatInterval: 500,
maxWaitTimeInMs: 100,
...this.consumerConfig
});
await consumer.connect();
try {
await consumer.subscribe({ topic });
}
catch (error) {
if (isKafkaJSProtocolError(error) && error.type === 'UNKNOWN_TOPIC_OR_PARTITION') {
await this.topicAdministrator.createTopic(topic);
}
else {
throw error;
}
}
this.consumers.set(topic, consumer);
await consumer.run({
autoCommit: true,
autoCommitThreshold: 1,
eachMessage: async ({ message }) => {
const action = transformKafkaMessageToAction(topic, message);
// if this is a transactionId we actually care about, broadcast
if (this.observersByTransaction.has(action.transaction_id)) {
this.broadcastAction(topic, action);
}
}
});
}
async disconnectConsumers() {
for (const consumer of this.consumers.values()) {
await consumer.stop();
await consumer.disconnect();
}
this.observersByTransaction.clear();
this.consumers.clear();
}
startTransaction(transactionId) {
if (this.observersByTransaction.has(transactionId)) {
throw new Error('Trying to start a transaction that has already started');
}
this.observersByTransaction.set(transactionId, new Map());
}
stopTransaction(transactionId) {
this.observersByTransaction.delete(transactionId);
}
registerTopicObserver({ transactionId, topic, observer }) {
const topicObserversForTransaction = this.observersByTransaction.get(transactionId) || new Map();
const topicObservers = topicObserversForTransaction.get(topic) || [];
topicObserversForTransaction.set(topic, [...topicObservers, observer]);
}
broadcastAction(topic, action) {
const topicObserversForTransaction = this.observersByTransaction.get(action.transaction_id);
if (!topicObserversForTransaction) {
return;
}
const topicObservers = topicObserversForTransaction.get(topic) || [];
topicObservers.forEach(notify => notify(action));
}
}
function createActionMessage({ action }) {
return {
value: JSON.stringify({
transaction_id: action.transaction_id,
payload: action.payload
}),
headers: action.headers
};
}
class ThrottledProducer {
constructor(kafka, producerConfig = {
maxOutgoingBatchSize: 10000,
flushIntervalMs: 1000
}, logger) {
this.kafka = kafka;
this.producerConfig = producerConfig;
this.recordsSent = 0;
this.isConnected = false;
this.recordQueue = [];
this.isFlushing = false;
// tslint:disable-next-line: cyclomatic-complexity
this.putAction = async (action, messageConfig = {
useTransactionIdAsKey: false
}) => {
if (!this.isConnected) {
throw new Error('You must .connect before producing actions');
}
return new Promise((resolve, reject) => {
const message = createActionMessage({ action });
if (messageConfig.useTransactionIdAsKey) {
message.key = action.transaction_id;
}
this.recordQueue = [
...this.recordQueue,
{
resolve,
reject,
record: {
topic: action.topic,
messages: [createActionMessage({ action })]
}
}
];
return;
});
};
this.connect = async () => {
if (this.isConnected) {
return;
}
const flushIntervalMs = this.producerConfig.flushIntervalMs || 1000;
this.logger.debug('Connecting producer');
await this.producer.connect();
this.logger.debug('Connected producer');
this.logger.debug({ flushIntervalMs }, 'Creating flush interval');
this.intervalTimeout = setInterval(this.flush, flushIntervalMs);
this.logger.debug('Created flush interval');
this.isConnected = true;
};
this.disconnect = async () => {
if (!this.isConnected) {
return;
}
this.logger.debug('Disconnecting');
clearInterval(this.intervalTimeout);
await this.producer.disconnect();
this.logger.debug('Disconnected');
this.isConnected = false;
};
this.createProducer = () => {
this.logger.debug('Creating a new producer');
this.producer = this.kafka.producer({
maxInFlightRequests: 1,
idempotent: true,
allowAutoTopicCreation: true,
...this.producerConfig
});
this.logger.debug('Created a new producer');
};
// tslint:disable-next-line: cyclomatic-complexity
this.flush = async (retryRecords, retryCounter = 0, retryBatchId) => {
if (!retryRecords && this.isFlushing) {
return;
}
if (retryCounter) {
/** Wait for a max of 30 seconds before retrying */
const retryDelay = Math.min(retryCounter * 1000, 30000);
this.logger.debug({ retryDelay }, 'Waiting before attempting retry');
await Bluebird.delay(retryDelay);
}
/**
* Ensures that if the interval call ends up being concurrent due latency in sendBatch,
* unintentinally overlapping cycles are deferred to the next interval.
*/
this.isFlushing = true;
const batchSize = this.producerConfig.maxOutgoingBatchSize || 1000;
const outgoingRecords = retryRecords || this.recordQueue.slice(0, batchSize);
this.recordQueue = this.recordQueue.slice(batchSize);
const batchId = retryBatchId || uuid.v4();
if (!outgoingRecords.length) {
this.isFlushing = false;
return;
}
this.logger.debug({
remaining: this.recordQueue.length,
records: outgoingRecords.length,
batchId
}, 'Flushing queue');
try {
await this.producer.sendBatch({
topicMessages: outgoingRecords.map(({ record }) => record),
acks: -1,
compression: kafkajs.CompressionTypes.GZIP
});
this.recordsSent += outgoingRecords.length;
this.logger.debug({ batchId }, 'Flushed queue');
outgoingRecords.map(({ resolve }) => resolve());
this.isFlushing = false;
return;
}
catch (error) {
/**
* If for some reason this producer is no longer recognized by the broker,
* create a new producer.
*/
if (isKafkaJSProtocolError(error) && error.type === 'UNKNOWN_PRODUCER_ID') {
await this.producer.disconnect();
this.createProducer();
await this.producer.connect();
this.logger.debug({ batchId }, 'Retrying failed flush attempt due to UNKNOWN_PRODUCER_ID');
await this.flush(outgoingRecords, retryCounter + 1, batchId);
return;
}
outgoingRecords.map(({ reject }) => reject(error));
this.isFlushing = false;
return;
}
};
this.logger = logger
? logger.child({ class: 'KafkaSagasThrottledProducer' })
: pino().child({ class: 'KafkaSagasThrottledProducer' });
this.createProducer();
}
}
class SagaRunner {
constructor(consumerPool, throttledProducer, middlewares = []) {
this.consumerPool = consumerPool;
this.throttledProducer = throttledProducer;
this.runSaga = async (initialAction, context, saga) => {
this.consumerPool.startTransaction(initialAction.transaction_id);
const result = await this.runGeneratorFsm(saga(initialAction, context), context);
this.consumerPool.stopTransaction(initialAction.transaction_id);
return result;
};
// tslint:disable-next-line: cyclomatic-complexity
this.runEffect = async (effectDescription, context) => {
if (isEffectCombinatorDescription(effectDescription)) {
const { effects, combinator } = effectDescription;
if (Array.isArray(effects)) {
const withRunningEffects = effects.map(effect => this.runEffect(effect, context));
return await combinator(withRunningEffects);
}
else if (typeof effects === 'object') {
const withRunningEffects = Object.keys(effects).reduce((obj, key) => {
return {
...obj,
[key]: this.runEffect(effects[key], context)
};
}, {});
return await combinator(withRunningEffects);
}
throw new Error('Incompatible effects passed into combinator. Must be an array or object of effects');
}
if (isActionChannelEffectDescription(effectDescription)) {
for (const topic of effectDescription.topics) {
this.consumerPool.registerTopicObserver({
transactionId: effectDescription.transactionId,
topic,
observer: effectDescription.observer
});
await this.consumerPool.streamActionsFromTopic(topic);
}
return effectDescription;
}
// If this effect already has a stream buffer for events matching the pattern,
// then just take from the buffer
if (isTakeActionChannelEffectDescription(effectDescription)) {
return await effectDescription.buffer.take();
}
if (isDelayEffectDescription(effectDescription)) {
const { delayInMilliseconds, payload } = effectDescription;
await Bluebird.delay(delayInMilliseconds);
return payload;
}
if (isTakeEffectDescription(effectDescription)) {
await Bluebird.map(effectDescription.topics, async (topic) => {
this.consumerPool.registerTopicObserver({
transactionId: effectDescription.transactionId,
topic,
observer: effectDescription.observer
});
await this.consumerPool.streamActionsFromTopic(topic);
});
return await effectDescription.buffer.take();
}
if (isPutEffectDescription(effectDescription)) {
const action = {
topic: effectDescription.pattern,
transaction_id: effectDescription.transactionId,
payload: effectDescription.payload
};
if (context.headers) {
action.headers = context.headers;
}
await this.throttledProducer.putAction(action);
return;
}
if (isCallEffectDescription(effectDescription)) {
const result = await effectDescription.effect(...(effectDescription.args || []));
if (isGenerator(result)) {
return this.runGeneratorFsm(result, context);
}
return result;
}
};
const initialNext = async (effect, ctx) => {
return this.runEffect(effect, ctx);
};
this.runEffectWithMiddleware = middlewares
.reduceRight((previousNext, middleware) => {
return middleware(previousNext);
}, initialNext)
.bind(this);
}
async runGeneratorFsm(machine, context, { previousGeneratorResponse = null, didThrow = false } = { previousGeneratorResponse: null, didThrow: false }) {
/**
* Dereferencing the receiver removes its context, so we need to bind it back to the machine.
*/
const receiver = didThrow ? machine.throw.bind(machine) : machine.next.bind(machine);
const { done, value } = receiver(previousGeneratorResponse);
if (done) {
return value;
}
try {
const result = await this.runEffectWithMiddleware(value, context);
return this.runGeneratorFsm(machine, context, {
previousGeneratorResponse: result,
didThrow: false
});
}
catch (error) {
return this.runGeneratorFsm(machine, context, {
previousGeneratorResponse: error,
didThrow: true
});
}
}
}
const getLoggerFromConfig = (config) => {
const loggerOptions = config && config.logOptions ? config.logOptions : {};
return config && config.logger ? config.logger : pino(loggerOptions);
};
class ConsumptionTimeoutError extends Error {
constructor(message) {
super(message);
this.name = 'ConsumptionTimeoutError';
this.message = message;
this.stack = new Error().stack;
}
}
class TopicSagaConsumer {
constructor({ kafka, topic, saga, getContext = async () => {
return {};
}, loggerConfig, middlewares = [], consumerConfig = {
/** How often should heartbeats be sent back to the broker? */
heartbeatInterval: 500,
/** Allows main consumer and action channel consumers to create new topics. */
allowAutoTopicCreation: false,
/**
* Is this a special consumer group?
* Use case: Provide a custom consumerGroup if this saga is not the primary consumer of an event.
* For instance, you may want to have multiple different reactions to an event aside from the primary work
* to kick off notifactions.
*/
groupId: topic,
/**
* How much time should be given to a saga to complete
* before a consumer is considered unhealthy and killed?
*
* Providing -1 will allow a saga to run indefinitely.
*/
consumptionTimeoutMs: 30000,
/**
* How long should the broker wait before responding in the case of too small a number of records to return?
*/
maxWaitTimeInMs: 100
}, producerConfig = {
/** Allows producer to create new topics. */
allowAutoTopicCreation: false,
/** How often should produced message batches be sent out? */
flushIntervalMs: 100,
/** When batching produced messages (with the PUT effect), how many should be flushed at a time? */
maxOutgoingBatchSize: 1000
}, topicAdministrator }) {
this.eventEmitter = new EventEmitter();
this.eachMessage = async (runner, { partition, message }) => {
const action = transformKafkaMessageToAction(this.topic, message);
this.logger.info({
partition,
offset: message.offset,
topic: action.topic,
transaction_id: action.transaction_id,
headers: action.headers,
timestamp: Date.now()
}, 'Beginning consumption of message');
try {
const externalContext = await this.getContext(message);
this.eventEmitter.emit('started_saga', {
headers: parseHeaders(message.headers),
...externalContext,
effects: new EffectBuilder(action.transaction_id),
originalMessage: {
key: message.key,
value: message.value,
offset: message.offset,
timestamp: message.timestamp,
partition
}
});
await runner.runSaga(action, {
headers: parseHeaders(message.headers),
...externalContext,
effects: new EffectBuilder(action.transaction_id),
originalMessage: {
key: message.key,
value: message.value,
offset: message.offset,
timestamp: message.timestamp,
partition
}
}, this.saga);
this.eventEmitter.emit('consumed_message', {
partition,
offset: message.offset,
payload: action.payload
});
this.logger.info({
partition,
offset: message.offset,
topic: action.topic,
transaction_id: action.transaction_id,
headers: action.headers,
timestamp: Date.now()
}, 'Successfully consumed message');
}
catch (error) {
this.consumerPool.stopTransaction(action.transaction_id);
this.logger.error({
transaction_id: action.transaction_id,
topic: action.topic,
headers: action.headers,
timestamp: Date.now(),
error
}, error.message
? `Error while running ${this.topic} saga: ${error.message}`
: `Error while running ${this.topic} saga`);
}
};
const { consumptionTimeoutMs, ...kafkaConsumerConfig } = consumerConfig;
this.consumerConfig = {
groupId: topic,
allowAutoTopicCreation: false,
retry: { retries: 0 },
heartbeatInterval: 500,
maxWaitTimeInMs: 100,
...kafkaConsumerConfig
};
this.consumptionTimeoutMs = consumptionTimeoutMs || 30000;
this.producerConfig = {
/** Allows producer to create new topics. */
allowAutoTopicCreation: false,
/** How often should produced message batches be sent out? */
flushIntervalMs: 100,
/** When batching produced messages (with the PUT effect), how many should be flushed at a time? */
maxOutgoingBatchSize: 1000,
...producerConfig
};
this.saga = saga;
this.topic = topic;
this.getContext = getContext;
this.middlewares = middlewares;
this.logger = getLoggerFromConfig(loggerConfig).child({
package: 'snpkg-snapi-kafka-sagas'
});
this.topicAdminstrator = topicAdministrator || new TopicAdministrator(kafka);
this.consumer = kafka.consumer(this.consumerConfig);
this.consumerPool = new ConsumerPool(kafka, topic, {
retry: { retries: 0 },
heartbeatInterval: kafkaConsumerConfig.heartbeatInterval || 500,
maxWaitTimeInMs: kafkaConsumerConfig.maxWaitTimeInMs || 100
}, this.topicAdminstrator);
this.throttledProducer = new ThrottledProducer(kafka, this.producerConfig, this.logger);
this.run = this.run.bind(this);
this.disconnect = this.disconnect.bind(this);
}
/**
* Catching and crashing is left to consumers of this class
* so that they can log as they see fit.
*/
async run() {
try {
await this.consumer.subscribe({
topic: this.topic,
fromBeginning: true
});
}
catch (error) {
if (isKafkaJSProtocolError(error) && error.type === 'UNKNOWN_TOPIC_OR_PARTITION') {
this.logger.info({ topic: this.topic }, 'Unknown topic. Creating topic and partitions.');
await this.topicAdminstrator.createTopic(this.topic);
}
else {
throw error;
}
}
await this.throttledProducer.connect();
const runner = new SagaRunner(this.consumerPool, this.throttledProducer, this.middlewares);
this.consumer.on('consumer.commit_offsets', (...args) => {
this.eventEmitter.emit('comitted_offsets', ...args);
});
this.consumer.on('consumer.end_batch_process', (...args) => {
this.eventEmitter.emit('completed_saga', ...args);
});
await this.consumer.run({
autoCommit: true,
autoCommitThreshold: 1,
eachBatchAutoResolve: true,
// tslint:disable-next-line: cyclomatic-complexity
eachBatch: async ({ batch: { topic, partition, messages }, commitOffsetsIfNecessary, heartbeat, resolveOffset, isRunning, isStale }) => {
for (const message of messages) {
if (!isRunning() || isStale()) {
break;
}
try {
await heartbeat();
this.backgroundHeartbeat = setInterval(async () => {
try {
await heartbeat();
}
catch (error) {
this.logger.error({ step: 'Heartbeat', ...error }, error.message);
if (this.backgroundHeartbeat) {
clearInterval(this.backgroundHeartbeat);
this.backgroundHeartbeat = undefined;
}
}
}, this.consumerConfig.heartbeatInterval || 500);
if (this.consumptionTimeoutMs === -1) {
await this.eachMessage(runner, { topic, partition, message });
}
else {
await Bluebird.resolve(this.eachMessage(runner, { topic, partition, message })).timeout(this.consumptionTimeoutMs, new ConsumptionTimeoutError(`Message consumption timed out after ${this.consumptionTimeoutMs} milliseconds.`));
}
clearInterval(this.backgroundHeartbeat);
this.backgroundHeartbeat = undefined;
await heartbeat();
}
catch (e) {
if (this.backgroundHeartbeat) {
clearInterval(this.backgroundHeartbeat);
this.backgroundHeartbeat = undefined;
}
await commitOffsetsIfNecessary();
throw e;
}
resolveOffset(message.offset);
await heartbeat();
await commitOffsetsIfNecessary();
}
if (this.backgroundHeartbeat) {
clearInterval(this.backgroundHeartbeat);
this.backgroundHeartbeat = undefined;
}
}
});
}
async disconnect() {
if (this.backgroundHeartbeat) {
clearInterval(this.backgroundHeartbeat);
this.backgroundHeartbeat = undefined;
}
await this.consumer.stop();
await this.consumer.disconnect();
await this.consumerPool.disconnectConsumers();
await this.throttledProducer.disconnect();
}
}
exports.ActionChannelBuffer = ActionChannelBuffer;
exports.ConsumerPool = ConsumerPool;
exports.ConsumptionTimeoutError = ConsumptionTimeoutError;
exports.EffectBuilder = EffectBuilder;
exports.EphemeralBuffer = EphemeralBuffer;
exports.SagaRunner = SagaRunner;
exports.ThrottledProducer = ThrottledProducer;
exports.TopicAdministrator = TopicAdministrator;
exports.TopicSagaConsumer = TopicSagaConsumer;
exports.TypeGuard = type_guard;
exports.createActionMessage = createActionMessage;
exports.parseHeaders = parseHeaders;
exports.transformKafkaMessageToAction = transformKafkaMessageToAction;