kafka-sagas/dist/index.es.js

Build sagas that consume from a kafka topic

import Bluebird from 'bluebird';
import EventEmitter from 'events';
import uuid from 'uuid';
import { CompressionTypes } from 'kafkajs';
import pino from 'pino';

class ActionChannelBuffer {
    constructor() {
        this.actions = [];
        this.observers = [];
    }
    put(action) {
        this.actions = [...this.actions, action];
        this.notifyObservers();
    }
    take() {
        const promise = new Promise(resolve => this.observers.push(resolve));
        this.notifyObservers();
        return promise;
    }
    // tslint:disable-next-line: cyclomatic-complexity
    notifyObservers() {
        const countObservers = this.observers.length;
        const countActions = this.actions.length;
        // if there are no observers and no actions, do nothing
        if (!countObservers || !countActions) {
            return;
        }
        let numberOfItemsToShift = Math.min(countObservers, countActions);
        while (numberOfItemsToShift) {
            const observer = this.observers.shift();
            const action = this.actions.shift();
            if (!observer || !action) {
                throw new Error('Possible mutation of observers or actions during notification loop');
            }
            observer(action);
            numberOfItemsToShift -= 1;
        }
    }
}

class EphemeralBuffer {
    constructor() {
        this.observers = [];
    }
    put(action) {
        for (const observerResolve of this.observers) {
            observerResolve(action);
        }
        delete this.observers;
        this.observers = [];
    }
    take() {
        const promise = new Promise(resolve => this.observers.push(resolve));
        return promise;
    }
}

var EffectDescriptionKind;
(function (EffectDescriptionKind) {
    EffectDescriptionKind["PUT"] = "PUT";
    EffectDescriptionKind["TAKE"] = "TAKE";
    EffectDescriptionKind["CALL"] = "CALL";
    EffectDescriptionKind["DELAY"] = "DELAY";
    EffectDescriptionKind["COMBINATOR"] = "COMBINATOR";
    EffectDescriptionKind["ACTION_CHANNEL"] = "ACTION_CHANNEL";
    EffectDescriptionKind["TAKE_ACTION_CHANNEL"] = "TAKE_ACTION_CHANNEL";
})(EffectDescriptionKind || (EffectDescriptionKind = {}));

function isTransactionMessage(messageValue) {
    return messageValue && messageValue.transaction_id;
}
function isTakeEffectDescription(effectDescription) {
    return effectDescription.kind === EffectDescriptionKind.TAKE;
}
function isPutEffectDescription(effectDescription) {
    return effectDescription.kind === EffectDescriptionKind.PUT;
}
function isCallEffectDescription(effectDescription) {
    return effectDescription.kind === EffectDescriptionKind.CALL;
}
function isEffectCombinatorDescription(effectDescription) {
    return !!(effectDescription.combinator &&
        effectDescription.effects);
}
function isActionChannelEffectDescription(effectDescription) {
    return effectDescription.kind === EffectDescriptionKind.ACTION_CHANNEL;
}
function isTakeActionChannelEffectDescription(effectDescription) {
    return effectDescription.kind === EffectDescriptionKind.TAKE_ACTION_CHANNEL;
}
function isDelayEffectDescription(effectDescription) {
    return effectDescription.kind === EffectDescriptionKind.DELAY;
}
function actionPatternIsPredicateRecord(pattern) {
    return !!(typeof pattern !== 'string' &&
        !Array.isArray(pattern) &&
        pattern.pattern &&
        pattern.predicate);
}
function isTakePatternActuallyActionChannelEffectDescription(effectDescription) {
    return (effectDescription.kind ===
        EffectDescriptionKind.ACTION_CHANNEL);
}
function takeInputIsActionPattern(takeInput) {
    return (typeof takeInput === 'string' ||
        Array.isArray(takeInput) ||
        actionPatternIsPredicateRecord(takeInput.pattern));
}
function takeInputIsActionChannelEffectDescription(input) {
    return (typeof input !== 'string' &&
        !Array.isArray(input) &&
        input.kind ===
            EffectDescriptionKind.ACTION_CHANNEL);
}
function isGenerator(possibleGenerator) {
    return (!!possibleGenerator &&
        !!possibleGenerator.next &&
        !!possibleGenerator.throw &&
        !!possibleGenerator.return);
}
function isKafkaJSProtocolError(error) {
    return error && typeof error === 'object' && error.name === 'KafkaJSProtocolError';
}

var type_guard = /*#__PURE__*/Object.freeze({
    __proto__: null,
    isTransactionMessage: isTransactionMessage,
    isTakeEffectDescription: isTakeEffectDescription,
    isPutEffectDescription: isPutEffectDescription,
    isCallEffectDescription: isCallEffectDescription,
    isEffectCombinatorDescription: isEffectCombinatorDescription,
    isActionChannelEffectDescription: isActionChannelEffectDescription,
    isTakeActionChannelEffectDescription: isTakeActionChannelEffectDescription,
    isDelayEffectDescription: isDelayEffectDescription,
    actionPatternIsPredicateRecord: actionPatternIsPredicateRecord,
    isTakePatternActuallyActionChannelEffectDescription: isTakePatternActuallyActionChannelEffectDescription,
    takeInputIsActionPattern: takeInputIsActionPattern,
    takeInputIsActionChannelEffectDescription: takeInputIsActionChannelEffectDescription,
    isGenerator: isGenerator,
    isKafkaJSProtocolError: isKafkaJSProtocolError
});

async function racePromiseRecord(raceable) {
    const [resolvedKey, resolvedValue] = await new Promise((resolve, reject) => {
        Object.entries(raceable).forEach(([key, promiseValue]) => {
            promiseValue.then(value => resolve([key, value]), err => reject(err));
        });
    });
    const withNullifiedValues = Object.keys(raceable).reduce((record, keyName) => {
        return {
            ...record,
            [keyName]: null
        };
    }, {});
    return {
        ...withNullifiedValues,
        [resolvedKey]: resolvedValue
    };
}
async function racePromise(raceable) {
    if (Array.isArray(raceable)) {
        return Bluebird.race(raceable);
    }
    else {
        return racePromiseRecord(raceable);
    }
}

function allPromise(promises) {
    if (Array.isArray(promises)) {
        return Bluebird.all(promises);
    }
    else if (
    // tslint:disable-next-line: triple-equals
    promises != null &&
        typeof promises === 'object') {
        return Bluebird.props(promises);
    }
    throw new Error(`allPromise cannot handle the type provided: ${promises}`);
}

class EffectBuilder {
    constructor(transactionId) {
        this.transactionId = transactionId;
        this.put = (...args) => {
            const pattern = args[0];
            const payload = args[1];
            return {
                pattern,
                topic: pattern,
                payload,
                transactionId: this.transactionId,
                kind: EffectDescriptionKind.PUT
            };
        };
        this.delay = (...[delayInMilliseconds, payload]) => {
            return {
                transactionId: this.transactionId,
                kind: EffectDescriptionKind.DELAY,
                delayInMilliseconds,
                payload
            };
        };
        this.put = this.put.bind(this);
        this.take = this.take.bind(this);
        this.callFn = this.callFn.bind(this);
        this.actionChannel = this.actionChannel.bind(this);
        this.all = this.all.bind(this);
        this.race = this.race.bind(this);
    }
    take(patterns) {
        if (takeInputIsActionChannelEffectDescription(patterns)) {
            return {
                transactionId: this.transactionId,
                patterns: patterns.pattern,
                kind: EffectDescriptionKind.TAKE_ACTION_CHANNEL,
                buffer: patterns.buffer,
                topics: this.generateTopics(patterns),
                observer: this.generateTopicStreamObserver(patterns.pattern, patterns.buffer)
            };
        }
        if (typeof patterns === 'string' ||
            Array.isArray(patterns) ||
            actionPatternIsPredicateRecord(patterns)) {
            const buffer = new EphemeralBuffer();
            const takeEffectDescription = {
                transactionId: this.transactionId,
                patterns,
                kind: EffectDescriptionKind.TAKE,
                buffer,
                topics: this.generateTopics(patterns),
                observer: this.generateTopicStreamObserver(patterns, buffer)
            };
            return takeEffectDescription;
        }
        throw new Error('Invalid input provided for take: expected string | string[] | IPredicateRecord | ActionChannelEffectDescription');
    }
    callFn(effect, args) {
        return {
            transactionId: this.transactionId,
            kind: EffectDescriptionKind.CALL,
            effect,
            args
        };
    }
    actionChannel(input, actionBuffer) {
        const defaultActionBuffer = new ActionChannelBuffer();
        const buffer = actionBuffer || defaultActionBuffer;
        return {
            transactionId: this.transactionId,
            pattern: input,
            buffer,
            kind: EffectDescriptionKind.ACTION_CHANNEL,
            topics: this.generateTopics(input),
            observer: this.generateTopicStreamObserver(input, buffer)
        };
    }
    all(effects) {
        return {
            transactionId: this.transactionId,
            kind: EffectDescriptionKind.COMBINATOR,
            combinator: allPromise,
            effects
        };
    }
    race(effects) {
        return {
            transactionId: this.transactionId,
            kind: EffectDescriptionKind.COMBINATOR,
            combinator: racePromise,
            effects
        };
    }
    // tslint:disable-next-line: cyclomatic-complexity
    generateTopics(input) {
        if (takeInputIsActionChannelEffectDescription(input)) {
            return input.topics;
        }
        if (actionPatternIsPredicateRecord(input)) {
            return this.generateTopics(input.pattern);
        }
        if (Array.isArray(input)) {
            return input;
        }
        if (typeof input === 'string') {
            return [input];
        }
        throw new Error('Cannot handle patterns of type ' + typeof input);
    }
    generateTopicStreamObserver(input, buffer) {
        return (action) => {
            if (actionPatternIsPredicateRecord(input)) {
                if (input.predicate(action)) {
                    buffer.put(action);
                }
            }
            else {
                buffer.put(action);
            }
        };
    }
}

function parseHeaders(headers) {
    if (!headers) {
        return {};
    }
    const keys = Object.keys(headers);
    return keys.reduce((parsed, key) => {
        const header = headers[key];
        return {
            ...parsed,
            [key.toString()]: header ? header.toString() : undefined
        };
    }, {});
}

function transformKafkaMessageToAction(topic, message, headerParser = parseHeaders, valueParser = JSON.parse) {
    const { headers, value } = message;
    const parsedValue = value ? valueParser(value.toString()) : value;
    if (!isTransactionMessage(parsedValue)) {
        throw new Error('Received a misshapen payload');
    }
    const action = {
        topic,
        transaction_id: parsedValue.transaction_id,
        payload: parsedValue.payload,
        headers: headerParser(headers)
    };
    return action;
}

class TopicAdministrator {
    constructor(kafka, topicConfig = {}, adminConfig = TopicAdministrator.adminClientConfiguration) {
        this.kafka = kafka;
        this.topicConfig = topicConfig;
        this.adminConfig = adminConfig;
        this.createTopic = this.createTopic.bind(this);
        this.deleteTopic = this.deleteTopic.bind(this);
    }
    async createTopic(topic) {
        const adminClient = this.kafka.admin(this.adminConfig);
        await adminClient.connect();
        await adminClient.createTopics({
            topics: [
                {
                    topic,
                    ...this.topicConfig
                }
            ],
            waitForLeaders: true
        });
        await adminClient.disconnect();
    }
    async deleteTopic(topic) {
        const adminClient = this.kafka.admin(this.adminConfig);
        await adminClient.connect();
        await adminClient.deleteTopics({
            topics: [topic]
        });
        await adminClient.disconnect();
    }
}
TopicAdministrator.adminClientConfiguration = {
    retry: {
        retries: 1,
        initialRetryTime: 300,
        maxRetryTime: 500
    }
};

class ConsumerPool {
    constructor(kafka, rootTopic, consumerConfig = {}, topicAdministrator) {
        this.kafka = kafka;
        this.rootTopic = rootTopic;
        this.consumerConfig = consumerConfig;
        this.consumers = new Map();
        this.observersByTransaction = new Map();
        this.topicAdministrator = topicAdministrator || new TopicAdministrator(kafka);
    }
    async streamActionsFromTopic(topic) {
        if (this.consumers.has(topic)) {
            return;
        }
        const consumer = this.kafka.consumer({
            groupId: `${this.rootTopic}-${uuid.v4()}`,
            allowAutoTopicCreation: false,
            heartbeatInterval: 500,
            maxWaitTimeInMs: 100,
            ...this.consumerConfig
        });
        await consumer.connect();
        try {
            await consumer.subscribe({ topic });
        }
        catch (error) {
            if (isKafkaJSProtocolError(error) && error.type === 'UNKNOWN_TOPIC_OR_PARTITION') {
                await this.topicAdministrator.createTopic(topic);
            }
            else {
                throw error;
            }
        }
        this.consumers.set(topic, consumer);
        await consumer.run({
            autoCommit: true,
            autoCommitThreshold: 1,
            eachMessage: async ({ message }) => {
                const action = transformKafkaMessageToAction(topic, message);
                // if this is a transactionId we actually care about, broadcast
                if (this.observersByTransaction.has(action.transaction_id)) {
                    this.broadcastAction(topic, action);
                }
            }
        });
    }
    async disconnectConsumers() {
        for (const consumer of this.consumers.values()) {
            await consumer.stop();
            await consumer.disconnect();
        }
        this.observersByTransaction.clear();
        this.consumers.clear();
    }
    startTransaction(transactionId) {
        if (this.observersByTransaction.has(transactionId)) {
            throw new Error('Trying to start a transaction that has already started');
        }
        this.observersByTransaction.set(transactionId, new Map());
    }
    stopTransaction(transactionId) {
        this.observersByTransaction.delete(transactionId);
    }
    registerTopicObserver({ transactionId, topic, observer }) {
        const topicObserversForTransaction = this.observersByTransaction.get(transactionId) || new Map();
        const topicObservers = topicObserversForTransaction.get(topic) || [];
        topicObserversForTransaction.set(topic, [...topicObservers, observer]);
    }
    broadcastAction(topic, action) {
        const topicObserversForTransaction = this.observersByTransaction.get(action.transaction_id);
        if (!topicObserversForTransaction) {
            return;
        }
        const topicObservers = topicObserversForTransaction.get(topic) || [];
        topicObservers.forEach(notify => notify(action));
    }
}

function createActionMessage({ action }) {
    return {
        value: JSON.stringify({
            transaction_id: action.transaction_id,
            payload: action.payload
        }),
        headers: action.headers
    };
}

class ThrottledProducer {
    constructor(kafka, producerConfig = {
        maxOutgoingBatchSize: 10000,
        flushIntervalMs: 1000
    }, logger) {
        this.kafka = kafka;
        this.producerConfig = producerConfig;
        this.recordsSent = 0;
        this.isConnected = false;
        this.recordQueue = [];
        this.isFlushing = false;
        // tslint:disable-next-line: cyclomatic-complexity
        this.putAction = async (action, messageConfig = {
            useTransactionIdAsKey: false
        }) => {
            if (!this.isConnected) {
                throw new Error('You must .connect before producing actions');
            }
            return new Promise((resolve, reject) => {
                const message = createActionMessage({ action });
                if (messageConfig.useTransactionIdAsKey) {
                    message.key = action.transaction_id;
                }
                this.recordQueue = [
                    ...this.recordQueue,
                    {
                        resolve,
                        reject,
                        record: {
                            topic: action.topic,
                            messages: [createActionMessage({ action })]
                        }
                    }
                ];
                return;
            });
        };
        this.connect = async () => {
            if (this.isConnected) {
                return;
            }
            const flushIntervalMs = this.producerConfig.flushIntervalMs || 1000;
            this.logger.debug('Connecting producer');
            await this.producer.connect();
            this.logger.debug('Connected producer');
            this.logger.debug({ flushIntervalMs }, 'Creating flush interval');
            this.intervalTimeout = setInterval(this.flush, flushIntervalMs);
            this.logger.debug('Created flush interval');
            this.isConnected = true;
        };
        this.disconnect = async () => {
            if (!this.isConnected) {
                return;
            }
            this.logger.debug('Disconnecting');
            clearInterval(this.intervalTimeout);
            await this.producer.disconnect();
            this.logger.debug('Disconnected');
            this.isConnected = false;
        };
        this.createProducer = () => {
            this.logger.debug('Creating a new producer');
            this.producer = this.kafka.producer({
                maxInFlightRequests: 1,
                idempotent: true,
                allowAutoTopicCreation: true,
                ...this.producerConfig
            });
            this.logger.debug('Created a new producer');
        };
        // tslint:disable-next-line: cyclomatic-complexity
        this.flush = async (retryRecords, retryCounter = 0, retryBatchId) => {
            if (!retryRecords && this.isFlushing) {
                return;
            }
            if (retryCounter) {
                /** Wait for a max of 30 seconds before retrying */
                const retryDelay = Math.min(retryCounter * 1000, 30000);
                this.logger.debug({ retryDelay }, 'Waiting before attempting retry');
                await Bluebird.delay(retryDelay);
            }
            /**
             * Ensures that if the interval call ends up being concurrent due latency in sendBatch,
             * unintentinally overlapping cycles are deferred to the next interval.
             */
            this.isFlushing = true;
            const batchSize = this.producerConfig.maxOutgoingBatchSize || 1000;
            const outgoingRecords = retryRecords || this.recordQueue.slice(0, batchSize);
            this.recordQueue = this.recordQueue.slice(batchSize);
            const batchId = retryBatchId || uuid.v4();
            if (!outgoingRecords.length) {
                this.isFlushing = false;
                return;
            }
            this.logger.debug({
                remaining: this.recordQueue.length,
                records: outgoingRecords.length,
                batchId
            }, 'Flushing queue');
            try {
                await this.producer.sendBatch({
                    topicMessages: outgoingRecords.map(({ record }) => record),
                    acks: -1,
                    compression: CompressionTypes.GZIP
                });
                this.recordsSent += outgoingRecords.length;
                this.logger.debug({ batchId }, 'Flushed queue');
                outgoingRecords.map(({ resolve }) => resolve());
                this.isFlushing = false;
                return;
            }
            catch (error) {
                /**
                 * If for some reason this producer is no longer recognized by the broker,
                 * create a new producer.
                 */
                if (isKafkaJSProtocolError(error) && error.type === 'UNKNOWN_PRODUCER_ID') {
                    await this.producer.disconnect();
                    this.createProducer();
                    await this.producer.connect();
                    this.logger.debug({ batchId }, 'Retrying failed flush attempt due to UNKNOWN_PRODUCER_ID');
                    await this.flush(outgoingRecords, retryCounter + 1, batchId);
                    return;
                }
                outgoingRecords.map(({ reject }) => reject(error));
                this.isFlushing = false;
                return;
            }
        };
        this.logger = logger
            ? logger.child({ class: 'KafkaSagasThrottledProducer' })
            : pino().child({ class: 'KafkaSagasThrottledProducer' });
        this.createProducer();
    }
}

class SagaRunner {
    constructor(consumerPool, throttledProducer, middlewares = []) {
        this.consumerPool = consumerPool;
        this.throttledProducer = throttledProducer;
        this.runSaga = async (initialAction, context, saga) => {
            this.consumerPool.startTransaction(initialAction.transaction_id);
            const result = await this.runGeneratorFsm(saga(initialAction, context), context);
            this.consumerPool.stopTransaction(initialAction.transaction_id);
            return result;
        };
        // tslint:disable-next-line: cyclomatic-complexity
        this.runEffect = async (effectDescription, context) => {
            if (isEffectCombinatorDescription(effectDescription)) {
                const { effects, combinator } = effectDescription;
                if (Array.isArray(effects)) {
                    const withRunningEffects = effects.map(effect => this.runEffect(effect, context));
                    return await combinator(withRunningEffects);
                }
                else if (typeof effects === 'object') {
                    const withRunningEffects = Object.keys(effects).reduce((obj, key) => {
                        return {
                            ...obj,
                            [key]: this.runEffect(effects[key], context)
                        };
                    }, {});
                    return await combinator(withRunningEffects);
                }
                throw new Error('Incompatible effects passed into combinator. Must be an array or object of effects');
            }
            if (isActionChannelEffectDescription(effectDescription)) {
                for (const topic of effectDescription.topics) {
                    this.consumerPool.registerTopicObserver({
                        transactionId: effectDescription.transactionId,
                        topic,
                        observer: effectDescription.observer
                    });
                    await this.consumerPool.streamActionsFromTopic(topic);
                }
                return effectDescription;
            }
            // If this effect already has a stream buffer for events matching the pattern,
            // then just take from the buffer
            if (isTakeActionChannelEffectDescription(effectDescription)) {
                return await effectDescription.buffer.take();
            }
            if (isDelayEffectDescription(effectDescription)) {
                const { delayInMilliseconds, payload } = effectDescription;
                await Bluebird.delay(delayInMilliseconds);
                return payload;
            }
            if (isTakeEffectDescription(effectDescription)) {
                await Bluebird.map(effectDescription.topics, async (topic) => {
                    this.consumerPool.registerTopicObserver({
                        transactionId: effectDescription.transactionId,
                        topic,
                        observer: effectDescription.observer
                    });
                    await this.consumerPool.streamActionsFromTopic(topic);
                });
                return await effectDescription.buffer.take();
            }
            if (isPutEffectDescription(effectDescription)) {
                const action = {
                    topic: effectDescription.pattern,
                    transaction_id: effectDescription.transactionId,
                    payload: effectDescription.payload
                };
                if (context.headers) {
                    action.headers = context.headers;
                }
                await this.throttledProducer.putAction(action);
                return;
            }
            if (isCallEffectDescription(effectDescription)) {
                const result = await effectDescription.effect(...(effectDescription.args || []));
                if (isGenerator(result)) {
                    return this.runGeneratorFsm(result, context);
                }
                return result;
            }
        };
        const initialNext = async (effect, ctx) => {
            return this.runEffect(effect, ctx);
        };
        this.runEffectWithMiddleware = middlewares
            .reduceRight((previousNext, middleware) => {
            return middleware(previousNext);
        }, initialNext)
            .bind(this);
    }
    async runGeneratorFsm(machine, context, { previousGeneratorResponse = null, didThrow = false } = { previousGeneratorResponse: null, didThrow: false }) {
        /**
         * Dereferencing the receiver removes its context, so we need to bind it back to the machine.
         */
        const receiver = didThrow ? machine.throw.bind(machine) : machine.next.bind(machine);
        const { done, value } = receiver(previousGeneratorResponse);
        if (done) {
            return value;
        }
        try {
            const result = await this.runEffectWithMiddleware(value, context);
            return this.runGeneratorFsm(machine, context, {
                previousGeneratorResponse: result,
                didThrow: false
            });
        }
        catch (error) {
            return this.runGeneratorFsm(machine, context, {
                previousGeneratorResponse: error,
                didThrow: true
            });
        }
    }
}

const getLoggerFromConfig = (config) => {
    const loggerOptions = config && config.logOptions ? config.logOptions : {};
    return config && config.logger ? config.logger : pino(loggerOptions);
};

class ConsumptionTimeoutError extends Error {
    constructor(message) {
        super(message);
        this.name = 'ConsumptionTimeoutError';
        this.message = message;
        this.stack = new Error().stack;
    }
}

class TopicSagaConsumer {
    constructor({ kafka, topic, saga, getContext = async () => {
        return {};
    }, loggerConfig, middlewares = [], consumerConfig = {
        /** How often should heartbeats be sent back to the broker? */
        heartbeatInterval: 500,
        /** Allows main consumer and action channel consumers to create new topics. */
        allowAutoTopicCreation: false,
        /**
         * Is this a special consumer group?
         * Use case: Provide a custom consumerGroup if this saga is not the primary consumer of an event.
         * For instance, you may want to have multiple different reactions to an event aside from the primary work
         * to kick off notifactions.
         */
        groupId: topic,
        /**
         * How much time should be given to a saga to complete
         * before a consumer is considered unhealthy and killed?
         *
         * Providing -1 will allow a saga to run indefinitely.
         */
        consumptionTimeoutMs: 30000,
        /**
         * How long should the broker wait before responding in the case of too small a number of records to return?
         */
        maxWaitTimeInMs: 100
    }, producerConfig = {
        /** Allows producer to create new topics. */
        allowAutoTopicCreation: false,
        /** How often should produced message batches be sent out? */
        flushIntervalMs: 100,
        /** When batching produced messages (with the PUT effect), how many should be flushed at a time? */
        maxOutgoingBatchSize: 1000
    }, topicAdministrator }) {
        this.eventEmitter = new EventEmitter();
        this.eachMessage = async (runner, { partition, message }) => {
            const action = transformKafkaMessageToAction(this.topic, message);
            this.logger.info({
                partition,
                offset: message.offset,
                topic: action.topic,
                transaction_id: action.transaction_id,
                headers: action.headers,
                timestamp: Date.now()
            }, 'Beginning consumption of message');
            try {
                const externalContext = await this.getContext(message);
                this.eventEmitter.emit('started_saga', {
                    headers: parseHeaders(message.headers),
                    ...externalContext,
                    effects: new EffectBuilder(action.transaction_id),
                    originalMessage: {
                        key: message.key,
                        value: message.value,
                        offset: message.offset,
                        timestamp: message.timestamp,
                        partition
                    }
                });
                await runner.runSaga(action, {
                    headers: parseHeaders(message.headers),
                    ...externalContext,
                    effects: new EffectBuilder(action.transaction_id),
                    originalMessage: {
                        key: message.key,
                        value: message.value,
                        offset: message.offset,
                        timestamp: message.timestamp,
                        partition
                    }
                }, this.saga);
                this.eventEmitter.emit('consumed_message', {
                    partition,
                    offset: message.offset,
                    payload: action.payload
                });
                this.logger.info({
                    partition,
                    offset: message.offset,
                    topic: action.topic,
                    transaction_id: action.transaction_id,
                    headers: action.headers,
                    timestamp: Date.now()
                }, 'Successfully consumed message');
            }
            catch (error) {
                this.consumerPool.stopTransaction(action.transaction_id);
                this.logger.error({
                    transaction_id: action.transaction_id,
                    topic: action.topic,
                    headers: action.headers,
                    timestamp: Date.now(),
                    error
                }, error.message
                    ? `Error while running ${this.topic} saga: ${error.message}`
                    : `Error while running ${this.topic} saga`);
            }
        };
        const { consumptionTimeoutMs, ...kafkaConsumerConfig } = consumerConfig;
        this.consumerConfig = {
            groupId: topic,
            allowAutoTopicCreation: false,
            retry: { retries: 0 },
            heartbeatInterval: 500,
            maxWaitTimeInMs: 100,
            ...kafkaConsumerConfig
        };
        this.consumptionTimeoutMs = consumptionTimeoutMs || 30000;
        this.producerConfig = {
            /** Allows producer to create new topics. */
            allowAutoTopicCreation: false,
            /** How often should produced message batches be sent out? */
            flushIntervalMs: 100,
            /** When batching produced messages (with the PUT effect), how many should be flushed at a time? */
            maxOutgoingBatchSize: 1000,
            ...producerConfig
        };
        this.saga = saga;
        this.topic = topic;
        this.getContext = getContext;
        this.middlewares = middlewares;
        this.logger = getLoggerFromConfig(loggerConfig).child({
            package: 'snpkg-snapi-kafka-sagas'
        });
        this.topicAdminstrator = topicAdministrator || new TopicAdministrator(kafka);
        this.consumer = kafka.consumer(this.consumerConfig);
        this.consumerPool = new ConsumerPool(kafka, topic, {
            retry: { retries: 0 },
            heartbeatInterval: kafkaConsumerConfig.heartbeatInterval || 500,
            maxWaitTimeInMs: kafkaConsumerConfig.maxWaitTimeInMs || 100
        }, this.topicAdminstrator);
        this.throttledProducer = new ThrottledProducer(kafka, this.producerConfig, this.logger);
        this.run = this.run.bind(this);
        this.disconnect = this.disconnect.bind(this);
    }
    /**
     * Catching and crashing is left to consumers of this class
     * so that they can log as they see fit.
     */
    async run() {
        try {
            await this.consumer.subscribe({
                topic: this.topic,
                fromBeginning: true
            });
        }
        catch (error) {
            if (isKafkaJSProtocolError(error) && error.type === 'UNKNOWN_TOPIC_OR_PARTITION') {
                this.logger.info({ topic: this.topic }, 'Unknown topic. Creating topic and partitions.');
                await this.topicAdminstrator.createTopic(this.topic);
            }
            else {
                throw error;
            }
        }
        await this.throttledProducer.connect();
        const runner = new SagaRunner(this.consumerPool, this.throttledProducer, this.middlewares);
        this.consumer.on('consumer.commit_offsets', (...args) => {
            this.eventEmitter.emit('comitted_offsets', ...args);
        });
        this.consumer.on('consumer.end_batch_process', (...args) => {
            this.eventEmitter.emit('completed_saga', ...args);
        });
        await this.consumer.run({
            autoCommit: true,
            autoCommitThreshold: 1,
            eachBatchAutoResolve: true,
            // tslint:disable-next-line: cyclomatic-complexity
            eachBatch: async ({ batch: { topic, partition, messages }, commitOffsetsIfNecessary, heartbeat, resolveOffset, isRunning, isStale }) => {
                for (const message of messages) {
                    if (!isRunning() || isStale()) {
                        break;
                    }
                    try {
                        await heartbeat();
                        this.backgroundHeartbeat = setInterval(async () => {
                            try {
                                await heartbeat();
                            }
                            catch (error) {
                                this.logger.error({ step: 'Heartbeat', ...error }, error.message);
                                if (this.backgroundHeartbeat) {
                                    clearInterval(this.backgroundHeartbeat);
                                    this.backgroundHeartbeat = undefined;
                                }
                            }
                        }, this.consumerConfig.heartbeatInterval || 500);
                        if (this.consumptionTimeoutMs === -1) {
                            await this.eachMessage(runner, { topic, partition, message });
                        }
                        else {
                            await Bluebird.resolve(this.eachMessage(runner, { topic, partition, message })).timeout(this.consumptionTimeoutMs, new ConsumptionTimeoutError(`Message consumption timed out after ${this.consumptionTimeoutMs} milliseconds.`));
                        }
                        clearInterval(this.backgroundHeartbeat);
                        this.backgroundHeartbeat = undefined;
                        await heartbeat();
                    }
                    catch (e) {
                        if (this.backgroundHeartbeat) {
                            clearInterval(this.backgroundHeartbeat);
                            this.backgroundHeartbeat = undefined;
                        }
                        await commitOffsetsIfNecessary();
                        throw e;
                    }
                    resolveOffset(message.offset);
                    await heartbeat();
                    await commitOffsetsIfNecessary();
                }
                if (this.backgroundHeartbeat) {
                    clearInterval(this.backgroundHeartbeat);
                    this.backgroundHeartbeat = undefined;
                }
            }
        });
    }
    async disconnect() {
        if (this.backgroundHeartbeat) {
            clearInterval(this.backgroundHeartbeat);
            this.backgroundHeartbeat = undefined;
        }
        await this.consumer.stop();
        await this.consumer.disconnect();
        await this.consumerPool.disconnectConsumers();
        await this.throttledProducer.disconnect();
    }
}

export { ActionChannelBuffer, ConsumerPool, ConsumptionTimeoutError, EffectBuilder, EffectDescriptionKind, EphemeralBuffer, SagaRunner, ThrottledProducer, TopicAdministrator, TopicSagaConsumer, type_guard as TypeGuard, createActionMessage, parseHeaders, transformKafkaMessageToAction };