kafka-node/lib/producer.js

Client for Apache Kafka v0.8+

'use strict';

var util = require('util');
var events = require('events');
var _ = require('lodash');
var protocol = require('./protocol');
var Message = protocol.Message;
var KeyedMessage = protocol.KeyedMessage;
var ProduceRequest = protocol.ProduceRequest;
var partitioner = require('./partitioner');
var DefaultPartitioner = partitioner.DefaultPartitioner;
var RandomPartitioner = partitioner.RandomPartitioner;
var CyclicPartitioner = partitioner.CyclicPartitioner;
var KeyedPartitioner = partitioner.KeyedPartitioner;

var PARTITIONER_TYPES = {
  default: 0,
  random: 1,
  cyclic: 2,
  keyed: 3
};

var DEFAULTS = {
  requireAcks: 1,
  ackTimeoutMs: 100,
  partitionerType: PARTITIONER_TYPES.default
};

/**
 * Provides common functionality for a kafka producer
 *
 * @param {Client} client A kafka client object to use for the producer
 * @param {Object} [options] An object containing configuration options
 * @param {Number} [options.requireAcks=1] Configuration for when to consider a message as acknowledged.
 *      <li>0 = No ack required</li>
 *      <li>1 = Leader ack required</li>
 *      <li>-1 = All in sync replicas ack required</li>
 *
 * @param {Number} [options.ackTimeoutMs=100] The amount of time in milliseconds to wait for all acks before considered
 *      the message as errored
 *
 * @constructor
 */
var Producer = function (client, options) {
  options = options || {};

  this.ready = false;
  this.client = client;

  this.requireAcks = options.requireAcks === undefined
    ? DEFAULTS.requireAcks
    : options.requireAcks;
  this.ackTimeoutMs = options.ackTimeoutMs === undefined
    ? DEFAULTS.ackTimeoutMs
    : options.ackTimeoutMs;

  if (options.partitionerType === PARTITIONER_TYPES.random) {
    this.partitioner = new RandomPartitioner();
  } else if (options.partitionerType === PARTITIONER_TYPES.cyclic) {
    this.partitioner = new CyclicPartitioner();
  } else if (options.partitionerType === PARTITIONER_TYPES.keyed) {
    this.partitioner = new KeyedPartitioner();
  } else {
    this.partitioner = new DefaultPartitioner();
  }

  this.connect();
};

util.inherits(Producer, events.EventEmitter);

Producer.prototype.connect = function () {
  // emiter...
  var self = this;
  this.ready = this.client.ready;
  if (this.ready) self.emit('ready');
  this.client.on('ready', function () {
    if (!self.ready) {
      self.ready = true;
      self.emit('ready');
    }
  });
  this.client.on('brokersChanged', function () {
    this.topicMetadata = {};
  });
  this.client.on('error', function (err) {
    self.emit('error', err);
  });
  this.client.on('close', function () {});
};

/**
 * Sends a new message or array of messages to a topic/partition
 * This will use the
 *
 * @see Client#sendProduceRequest for a more low level way to send messages to kafka
 *
 * @param {Array.<Producer~sendPayload>} payloads An array of topic payloads
 * @param {Producer~sendCallback} cb A function to call once the send has completed
 */
Producer.prototype.send = function (payloads, cb) {
  var client = this.client;
  var requireAcks = this.requireAcks;
  var ackTimeoutMs = this.ackTimeoutMs;

  client.sendProduceRequest(this.buildPayloads(payloads, client.topicMetadata), requireAcks, ackTimeoutMs, cb);
};

Producer.prototype.buildPayloads = function (payloads, topicMetadata) {
  var self = this;
  return payloads.map(function (p) {
    var topicPartitions = _.pluck(topicMetadata[p.topic], 'partition');
    p.partition = p.hasOwnProperty('partition') ? p.partition : self.partitioner.getPartition(topicPartitions, p.key);
    p.attributes = p.hasOwnProperty('attributes') ? p.attributes : 0;
    var messages = _.isArray(p.messages) ? p.messages : [p.messages];
    messages = messages.map(function (message) {
      if (message instanceof KeyedMessage) {
        return message;
      }
      return new Message(0, 0, '', message);
    });
    return new ProduceRequest(p.topic, p.partition, messages, p.attributes);
  });
};

Producer.prototype.createTopics = function (topics, async, cb) {
  if (!this.ready) {
    return cb(new Error('Producer not ready!'));
  }

  this.client.createTopics(topics, async, cb);
};

Producer.prototype.close = function (cb) {
  this.client.close(cb);
};

Producer.PARTITIONER_TYPES = PARTITIONER_TYPES;

module.exports = Producer;