ductile/lib/writable-bulk.js

Elasticsearch Bulk Loader

// Generated by CoffeeScript 1.11.1

/**
 * Expose a writeable stream and execute it as a set of bulk requests.
 */


/**
 * @param bulkExec closure invoked with the bulk cmds as an array and a callback
 * @param highWaterMark number of bulk commands executed at once. 128 by default.
 */

(function() {
  var Writable, WritableBulk;

  WritableBulk = function(bulkExec, highWaterMark) {
    if (!(this instanceof WritableBulk)) {
      return new WritableBulk(bulkExec, highWaterMark);
    }
    Writable.call(this, {
      objectMode: true
    });
    this.bulkExec = bulkExec;
    this.highWaterMark = highWaterMark || 128;
    this.bulk = [];
    this.bulkCount = 0;
    this.expectingPayload = false;
    this.on('finish', (function() {
      this._flushBulk((function() {
        this.emit('close');
      }).bind(this));
    }).bind(this));
  };

  'use strict';

  Writable = require('stream').Writable;

  module.exports = WritableBulk;

  WritableBulk.prototype = Object.create(Writable.prototype, {
    constructor: {
      value: WritableBulk
    }
  });


  /**
   * @param chunk a piece of a bulk request as json.
   */

  WritableBulk.prototype._write = function(chunk, enc, next) {
    var i, willExpectPayload;
    if (this.expectingPayload) {
      this.bulkCount++;
      this.expectingPayload = false;
    } else {
      willExpectPayload = ['index', 'create', 'update'];
      i = 0;
      while (i < willExpectPayload.length) {
        if (chunk.hasOwnProperty(willExpectPayload[i])) {
          this.expectingPayload = willExpectPayload[i];
          break;
        }
        i++;
      }
      if (!this.expectingPayload) {
        if (!chunk.hasOwnProperty('delete') && !chunk.hasOwnProperty('alias') && !chunk.hasOwnProperty('mapping') && !chunk.hasOwnProperty('settings') && !chunk.hasOwnProperty('template')) {
          this.emit('error', new Error('Unexpected chunk, not an ' + 'index/create/update/delete/alias/mapping/settings/template command and ' + 'not a document to index either'));
          return next();
        }
        this.bulkCount++;
      }
    }
    this.bulk.push(chunk);
    if (this.highWaterMark <= this.bulkCount) {
      return this._flushBulk(next);
    }
    next();
  };

  WritableBulk.prototype._flushBulk = function(callback) {
    var self;
    if (!this.bulkCount) {
      return setImmediate(callback);
    }
    self = this;
    this.bulkExec(this.bulk, function(e, resp) {
      var bulkItemResp, i, key;
      if (e) {
        self.emit('error', e);
      }
      if (resp.errors && resp.items) {
        i = 0;
        while (i < resp.items.length) {
          bulkItemResp = resp.items[i];
          key = Object.keys(bulkItemResp)[0];
          if (bulkItemResp[key].error) {
            self.emit('error', new Error(bulkItemResp[key].error));
          }
          i++;
        }
      }
      self.bulk = [];
      self.bulkCount = 0;
      self.expectingPayload = false;
      callback();
    });
  };

}).call(this);

//# sourceMappingURL=writable-bulk.js.map