ductile/lib/writer.js

Elasticsearch Bulk Loader

// Generated by CoffeeScript 1.11.1
(function() {
  var CLEAN_PIPE, OPERS, WritableBulk, byline, fromJson, isNonStandard, isTwoRow, mixin, through2, toDoc, transform,
    slice = [].slice;

  WritableBulk = require('./writable-bulk');

  through2 = require('through2');

  byline = require('byline');

  mixin = require('./mixin');

  isTwoRow = function(t) {
    if (typeof t === 'string') {
      return t === 'index' || t === 'update' || t === 'create';
    } else {
      return t && (t.index || t.update || t.create);
    }
  };

  OPERS = require('./opers');

  isNonStandard = function(bulk) {
    var b, i, len;
    for (i = 0, len = bulk.length; i < len; i++) {
      b = bulk[i];
      if (b.alias || b.mapping || b.settings || b.template) {
        return true;
      }
    }
    return false;
  };

  CLEAN_PIPE = {
    clean: true
  };

  fromJson = function(emit) {
    var find123, saved;
    saved = null;
    find123 = false;
    return through2.obj(function(line, enc, callback) {
      var ex, json, message;
      if (find123) {
        if (line[0] !== 123) {
          return callback();
        } else {
          find123 = false;
        }
      }
      json = (function() {
        try {
          return JSON.parse(line);
        } catch (error) {
          ex = error;
          if (saved) {
            message = ex.message;
            emit('info', "Skipping record, JSON parse failed (" + message + ") on line after:\n" + (JSON.stringify(saved)));
            saved = null;
            find123 = true;
            return null;
          } else {
            throw ex;
          }
        }
      })();
      if (json === null) {
        this.push(CLEAN_PIPE);
      } else if (saved !== null) {
        this.push(saved);
        this.push(json);
        saved = null;
      } else if (isTwoRow(json)) {
        saved = json;
      } else if (json) {
        this.push(json);
      }
      return callback();
    });
  };

  toDoc = function() {
    var d, saved;
    d = function(oper, source) {
      var head, opername;
      opername = OPERS.find(function(o) {
        return oper[o];
      });
      head = oper[opername];
      return mixin(head, {
        _oper: opername,
        _source: source
      });
    };
    saved = null;
    return through2.obj(function(row, enc, callback) {
      if (row === CLEAN_PIPE) {
        saved = null;
      } else if (saved) {
        this.push(d(saved, row));
        saved = null;
      } else {
        if (isTwoRow(row)) {
          saved = row;
        } else {
          this.push(d(row));
        }
      }
      return callback();
    });
  };

  transform = function(operdelete, trans, index, type) {
    return through2.obj(function(row, enc, callback) {
      var oper, t;
      if (t = trans(row)) {
        if (operdelete) {
          t._oper = 'delete';
        }
        oper = {};
        if (t._oper === 'alias') {
          oper.alias = {
            _name: t._name,
            _index: t._index
          };
        } else if (t._oper === 'mapping') {
          oper.mapping = {
            _index: index != null ? index : t._index,
            _type: type != null ? type : t._type,
            _mapping: t._mapping
          };
        } else if (t._oper === 'settings') {
          oper.settings = {
            _index: index != null ? index : t._index,
            _settings: t._settings
          };
        } else if (t._oper === 'template') {
          oper.template = {
            _name: t._name,
            _template: t._template
          };
        } else {
          oper[t._oper] = {
            _id: t._id,
            _index: index != null ? index : t._index,
            _type: type != null ? type : t._type
          };
        }
        this.push(oper);
        if (isTwoRow(t._oper)) {
          this.push(t._source);
        }
      }
      return callback();
    });
  };

  module.exports = function(client, _opts, operdelete, trans, instream) {
    var bulkExec, count, fromJsonS, stream, writeAlias, writeBulk, writeMapping, writeSettings, writeTemplate;
    writeAlias = require('./write-alias')(client);
    writeMapping = require('./write-mapping')(client);
    writeSettings = require('./write-settings')(client);
    writeTemplate = require('./write-template')(client);
    writeBulk = require('./write-bulk')(client, _opts);
    bulkExec = function(bulk, callback) {
      var a, b, i, item, len, m, s, t;
      if (isNonStandard(bulk)) {
        a = [];
        m = [];
        s = [];
        t = [];
        b = [];
        for (i = 0, len = bulk.length; i < len; i++) {
          item = bulk[i];
          if (item.alias) {
            a.push(item);
          } else if (item.mapping) {
            m.push(item);
          } else if (item.settings) {
            s.push(item);
          } else if (item.template) {
            t.push(item);
          } else {
            b.push(item);
          }
        }
        if (a.length) {
          writeAlias(a, callback);
        }
        if (m.length) {
          writeMapping(m, callback);
        }
        if (s.length) {
          writeSettings(s, callback);
        }
        if (t.length) {
          writeTemplate(t, callback);
        }
        if (b.length) {
          return writeBulk(b, callback);
        }
      } else {
        return writeBulk(bulk, callback);
      }
    };
    count = 0;
    fromJsonS = fromJson(function() {
      var as;
      as = 1 <= arguments.length ? slice.call(arguments, 0) : [];
      return stream.emit.apply(stream, as);
    });
    stream = instream.pipe(byline.createStream()).pipe(fromJsonS).pipe(toDoc()).pipe(through2.obj(function(doc, enc, callback) {
      count++;
      this.push(doc);
      return callback();
    })).pipe(transform(operdelete, trans, _opts.index, _opts.type)).pipe(through2.obj(function(doc, enc, callback) {
      this.push(doc);
      return callback();
    })).pipe(new WritableBulk(function(bulk, callback) {
      stream.emit('progress', {
        count: count
      });
      return bulkExec(bulk, callback);
    }));
    return stream;
  };

}).call(this);

//# sourceMappingURL=writer.js.map