ductile/lib/readable-search.js

Elasticsearch Bulk Loader

// Generated by CoffeeScript 1.11.1

/**
 * Expose an elasticsearch query that returns hits or docs as a stream of hits or docs.
 *
 * Expect the query to be a JSON object where the from property defines the offset
 * and the limit defines the page size.
 * Expect the client to return a parsed JSON.
 */


/**
 * @param queryExec an executable query functions that takes 2 arguments: the offset and a callback.
 */

(function() {
  var Readable, ReadableHits, identity;

  ReadableHits = function(queryExec, parseHit) {
    if (!(this instanceof ReadableHits)) {
      return new ReadableHits(queryExec);
    }
    Readable.call(this, {
      objectMode: true
    });
    this.queryExec = queryExec;
    this.total = -1;
    this.from = 0;
    this._next = true;
    this._hits = [];
    this._current = 0;
    this.parseHit = parseHit || identity;
  };

  identity = function(hit) {
    return hit;
  };

  'use strict';

  Readable = require('stream').Readable;

  module.exports = ReadableHits;

  ReadableHits.prototype = Object.create(Readable.prototype, {
    constructor: {
      value: ReadableHits
    }
  });

  ReadableHits.prototype._read = function() {
    this._current++;
    if (this._current >= this._hits.length) {
      if (!this._next) {
        return this.push(null);
      }
      this._fetchNextPage();
    } else {
      this._shift();
    }
  };

  ReadableHits.prototype._fetchNextPage = function() {
    var self;
    self = this;
    this.queryExec(this.from, function(e, resp) {
      self._current = 0;
      if (e) {
        self.hits = [];
        self._next = false;
        self.emit('error', e);
        return self.push(null);
      }
      self.total = resp.hits.total;
      self._hits = resp.hits.hits;
      self.from += self._hits.length;
      if (self.from >= self.total) {
        self._next = false;
      }
      if (!self._hits.length) {
        return self.push(null);
      }
      self._shift();
    });
  };

  ReadableHits.prototype._shift = function() {
    this.push(this.parseHit(this._hits[this._current]));
  };

  ReadableHits.prototype.destroy = function() {
    if (this.destroyed) {
      return;
    }
    this.destroyed = true;
    this._next = false;
    this.unpipe();
  };

}).call(this);

//# sourceMappingURL=readable-search.js.map