castor-load/lib/mount.js

Traverse a directory to build a MongoDB collection with the found files. Then it's enable to keep directory and collection synchronised.

/*jshint node:true, laxcomma:true*/
'use strict';
var path = require('path')
  , basename = path.basename(__filename, '.js')
  , debug = require('debug')('castor:load:' + basename)
  , util      = require('util')
  , crypto    = require('crypto')
  , assert    = require('assert')
  , async     = require('async')
  , minimatch = require('minimatch')
  , clone     = require('clone')
  , File      = require('./file.js')
  , EventEmitter = require('events').EventEmitter
  ;





function Mount(options)
{
  if (!(this instanceof Mount)) {
    return new Mount(options);
  }
  var self = this;
  options = options || {};
  self.options = {};
  self.options.concurrency = options.concurrency || 1;
  self.options.delay = options.delay || 1000;
  self.options.match = options.match || { matchBase: true };
  self.handlers = [];
  var worker = function (doc, done) {
    var h = [function(callback){ callback(null, doc); }];
    async.waterfall(h.concat(self.handlers), function(err, d) {
      if (err === null && d === null) {
        debug('The waterfall is deliberately broken');
      }
      done(err, d !== undefined ? d : {});
    });
  };

  self.queue = async.queue(worker, self.options.concurrency);
}

util.inherits(Mount, EventEmitter);

Mount.prototype.use = function (pattern, fn) {

  var self = this;

  if (fn === undefined) {
    fn = pattern;
    pattern = '*';
  }
  assert.equal(typeof pattern, 'string');
  assert.equal(typeof fn, 'function');

  var fnid = crypto.createHash('md5').update(fn.toString().replace(/\s*/g, '')).digest("hex");

  var mark = function(o) {
    if (o.mountBy === undefined) {
      o.mountBy = [fnid];
    }
    o.mountBy.push(fnid);
    return o;
  };

  self.handlers.push(function(iDoc, next) {
     if (!iDoc) {
       return next();
    }
    if (Array.isArray(iDoc)) {
      return next(null, iDoc);
    }
    if (iDoc.mountBy && iDoc.mountBy.indexOf(fnid) >= 0) {
      return next(null, iDoc);
    }
    if (minimatch(iDoc.filename, pattern, self.options.match)) {
      mark(iDoc);
      var oDoc = clone(iDoc, false)
        , numb = 0
        , nump = 0
        , wrk = function (arg1, callback) {
            self.appendDoc(arg1, arg1.memorizeWorkerForSubdoc, callback);
          }
        , last = false
        , qe = async.queue(wrk, self.options.concurrency)
        , resolve = null
        ;
      debug('apply ' + pattern, fnid, iDoc.filename.concat('#').concat(iDoc.number || '0'));
      fn(oDoc,
        function submit(arg1, arg2) {
          numb++;
          if (arg1 instanceof Error) {
            self.emit('loadError', arg1, iDoc.location, iDoc.number);
            if (numb === 1) {
              next(arg1);
            }
            else {
              debug('fail ' + pattern, fnid, iDoc.filename.concat('#').concat(iDoc.number || '0'), arg1);
            }
          }
          else if (!arg1 && arg2) {
            if (numb === 1) {
              next(null, arg2);
            }
          }
          else if (!arg1 && !arg2) {
            // last call
            last = true;
            if (qe.idle() && typeof resolve === 'function') { resolve(); }
            return;
          }
          else if (typeof arg1 === 'object') {
            if (numb === 1) {
              var se = new Error('File Exploded');
              se.wait = function(cbd) {
                debug('apply ' + pattern, fnid, iDoc.filename.concat('#').concat(iDoc.number || '0'));
                resolve = cbd;
                qe.drain = function() {
                  if (last) {
                    cbd();
                    // debug('finalize', fnid, numb, nump, qe.length());
                  }
                  // else {
                    // debug('partial', fnid, numb, nump, qe.length());
                  // }
                };
              };
              next(se);
            }

            arg1.number = numb;

            Object.defineProperty(arg1, '_exploded', {
              value: true,
              enumerable: false
            });

            // sub Document
            // debug('explode', arg1.fid, arg1.filename);
            qe.push(arg1, function(e) {
              nump++;
              // debug('terminated', fnid, arg1.filename);
              if (typeof arg2 === 'function') {
                arg2(e);
              }
            });
            return qe;
          }
          else {
            if (numb === 1) {
              next(null, oDoc);
            }
          }
        },
        self.options);
      }
      else {
        next(null, iDoc);
      }
    });
};

Mount.prototype.append = function (file, worker, done) {

  assert(file instanceof File);
  var self = this;
  file.analyze(function(err, doc) {
    if (err) {
      return done(err);
    }
    self.appendDoc(doc, worker, function(err) {
      if (err && err.wait) {
        err.wait(done);
      }
      else {
        done(err);
      }
    });
  });
};

Mount.prototype.appendDoc = function (doc, worker, done) {
  var self = this;
  doc.memorizeWorkerForSubdoc = worker;
  self.queue.push(doc, function(err, dta) {
    if (err) {
      return done(err);
    }
    if (Array.isArray(dta)) {
      async.mapLimit(dta, self.options.concurrency, function iterator(item, callback) {
        self.appendIDoc(item, function(err, idoc) {
          worker(idoc, callback);
        });
      }, function(err, res) {
        done(err);
      });
    }
    else if (!dta) {
      done(new Error('The document was drained'));
    }
    else {
      worker(dta, function(err) {
        return done(err);
      });
    }
  });
};

Mount.prototype.appendIDoc = function (doc, next) {
  var self = this;
  self.queue.push(doc, function(err, dta) {
    if (err) {
      return next(err, dta);
    }
    else {
      next(null, dta);
    }
  });
};


module.exports = Mount;