castor-load/lib/load.js

Traverse a directory to build a MongoDB collection with the found files. Then it's enable to keep directory and collection synchronised.

'use strict';
var path = require('path')
  , basename = path.basename(__filename, '.js')
  , debug = require('debug')('castor:load:' + basename)
  , assert = require('assert')
  , util = require('util')
  , fs = require('fs')
  , events = require('events')
  , crypto = require('crypto')
  , url = require('url')
  , minimatch = require('minimatch')
  , mkdirp = require('mkdirp')
  , chokidar = require('chokidar')
  , findit = require('findit')
  , Mount = require('./mount.js')
  , Sync = require('./sync.js')
  , File = require('./file.js')
;


function Filerake(dir, options) {

  assert.equal(typeof dir, 'string');
  assert.ok(fs.existsSync(dir), 'Directory should exist to use Filerake');
  options = options || {};

  if (!(this instanceof Filerake)) {
    return new Filerake(dir, options);
  }
  events.EventEmitter.call(this);
  var self = this;
  self.directory = path.normalize(dir);
  self.options = {};
  self.options.ignore = options.ignore || [];
  self.options.persistent = options.persistent || true;
  self.options.include = options.include;
  self.options.collectionName = options.collectionName || crypto.createHash('sha1').update(self.directory).digest('hex');
  self.options.connexionURI = options.connexionURI || process.env.MONGO_URL;
  self.options.concurrency = options.concurrency || 1;
  self.options.directory = self.directory; // for tingodb
  self.options.dateConfig = options.dateConfig || null;
  if (!Array.isArray(self.options.ignore)) {
     self.options.ignore = [];
  }
  if (!Array.isArray(self.options.include)) {
     self.options.include = ['file'];
  }
  self.basedoc = {
    dateConfig : self.options.dateConfig,
    dateLoaded : new Date(),
    basedir: self.directory
  };

  self.mount = new Mount(self.options);
  self.syncr = new Sync(self.mount, self.options);
}

util.inherits(Filerake, events.EventEmitter);


Filerake.prototype.use = function (pattern, fn)
{
  var self = this;
  self.mount.use(pattern, fn);
}

Filerake.prototype.ignore = function (file) {
  var self = this;
  var match = function (x) {
    return minimatch(file, x, { matchBase: true });
  };
  var check = function (prev, cur) {
    return (cur ? cur : prev)
  }
  return self.options.ignore.map(match).reduce(check, false);
}

Filerake.prototype.include = function (type) {
  var self = this;
  return self.options.include.indexOf(type) === -1 ? false : true;
}


Filerake.prototype.watch = function () {
  var self = this;

  var watcher = chokidar.watch(self.directory, {
      ignored: function(s) { return self.ignore(s); }
    , ignoreInitial : true
    , persistent: true
    }
  );
  debug('Watcher is started');
  watcher
  .on('add', function(path, stats) {
      var f = new File(path, stats, self.basedoc);
      self.syncr.check(f, function(err) {
          debug('checked - watch(add)', f.doc.filename);
          self.emit('added', err, f.get());
      });
  })
  .on('change', function(path, stats) {
      var f = new File(path, stats, self.basedoc);
      if (self.ignore(path)) {
        self.syncr.cancel(f, function(err) {
            debug('cancelled - watch(change)', f.doc.filename);
            self.emit('cancelled', err, f.get());
        });
      }
      self.syncr.check(f, function(err) {
          debug('checked - watch(change)', f.doc.filename);
          self.emit('changed', err, f.get());
      });
  })
  .on('unlink', function(path) {
      var f = new File(path, undefined, self.basedoc);
      self.syncr.drop(f, function(err) {
          debug('droped - watch(unlink)', f.doc.filename);
          self.emit('dropped', err, f.get());
      });
  })
  .on('error', function(error) {
      debug('Error happened', error);
  })

 }


Filerake.prototype.sync = function (done)
{
  var self = this, i = 0, o = 0;
  function conclude(o) {
    done(o);
    process.nextTick(function() {
      self.emit('watching');
      self.watch();
    });
  }
  var findr = findit(self.directory);
  findr.on('directory', function (dir, stats, stop) {
      var f = new File(dir, stats, self.basedoc);
      if (self.ignore(dir)) {
        stop();
        i++;
        self.syncr.cancel(f, function(err) {
            o++;
            debug('cancelled - sync(directory)', f.doc.filename);
            self.emit('cancelled', err, f);
            if (i === o) {
              conclude(o);
            }
        });
      }
      else {
        if (self.include('directory')) {
          i++;
          self.syncr.check(f, function(err) {
              o++;
              debug('checked - sync(directory)', f.doc.filename);
              self.emit('checked', err, f);
              if (i === o) {
                conclude(o);
              }
          });
        }
      }
    }
  );
  findr.on('file', function (file, stats) {
      var f = new File(file, stats, self.basedoc);
      if (self.ignore(file)) {
        i++;
        self.syncr.cancel(f, function(err) {
            o++;
            debug('cancelled - sync(file)', f.doc.filename);
            self.emit('cancelled', err, f);
            if (i === o) {
              conclude(o);
            }
        });
      }
      else {
        if (self.include('file')) {
          i++;
          self.syncr.check(f, function(err) {
              o++;
              debug('checked - sync(file)', f.doc.filename);
              self.emit('checked', err, f);
              if (i === o) {
                conclude(o);
              }
          });
        }
      }
    }
  );
}


module.exports = Filerake;