castor-load/lib/load.js

Traverse a directory to build a MongoDB collection with the found files. Then it's enable to keep directory and collection synchronised.

/*jshint node:true,laxcomma:true*/
'use strict';
var path = require('path')
  , basename = path.basename(__filename, '.js')
  , debug = require('debug')('castor:load:' + basename)
  , assert = require('assert')
  , util = require('util')
  , fs = require('fs')
  , events = require('events')
  , crypto = require('crypto')
  , url = require('url')
  , minimatch = require('minimatch')
  , mkdirp = require('mkdirp')
  , chokidar = require('chokidar')
  , findit = require('findit')
  , Mount = require('./mount.js')
  , Sync = require('./sync.js')
  , File = require('./file.js')
  , readline = require('readline')
;


function Filerake(dir, options) {

  assert.equal(typeof dir, 'string');
  assert.ok(fs.existsSync(dir), 'Directory should exist to use Filerake');
  options = options || {};

  if (!(this instanceof Filerake)) {
    return new Filerake(dir, options);
  }
  events.EventEmitter.call(this);
  var self = this;
  self.directory = path.normalize(dir);
  self.options = {};
  self.options.watch = options.hasOwnProperty('watch') ? options.watch : true;
  self.options.strictCompare = options.strictCompare || false;
  self.options.ignore = options.ignore || [];
  self.options.include = options.include;
  self.options.collectionName = options.collectionName || crypto.createHash('sha1').update(self.directory).digest('hex');
  self.options.connexionURI = options.connexionURI || process.env.MONGO_URL;
  self.options.concurrency = options.concurrency || 1;
  self.options.delay = options.delay || undefined;
  self.options.modifier = options.modifier || undefined;
  self.options.writeConcern = options.writeConcern || undefined;
  self.options.maxFileSize = options.maxFileSize || undefined;
  self.options.directory = self.directory; // for tingodb
  self.options.dateConfig = options.dateConfig || null;
  if (!Array.isArray(self.options.ignore)) {
     self.options.ignore = [];
  }
  if (!Array.isArray(self.options.include)) {
     self.options.include = ['file'];
  }
  self.basedoc = {
    dateConfig : self.options.dateConfig,
    dateLoaded : new Date(),
    basedir: self.directory
  };

  self.mount = new Mount(self.options);
  self.syncr = new Sync(self.mount, self.options);

  function onSaved(doc) {
    self.emit('saved',doc);
  }
  self.syncr.on('saved', onSaved);
  self.mount.on('loadError', function (err, location, number) {
    self.emit('loadError', err, location, number);
  });
}

util.inherits(Filerake, events.EventEmitter);


Filerake.prototype.use = function (pattern, fn)
{
  var self = this;
  self.mount.use(pattern, fn);
};

Filerake.prototype.ignore = function (file) {

  var ignored = this.options.ignore.some(function (check) {
    if (check instanceof RegExp) {
      return check.test(file);
    } else {
      return minimatch(file, check, { matchBase: true });
    }
  });

  if (ignored) { debug('ignore', file); }

  return ignored;
};

Filerake.prototype.include = function (type) {
  var self = this;
  return self.options.include.indexOf(type) === -1 ? false : true;
};


Filerake.prototype.watch = function (callback) {
  var self = this;

  var watcher = chokidar.watch(self.directory, {
      ignored: function(s) { return self.ignore(s); }
    , ignoreInitial : true
    , persistent: true
    }
  );
  debug('Watcher is started');
  watcher
  .on('ready', function () {
    if (typeof callback === 'function') { callback(); }
  })
  .on('add', function(path, stats) {
    var f = new File(path, stats, self.basedoc, self.options.modifier);
    if (self.ignore(path)) {
      self.emit('preCancel', f.get());
      return self.syncr.cancel(f, function(err) {
        debug('cancelled - watch(add)', f.doc.filename);
        self.emit('cancelled', err, f.get());
      });
    }
    self.emit('preCheck', f.get());
    self.syncr.check(f, function(err) {
      debug('checked - watch(add)', f.doc.filename);
      self.emit('added', err, f.get());
    });
  })
  .on('change', function(path, stats) {
    var f = new File(path, stats, self.basedoc, self.options.modifier);
    if (self.ignore(path)) {
      self.emit('preCancel', f.get());
      return self.syncr.cancel(f, function(err) {
        debug('cancelled - watch(change)', f.doc.filename);
        self.emit('cancelled', err, f.get());
      });
    }
    self.emit('preCheck', f.get());
    self.syncr.check(f, function(err) {
      debug('checked - watch(change)', f.doc.filename);
      self.emit('changed', err, f.get());
    });
  })
  .on('unlink', function(path) {
    var f = new File(path, undefined, self.basedoc, self.options.modifier);
    self.emit('preDrop', f.get());
    self.syncr.drop(f, function(err) {
        debug('droped - watch(unlink)', f.doc.filename);
        self.emit('dropped', err, f.get());
    });
  })
  .on('error', function(error) {
      debug('Error happened', error);
  });
};


Filerake.prototype.sync = function (done)
{
  var self = this, i = 0, o = 0, browsing = true;
  self.syncr.refresh();

  function conclude(o) {
    if (browsing) { return; }

    self.syncr.clean(function (err) {
      debug('clean', err);
      if (typeof done === 'function') { done(o); }
      process.nextTick(function() {
        if (self.options.watch) {
          self.watch(function () { self.emit('watching'); });
        }
      });
    });
  }
  var findr = findit(self.directory);
  findr.on('end', function () {
    browsing = false;
    self.emit('browseOver', i);
    if (i === o) { conclude(o); }
  });
  findr.on('directory', function (dir, stats, stop) {
      var f = new File(dir, stats, self.basedoc, self.options.modifier);
      if (self.ignore(dir)) {
        stop();
        i++;
        self.emit('preCancel', f.get());
        self.syncr.cancel(f, function(err) {
            o++;
            debug('cancelled - sync(directory)', f.doc.filename);
            self.emit('cancelled', err, f.get());
            if (i === o) {
              conclude(o);
            }
        });
      }
      else {
        if (self.include('directory')) {
          i++;
          self.emit('preCheck', f.get());
          self.syncr.check(f, function(err) {
              o++;
              debug('checked - sync(directory)', f.doc.filename);
              self.emit('checked', err, f.get());
              if (i === o) {
                conclude(o);
              }
          });
        }
      }
    }
  );
  findr.on('file', function (file, stats) {
      var f = new File(file, stats, self.basedoc, self.options.modifier);
      if (self.ignore(file)) {
        i++;
        self.emit('preCancel', f.get());
        self.syncr.cancel(f, function(err) {
            o++;
            debug('cancelled - sync(file)', f.doc.filename);
            self.emit('cancelled', err, f.get());
            if (i === o) {
              conclude(o);
            }
        });
      }
      else {
        if (self.include('file')) {
          i++;
          self.emit('preCheck', f.get());
          self.syncr.check(f, function(err) {
              o++;
              debug('checked - sync(file)', f.doc.filename);
              self.emit('checked', err, f.get());
              if (i === o) {
                conclude(o);
              }
          });
        }
      }
    }
  );
};

Filerake.prototype.push = function (file, stats, basedoc, modifier)
{
  var self = this;
  var f = new File(file, stats, basedoc || self.basedoc, modifier || self.options.modifier);
  if (self.ignore(file)) {
    self.emit('preCancel', f.get());
    self.syncr.cancel(f, function(err) {
      debug('cancelled - push(file)', f.doc.filename);
      self.emit('cancelled', err, f.get());
    });
  }
  else {
    if (self.include('file')) {
      self.emit('preCheck', f.get());
      self.syncr.check(f, function(err) {
        debug('checked - push(file)', f.doc.filename);
        self.emit('checked', err, f.get());
      });
    }
  }
};


Filerake.prototype.submit = function (file, done)
{
  var ret = false;
  var self = this;
  var f = new File(file, {}, self.basedoc, self.options.modifier);
  if (self.ignore(file)) {
    self.emit('preCancel', f.get());
    self.syncr.cancel(f, function(err) {
      debug('cancelled - submit(file)', f.doc.filename);
      self.emit('cancelled', err, f.get());
      if (typeof done === 'function') {
        done(new Error('Cancelled'), f.get()); 
      }
    });
  }
  else {
    if (self.include('file')) {
      self.emit('preCheck', f.get());
      self.syncr.check(f, function(err) {
        debug('checked - submit(file)', f.doc.filename);
        self.emit('checked', err, f.get());
        if (typeof done === 'function') {
          done(null, f.get()); 
        }
      });
      ret = true;
    }
  }
  return ret;
};




module.exports = Filerake;