castor-load/lib/sync.js

Traverse a directory to build a MongoDB collection with the found files. Then it's enable to keep directory and collection synchronised.

'use strict';

var path = require('path')
  , basename = path.basename(__filename, '.js')
  , debug = require('debug')('castor:load:' + basename)
  , util = require('util')
  , assert = require('assert')
  , async = require('async')
  , extend = require('extend')
  , File = require('./file.js')
  , mkdirp = require('mkdirp')
  , parseBytes = require('bytes')
  ;





function Sync(mountor, options)
{
  if (!(this instanceof Sync)) {
    return new Sync(mountor, options);
  }
  var self = this;
  self.options = {};
  self.options.collectionName = options.collectionName || 'filerake';
  self.options.connexionURI = options.connexionURI || 'mongodb://localhost:27017/test/';
  self.options.connexion = require('url').parse(self.options.connexionURI);
  self.options.concurrency = options.concurrency || 1;
  self.options.maxFileSize = options.maxFileSize || '128mb';
  self.options.delay = options.delay || 1000;
  self.options.synckey = options.synckey || null;
  self.dateSynchronised = new Date();
  self.mountor = mountor;
  self.checker = [];

  function worker(file, done)  {
    self.compare(file, function(err) {
      if (err) {
        debug('error', file.doc.filename, err);
      }
      done(err);
    });
  }
  self.queue = async.queue(worker, self.options.concurrency);
  self.queue.drain = function () {
    debug('queue drain', self.dateSynchronised);
  }
}

Sync.prototype.connect = function (fn) {
  var self = this;

  if (self.coll) {
    return fn(null, self.coll, self.db);
  }

  var func = function (err, db) {
    assert.ifError(err, 'connect failed');
    assert.notEqual(db, null, 'connect failed');
    db.collection(self.options.collectionName, function(err, coll) {
      self.db = db;
      self.coll = coll;
      self.coll.ensureIndex({ fid: 1 }, {unique: true, dropDups:true, w: 1 }, function(err, indexName) {
        fn(err, self.coll, self.db);
      })
    });
  }

  if (self.options.connexion.protocol === 'mongodb:') {
    require('mongodb').MongoClient.connect(self.options.connexionURI, func);
  }
  else if (self.options.connexion.protocol === 'tingodb:') {
    var colldir = path.join(self.options.directory, self.options.connexion.path, 'filerake');
    debug('Tingodb\'s path', colldir);

    mkdirp(colldir, function (err) {
      assert.ifError(err);
      var tingodb = require('tingodb')();
      var handle = new tingodb.Db(colldir, {});
      handle.open(func);
    }
  )
}
else {
  throw new Error('Unsupported engine : ' + self.options.connexion.protocol)
}

}

Sync.prototype.check = function (file, next)
{
  assert(file instanceof File);
  var self = this;

  if (self.options.maxFileSize && self.options.maxFileSize < file.doc.filesize) {
    debug('reject', file.doc.filename, file.doc.fid);
    next(new Error('File exceeds the maximum size.'));
  }
  else if (self.checker.indexOf(file.doc.fid) === -1) {
    if (self.queue.length() >= self.options.concurrency) {
      debug('delayed', file.doc.filename, file.doc.fid);
      setTimeout(function() { self.check(file, next);}, self.options.delay);
    }
    else {
      debug('check', file.doc.filename, file.doc.fid);
      self.checker.push(file.doc.fid);
      self.queue.push(file, function(err) {
        self.checker = self.checker.filter(function(i) { return (i !== file.doc.fid) });
        next(err);
      });
    }
  }
  else {
    debug('in progress', file.doc.filename, file.doc.fid);
    // next()
  }
}

Sync.prototype.drop = function (file, next)
{
  assert(file instanceof File);
  var self = this;
  var doc =  file.get();
  var selector = {fid: doc.fid};
  var sortor = [['state', 1]];

  self.connect(function(err, collection) {
    if (err) {
      return next(err);
    }
    collection.findAndModify(selector, sortor, { $set : { state : 'deleted' } }, function (err, doc) {
      if (err) {
        return next(err);
      }
      if (doc) {
        debug('delete', doc.filename);
      }
      next();
    });
  });
}

Sync.prototype.cancel = function (file, next)
{
  assert(file instanceof File);
  var self = this;
  var doc =  file.get();
  var selector = {fid: doc.fid};
  self.connect(function(err, collection) {
    if (err) {
      return next(err);
    }
    collection.findOne(selector, function (err, res) {
      if (err) {
        return next(err);
      }
      if (res) {
        debug('cancel', doc.filename);
        collection.remove(doc, {w: 0});
      }
      next();
    }
  );
}
);
}

Sync.prototype.clean = function (condition, next)
{
  var self = this;
  var selector = {
    $and : [
      { dateSynchronised : { $ne: self.dateSynchronised } },
      condition
    ]
  };
  var sortor = [['state', 1]];
  self.connect(function(err, collection) {
    if (err) {
      return next(err);
    }
    collection.findAndModify(selector, sortor, { $set : { state : 'deleted', dateSynchronised : self.dateSynchronised } }, function (err, doc) {
      if (err) {
        return next(err);
      }
      if (doc) {
        debug('clean', doc.filename);
      }
      next();
    });
  }
);
}

Sync.prototype.update = function (fid, doc, next)
{
  var self = this;
  var selector = {
    fid : doc.fid
  };
  self.connect(function(err, collection) {
    if (err) {
      return next(err);
    }
    collection.update(selector, doc, next);
  });
}

Sync.prototype.insert = function (doc, next)
{
  var self = this;
  var selector = {
    sha1 : doc.sha1,
    dateModified: doc.dateModified,
    state: 'deleted'
  };
  self.connect(function(err, collection) {
    if (err) {
      return next(err);
    }
    collection.findOne(selector, {limit: 1}, function (err, dta) {
      if (!err && dta) {
        self.rename(dta.fid, next);
      }
      else {
        doc.state = 'inserted';
        doc.dateSynchronised = self.dateSynchronised;
        collection.insert(doc, next);
      }
    });
  });
}

Sync.prototype.updateAll = function (docs, next)
{
  var self = this;
  async.eachSerie(docs, function(doc, callback) {
    doc.state = 'updated';
    doc.dateSynchronised = self.dateSynchronised;
    self.update(doc.fid, doc, callback);
  },
  function (err) {
    self.clean({ fid: docs[0].fid }, next);
  }
);
}


Sync.prototype.insertAll = function (docs, next)
{
  var self = this;
  var selector = {
    sha1 : docs[0].sha1,
    dateModified: docs[0].dateModified,
    state: 'deleted'
  };
  self.connect(function(err, collection) {
    if (err) {
      return next(err);
    }
    collection.findOne(selector, {limit: 1}, function (err, dta) {
      if (!err && dta) {
        self.rename(dta.fid, next);
      }
      else {
        async.mapLimit(docs.map(function(i) {
          i.state = 'inserted';
          i.dateSynchronised = self.dateSynchronised;
          return i;
        })
      , self.options.concurrency
      , function (item, callback) {
          debug('insert item', item.filename);
          collection.insert(item, callback);
        }
      , function(err, results) {
          next();
          // results is now an array of stats for each file
        });
      }
    });
  });
}


Sync.prototype.rename = function (fid, next)
{
  var self = this;
  var selector = {
    fid : fid
  };
  var doc = {
    state : 'renamed',
    dateSynchronised : self.dateSynchronised
  }
  self.connect(function(err, collection) {
    if (err) {
      return next(err);
    }
    collection.update(selector, doc, next);
  }
);
}



Sync.prototype.compare = function (file, next)
{
  assert(file instanceof File);
  var self = this;
  var doc = file.get();
  var selector = {fid: doc.fid};

  self.connect(function(err, collection) {
    collection.findOne(selector, {limit: 1}, function (err, dta) {
      if (err) {
        return next(err);
      }
      if (!dta) {
        dta = {};
      }
      self.compareFileAndDoc(file, dta, next);
    });
  });
}

Sync.prototype.compareFileAndDoc = function (file, d, next)
{
  assert(file instanceof File);
  var doc = file.get();
  var self = this;

  if ( (
    d.dateModified && doc.dateModified && d.dateModified.getTime() === doc.dateModified.getTime()
  )
  &&
  (
    ! doc.dateConfig || (
      d.dateConfig && doc.dateConfig && d.dateConfig.getTime() === doc.dateConfig.getTime()
    )
  ) ) {
      debug('unchange', doc.fid);
    next();
  }
  else {
    self.mountor.append(file, function worker(dta, cb) {
      self.compareDocAndDoc(d, dta, cb);
    }, next);
  }
}

Sync.prototype.compareDocAndDoc = function (logical, physical, next)
{
  var self = this
    , physicalSHA1
    , logicalSHA1 = logical.sha1 ? logical.sha1 : false;

  if (Array.isArray(physical)) {
    physicalSHA1 = physical[0].sha1 ? physical[0].sha1 : false;
  }
  else {
    physicalSHA1 = physical.sha1 ? physical.sha1 : false;
  }
  if (logicalSHA1) {
    var state = logicalSHA1 === physicalSHA1 ? 'unmodified' : 'updated';
    debug(state, physical.filename);
    if (Array.isArray(physical)) {
      self.updateAll(logical.fid, physical, next);
    }
    else {
      var newdoc = {};
      extend(true, newdoc, logical, physical);
      newdoc.dateSynchronised = self.dateSynchronised;
      newdoc.state = state;
      self.update(logical.fid, newdoc, next);
    }
  }
  else {
    if (Array.isArray(physical)) {
      debug('inserted', physical[0].filename, '(Exploded)');
      self.insertAll(physical, next);
    }
    else {
      debug('inserted', physical.filename);
      self.insert(physical, next);
    }
  }
}


module.exports = Sync;