castor-load
Version:
Traverse a directory to build a MongoDB collection with the found files. Then it's enable to keep directory and collection synchronised.
651 lines (591 loc) • 23.9 kB
JavaScript
/* global describe, it, before */
;
var path = require('path')
, basename = path.basename(__filename, '.js')
, debug = require('debug')('filerake:' + basename)
, util = require('util')
, extend = require('extend')
, clone = require('clone')
, assert = require('assert')
, fs = require('fs')
, MongoClient = require('mongodb').MongoClient
, Filerake = require('../');
var shareopt = {
// "connexionURI": "tingodb://localhost/.tingodb",
"connexionURI" : "mongodb://localhost:27017/test",
"ignore" : [ "**/.*", "*~", "*.sw+(o|p)", "*.old", "*.bak", "**/node_modules"]
};
/**
* Basic middleware that create one subdocument for each line
*/
var oneDocPerLine = function(input, submit) {
fs.readFile(input.location, function (err, data) {
if (err) { return submit(err); }
data.toString().trim().split('\n').forEach(function (line) {
var d = { line : line };
extend(true, d, input);
submit(d);
});
submit();
});
};
describe('Filerake', function () {
// drop test database before doing anything
before(function (done) {
MongoClient.connect(shareopt.connexionURI, function (err, db) {
assert.ifError(err);
db.dropDatabase(function (err) {
assert.ifError(err);
done();
});
});
});
it('01 - should correctly ignore a file matching a glob', function (done) {
var options = {};
extend(true, options, shareopt);
var fina = path.resolve(__dirname, '../dataset/1/');
options.ignore.push("*.ignore");
options.collectionName = 'test01';
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 1);
var doc = docs[0];
assert.equal(docs[0].basename, 'to.load');
col.drop(function(err) {
assert.ifError(err);
done();
});
}
);
}
);
}
);
}
);
it('01b - should correctly ignore a file matching a regexp', function (done) {
var options = {};
extend(true, options, shareopt);
var fina = path.resolve(__dirname, '../dataset/1/');
options.ignore.push(/\.ignore$/);
options.collectionName = 'test01';
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 1);
var doc = docs[0];
assert.equal(docs[0].basename, 'to.load');
col.drop(function(err) {
assert.ifError(err);
done();
});
}
);
}
);
}
);
}
);
it('02 - should return one more file', function (done) {
var options = {};
extend(true, options, shareopt);
options.collectionName = 'test02';
options.concurrency = 1000;
var fina = path.resolve(__dirname, '../dataset/2/');
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 7);
done();
}
);
}
);
}
);
}
);
it('03 - should return one file & one dir', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/3/');
extend(true, options, shareopt);
options.collectionName = 'test03';
options.concurrency = 1000;
options.include = [ 'file', 'directory'];
options.ignore.push("*__metadata.yml");
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
var doc = docs[0];
assert(docs.some(function(c) { return c.basename === 'directory1' && c.extension === ''; }));
assert(docs.some(function(c) { return c.basename === 'file1.txt' && c.extension === 'txt'; }));
assert(docs.some(function(c) { return c.basename === 'file2.txt' && c.extension === 'txt'; }));
done();
}
);
}
);
}
);
}
);
it('04 - should return n lines for one file', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/4');
extend(true, options, shareopt);
options.collectionName = 'test04';
var fr = new Filerake(fina, options);
fr.use('*', function(input, submit) {
fs.readFile(input.location, function (err, data) {
if (err) {
return submit(err);
}
data.toString().trim().split("\n").map(function(line) {
var d = { line : line };
extend(true, d, input);
submit(d);
return d;
});
submit(null);
});
});
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
fs.readFile(fina + '/3lines.text', function (err, data) {
assert.ifError(err);
var arr = data.toString().trim().split("\n");
assert.equal(arr.length, docs.length);
assert.equal(docs.length, 3);
assert(docs.some(function(c) { return c.line === 'Line #1'; }));
assert(docs.some(function(c) { return c.line === 'Line #2'; }));
assert(docs.some(function(c) { return c.line === 'Line #3'; }));
done();
});
}
);
}
);
}
);
}
);
it('05 - should return n lines for one file with errors', function (done) {
var options = {};
extend(true, options, shareopt);
options.ignore.push("*__metadata.yml");
options.collectionName = 'test05';
var fr = new Filerake(__dirname, options);
fr.use(function(input, submit) {
fs.readFile(input.filename, function (err, data) {
if (err) {
return submit(err);
}
data.toString().split("\n").map(function(line) {
var d = { line : line };
extend(true, d, input);
submit(d);
return d;
});
submit(null);
});
});
fr.sync(function (c) {
done();
}
);
}
);
it('05b- should return n lines for one file with errors', function (done) {
var options = {};
extend(true, options, shareopt);
options.ignore.push("*__metadata.yml");
options.collectionName = 'test05b';
var fr = new Filerake(__dirname, options);
fr.use(function(input, submit) {
var readable = input.openStream()
, buffer = '';
readable.on('data', function(chunk) {
buffer += chunk;
});
readable.on('error', function(err) {
submit(err);
});
readable.on('end', function() {
buffer.toString().split("\n").map(function(line) {
var d = { line : line };
extend(true, d, input);
submit(d);
return d;
});
submit(null);
});
});
fr.sync(function (c) {
done();
}
);
}
);
it('06 - should return n lines for one file and pass to another', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/4');
extend(true, options, shareopt);
options.collectionName = 'test06';
var fr = new Filerake(fina, options);
fr.use('*', oneDocPerLine);
fr.use('*', function(input, submit) {
input.test1 = true;
submit(null, input);
}
)
fr.use('*', function(input, submit) {
input.test2 = true;
submit(null, input);
}
)
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
fs.readFile(fina + '/3lines.text', function (err, data) {
assert.ifError(err);
var arr = data.toString().trim().split("\n");
assert.equal(arr.length, docs.length);
assert.equal(docs.length, 3);
assert(docs.some(function(c) { return c.line === 'Line #1' }));
assert(docs.some(function(c) { return c.line === 'Line #2' }));
assert(docs.some(function(c) { return c.line === 'Line #3' }));
assert(docs.every(function(c) { return c.test1 === true; }));
assert(docs.every(function(c) { return c.test2 === true; }));
done();
});
}
);
}
);
}
);
}
);
it('07 - should return one file with enrich', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/2');
extend(true, options, shareopt);
options.collectionName = 'test07';
options.concurrency = 1000;
var fr = new Filerake(fina, options);
fr.use('*', function(input, submit) {
input.test1 = true;
submit(null, input);
}
)
fr.use('*', function(input, submit) {
input.test2 = true;
submit(null, input);
}
)
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 7);
assert(docs.every(function(c) { return c.test1 === true; }));
assert(docs.every(function(c) { return c.test2 === true; }));
done();
}
);
}
);
}
);
}
);
it('08 - should mark documents as deleted when a file has been deleted', function (done) {
var options = {};
var dir = path.resolve(__dirname, '../dataset/8');
var filename = '4lines.txt';
var filepath = path.join(dir, filename);
extend(true, options, shareopt);
options.collectionName = 'test08';
options.concurrency = 1000;
fs.mkdir(dir, function (err) {
if (err && err.code !== 'EEXIST') { throw err; }
var fr = new Filerake(dir, options);
fr.use('*', oneDocPerLine);
fr.on('watching', function () {
fs.unlink(filepath, function (err) {
assert.ifError(err);
});
});
fr.once('dropped', function (err, doc) {
assert.ifError(err);
assert.equal(doc.basename, filename);
fr.syncr.connect(function (err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
assert(docs.every(function (doc) { return doc.state === 'deleted'; }));
done();
});
});
});
fs.writeFile(filepath, '#1\n#2\n#3\n#4\n', function (err) {
assert.ifError(err);
fr.sync();
});
});
});
it('09 - should remove documents of ignored files from the DB', function (done) {
var options = {};
var dir = path.resolve(__dirname, '../dataset/4');
extend(true, options, shareopt);
options.collectionName = 'test09';
options.concurrency = 1000;
var fr = new Filerake(dir, options);
fr.use('*', oneDocPerLine);
fr.sync(function () {
fr.syncr.connect(function (err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 3);
assert(docs.every(function (doc) { return doc.state === 'inserted'; }));
fr.options.ignore.push('**/*.text');
fr.sync(function () {
assert.ifError(err);
col.count(function (err, count) {
assert.ifError(err);
assert.equal(count, 0);
done();
});
});
});
});
});
});
it('10 - should mark documents as renamed when a file has been renamed', function (done) {
var options = {};
var dir = path.resolve(__dirname, '../dataset/10');
var filename = '4lines.txt';
var newName = '4lines.log';
var filepath = path.join(dir, filename);
var newPath = path.join(dir, newName);
extend(true, options, shareopt);
options.collectionName = 'test10';
options.concurrency = 1000;
fs.mkdir(dir, function (err) {
if (err && err.code !== 'EEXIST') { throw err; }
var fr = new Filerake(dir, options);
fr.use('*', oneDocPerLine);
fr.once('watching', function () {
fr.once('added', function (err, doc) {
assert.ifError(err);
assert.equal(doc.basename, newName);
fr.syncr.connect(function (err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
assert(docs.every(function (doc) {
return (doc.state == 'renamed');
}), 'some documents were not marked as renamed');
assert(docs.every(function (doc) {
return (doc.basename == newName);
}), 'some documents were not renamed (expected ' + newName + ')');
done();
});
});
});
fs.rename(filepath, newPath, function (err) {
assert.ifError(err);
});
});
fs.unlink(newPath, function (err) {
if (err && err.code != 'ENOENT') { throw err; }
fs.writeFile(filepath, '#1\n#2\n#3\n#4\n', function (err) {
assert.ifError(err);
fr.sync();
});
});
});
});
it('11 - should mark documents as deleted when a file has been deleted between two synchronizations', function (done) {
var options = { watch: false };
var dir = path.resolve(__dirname, '../dataset/11');
var filename = '4lines.txt';
var filepath = path.join(dir, filename);
extend(true, options, shareopt);
options.collectionName = 'test11';
options.concurrency = 1000;
fs.mkdir(dir, function (err) {
if (err && err.code !== 'EEXIST') { throw err; }
fs.writeFile(filepath, '#1\n#2\n#3\n#4\n', function (err) {
assert.ifError(err);
var fr = new Filerake(dir, options);
fr.use('*', oneDocPerLine);
fr.syncr.connect(function (err, col) {
assert.ifError(err);
fr.sync(function () {
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
assert(docs.every(function (doc) {
return (doc.state == 'inserted');
}), 'some documents was not marked as inserted');
fs.unlink(filepath, function (err) {
assert.ifError(err);
fr.sync(function () {
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
assert(docs.every(function (doc) {
return (doc.state == 'deleted');
}), 'some documents were not marked as deleted');
done();
});
});
});
});
});
});
});
});
});
it('12 - should correctly emit events during sync', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/3');
extend(true, options, shareopt);
options.collectionName = 'test12';
options.concurrency = 1000;
options.include = [ 'file', 'directory'];
options.ignore.push("file1.txt");
var fr = new Filerake(fina, options);
var browseOver = false;
var nbChecked = 0;
var nbCancelled = 0;
var nbPreCheck = 0;
var nbPreCancel = 0;
fr.on('browseOver', function (found) {
browseOver = true;
assert.equal(found, 5);
});
fr.on('preCheck', function () { nbPreCheck++; });
fr.on('preCancel', function () { nbPreCancel++; });
fr.on('checked', function () { nbChecked++; });
fr.on('cancelled', function () { nbCancelled++; });
fr.sync(function (nbSynchronized) {
assert(browseOver);
assert.equal(nbSynchronized, 5);
assert.equal(nbChecked, 3);
assert.equal(nbCancelled, 2);
assert.equal(nbPreCheck, nbChecked);
assert.equal(nbPreCancel, nbCancelled);
fr.syncr.connect(function (err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 3);
done();
});
});
});
});
it('13 - should mark old documents as deleted when a file has been changed', function (done) {
var options = { strictCompare: true };
var dir = path.resolve(__dirname, '../dataset/13');
var filename = '4lines.txt';
var filepath = path.join(dir, filename);
extend(true, options, shareopt);
options.collectionName = 'test13';
options.concurrency = 1000;
fs.mkdir(dir, function (err) {
if (err && err.code !== 'EEXIST') { throw err; }
var fr = new Filerake(dir, options);
fr.use('*', oneDocPerLine);
fr.syncr.connect(function (err, col) {
assert.ifError(err);
fr.once('watching', function () {
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
assert(docs.some(function (doc) { return doc.number === 1; }));
assert(docs.some(function (doc) { return doc.number === 2; }));
assert(docs.some(function (doc) { return doc.number === 3; }));
assert(docs.some(function (doc) { return doc.number === 4; }));
assert(docs.every(function (doc) { return doc.state === 'inserted'; }));
fs.writeFile(filepath, '#1\n#2\n', function (err) {
assert.ifError(err);
});
});
})
fr.once('changed', function (err, doc) {
assert.ifError(err);
assert.equal(doc.basename, filename);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
assert(docs.some(function (doc) { return (doc.state === 'updated' && doc.number == 1); }));
assert(docs.some(function (doc) { return (doc.state === 'updated' && doc.number == 2); }));
assert(docs.some(function (doc) { return (doc.state === 'deleted' && doc.number == 3); }));
assert(docs.some(function (doc) { return (doc.state === 'deleted' && doc.number == 4); }));
done();
});
});
fs.writeFile(filepath, '#1\n#2\n#3\n#4\n', function (err) {
assert.ifError(err);
fr.sync();
});
});
});
});
it('14 - should correctly change base documents with the modifier', function (done) {
var options = {
modifier: function (doc) { doc.fid = 'arbitrary'; }
};
extend(true, options, shareopt);
var fina = path.resolve(__dirname, '../dataset/4/');
options.ignore.push("*.ignore");
options.collectionName = 'test14';
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 1);
assert.equal(docs[0].fid, 'arbitrary');
col.drop(function(err) {
assert.ifError(err);
done();
});
}
);
}
);
}
);
}
);
}
)