castor-load
Version:
Traverse a directory to build a MongoDB collection with the found files. Then it's enable to keep directory and collection synchronised.
276 lines (272 loc) • 10.2 kB
JavaScript
/* global describe, it */
;
var path = require('path')
, basename = path.basename(__filename, '.js')
, debug = require('debug')('filerake:' + basename)
, util = require('util')
, extend = require('extend')
, clone = require('clone')
, assert = require('assert')
, fs = require('fs')
, Filerake = require('../');
var shareopt = {
"mongo": {
// "url" : "tingodb://localhost/.tingodb"
"url" : "mongodb://localhost:27017/test/"
},
"ignore" : [ "**/.*", "*~", "*.sw+(o|p)", "*.old", "*.bak", "**/node_modules"]
};
describe('Filerake', function () {
it('01 - should return one file', function (done) {
var options = {};
extend(options, shareopt);
var fina = path.resolve(__dirname, '../dataset/1/');
options.ignore.push("*.ignore");
options.collname = 'test01';
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 1);
var doc = docs[0];
assert.equal(docs[0].basename, 'to.load');
col.drop(function(err) {
assert.ifError(err);
done();
});
}
);
}
);
}
);
}
);
it('02 - should return one more file', function (done) {
var options = {};
extend(options, shareopt);
options.collname = 'test02';
options.concurrency = 1000;
var fina = path.resolve(__dirname, '../dataset/2/');
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 7);
col.drop(function(err) {
assert.ifError(err);
done();
});
}
);
}
);
}
);
}
);
it('03 - should return one file & one dir', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/3/');
extend(options, shareopt);
options.collname = 'test03';
options.concurrency = 1000;
options.include = [ 'file', 'directory'];
options.ignore.push("*__metadata.yml");
var fr = new Filerake(fina, options);
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 4);
var doc = docs[0];
assert(docs.reduce(function(p, c) { return c.basename === 'directory1' && c.extension === '' ? true : p; }, false));
assert(docs.reduce(function(p, c) { return c.basename === 'file1.txt' && c.extension === 'txt' ? true : p; }, false));
assert(docs.reduce(function(p, c) { return c.basename === 'file2.txt' && c.extension === 'txt'? true : p; }, false));
col.drop(function(err) {
assert.ifError(err);
done();
});
}
);
}
);
}
);
}
);
it('04 - should return n lines for one file', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/4');
extend(options, shareopt);
options.collname = 'test04';
var fr = new Filerake(fina, options);
fr.use('*', function(input, submit) {
fs.readFile(input.location, function (err, data) {
if (err) {
return submit(err);
}
data.toString().trim().split("\n").map(function(line) {
var d = { line : line };
extend(d, input);
submit(d);
return d;
});
submit(null);
});
});
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
fs.readFile(fina + '/3lines.text', function (err, data) {
assert.ifError(err);
var arr = data.toString().trim().split("\n");
assert.equal(arr.length, docs.length);
assert.equal(docs.length, 3);
assert(docs.reduce(function(p, c) { return c.line === 'Line #1' ? true : p; }, false));
assert(docs.reduce(function(p, c) { return c.line === 'Line #2' ? true : p; }, false));
assert(docs.reduce(function(p, c) { return c.line === 'Line #3' ? true : p; }, false));
col.drop(function(err) {
assert.ifError(err);
done();
});
});
}
);
}
);
}
);
}
);
it('05 - should return n lines for one file with errors', function (done) {
var options = {};
extend(options, shareopt);
options.ignore.push("*__metadata.yml");
options.collname = 'test04';
var fr = new Filerake(__dirname, options);
fr.use(function(input, submit) {
fs.readFile(input.filename, function (err, data) {
if (err) {
return submit(err);
}
data.toString().split("\n").map(function(line) {
var d = { line : line };
extend(d, input);
submit(d);
return d;
});
submit(null);
});
});
fr.sync(function (c) {
done();
}
);
}
);
it('06 - should return n lines for one file and pass to another', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/4');
extend(options, shareopt);
options.collname = 'test04';
var fr = new Filerake(fina, options);
fr.use('*', function(input, submit) {
fs.readFile(input.location, function (err, data) {
if (err) {
return submit(err);
}
data.toString().trim().split("\n").map(function(line) {
var d = clone(input);
d.line = line;
submit(d);
return d;
});
submit(null, submit);
});
});
fr.use('*', function(input, submit) {
input.test1 = true;
submit(null, input);
}
)
fr.use('*', function(input, submit) {
input.test2 = true;
submit(null, input);
}
)
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
fs.readFile(fina + '/3lines.text', function (err, data) {
assert.ifError(err);
var arr = data.toString().trim().split("\n");
assert.equal(arr.length, docs.length);
assert.equal(docs.length, 3);
assert(docs.reduce(function(p, c) { return c.line === 'Line #1' ? true : p; }, false));
assert(docs.reduce(function(p, c) { return c.line === 'Line #2' ? true : p; }, false));
assert(docs.reduce(function(p, c) { return c.line === 'Line #3' ? true : p; }, false));
assert(docs.reduce(function(p, c) { return p === true ? c.test1 : p; }, true));
assert(docs.reduce(function(p, c) { return p === true ? c.test2 : p; }, true));
col.drop(function(err) {
assert.ifError(err);
done();
});
});
}
);
}
);
}
);
}
);
it('07 - should return one file with enrich', function (done) {
var options = {};
var fina = path.resolve(__dirname, '../dataset/2');
extend(options, shareopt);
options.collname = 'test04';
options.concurrency = 1000;
var fr = new Filerake(fina, options);
fr.use('*', function(input, submit) {
input.test1 = true;
submit(null, input);
}
)
fr.use('*', function(input, submit) {
input.test2 = true;
submit(null, input);
}
)
fr.sync(function (c) {
fr.syncr.connect(function(err, col) {
assert.ifError(err);
col.find().toArray(function (err, docs) {
assert.ifError(err);
assert.equal(docs.length, 7);
assert(docs.reduce(function(p, c) { return p === true ? c.test1 : p; }, true));
assert(docs.reduce(function(p, c) { return p === true ? c.test2 : p; }, true));
col.drop(function(err) {
assert.ifError(err);
done();
});
}
);
}
);
}
);
}
);
/* */
}
)