qlobber-fsq
Version:
Shared file system queue. Supports pub-sub, work queues, wildcards, multi-process and distributed use.
1,839 lines (1,552 loc) • 134 kB
JavaScript
/*global fsq: true,
fs: false,
get_message_files: false,
expect: false,
msg_dir: false,
QlobberFSQ: false,
fsq_dir: false,
default_options: false,
check_empty: false,
async: false,
wu: false,
path: false,
lsof: false,
rimraf: false,
ignore_ebusy: false,
util: false,
single_supported: false,
os: false,
argv: false*/
"use strict";
const crypto = require('crypto');
/*var orig_error = console.error;
console.error = function wup()
{
console.log("WUP");
console.error = orig_error;
console.trace();
console.error = wup;
return orig_error.apply(this, arguments);
};*/
//process.on('uncaughtException', console.error);
function read_all(s, cb)
{
var bufs = [];
s.on('end', function ()
{
cb(Buffer.concat(bufs));
});
s.on('readable', function ()
{
while (true) // eslint-disable-line no-constant-condition
{
var data = this.read();
if (data === null) { break; }
bufs.push(data);
}
});
}
function test(getdents_size, use_disruptor, ephemeral)
{
describe('qlobber-fsq (getdents_size=' + getdents_size + ', use_disruptor=' + use_disruptor + ', ephemeral=' + ephemeral + ')', function ()
{
this.timeout(60 * 1000);
var test_options = {},
Disruptor;
function maybe_ephemeral(fsq)
{
if (!ephemeral) { return; }
var orig_publish = fsq.publish;
fsq.publish = function (topic, payload, options, cb)
{
if ((typeof payload !== 'string') && !Buffer.isBuffer(payload))
{
cb = options;
options = payload;
payload = undefined;
}
if (typeof options === 'function')
{
cb = options;
options = undefined;
}
options = options || {};
options = Object.assign({}, options);
options.ephemeral = true;
var r = orig_publish.call(this, topic, payload, options, cb);
if ((typeof payload !== 'string') &&
(!Buffer.isBuffer(payload)) &&
!options.direct)
{
expect(r.ephemeral_size).to.be.above(0);
}
return r;
};
}
function make_fsq(n, i, extra_options)
{
var r = new QlobberFSQ(Object.assign({},
default_options,
test_options,
extra_options));
r.N = n;
r.I = i;
maybe_ephemeral(r);
if (use_disruptor)
{
const make_buckets = () =>
{
for (var b = 0; b < r.num_buckets; b += 1)
{
new Disruptor('/test' + b,
20 * 1024,
2048,
n,
0,
true,
false).release();
}
};
if (i === 0)
{
make_buckets();
}
else if ((i === 1) && !fsq.stopped)
{
var orig_on = r.on,
fsq_stopped = false,
fsq_started = false,
onstart = null;
const check = () =>
{
if (fsq_stopped && fsq_started && onstart)
{
process.nextTick(function ()
{
onstart.call(r);
});
}
};
fsq.stop_watching(function ()
{
fsq_stopped = true;
check();
});
fsq = new QlobberFSQ(Object.assign({},
default_options,
test_options));
fsq.N = n;
fsq.I = 0;
maybe_ephemeral(fsq);
make_buckets();
ignore_ebusy(fsq);
fsq.on('start', function ()
{
fsq_started = true;
check();
});
r.on = function (evname, f)
{
if (evname !== 'start')
{
return orig_on.call(this, evname, f);
}
orig_on.call(this, 'start', function ()
{
onstart = f;
check();
});
};
}
}
return r;
}
if (getdents_size > 0)
{
require('../lib/process_all_getdents'); // account for file handle
}
if (use_disruptor)
{
Disruptor = require('shared-memory-disruptor').Disruptor;
}
beforeEach(function (cb)
{
if (getdents_size > 0)
{
test_options.getdents_size = getdents_size;
}
if (use_disruptor)
{
test_options.get_disruptor = function (bucket)
{
return new Disruptor('/test' + bucket,
20 * 1024,
2048,
this.N,
this.I,
false,
false);
};
}
if ((getdents_size > 0) || use_disruptor)
{
return fsq.stop_watching(function ()
{
fsq = make_fsq(1, 0);
if (getdents_size > 0)
{
fsq.on('getdents_disabled', function (err)
{
cb(err);
});
}
ignore_ebusy(fsq);
fsq.on('start', cb);
});
}
cb();
});
var orig_ftruncate, orig_rename, orig_close, orig_flock;
function restore()
{
fsq._fs.ftruncate = orig_ftruncate;
fsq._fs.rename = orig_rename;
fsq._fs.close = orig_close;
fsq._fsext.flock = orig_flock;
}
beforeEach(function ()
{
orig_ftruncate = fsq._fs.ftruncate;
orig_rename = fsq._fs.rename;
orig_close = fsq._fs.close;
orig_flock = fsq._fsext.flock;
var busied_ftruncate = false,
busied_rename = false,
busied_close = false,
busied_flock = false;
fsq._fs.ftruncate = function (fd, size, cb)
{
if (busied_ftruncate)
{
busied_ftruncate = false;
return orig_ftruncate.apply(this, arguments);
}
busied_ftruncate = true;
cb({ code: 'EBUSY' });
};
fsq._fs.rename = function (src, dest, cb)
{
if (busied_rename)
{
busied_rename = false;
return orig_rename.apply(this, arguments);
}
busied_rename = true;
cb({ code: 'EBUSY' });
};
// fs.WriteStream calls fs.close when it ends so if we're not using
// fs-ext then don't overwrite fs.close otherwise publish will error
if (!single_supported)
{
return;
}
fsq._fs.close = function (fd, cb)
{
if (busied_close)
{
busied_close = false;
return orig_close.apply(this, arguments);
}
busied_close = true;
cb({ code: 'EBUSY' });
};
fsq._fsext.flock = function (fd, type, cb)
{
if (busied_flock)
{
busied_flock = false;
return orig_flock.apply(this, arguments);
}
busied_flock = true;
cb({ code: 'EBUSY' });
};
});
afterEach(restore);
it('should subscribe and publish to a simple topic', function (done)
{
var pub_info;
fsq.subscribe('foo', function handler(data, info, cb)
{
expect(info.topic).to.equal('foo');
expect(info.single).to.equal(false);
expect(info.direct).to.equal(false);
expect(info.path.lastIndexOf(msg_dir, 0)).to.equal(0);
expect(info.fname.lastIndexOf(Buffer.from('foo').toString('hex') + '@', 0)).to.equal(0);
expect(info.topic_path).to.equal(undefined);
if (info.data !== undefined)
{
pub_info.data = info.data;
}
if (info.new !== undefined)
{
pub_info.new = info.new;
}
expect(info).to.eql(pub_info);
expect(data.toString('utf8')).to.equal('bar');
expect(cb.num_handlers).to.equal(1);
fs.readFile(info.path, function (err, data)
{
if (ephemeral)
{
expect(err.code).to.equal('ENOENT');
}
else
{
if (err) { return done(err); }
expect(data.toString()).to.equal('\u0000bar');
}
done();
});
});
fsq.publish('foo', 'bar', function (err, info)
{
if (err) { done(err); }
pub_info = info;
});
});
it('should construct received data only once', function (done)
{
var the_data = { foo: 0.435, bar: 'hello' },
called1 = false,
called2 = false,
received_data;
fsq.subscribe('test', function (data, info, cb)
{
expect(info.topic).to.equal('test');
expect(JSON.parse(data)).to.eql(the_data);
if (received_data)
{
expect(data === received_data).to.equal(true);
}
else
{
received_data = data;
}
called1 = true;
if (called1 && called2)
{
cb(null, done);
}
else
{
cb();
}
});
fsq.subscribe('test', function (data, info, cb)
{
expect(info.topic).to.equal('test');
expect(JSON.parse(data)).to.eql(the_data);
if (received_data)
{
expect(data === received_data).to.equal(true);
}
else
{
received_data = data;
}
called2 = true;
if (called1 && called2)
{
cb(null, done);
}
else
{
cb();
}
});
fsq.publish('test', JSON.stringify(the_data), function (err)
{
if (err) { done(err); }
});
});
it('should support more than 10 subscribers', function (done)
{
var the_data = { foo: 0.435, bar: 'hello' },
counter = 11,
received_data,
a = [],
i;
function subscribe(cb)
{
fsq.subscribe('test', function (data, info, cb)
{
expect(info.topic).to.equal('test');
expect(JSON.parse(data)).to.eql(the_data);
if (received_data)
{
expect(data === received_data).to.equal(true);
}
else
{
received_data = data;
}
counter -= 1;
if (counter === 0)
{
cb(null, done);
}
else
{
cb();
}
}, cb);
}
for (i = counter; i > 0; i -= 1)
{
a.push(subscribe);
}
async.parallel(a, function (err)
{
if (err) { return done(err); }
fsq.publish('test', JSON.stringify(the_data), function (err)
{
if (err) { done(err); }
});
});
});
it('should support more than 10 subscribers with same handler', function (done)
{
fsq.stop_watching(function ()
{
var fsq2 = make_fsq(1, 0, { dedup: false }),
the_data = { foo: 0.435, bar: 'hello' },
counter = 11,
received_data,
a = [],
i;
ignore_ebusy(fsq2);
function handler(data, info, cb)
{
expect(info.topic).to.equal('test');
expect(JSON.parse(data)).to.eql(the_data);
if (received_data)
{
expect(data === received_data).to.equal(true);
}
else
{
received_data = data;
}
counter -= 1;
if (counter === 0)
{
cb(null, function (err)
{
fsq2.stop_watching(function ()
{
done(err);
});
});
}
else
{
cb();
}
}
function subscribe(cb)
{
fsq2.subscribe('test', handler, cb);
}
for (i = counter; i > 0; i -= 1)
{
a.push(subscribe);
}
fsq2.on('start', function ()
{
async.parallel(a, function (err)
{
if (err) { return done(err); }
fsq2.publish('test', JSON.stringify(the_data), function (err)
{
if (err) { done(err); }
});
});
});
});
});
it('should subscribe to wildcards', function (done)
{
var count = 0;
function received()
{
count += 1;
if (count === 2) { done(); }
}
fsq.subscribe('*', function (data, info)
{
expect(info.topic).to.equal('foo');
expect(data.toString('utf8')).to.equal('bar');
received();
});
fsq.subscribe('#', function (data, info)
{
expect(info.topic).to.equal('foo');
expect(data.toString('utf8')).to.equal('bar');
received();
});
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should only call each handler once', function (done)
{
var handler = function (data, info)
{
expect(info.topic).to.equal('foo');
expect(data.toString('utf8')).to.equal('bar');
done();
};
fsq.subscribe('*', handler);
fsq.subscribe('#', handler);
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should be able to disable handler dedup', function (done)
{
fsq.stop_watching(function ()
{
var fsq2 = make_fsq(1, 0, { dedup: false }),
count_multi = 0,
count_single = 0;
ignore_ebusy(fsq2);
function handler(data, info, cb)
{
expect(info.topic).to.equal('foo');
expect(data.toString('utf8')).to.equal('bar');
cb(null, function (err)
{
if (info.single)
{
count_single += 1;
}
else
{
count_multi += 1;
}
if ((count_single === (single_supported ? 1 : 0)) &&
(count_multi === 2))
{
fsq2.stop_watching(function ()
{
done(err);
});
}
else
{
if ((count_single > 1) || (count_multi > 2))
{
throw new Error('called too many times');
}
}
});
}
fsq2.on('start', function ()
{
fsq2.subscribe('*', handler);
fsq2.subscribe('#', handler);
fsq2.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
fsq2.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
});
});
});
it('should call all handlers on a topic for pubsub', function (done)
{
var count = 0;
function received()
{
count += 1;
if (count === 2) { done(); }
}
function handler(data, info)
{
expect(info.topic).to.equal('foo');
expect(data.toString('utf8')).to.equal('bar');
received();
}
fsq.subscribe('foo', function () { handler.apply(this, arguments); });
fsq.subscribe('foo', function () { handler.apply(this, arguments); });
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
if (single_supported)
{
it('should support a work queue', function (done)
{
fsq.subscribe('foo', function (data, info, cb)
{
expect(info.topic).to.equal('foo');
expect(info.single).to.equal(true);
expect(info.direct).to.equal(false);
expect(info.path.lastIndexOf(msg_dir, 0)).to.equal(0);
expect(info.fname.lastIndexOf(Buffer.from('foo').toString('hex') + '@', 0)).to.equal(0);
expect(data.toString('utf8')).to.equal('bar');
fs.stat(info.path, function (err)
{
expect(err).to.equal(null);
fs.open(info.path, 'r+', function (err, fd)
{
expect(err).to.equal(null);
orig_flock(fd, 'exnb', function (err)
{
expect(err.code).to.be.oneOf(['EAGAIN', 'EWOULDBLOCK']);
orig_close(fd, function (err)
{
expect(err).to.equal(null);
cb(null, function ()
{
fs.stat(info.fname, function (err)
{
expect(err.code).to.equal('ENOENT');
done();
});
});
});
});
});
});
});
fsq.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
});
}
it('should guard against calling subscribe callback twice', function (done)
{
fsq.on('warning', function (err)
{
if (err && (err.code !== 'EBUSY'))
{
throw new Error('should not be called');
}
});
fsq.subscribe('foo', function (data, info, cb)
{
expect(info.single).to.equal(single_supported);
cb(null, function (err)
{
if (err) { return done(err); }
setTimeout(function ()
{
cb(null, done);
}, 2000);
});
}, function (err)
{
if (err) { return done(err); }
fsq.publish('foo', 'bar', { single: single_supported }, function (err)
{
if (err) { done(err); }
});
});
});
if (single_supported)
{
it('should only give work to one worker', function (done)
{
this.timeout(30000);
var fsq2 = make_fsq(2, 1),
called = false;
ignore_ebusy(fsq2);
function handler (data, info, cb)
{
expect(called).to.equal(false);
called = true;
expect(info.topic).to.equal('foo');
expect(info.single).to.equal(true);
expect(data.toString('utf8')).to.equal('bar');
setTimeout(function ()
{
cb(null, function (err)
{
fsq2.stop_watching(function ()
{
done(err);
});
});
}, 2000);
}
fsq.subscribe('foo', function () { handler.apply(this, arguments); });
fsq.subscribe('foo', function () { handler.apply(this, arguments); });
fsq2.subscribe('foo', function () { handler.apply(this, arguments); });
fsq2.subscribe('foo', function () { handler.apply(this, arguments); });
fsq2.on('start', function ()
{
fsq.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
});
});
it('should put work back on the queue', function (done)
{
var count = 0;
fsq.subscribe('foo', function (data, info, cb)
{
count += 1;
if (count === 1)
{
cb('dummy failure');
}
else
{
cb(null, done);
}
});
fsq.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
});
}
it('should allow handlers to refuse work', function (done)
{
fsq.stop_watching(function ()
{
function handler1()
{
throw new Error('should not be called');
}
var fsq2;
function handler2(data, info, cb)
{
cb(null, function (err)
{
fsq2.stop_watching(function ()
{
done(err);
});
});
}
fsq2 = make_fsq(1, 0,
{
filter: function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
handlers.delete(handler1);
cb(null, true, handlers);
}
});
ignore_ebusy(fsq2);
fsq2.subscribe('foo', handler1);
fsq2.subscribe('foo', handler2);
fsq2.on('start', function ()
{
fsq2.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
});
});
it('should not allow filters to modify qlobber matches', function (done)
{
fsq.stop_watching(function ()
{
var fsq2, count = 0;
function handler(data, info, cb)
{
expect(count).to.equal(2);
cb(null, function (err)
{
fsq2.stop_watching(function ()
{
done(err);
});
});
}
fsq2 = make_fsq(1, 0,
{
filter: function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
if (++count === 1)
{
handlers.delete(handler);
return cb(null, false);
}
cb(null, true, handlers);
}
});
ignore_ebusy(fsq2);
fsq2.subscribe('foo', handler);
fsq2.on('start', function ()
{
fsq2.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
});
});
it('should be able to set filter by property', function (done)
{
function handler1()
{
throw new Error('should not be called');
}
function handler2(data, info, cb)
{
cb(null, done);
}
fsq.filters.push(function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
handlers.delete(handler1);
cb(null, true, handlers);
});
fsq.subscribe('foo', handler1);
fsq.subscribe('foo', handler2);
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should be able to pass filtered handlers as iterator (Set)', function (done)
{
function handler1()
{
throw new Error('should not be called');
}
function handler2(data, info, cb)
{
cb(null, done);
}
fsq.filters.push(function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
cb(null, true, wu(handlers).filter(function (h)
{
return h !== handler1;
}));
});
fsq.subscribe('foo', handler1);
fsq.subscribe('foo', handler2);
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should be able to pass filtered handlers as iterator (Array)', function (done)
{
fsq.stop_watching(function ()
{
function handler1()
{
throw new Error('should not be called');
}
var fsq2;
function handler2(data, info, cb)
{
cb(null, function (err)
{
fsq2.stop_watching(function ()
{
done(err);
});
});
}
fsq2 = make_fsq(1, 0,
{
filter: function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
cb(null, true, wu(handlers).filter(function (h)
{
return h !== handler1;
}));
},
dedup: false
});
ignore_ebusy(fsq2);
fsq2.subscribe('foo', handler1);
fsq2.subscribe('foo', handler2);
fsq2.on('start', function ()
{
fsq2.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
});
});
it('should support multiple filters', function (done)
{
function handler1()
{
throw new Error('should not be called');
}
function handler2()
{
throw new Error('should not be called');
}
function handler3(data, info, cb)
{
cb(null, done);
}
fsq.filters.push(
function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
handlers.delete(handler1);
cb(null, true, handlers);
},
function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
handlers.delete(handler2);
cb(null, true, handlers);
}
);
fsq.subscribe('foo', handler1);
fsq.subscribe('foo', handler2);
fsq.subscribe('foo', handler3);
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should not call other filters if error', function (done)
{
var called = false;
fsq.filters.push(
function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
cb(new Error('dummy'));
if (called)
{
return done();
}
called = true;
},
function (unused_info, unused_handlers, unused_cb)
{
throw new Error('should not be called');
}
);
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should not call other filters if not ready', function (done)
{
var called = false;
fsq.filters.push(
function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
cb(null, false);
if (called)
{
return done();
}
called = true;
},
function (unused_info, unused_handlers, unused_cb)
{
throw new Error('should not be called');
}
);
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
if (single_supported)
{
it('should put work back on queue for another handler', function (done)
{
fsq.stop_watching(function ()
{
function handler(data, info, cb)
{
cb('dummy failure');
}
var filter_called = false,
fsq2 = make_fsq(1, 0,
{
filter: function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
expect(info.single).to.equal(true);
if (filter_called)
{
handlers.delete(handler);
return cb(null, true, handlers);
}
filter_called = true;
cb(null, true, handlers);
}
});
ignore_ebusy(fsq2);
fsq2.subscribe('foo', handler);
fsq2.subscribe('foo', function (data, info, cb)
{
if (filter_called)
{
cb(null, function (err)
{
fsq2.stop_watching(function ()
{
done(err);
});
});
}
else
{
cb('dummy failure2');
}
});
fsq2.on('start', function ()
{
fsq2.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
});
});
});
it('should put work back on queue for a handler on another queue', function (done)
{
this.timeout(30000);
fsq.stop_watching(function ()
{
function handler(data, info, cb)
{
cb('dummy failure');
}
var filter_called = false,
fsq2 = make_fsq(2, 0,
{
filter: function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
expect(info.single).to.equal(true);
if (filter_called)
{
handlers.delete(handler);
}
filter_called = true;
cb(null, true, handlers);
}
}),
fsq3 = make_fsq(2, 1),
started2 = false,
started3 = false;
ignore_ebusy(fsq2);
ignore_ebusy(fsq3);
fsq2.subscribe('foo', handler);
fsq3.subscribe('foo', function (data, info, cb)
{
if (filter_called)
{
cb(null, function (err)
{
fsq2.stop_watching(function ()
{
fsq3.stop_watching(function ()
{
done(err);
});
});
});
}
else
{
cb('dummy failure2');
}
});
function start()
{
if (!(started2 && started3)) { return; }
fsq2.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
}
fsq2.on('start', function ()
{
started2 = true;
start();
});
fsq3.on('start', function ()
{
started3 = true;
start();
});
});
});
}
if (!ephemeral)
{
it('should allow handlers to delay a message', function (done)
{
restore();
fsq.stop_watching(function ()
{
var ready_multi = false,
ready_single = !single_supported,
got_multi = false,
got_single = !single_supported,
count = 0,
fsq2 = make_fsq(1, 0,
{
filter: function (info, handlers, cb)
{
expect(info.topic).to.equal('foo');
if (info.single)
{
ready_single = true;
}
else
{
ready_multi = true;
}
if (!single_supported)
{
expect(info.single).to.equal(false);
}
count += 1;
cb(null, (count % 5) === 0, handlers);
}
});
ignore_ebusy(fsq2);
function handler(data, info, cb)
{
expect(data.toString('utf8')).to.equal('bar');
cb(null, function (err)
{
if (info.single)
{
expect(got_single).to.equal(false);
got_single = true;
}
else
{
expect(got_multi).to.equal(false);
got_multi = true;
}
if (!single_supported)
{
expect(info.single).to.equal(false);
}
if (got_single && got_multi && ready_single && ready_multi)
{
expect(count).to.equal(single_supported ? 10 : 5);
fsq2.stop_watching(function ()
{
done(err);
});
}
});
}
fsq2.subscribe('foo', handler);
fsq2.on('start', function ()
{
fsq2.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
fsq2.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
});
});
});
}
it('should emit start and stop events', function (done)
{
this.timeout(30000);
var fsq2 = make_fsq(2, 1);
ignore_ebusy(fsq2);
fsq2.on('start', function ()
{
fsq2.stop_watching();
fsq2.on('stop', done);
});
});
it('should support per-message time-to-live', function (done)
{
this.timeout(20000);
restore();
fsq.subscribe('foo', function ()
{
setTimeout(function ()
{
fsq.force_refresh();
setTimeout(function ()
{
check_empty(msg_dir, done, done);
}, 500);
}, 500);
});
fsq.publish('foo', 'bar', { ttl: 500 }, function (err)
{
if (err) { done(err); }
});
});
it('should call error function', function (done)
{
restore();
fsq.on('warning', function (err)
{
expect(err).to.equal('dummy failure');
done();
});
fsq.subscribe('foo', function (data, info, cb)
{
cb('dummy failure');
});
fsq.publish('foo', 'bar', { single : single_supported }, function (err)
{
if (err) { done(err); }
});
});
if (single_supported)
{
it('should support custom polling interval', function (done)
{
this.timeout(30000);
restore();
var time, count = 0, fsq2 = make_fsq(2, 1, { poll_interval: 50 });
ignore_ebusy(fsq2);
fsq2.subscribe('foo', function (data, info, cb)
{
count += 1;
var time2 = new Date().getTime();
expect(time2 - time).to.be.below(900);
time = time2;
if (count === 10)
{
cb(null, function ()
{
fsq2.stop_watching(done);
});
}
else
{
cb('dummy failure');
}
});
fsq2.on('start', function ()
{
time = new Date().getTime();
fsq.publish('foo', 'bar', {single : true}, function (err)
{
if (err) { done(err); }
});
});
});
}
it('should support unsubscribing', function (done)
{
this.timeout(5000);
var count = 0;
function handler(data, info, cb)
{
count += 1;
if (count > 1)
{
throw new Error('should not be called');
}
fsq.unsubscribe('foo', handler, function ()
{
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
setTimeout(function ()
{
cb(null, done);
}, 2000);
});
}
fsq.subscribe('foo', handler);
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should support unsubscribing to all handlers for a topic', function (done)
{
this.timeout(5000);
var count = 0;
function handler(data, info, cb)
{
count += 1;
if (count > 2)
{
throw new Error('should not be called');
}
if (count === 2)
{
fsq.unsubscribe('foo', undefined, function ()
{
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
setTimeout(function ()
{
cb(null, done);
}, 2000);
});
}
}
fsq.subscribe('foo', function (data, info, cb)
{
handler(data, info, cb);
});
fsq.subscribe('foo', function (data, info, cb)
{
handler(data, info, cb);
});
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should support unsubscribing to all handlers', function (done)
{
this.timeout(5000);
var count = 0;
function handler(data, info, cb)
{
count += 1;
if (count > 2)
{
throw new Error('should not be called');
}
if (count === 2)
{
fsq.subscribe('foo2', function ()
{
throw new Error('should not be called');
});
fsq.unsubscribe(function ()
{
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
fsq.publish('foo2', 'bar2', function (err)
{
if (err) { done(err); }
});
setTimeout(function ()
{
cb(null, done);
}, 2000);
});
}
}
fsq.subscribe('foo', function (data, info, cb)
{
handler(data, info, cb);
});
fsq.subscribe('foo', function (data, info, cb)
{
handler(data, info, cb);
});
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
});
it('should support changing the default time-to-live', function (done)
{
this.timeout(30000);
restore();
fsq.stop_watching(function () // stop fsq dequeuing
{
var got_single = !single_supported,
got_multi = false,
fsq2 = make_fsq(1, 0,
{
multi_ttl: 1000,
single_ttl: 1000
});
ignore_ebusy(fsq2);
fsq2.subscribe('foo', function (data, info, cb)
{
cb(null, function ()
{
if (info.single)
{
got_single = true;
}
else
{
got_multi = true;
}
if (got_single && got_multi)
{
setTimeout(function ()
{
fsq2.force_refresh();
setTimeout(function ()
{
check_empty(msg_dir, done, function ()
{
fsq2.stop_watching(done);
});
}, 1000);
}, 1000);
}
});
});
fsq2.on('start', function ()
{
fsq2.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
});
if (single_supported)
{
fsq2.publish('foo', 'bar', { single: true }, function (err)
{
if (err) { done(err); }
});
}
});
});
});
it('should publish and receive twice', function (done)
{
var count_multi = 0,
count_single = single_supported ? 0 : 2;
fsq.subscribe('foo', function (data, info, cb)
{
cb(null, function ()
{
if (info.single)
{
count_single += 1;
}
else
{
count_multi += 1;
}
if ((count_single === 2) && (count_multi === 2))
{
done();
}
else if ((count_single > 2) || (count_multi > 2))
{
throw new Error('called too many times');
}
});
});
async.timesSeries(2, function (n, cb)
{
async.eachSeries([true, false], function (single, cb)
{
fsq.publish('foo', 'bar', { single: single }, function (err)
{
cb(err);
});
}, cb);
}, function (err)
{
if (err) { done(err); }
});
});
it('should default to putting messages in module directory', function (done)
{
var fsq2 = make_fsq(2, 1, { fsq_dir: undefined });
ignore_ebusy(fsq2);
fsq2.subscribe('foo', function (data, info)
{
if (use_disruptor)
{
// fsq and fsq2 use same disruptors
expect(info.new).to.be.true;
expect(info.data.toString('utf8')).to.equal('bar');
}
else
{
throw new Error('should not be called');
}
});
fsq2.subscribe('foo2', function (data, info, cb)
{
expect(data.toString('utf8')).to.equal('bar2');
expect(info.path.lastIndexOf(path.join(__dirname, '..', 'fsq', 'messages'), 0)).to.equal(0);
cb(null, function ()
{
fsq2.stop_watching(done);
});
});
fsq2.on('start', function ()
{
fsq.publish('foo', 'bar', function (err)
{
if (err) { done(err); }
// wait for publish so EBUSY isn't retrying while fsq is being cleaned up
fsq2.publish('foo2', 'bar2', function (err)
{
if (err) { done(err); }
});
});
});
});
it('should publish and subscribe to messages with long topics (multi)', function (done)
{
var arr = [], topic;
arr.length = 64 * 1024 + 1;
topic = arr.join('a');
fsq.subscribe(topic, function (data, info)
{
expect(info.topic).to.equal(topic);
expect(info.single).to.equal(false);
expect(info.path.lastIndexOf(msg_dir, 0)).to.equal(0);
expect(info.fname.lastIndexOf(Buffer.from(topic).toString('hex').substr(0, fsq._split_topic_at) + '@', 0)).to.equal(0);
expect(data.toString('utf8')).to.equal('bar');
var topic_dir = path.dirname(path.dirname(info.topic_path));
expect(topic_dir).to.equal(path.join(msg_dir, '..', 'topics'));
fs.readFile(info.topic_path, function (err, split)
{
if (err) { return done(err); }
expect(split.toString('utf8')).to.equal(Buffer.from(topic).toString('hex').substr(fsq._split_topic_at));
setTimeout(function ()
{
fsq.force_refresh();
setTimeout(function ()
{
check_empty(msg_dir, done, function ()
{
check_empty(topic_dir, done, done);
});
}, 500);
}, 1000);
});
});
fsq.publish(topic, 'bar', { ttl: 1000 }, function (err)
{
if (ephemeral)
{
expec