@getanthill/datastore
Version:
Event-Sourced Datastore
1,282 lines (1,095 loc) • 30.6 kB
text/typescript
import setup from '../../test/setup';
import request from 'supertest';
import * as telemetry from '@getanthill/telemetry';
import * as runner from './runner';
import * as utils from './utils';
import { HandlerConfig, RunnerServices } from '../typings';
describe('sdk/runner', () => {
let _config;
let mongodb;
let models;
let app;
let instance;
let sdk;
let services: RunnerServices = {
datastores: {},
};
let stopHandler;
const options = {
pageSize: 10,
exitTimeout: 100,
verbose: true,
cwd: '',
};
const mocks: any = {};
beforeEach(async () => {
[_config, mongodb, models, app, , sdk, , instance] = await setup.startApi({
mode: 'development',
features: { api: { admin: true } },
});
services = {
datastores: {},
};
stopHandler = jest.fn();
mocks.consoleError = jest
.spyOn(console, 'error')
.mockImplementation(() => null);
mocks.loggerError = jest
.spyOn(telemetry.logger, 'error')
.mockImplementation(() => null);
mocks.loggerInfo = jest
.spyOn(telemetry.logger, 'info')
.mockImplementation(() => null);
mocks.processExit = jest
.spyOn(process, 'exit')
// @ts-ignore
.mockImplementation(() => null);
});
afterEach(async () => {
jest.restoreAllMocks();
await setup.stopApi(instance);
await setup.teardownDb(mongodb);
});
afterAll(async () => {
await new Promise((resolve) => setTimeout(resolve, 200));
jest.restoreAllMocks();
});
describe('#init', () => {
it('binds events for process termination', () => {
const processOnceMock = jest.spyOn(process, 'once');
runner.init(services, stopHandler, options);
expect(processOnceMock).toHaveBeenCalledTimes(4);
expect(processOnceMock.mock.calls[0][0]).toEqual('SIGTERM');
expect(processOnceMock.mock.calls[1][0]).toEqual('SIGINT');
expect(processOnceMock.mock.calls[2][0]).toEqual('uncaughtException');
expect(processOnceMock.mock.calls[3][0]).toEqual('unhandledRejection');
});
});
describe('#signalHandler', () => {
it('returns a handler to call destroy', () => {
const destroyMock = jest.fn();
const handler = runner.signalHandler(
services,
stopHandler,
options,
'SIGTERM',
destroyMock,
);
expect(typeof handler).toEqual('function');
handler();
expect(destroyMock).toHaveBeenCalledWith(
services,
stopHandler,
options,
'SIGTERM',
);
});
});
describe('#errorHandler', () => {
it('returns a handler to call destroy', () => {
const destroyMock = jest.fn();
const handler = runner.errorHandler(
services,
stopHandler,
options,
'uncaughtException',
destroyMock,
);
expect(typeof handler).toEqual('function');
const err = new Error('Ooops');
handler(err);
expect(destroyMock).toHaveBeenCalledWith(
services,
stopHandler,
options,
'uncaughtException',
err,
);
});
});
describe('#destroy', () => {
it('calls the process.exit gracefully after a short delay', async () => {
await runner.destroy(services, stopHandler, options, 'SIGINT');
expect(mocks.processExit).toHaveBeenCalledWith(0);
});
it('calls the process.exit on error gracefully after a short delay', async () => {
const err = new Error('Ooops');
await runner.destroy(
{
...services,
telemetry: {
// @ts-ignore
logger: {
info: jest.fn(),
error: jest.fn(),
},
},
},
stopHandler,
options,
'uncaughtException',
err,
);
expect(mocks.processExit).toHaveBeenCalledWith(1);
});
it('calls the process.exit on shutdown error gracefully after a short delay', async () => {
const err = new Error('Ooops');
const loggerInfo = jest
.spyOn(telemetry.logger, 'info')
.mockImplementationOnce(() => null)
.mockImplementation(() => {
throw err;
});
const timeout = await runner.destroy(
{
...services,
telemetry: {
logger: {
// @ts-ignore
info: loggerInfo,
debug: jest.fn(),
error: jest.fn(),
},
},
},
stopHandler,
options,
'SIGINT',
);
expect(mocks.processExit).toHaveBeenCalledWith(0);
});
});
describe('#stop', () => {
const OLD_ENV = process.env;
beforeEach(() => {
jest.resetModules();
process.env = { ...OLD_ENV };
});
afterEach(() => {
process.env = OLD_ENV;
});
it('stops the server if running', async () => {
process.env.PORT = '0';
await runner.heartbeat();
const server = runner.getServer();
await server.close();
const _server = {
close: jest.fn(),
};
runner.setServer(_server);
await runner.stop(null, options, null);
expect(_server.close).toHaveBeenCalledTimes(1);
});
it('invokes the stopHandler if available and of type function', async () => {
await runner.stop(null, options, stopHandler);
expect(stopHandler).toHaveBeenCalledTimes(1);
});
it('closes all datastores stream handlers if available', async () => {
const services = {
datastores: {
alice: { streams: { closeAll: jest.fn() } },
bernard: { streams: { closeAll: jest.fn() } },
},
};
await runner.stop(services);
expect(services.datastores.alice.streams.closeAll).toHaveBeenCalledTimes(
1,
);
expect(
services.datastores.bernard.streams.closeAll,
).toHaveBeenCalledTimes(1);
});
});
describe('#heartbeat', () => {
afterEach(async () => {
await runner.stop();
});
it('starts a heartbeat on the given port defined in environment variable', async () => {
const { app } = await runner.heartbeat('0');
const res = await request(app).get('/heartbeat');
expect(res.statusCode).toEqual(200);
expect(res.body).toEqual({
is_alive: true,
});
});
});
describe('#localEventHandler', () => {
it('invokes the handler with the data transmitted in input', async () => {
const handler = jest.fn();
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
false,
);
const entity = { hello: 'world' };
await localHandler(entity);
expect(handler).toHaveBeenCalledWith(entity, {
handlerId: '/utils#log',
path: '/utils',
datastore: 'models',
model: 'users',
source: 'entities',
raw: false,
});
});
it('invokes the ack function if available', async () => {
const handler = jest.fn();
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
false,
);
const entity = { hello: 'world' };
const ack = jest.fn();
await localHandler(
entity,
{},
{},
{
ack,
},
);
expect(ack).toHaveBeenCalledTimes(1);
});
it('invokes the handler with the parsed data transmitted if raw=false', async () => {
const handler = jest.fn();
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
false,
);
const entity = { hello: 'world' };
await localHandler(JSON.stringify(entity));
expect(handler).toHaveBeenCalledWith(entity, {
handlerId: '/utils#log',
path: '/utils',
datastore: 'models',
model: 'users',
source: 'entities',
raw: false,
});
});
it('invokes the handler with the stringified data transmitted if raw=true and the data is not string', async () => {
const handler = jest.fn();
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
);
const entity = { hello: 'world' };
await localHandler(entity);
expect(handler).toHaveBeenCalledWith(JSON.stringify(entity), {
handlerId: '/utils#log',
path: '/utils',
datastore: 'models',
model: 'users',
source: 'entities',
raw: true,
});
});
it('invokes the handler with the stringified data transmitted if raw=true and the data is string', async () => {
const handler = jest.fn();
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
);
const entity = { hello: 'world' };
await localHandler(JSON.stringify(entity));
expect(handler).toHaveBeenCalledWith(JSON.stringify(entity), {
handlerId: '/utils#log',
path: '/utils',
datastore: 'models',
model: 'users',
source: 'entities',
raw: true,
});
});
it('logs an error message in case of exception ', async () => {
const error = new Error('Ooops');
const handler = jest.fn().mockImplementation(() => {
throw error;
});
// @ts-ignore
services.telemetry = { logger: { info: jest.fn(), error: jest.fn() } };
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
);
const entity = { hello: 'world' };
await localHandler(entity);
expect(services.telemetry!.logger.error).toHaveBeenCalledWith(
'Event handler error',
{
message: error.message,
response: undefined,
details: undefined,
msg: entity,
path: '/utils',
handlerId: '/utils#log',
datastore: 'models',
model: 'users',
source: 'entities',
raw: true,
},
);
});
it('invokes the nack function in case of exception ', async () => {
const error = new Error('Ooops');
const handler = jest.fn().mockImplementation(() => {
throw error;
});
// @ts-ignore
services.telemetry = { logger: { info: jest.fn(), error: jest.fn() } };
const nack = jest.fn();
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
);
const entity = { hello: 'world' };
await localHandler(
entity,
{},
{},
{
nack,
},
);
expect(nack).toHaveBeenCalledTimes(1);
});
it('invokes the ack function in case event delivery retry ', async () => {
const error = new Error('Ooops');
const handler = jest.fn().mockImplementation(() => {
throw error;
});
// @ts-ignore
services.telemetry = { logger: { info: jest.fn(), warn: jest.fn() } };
const ack = jest.fn();
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
);
const entity = { hello: 'world' };
await localHandler(
entity,
{},
{},
{
delivery: 1,
ack,
},
);
expect(ack).toHaveBeenCalledTimes(1);
expect(services.telemetry!.logger.warn).toHaveBeenCalledTimes(1);
});
it('is safe event with the telemetry logger error available', async () => {
const error = new Error('Ooops');
const handler = jest.fn().mockImplementation(() => {
throw error;
});
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
);
const entity = { hello: 'world' };
await localHandler(entity);
expect(handler).toHaveBeenCalledTimes(1);
});
it('is handling back pressure on input events', async () => {
const handler = jest.fn().mockImplementation(
// 110ms processing duration leads to max 2 req / 300 ms
() => new Promise((resolve) => setTimeout(resolve, 110)),
);
const stats = {
/**
* 300 ms time window with a processing duration of
* 100 ms leads to a max parallel requests of 2
*/
processingTimeWindowInMilliseconds: 300,
progress: 1,
queuing: 0,
waiting: 0,
waited: 0,
processing: 0,
processed: 0,
totalWaitingDurationInMilliseconds: 0,
averageWaitingDurationInMilliseconds: 0,
totalProcessingDurationInMilliseconds: 0,
averageProcessingDurationInMilliseconds: 0,
maxParallelEvents: Infinity,
};
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
stats,
);
const entity = { hello: 'world' };
await localHandler(entity);
await Promise.all([
localHandler(entity),
localHandler(entity),
localHandler(entity),
localHandler(entity),
localHandler(entity),
]);
expect(stats).toMatchObject({
waited: 3,
processed: 6,
});
expect(handler).toHaveBeenCalledTimes(6);
});
it('is handling back pressure on input events without default stats provided', async () => {
const handler = jest.fn().mockImplementation(
// 110ms processing duration leads to max 2 req / 300 ms
() => new Promise((resolve) => setTimeout(resolve, 110)),
);
const localHandler = runner.localEventHandler(
services,
handler,
'/utils#log',
'models',
'users',
'entities',
true,
);
const entity = { hello: 'world' };
await localHandler(entity);
await Promise.all([
localHandler(entity),
localHandler(entity),
localHandler(entity),
localHandler(entity),
localHandler(entity),
]);
expect(handler).toHaveBeenCalledTimes(6);
});
});
describe('#buildHandler', () => {
let handlerConfig: HandlerConfig;
const OLD_ENV = process.env;
beforeEach(() => {
jest.resetModules();
process.env = { ...OLD_ENV };
handlerConfig = {
datastore: 'models',
model: 'all',
source: 'entities',
start: async () => services,
stop: async () => null,
handler: async (obj: any) => null,
};
});
afterEach(() => {
process.env = OLD_ENV;
jest.restoreAllMocks();
});
it('returns the handler configuration from `handlerId`', async () => {
const config = await runner.buildHandler(
'/utils#log',
{
cwd: '',
},
{
log: async () => handlerConfig,
},
);
expect(config).toEqual({
services,
triggers: [
{
datastore: 'models',
model: 'all',
query: {},
raw: false,
source: 'entities',
headers: {},
queryAsJSONSchema: false,
},
],
handler: handlerConfig.handler,
stop: handlerConfig.stop,
});
});
it('maps Map datastores to object', async () => {
const _services = {
datastores: new Map([
['alice', { streams: { closeAll: jest.fn() } }],
['bernard', { streams: { closeAll: jest.fn() } }],
]),
};
handlerConfig = {
datastore: 'models',
model: 'all',
source: 'entities',
start: async () => _services,
stop: async () => null,
handler: async (obj: any) => null,
};
const config = await runner.buildHandler(
'/utils#log',
{
cwd: '',
},
{
log: async () => handlerConfig,
},
);
expect(config).toEqual({
services: {
datastores: {
alice: _services.datastores.alice,
bernard: _services.datastores.bernard,
},
},
triggers: [
{
datastore: 'models',
model: 'all',
query: {},
raw: false,
source: 'entities',
headers: {},
queryAsJSONSchema: false,
},
],
handler: handlerConfig.handler,
stop: handlerConfig.stop,
});
});
it('returns the handler configuration from `handlerId` with multiple triggers', async () => {
const config = await runner.buildHandler(
'/utils#log',
{
cwd: '',
},
{
log: async () => ({
...handlerConfig,
triggers: [
{
datastore: 'models',
model: 'all',
source: 'entities',
},
],
}),
},
);
expect(config).toEqual({
services,
triggers: [
{
datastore: 'models',
model: 'all',
query: {},
raw: false,
source: 'entities',
headers: {},
queryAsJSONSchema: false,
},
],
handler: handlerConfig.handler,
stop: handlerConfig.stop,
});
});
it('returns the `main` handler if no hash is specified', async () => {
const config = await runner.buildHandler(
'/utils',
{
cwd: '',
},
{
main: async () => handlerConfig,
},
);
expect(config).toEqual({
services,
triggers: [
{
datastore: 'models',
model: 'all',
query: {},
raw: false,
source: 'entities',
headers: {},
queryAsJSONSchema: false,
},
],
handler: handlerConfig.handler,
stop: handlerConfig.stop,
});
});
it('loads the handler from `handlerId`', async () => {
process.env.DATASTORE_API_URL = `http://localhost:${_config.port}`;
const config = await runner.buildHandler(
__dirname + '/__fixtures__/handlers.fixtures.ts#log',
{
cwd: '/',
},
);
expect(config).toMatchObject({
triggers: [
{
model: 'all',
query: {},
raw: false,
source: 'events',
},
],
});
});
});
describe('#start', () => {
let handlerConfig: HandlerConfig;
beforeEach(() => {
jest.setTimeout(30000);
jest.resetModules();
services.datastores.models = {
heartbeat: jest.fn(),
// @ts-ignore
core: {
setTimeout: jest.fn(),
},
// @ts-ignore
streams: {
getStreamId: jest.fn().mockImplementation(() => 'streamId'),
listen: jest.fn(),
on: jest.fn(),
},
};
handlerConfig = {
datastore: 'models',
model: 'users',
source: 'entities',
start: async () => services,
stop: async () => null,
handler: async (obj: any) => null,
};
});
afterEach(async () => {
await runner.stop();
jest.restoreAllMocks();
});
it('starts listening on the handlers provided in input', async () => {
const handler = await runner.start();
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => handlerConfig,
},
);
expect(
services.datastores.models.streams.getStreamId,
).toHaveBeenCalledWith('users', 'entities', {});
expect(services.datastores.models.streams.on.mock.calls[0][0]).toEqual(
'streamId',
);
expect(services.datastores.models.streams.listen).toHaveBeenCalledWith(
'users',
'entities',
{},
{
reconnectionMaxAttempts: undefined,
reconnectionInterval: undefined,
connectionMaxLifeSpanInSeconds: undefined,
queueName: '/utils#log',
queryAsJSONSchema: false,
},
);
});
it('starts the heartbeat if requested', async () => {
const handler = await runner.start();
// @ts-ignore
options.heartbeat = true;
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => handlerConfig,
},
);
expect(runner.getServer()).not.toBeUndefined();
});
it('logs an error if no model is defined in the trigger', async () => {
const handler = await runner.start();
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => ({
datastore: 'models',
// model: 'users', // <- missing
source: 'entities',
start: async () => services,
stop: async () => null,
handler: async (obj: any) => null,
}),
},
);
expect(mocks.loggerError.mock.calls[0][1].message).toEqual(
'Model is not defined',
);
});
it('logs an error if the source is not `entities` or `events`', async () => {
const handler = await runner.start();
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => ({
source: 'invalid',
// ---
datastore: 'models',
model: 'users',
start: async () => services,
stop: async () => null,
handler: async (obj: any) => null,
}),
},
);
expect(mocks.loggerError.mock.calls[0][1].message).toEqual(
'Source must be either `entities` or `events`',
);
});
it('logs an error in case of initialization error', async () => {
const handler = await runner.start();
const error = new Error('Ooops');
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => {
throw error;
},
},
);
expect(mocks.loggerError).toHaveBeenCalledWith(
'Initialization error',
error,
);
});
});
describe('#replay', () => {
let handlerConfig: HandlerConfig;
let walkMultiMock;
beforeEach(() => {
jest.resetModules();
walkMultiMock = jest
.spyOn(utils, 'walkMulti')
.mockImplementation(() => null);
services.datastores.models = {
config: {},
heartbeat: jest.fn(),
walk: jest.fn(),
// @ts-ignore
core: {
setTimeout: jest.fn(),
},
};
handlerConfig = {
datastore: 'models',
model: 'users',
source: 'entities',
start: async () => services,
stop: async () => null,
handler: jest.fn(),
};
});
afterEach(async () => {
await runner.stop();
jest.restoreAllMocks();
});
it('replays all entities on the handlers provided in input', async () => {
const handler = await runner.replay();
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => handlerConfig,
},
);
expect(walkMultiMock.mock.calls[0][1]).toEqual([
{
datastore: 'models',
model: 'users',
query: {},
source: 'entities',
headers: {},
},
]);
expect(walkMultiMock.mock.calls[0][2]).toEqual(10);
});
it('replays only for one entity in `debug` mode', async () => {
const handler = await runner.replay();
walkMultiMock = jest
.spyOn(utils, 'walkMulti')
.mockImplementation(async (_a, _b, _c, handler) => {
const query = {
datastore: 'models',
model: 'users',
source: 'entities',
query: {},
raw: false,
};
await handler({ a: 1 }, query);
await handler({ a: 2 }, query);
await handler({ a: 3 }, query);
await handler({ a: 4 }, query);
await handler({ a: 5 }, query);
});
const run = await handler(
['/utils#log'],
{
...options,
debug: true,
},
{},
{
log: async () => handlerConfig,
},
);
expect(walkMultiMock.mock.calls[0][1]).toEqual([
{
datastore: 'models',
model: 'users',
query: {},
source: 'entities',
headers: {},
},
]);
// Page size forced to 1:
expect(walkMultiMock.mock.calls[0][2]).toEqual(1);
expect(handlerConfig.handler).toHaveBeenCalledTimes(1);
expect(handlerConfig.handler).toHaveBeenCalledWith(
{
a: 1,
},
{
datastore: 'models',
handlerId: '/utils#log',
model: 'users',
path: '/utils',
raw: false,
source: 'entities',
},
);
});
it('logs a message if --verbose is set', async () => {
const handler = await runner.replay();
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => handlerConfig,
},
);
expect(mocks.loggerInfo).toHaveBeenCalledWith(
'[runner] Starting replay',
{
handler_ids: ['/utils#log'],
options,
},
);
});
it('throws an error in case of handler replay exception', async () => {
const handler = await runner.replay();
services.telemetry = telemetry;
handlerConfig = {
datastore: 'models',
model: 'users',
source: 'entities',
start: async () => services,
stop: async () => null,
handler: async () => {
throw new Error('Ooops');
},
};
walkMultiMock = jest
.spyOn(utils, 'walkMulti')
.mockImplementation(async (_a, _b, _c, handler) => {
const query = {
datastore: 'models',
model: 'users',
source: 'entities',
query: {},
raw: false,
};
await handler({ a: 1 }, query);
});
const run = await handler(
['/utils#log', '/utils#log'],
options,
{
safe: true,
},
{
log: async () => handlerConfig,
},
);
expect(mocks.loggerError).toHaveBeenCalledTimes(2);
expect(mocks.loggerError).toHaveBeenCalledWith('Event handler error', {
datastore: 'models',
details: undefined,
handlerId: '/utils#log',
message: 'Ooops',
model: 'users',
msg: {
a: 1,
},
path: '/utils',
raw: false,
response: undefined,
source: 'entities',
});
});
it('skips logging if --verbose is not set', async () => {
const handler = await runner.replay();
/**
* @fixme don't get why the standard way is not working
* here
*/
const initialCallsCount = mocks.loggerInfo.mock.calls.length;
const run = await handler(
['/utils#log'],
{
...options,
verbose: false,
},
{},
{
log: async () => handlerConfig,
},
);
expect(mocks.loggerInfo).toHaveBeenCalledTimes(initialCallsCount);
});
it('starts an heartbeat if required', async () => {
const handler = await runner.replay();
const run = await handler(
['/utils#log'],
{
...options,
heartbeat: true,
},
{},
{
log: async () => handlerConfig,
},
);
expect(runner.getServer()).not.toBeUndefined();
});
it.skip('logs an error if no model is defined in the trigger', async () => {
const handler = await runner.replay();
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => ({
datastore: 'models',
// model: 'users', // <- missing
source: 'entities',
start: async () => services,
stop: async () => null,
handler: async (obj: any) => null,
}),
},
);
expect(mocks.loggerError.mock.calls[0][1].err.message).toEqual(
'Model is not defined',
);
});
it.skip('logs an error if no model is defined in the trigger', async () => {
const handler = await runner.replay();
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => ({
source: 'invalid',
// ---
datastore: 'models',
model: 'users',
start: async () => services,
stop: async () => null,
handler: async (obj: any) => null,
}),
},
);
expect(mocks.loggerError.mock.calls[0][1].err.message).toEqual(
'Source must be either `entities` or `events`',
);
});
it('logs an error in case of initialization error', async () => {
const handler = await runner.replay();
const _error = new Error('Ooops');
let error;
const run = await handler(
['/utils#log'],
options,
{},
{
log: async () => {
throw _error;
},
},
);
expect(mocks.loggerError).toHaveBeenCalledWith('Replay error', {
err: _error,
});
});
});
});