@getanthill/datastore
Version:
Event-Sourced Datastore
551 lines • 21.2 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DEFAULT_STEP_TYPES = exports.STEP_TYPE_FROM = exports.STEP_TYPE_PERSIST = exports.STEP_TYPE_OP = exports.STEP_TYPE_FILTER = exports.STEP_TYPE_IF = exports.STEP_TYPE_EACH = exports.STEP_TYPE_VALIDATE = exports.STEP_TYPE_UNSET = exports.STEP_TYPE_MAP = exports.STEP_TYPE_JSON_PATCH = exports.STEP_TYPE_FETCH = void 0;
const crypto_1 = __importDefault(require("crypto"));
const ajv_1 = __importDefault(require("ajv"));
const ajv_formats_1 = __importDefault(require("ajv-formats"));
const get_1 = __importDefault(require("lodash/get"));
const set_1 = __importDefault(require("lodash/set"));
const unset_1 = __importDefault(require("lodash/unset"));
const mergeWith_1 = __importDefault(require("lodash/mergeWith"));
const cloneDeep_1 = __importDefault(require("lodash/cloneDeep"));
const jsonpatch = __importStar(require("fast-json-patch"));
const schemas_json_1 = __importDefault(require("../schemas.json"));
const lodash_1 = __importDefault(require("lodash"));
const utils = __importStar(require("../../utils"));
const validator = new ajv_1.default({
useDefaults: false,
coerceTypes: true,
strict: false,
});
// @ts-ignore
(0, ajv_formats_1.default)(validator);
/**
* Type of steps
* - fetch
* - map
* - unset
* - validate
* - each
* - filter
* - op
*
* - parallel
* - request
* - json_patch
*/
exports.STEP_TYPE_FETCH = 'fetch';
exports.STEP_TYPE_JSON_PATCH = 'json_patch';
exports.STEP_TYPE_MAP = 'map';
exports.STEP_TYPE_UNSET = 'unset';
exports.STEP_TYPE_VALIDATE = 'validate';
exports.STEP_TYPE_EACH = 'each';
exports.STEP_TYPE_IF = 'if';
exports.STEP_TYPE_FILTER = 'filter';
exports.STEP_TYPE_OP = 'op';
exports.STEP_TYPE_PERSIST = 'persist';
exports.STEP_TYPE_FROM = 'from';
exports.DEFAULT_STEP_TYPES = Object.freeze([
exports.STEP_TYPE_FETCH,
exports.STEP_TYPE_JSON_PATCH,
exports.STEP_TYPE_MAP,
exports.STEP_TYPE_UNSET,
exports.STEP_TYPE_VALIDATE,
exports.STEP_TYPE_EACH,
exports.STEP_TYPE_IF,
exports.STEP_TYPE_FILTER,
exports.STEP_TYPE_OP,
exports.STEP_TYPE_PERSIST,
exports.STEP_TYPE_FROM,
]);
class Aggregator {
constructor(datastores, config = {
max_retry: 0,
}) {
this.steps = new Map();
this.logs = [];
this._pipelineValidator = Aggregator.pipelineValidator;
this.datastores = datastores;
this.config = config;
this.metrics = {};
}
static _customizer(objValue, srcValue) {
if (Array.isArray(objValue)) {
return objValue.concat(srcValue);
}
}
get pipelineValidator() {
return this._pipelineValidator;
}
log(level, msg, context) {
this.logs.push({
ts: Date.now(),
level,
msg,
context,
});
}
addStepType(stepType, stepDefinition) {
if (this.steps.has(stepType)) {
throw Aggregator.ERROR_CONFLICT_STEP_TYPE;
}
this.steps.set(stepType, stepDefinition);
this.updateValidator();
return this;
}
removeStepType(stepType) {
this.steps.delete(stepType);
this.updateValidator();
return this;
}
updateValidator() {
const _schemas = (0, cloneDeep_1.default)(schemas_json_1.default);
for (const key of this.steps.keys()) {
_schemas[0].items?.oneOf?.push({
$ref: `/schemas/datastore/aggregator/step/${key}`,
});
_schemas.push({
$id: `/schemas/datastore/aggregator/step/${key}`,
type: 'object',
required: ['type'],
// @ts-ignore
properties: {
type: {
$ref: '/schemas/datastore/aggregator/components/type',
enum: [key],
},
},
});
}
this._pipelineValidator = new ajv_1.default({
schemas: _schemas,
useDefaults: false,
coerceTypes: false,
strict: true,
});
}
mergeData(step, data = {}, results = step.default || [], destination = step.destination) {
if (!destination) {
throw Aggregator.ERROR_DESTINATION_UNDEFINED;
}
const res = step.as_entity === true ? results[0] || step.default : results;
if (step.as_entity === true && res === undefined) {
throw Aggregator.ERROR_ENTITY_NOT_FOUND;
}
return (0, mergeWith_1.default)({}, data, step.as_entity === true && destination === '.'
? res
: (0, set_1.default)({}, destination, res), Aggregator._customizer);
}
static ok(condition, message) {
if (condition !== true) {
const err = new Error(message);
throw err;
}
}
static hash(value) {
const hash = crypto_1.default.createHash('sha512');
hash.update(value);
return hash.digest('hex');
}
applyPatch(data, patch) {
const { newDocument } = jsonpatch.applyPatch(data, patch, true);
return newDocument;
}
applyMap(source = {}, map = [], data = null) {
map.forEach(({ from, to, default: defaultValue, must_hash: mustHash, json_stringify: jsonStringify, relative_date_in_seconds: relativeDateInSeconds, }) => {
let value = from === '.' ? data ?? defaultValue : (0, get_1.default)(data, from, defaultValue);
if (relativeDateInSeconds) {
const start = value ? new Date(value).getTime() : Date.now();
value = new Date(start + relativeDateInSeconds * 1000);
}
if (jsonStringify === true) {
value = JSON.stringify(value);
}
if (mustHash === true) {
value = Aggregator.hash(value);
}
if (to === '.') {
source = value;
}
else {
(0, set_1.default)(source, to, value);
}
});
return source;
}
async fetch(step, data = {}) {
const datastore = this.getDatastore(step.datastore);
Aggregator.ok(typeof datastore.walk === 'function', 'Invalid datastore');
let results = [];
const query = this.applyMap(step.query, step.map, data);
this.log('debug', 'Fetch query', {
model: step.model,
source: step.source,
headers: step.headers,
query,
});
if ('page' in step || 'page_size' in step) {
if (step.source === 'events') {
const { data } = await datastore.allEvents(step.model, query, step.page, step.page_size, step.headers);
results = data;
}
else {
const { data } = await datastore.find(step.model, query, step.page, step.page_size, step.headers);
results = data;
}
}
else {
await datastore.walk(step.model, query, (entity) => {
results.push(entity);
}, 10, step.source, step.headers);
}
const timetravelDate = (0, get_1.default)(data, step.timetravel ?? '');
Aggregator.ok(step.timetravel === undefined ||
(!!timetravelDate && !!step.correlation_field), 'Invalid timetravel condition');
if (!!timetravelDate && !!step.correlation_field) {
const res = await Promise.all(results.map((r) => datastore.at(step.model, r[step.correlation_field], timetravelDate)));
results = res.map((r) => r.data).filter((v) => v);
}
if (step.must_decrypt === true) {
const res = await datastore.decrypt(step.model, results);
results = res.data;
}
return results;
}
getDatastore(datastore) {
if (datastore) {
const _datastore = this.datastores.get(datastore);
if (_datastore) {
return _datastore;
}
}
const [_datastore] = this.datastores.values();
return _datastore;
}
async persist(step, data = {}) {
const datastore = this.getDatastore(step.datastore);
Aggregator.ok(typeof datastore.walk === 'function', 'Invalid datastore');
const payload = this.applyMap(step.payload, step.map, data);
const headers = {
...(step.headers || {}),
};
if ('imperative_version_next' in step) {
headers.version = ((0, get_1.default)(data, step.imperative_version_next) ?? -1) + 1;
}
const correlationField = step.correlation_field ?? null;
const correlationId = correlationField !== null ? payload[correlationField] : null;
Aggregator.ok(!!correlationId || correlationId === null, 'Correlation ID must be null or exist');
let result;
if (correlationId === null) {
if (payload.created_at) {
headers['created-at'] = payload.created_at;
delete payload.created_at;
delete payload.updated_at;
}
const res = await datastore.create(step.model, payload, headers);
result = res.data;
}
else {
if ('updated_at' in payload || 'created_at' in payload) {
headers['created-at'] = payload.updated_at || payload.created_at;
delete payload.created_at;
delete payload.updated_at;
}
const res = await datastore.update(step.model, correlationId, payload, headers);
result = res.data;
}
return result;
}
async runStepFetch(step, data = {}) {
const results = await this.fetch(step, data);
return this.mergeData(step, data, results, step.destination ?? step.model);
}
async runStepPersist(step, data = {}) {
const res = await this.persist(step, data);
const destination = step.destination || 'persist';
(0, set_1.default)(data, destination, res);
return data;
}
async runStepJsonPatch(step, data = {}) {
return this.applyPatch(data, step.patch);
}
async runStepMap(step, data = {}) {
return this.applyMap(data, step.map, data);
}
async runStepUnset(step, data = {}) {
(0, unset_1.default)(data, step.path);
return data;
}
async runStepValidate(step, data = {}) {
const value = step.path ? (0, get_1.default)(data, step.path) : data;
const isValid = validator.validate(step.schema, value);
this.log('debug', 'Validate', {
step,
value,
is_valid: isValid,
});
if (isValid === false && step.must_throw === true) {
const err = new Error(Aggregator.ERROR_VALIDATE_STEP_FAILED.message);
// @ts-ignore
err.step = step;
// @ts-ignore
err.schema = step.schema;
// @ts-ignore
err.value = value;
// @ts-ignore
err.errors = validator.errors;
throw err;
}
const destination = step.destination ?? 'validation';
const res = {
is_valid: isValid,
errors: validator.errors,
};
(0, set_1.default)(data, destination, res);
return data;
}
async runStepEach(step, data = {}) {
const items = (0, get_1.default)(data, step.path, null);
if (!Array.isArray(items)) {
throw Aggregator.ERROR_ITERATE_STEP_IS_NOT_ARRAY;
}
const aggregates = await Promise.all(items.map((item) => this.aggregate(step.pipeline, {
...item,
_: data,
})));
const destination = step.destination ?? step.path;
(0, set_1.default)(data, destination, aggregates.map((agg) => {
delete agg._;
return agg;
}));
return data;
}
async runStepIf(step, data = {}, i = 0) {
const maxIterationCount = step.max_iteration_count ?? 100;
if (i >= maxIterationCount) {
return data;
}
const value = step.path ? (0, get_1.default)(data, step.path) : data;
const isValid = validator.validate(step.schema, value);
if (isValid === false) {
return data;
}
const _data = await this.aggregate(step.pipeline, data);
if (step.repeat_while_true === true) {
return this.runStepIf(step, _data, i + 1);
}
return _data;
}
async runStepFilter(step, data = {}) {
const items = (0, get_1.default)(data, step.path, null);
if (!Array.isArray(items)) {
throw Aggregator.ERROR_ITERATE_STEP_IS_NOT_ARRAY;
}
const schema = this.applyMap(step.schema, step.map, data);
const results = items.filter((item) => validator.validate(schema, item));
if (!step.destination) {
(0, unset_1.default)(data, step.path);
}
return this.mergeData(step, data, results, step.destination ?? step.path);
}
/**
* @alpha
*/
async runStepOp(step, data = null) {
const value = step.path
? (0, get_1.default)(data, step.path, step.default)
: step.default ?? data;
let args = 'args' in step
? step.args.map((arg) => {
if (typeof arg !== 'object') {
return arg;
}
if (arg.func) {
// @ts-ignore
return lodash_1.default[arg.func];
}
if (arg.path === '.') {
return value;
}
return (0, get_1.default)(value, arg.path, arg.default);
})
: [value];
if (step.args_as_array === true) {
args = [args];
}
let res = null;
if (step.func === 'length') {
res = args[0].length;
}
else if (step.func === 'date') {
res = utils.getDate(args[0]).toISOString();
}
else if (step.func === 'timestamp') {
res = utils.getDate(args[0]).getTime();
}
else {
/* @ts-ignore */
res = lodash_1.default[step.func].apply(null, args);
}
const destination = step.destination ?? step.func;
data = data === null ? {} : data;
if (destination === '.') {
return res;
}
(0, set_1.default)(data, destination, res);
return data;
}
async runStepFrom(step, data = {}) {
const steps = (0, get_1.default)(data, step.path, []);
return this.aggregate(steps, data);
}
async runStep(step, data = {}) {
utils.mapValuesDeep(step, (v) => typeof v === 'string' && v.startsWith('{') && v.endsWith('}')
? (0, cloneDeep_1.default)((0, get_1.default)(data, v.slice(1, -1), v))
: v);
const stepDefinition = this.steps.get(step.type);
if (stepDefinition !== undefined) {
const stepHandler = stepDefinition.handler;
Aggregator.ok(typeof stepHandler === 'function', 'Invalid step handler defined');
if (!!stepDefinition.schema &&
validator.validate(stepDefinition.schema, step) !== true) {
throw Aggregator.ERROR_CONTRACT_ERROR_STEP_TYPE;
}
return stepHandler.call(this, step, data);
}
if (step.type === exports.STEP_TYPE_FETCH) {
return this.runStepFetch(step, data);
}
if (step.type === exports.STEP_TYPE_JSON_PATCH) {
return this.runStepJsonPatch(step, data);
}
if (step.type === exports.STEP_TYPE_MAP) {
return this.runStepMap(step, data);
}
if (step.type === exports.STEP_TYPE_UNSET) {
return this.runStepUnset(step, data);
}
if (step.type === exports.STEP_TYPE_VALIDATE) {
return this.runStepValidate(step, data);
}
if (step.type === exports.STEP_TYPE_EACH) {
return this.runStepEach(step, data);
}
if (step.type === exports.STEP_TYPE_IF) {
return this.runStepIf(step, data);
}
if (step.type === exports.STEP_TYPE_FILTER) {
return this.runStepFilter(step, data);
}
if (step.type === exports.STEP_TYPE_OP) {
return this.runStepOp(step, data);
}
if (step.type === exports.STEP_TYPE_PERSIST) {
return this.runStepPersist(step, data);
}
if (step.type === exports.STEP_TYPE_FROM) {
return this.runStepFrom(step, data);
}
throw Aggregator.ERROR_INVALID_STEP_TYPE;
}
validate(pipeline) {
const isValid = this.pipelineValidator.validate('/schemas/datastore/aggregator/pipeline', pipeline);
if (isValid === false) {
const err = Aggregator.ERROR_INVALID_PIPELINE_DEFINITION;
// @ts-ignore
err.details = this.pipelineValidator.errors;
throw err;
}
}
async aggregate(pipeline, initialData = {}, iteration = 0) {
this.validate(pipeline);
this.logs = [];
const _pipeline = (0, cloneDeep_1.default)(pipeline);
let data = (0, cloneDeep_1.default)(initialData);
this.metrics.started_at_in_ms = this.metrics.started_at_in_ms ?? Date.now();
this.metrics.iteration = iteration;
try {
for (const step of _pipeline) {
this.log('debug', 'Before step', { step, data });
data = await this.runStep(step, data);
this.log('debug', 'After step', { step, data });
}
}
catch (err) {
if (iteration > (this.config.max_retry || 0)) {
this.log('error', 'Max retry reached', {
err,
iteration,
max_retry: this.config.max_retry || 0,
});
throw err;
}
this.log('error', 'Retrying the aggregation', {
err,
iteration,
max_retry: this.config.max_retry || 0,
});
return this.aggregate(pipeline, initialData, iteration + 1);
}
this.metrics.ended_at_in_ms = Date.now();
this.metrics.elapsed_time_in_ms =
this.metrics.ended_at_in_ms - this.metrics.started_at_in_ms;
this.log('debug', 'Aggregation ended successfully', {
metrics: this.metrics,
});
return data;
}
}
Aggregator.pipelineValidator = new ajv_1.default({
schemas: schemas_json_1.default,
useDefaults: false,
coerceTypes: false,
strict: true,
});
Aggregator.ERROR_INVALID_PIPELINE_DEFINITION = new Error('Invalid pipeline definition');
Aggregator.ERROR_INVALID_STEP_TYPE = new Error('Invalid step type');
Aggregator.ERROR_ENTITY_NOT_FOUND = new Error('Entity not found');
Aggregator.ERROR_DESTINATION_UNDEFINED = new Error('Destination must be defined');
Aggregator.ERROR_CONFLICT_STEP_TYPE = new Error('Conflict on step type');
Aggregator.ERROR_CONTRACT_ERROR_STEP_TYPE = new Error('Contract error on step type definition');
Aggregator.ERROR_VALIDATE_STEP_FAILED = new Error('Validate step failed');
Aggregator.ERROR_ITERATE_STEP_IS_NOT_ARRAY = new Error('Iterate step not on array');
Aggregator.ERROR_OP_STEP_INVALID_VALUE = new Error('Op step value is invalid');
Aggregator.ERRORS = [
Aggregator.ERROR_INVALID_STEP_TYPE,
Aggregator.ERROR_ENTITY_NOT_FOUND,
Aggregator.ERROR_DESTINATION_UNDEFINED,
Aggregator.ERROR_CONFLICT_STEP_TYPE,
Aggregator.ERROR_CONTRACT_ERROR_STEP_TYPE,
Aggregator.ERROR_VALIDATE_STEP_FAILED,
Aggregator.ERROR_ITERATE_STEP_IS_NOT_ARRAY,
Aggregator.ERROR_OP_STEP_INVALID_VALUE,
];
exports.default = Aggregator;
//# sourceMappingURL=Aggregator.js.map