@getanthill/datastore
Version:
Event-Sourced Datastore
331 lines • 14.4 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ERRORS = void 0;
exports.objToJsonSchema = objToJsonSchema;
exports.defaultWalkMultiSortHandler = defaultWalkMultiSortHandler;
exports.sortResults = sortResults;
exports.getMinVersions = getMinVersions;
exports.getMaxVersions = getMaxVersions;
exports.fetchResultsForQuery = fetchResultsForQuery;
exports.handleResults = handleResults;
exports.handleIterationWithMutation = handleIterationWithMutation;
exports.getLastMatchingCorrelationIdIndex = getLastMatchingCorrelationIdIndex;
exports.checkOptions = checkOptions;
exports.walkMulti = walkMulti;
const chunk_1 = __importDefault(require("lodash/chunk"));
const lodash_1 = require("lodash");
exports.ERRORS = {
INCOMPATIBLE_MULTIPLE_HANDLE_OPTIONS: 'Only one "handle in" option must be defined',
INCOMPATIBLE_MUTATION_ON_EVENTS: 'Options are incompatible: mutation on events source is not supported',
};
function objToJsonSchema(obj) {
var _a, _b;
if (obj === null) {
return {
type: 'null',
nullable: true,
};
}
if (typeof obj === 'number') {
return {
type: 'number',
enum: [obj],
};
}
if (typeof obj === 'string') {
return {
type: 'string',
enum: [obj],
};
}
if (obj instanceof Date) {
return {
type: 'string',
enum: [obj.toISOString()],
};
}
if (typeof obj === 'boolean') {
return {
type: 'boolean',
enum: [obj],
};
}
if (Array.isArray(obj)) {
// @ts-ignore
return {
type: 'array',
items: obj.map(objToJsonSchema),
};
}
const schema = {
type: 'object',
required: [],
properties: {},
};
for (const key of Object.keys(obj)) {
schema.properties[key] = objToJsonSchema(obj[key]);
obj[key] !== null && ((_a = schema.required) === null || _a === void 0 ? void 0 : _a.push(key));
}
if (((_b = schema.required) === null || _b === void 0 ? void 0 : _b.length) === 0) {
delete schema.required;
}
return schema;
}
function defaultWalkMultiSortHandler(a, b) {
var _a, _b, _c, _d;
const aComparisonDate = (_a = a.updated_at) !== null && _a !== void 0 ? _a : a.created_at;
const bComparisonDate = (_b = b.updated_at) !== null && _b !== void 0 ? _b : b.created_at;
const dateComparisonValue = aComparisonDate.localeCompare(bComparisonDate);
return dateComparisonValue === 0
? ((_c = a.version) !== null && _c !== void 0 ? _c : 0) - ((_d = b.version) !== null && _d !== void 0 ? _d : 0)
: dateComparisonValue;
}
function sortResults(results, sortHandler = defaultWalkMultiSortHandler) {
return results.sort(sortHandler);
}
async function getMinVersions(datastores, queries) {
return Promise.all(queries.map((q) => {
var _a, _b;
return (_b = (_a = datastores
.get(q.datastore)) === null || _a === void 0 ? void 0 : _a.minEventsVersion(q.model, q.query, q.headers)) !== null && _b !== void 0 ? _b : 0;
}));
}
async function getMaxVersions(datastores, queries) {
return Promise.all(queries.map((q) => {
var _a, _b;
return (_b = (_a = datastores
.get(q.datastore)) === null || _a === void 0 ? void 0 : _a.maxEventsVersion(q.model, q.query, q.headers)) !== null && _b !== void 0 ? _b : -1;
}));
}
async function fetchResultsForQuery(datastores, query, pageSize, queryIteration, opts) {
var _a, _b, _c;
const isVersionOrdered = (_a = opts === null || opts === void 0 ? void 0 : opts.version_ordered) !== null && _a !== void 0 ? _a : false;
const { data: results, headers } = await datastores
.get(query.datastore)
.walkNext(query.model, query.query, query.source, queryIteration.page, 2 * pageSize, {
cursor_last_id: queryIteration.cursor_last_id,
cursor_last_correlation_id: queryIteration.cursor_last_correlation_id,
current_version: queryIteration.version,
version_ordered: isVersionOrdered,
headers: query.headers,
});
query.correlationField = headers['correlation-field'];
if (results.length < pageSize && isVersionOrdered === false) {
queryIteration.is_exhausted = true;
queryIteration.results.push(...results);
return results;
}
const cursorLastId = (_b = headers === null || headers === void 0 ? void 0 : headers['cursor-last-id']) !== null && _b !== void 0 ? _b : '';
const cursorLastCorrelationId = (_c = headers === null || headers === void 0 ? void 0 : headers['cursor-last-correlation-id']) !== null && _c !== void 0 ? _c : '';
if (!!cursorLastId &&
!!queryIteration.cursor_last_id &&
cursorLastId === queryIteration.cursor_last_id &&
cursorLastCorrelationId === queryIteration.cursor_last_correlation_id) {
throw new Error('Same cursor last id after iteration');
}
queryIteration.results.push(...results);
queryIteration.page += 1;
queryIteration.cursor_last_id = cursorLastId;
queryIteration.cursor_last_correlation_id = cursorLastCorrelationId;
if (results.length < pageSize &&
queryIteration.version >= queryIteration.max_version) {
queryIteration.is_exhausted = true;
}
if (results.length < pageSize &&
queryIteration.version < queryIteration.max_version) {
const nextVersion = await datastores.get(query.datastore).minEventsVersion(query.model, {
...query.query,
version: {
$gt: queryIteration.version,
},
}, query.headers);
queryIteration.version = nextVersion;
queryIteration.cursor_last_id = '';
queryIteration.cursor_last_correlation_id = '';
queryIteration.page = 0;
}
return results;
}
async function handleResults(results, queries, iteration, handler, opts) {
let batch = [];
let batchId = 0;
let ids = new Set();
for (const [ri, r] of results.entries()) {
if ((opts === null || opts === void 0 ? void 0 : opts.handle_in_order) === true) {
await handler(r.result, queries[r.queryIndex], iteration.get(r.queryIndex), batchId, 0);
batchId += 1;
continue;
}
const query = queries[r.queryIndex];
const correlationId = r.result[query.correlationField];
const resultId = (opts === null || opts === void 0 ? void 0 : opts.handle_in_parallel) === true
? ri
: `${query.datastore}/${query.model}/${correlationId}`;
if (ids.has(resultId)) {
await Promise.all(batch.map(({ result, queryIndex }, index) => {
return handler(result, queries[queryIndex], iteration.get(queryIndex), batchId, index);
}));
batchId += 1;
batch = [];
ids = new Set();
}
ids.add(resultId);
batch.push(r);
}
await Promise.all(batch.map(({ result, queryIndex }, index) => {
return handler(result, queries[queryIndex], iteration.get(queryIndex), batchId, index);
}));
}
async function fetchResultsForQueries(datastores, iteration, queries, pageSize, opts) {
var _a;
const batches = (0, chunk_1.default)(queries, (_a = opts === null || opts === void 0 ? void 0 : opts.chunk_size) !== null && _a !== void 0 ? _a : 5);
for (const [i, batch] of batches.entries()) {
await Promise.all(batch.map((q, j) => {
var _a;
const queryIteration = iteration.get(i * ((_a = opts === null || opts === void 0 ? void 0 : opts.chunk_size) !== null && _a !== void 0 ? _a : 5) + j);
if (queryIteration.is_exhausted === true) {
return Promise.resolve({ data: [] });
}
if (queryIteration.results.length >= pageSize) {
return Promise.resolve(queryIteration.results);
}
return fetchResultsForQuery(datastores, q, pageSize, queryIteration, {
version_ordered: opts === null || opts === void 0 ? void 0 : opts.version_ordered,
});
}));
}
}
async function fetchSortedResults(datastores, iteration, queries, pageSize, sortHandler, opts) {
const results = [];
await fetchResultsForQueries(datastores, iteration, queries, pageSize, opts);
const sortableResults = [];
iteration.forEach((queryIteration, queryIndex) => {
sortResults(queryIteration.results, sortHandler);
sortableResults.push(...queryIteration.results.map((result, resultIndex) => ({
result,
created_at: result.created_at,
updated_at: result.updated_at,
queryIndex,
resultIndex,
})));
});
sortResults(sortableResults, sortHandler);
while (sortableResults.length > 0 && results.length < pageSize) {
const result = sortableResults.shift();
iteration.get(result.queryIndex).results.shift();
results.push(result);
}
return results;
}
async function handleIterationWithMutation(datastores, iteration, clonedIteration, queries, pageSize, opts) {
// Clean previous results
clonedIteration.forEach((queryIteration) => {
queryIteration.results = [];
});
await fetchResultsForQueries(datastores, clonedIteration, queries, pageSize, opts);
/**
* @todo check same state to avoid processing
*/
// if (!isEqual(resultsAfterMutation, results)) {
// }
iteration.forEach((queryIteration, index) => {
const clonedQueryIteration = clonedIteration.get(index);
const lastMatchingCorrelationIdIndex = getLastMatchingCorrelationIdIndex(queryIteration.results.map((r) => r[queries[index].correlationField]), clonedQueryIteration.results.map((r) => r[queries[index].correlationField]));
const resultsToPush = clonedQueryIteration.results.slice(lastMatchingCorrelationIdIndex + 1);
queryIteration.results.push(...resultsToPush);
queryIteration.cursor_last_correlation_id =
clonedQueryIteration.cursor_last_correlation_id;
queryIteration.cursor_last_id = clonedQueryIteration.cursor_last_id;
queryIteration.page = clonedQueryIteration.page;
queryIteration.version = clonedQueryIteration.version;
queryIteration.is_exhausted = clonedQueryIteration.is_exhausted;
});
}
function getLastMatchingCorrelationIdIndex(before, after) {
const lastCorrelationId = before[before.length - 1];
return after.lastIndexOf(lastCorrelationId);
}
function checkOptions(queries, opts) {
if ((opts === null || opts === void 0 ? void 0 : opts.handle_in_order) === true && (opts === null || opts === void 0 ? void 0 : opts.handle_in_parallel) === true) {
throw new Error(exports.ERRORS.INCOMPATIBLE_MULTIPLE_HANDLE_OPTIONS);
}
for (const query of queries) {
if (query.source === 'events' && (opts === null || opts === void 0 ? void 0 : opts.is_mutating) === true) {
throw new Error(exports.ERRORS.INCOMPATIBLE_MUTATION_ON_EVENTS);
}
}
}
async function walkMulti(datastores, queries, pageSize = 100, handler, opts, sortHandler = defaultWalkMultiSortHandler) {
checkOptions(queries, opts);
queries.forEach((q) => {
if ('_fields' in q.query) {
q.query._fields = {
...q.query._fields,
created_at: 1,
updated_at: 1,
};
}
return q;
});
const minVersions = (opts === null || opts === void 0 ? void 0 : opts.version_ordered) === true
? await getMinVersions(datastores, queries)
: queries.map(() => -1);
const maxVersions = (opts === null || opts === void 0 ? void 0 : opts.version_ordered) === true
? await getMaxVersions(datastores, queries)
: queries.map(() => -1);
const iteration = new Map(queries.map((_, i) => [
i,
{
query_index: i,
page: 0,
version: minVersions[i],
max_version: maxVersions[i],
is_exhausted: false,
cursor_last_id: '',
cursor_last_correlation_id: '',
results: [],
},
]));
const handledCorrelatoniIds = new Set();
let fullyExhausted;
let clonedIteration = new Map();
do {
fullyExhausted = true;
if ((opts === null || opts === void 0 ? void 0 : opts.is_mutating) === true) {
clonedIteration = (0, lodash_1.cloneDeep)(iteration);
}
let results = await fetchSortedResults(datastores, iteration, queries, pageSize, sortHandler, opts);
if ((opts === null || opts === void 0 ? void 0 : opts.is_mutating) === true) {
results = results.filter((result) => {
const query = queries[result.queryIndex];
const correlationId = `${query.datastore}:${query.model}:${result.result[query.correlationField]}`;
if (handledCorrelatoniIds.has(correlationId)) {
return false;
}
handledCorrelatoniIds.add(correlationId);
return true;
});
}
await handleResults(results, queries, iteration, handler, {
handle_in_order: opts === null || opts === void 0 ? void 0 : opts.handle_in_order,
handle_in_parallel: opts === null || opts === void 0 ? void 0 : opts.handle_in_parallel,
});
if ((opts === null || opts === void 0 ? void 0 : opts.sleep) && opts.sleep > 0) {
await new Promise((resolve) => setTimeout(resolve, opts === null || opts === void 0 ? void 0 : opts.sleep));
}
Array.from(iteration.values()).every((it) => {
if (it.is_exhausted === false || it.results.length > 0) {
fullyExhausted = false;
return false;
}
return true;
});
if ((opts === null || opts === void 0 ? void 0 : opts.is_mutating) === true) {
await handleIterationWithMutation(datastores, iteration, clonedIteration, queries, pageSize, opts);
}
} while (fullyExhausted === false);
}
//# sourceMappingURL=utils.js.map