@getanthill/datastore
Version:
Event-Sourced Datastore
322 lines • 13.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.walkMulti = exports.checkOptions = exports.getLastMatchingCorrelationIdIndex = exports.handleIterationWithMutation = exports.handleResults = exports.fetchResultsForQuery = exports.getMaxVersions = exports.getMinVersions = exports.sortResults = exports.defaultWalkMultiSortHandler = exports.objToJsonSchema = exports.ERRORS = void 0;
const chunk_1 = __importDefault(require("lodash/chunk"));
const lodash_1 = require("lodash");
exports.ERRORS = {
INCOMPATIBLE_MULTIPLE_HANDLE_OPTIONS: 'Only one "handle in" option must be defined',
INCOMPATIBLE_MUTATION_ON_EVENTS: 'Options are incompatible: mutation on events source is not supported',
};
function objToJsonSchema(obj) {
if (obj === null) {
return {
type: 'null',
nullable: true,
};
}
if (typeof obj === 'number') {
return {
type: 'number',
enum: [obj],
};
}
if (typeof obj === 'string') {
return {
type: 'string',
enum: [obj],
};
}
if (obj instanceof Date) {
return {
type: 'string',
enum: [obj.toISOString()],
};
}
if (typeof obj === 'boolean') {
return {
type: 'boolean',
enum: [obj],
};
}
if (Array.isArray(obj)) {
// @ts-ignore
return {
type: 'array',
items: obj.map(objToJsonSchema),
};
}
const schema = {
type: 'object',
required: [],
properties: {},
};
for (const key of Object.keys(obj)) {
schema.properties[key] = objToJsonSchema(obj[key]);
obj[key] !== null && schema.required?.push(key);
}
if (schema.required?.length === 0) {
delete schema.required;
}
return schema;
}
exports.objToJsonSchema = objToJsonSchema;
function defaultWalkMultiSortHandler(a, b) {
const aComparisonDate = a.updated_at ?? a.created_at;
const bComparisonDate = b.updated_at ?? b.created_at;
const dateComparisonValue = aComparisonDate.localeCompare(bComparisonDate);
return dateComparisonValue === 0
? (a.version ?? 0) - (b.version ?? 0)
: dateComparisonValue;
}
exports.defaultWalkMultiSortHandler = defaultWalkMultiSortHandler;
function sortResults(results, sortHandler = defaultWalkMultiSortHandler) {
return results.sort(sortHandler);
}
exports.sortResults = sortResults;
async function getMinVersions(datastores, queries) {
return Promise.all(queries.map((q) => datastores
.get(q.datastore)
?.minEventsVersion(q.model, q.query, q.headers) ?? 0));
}
exports.getMinVersions = getMinVersions;
async function getMaxVersions(datastores, queries) {
return Promise.all(queries.map((q) => datastores
.get(q.datastore)
?.maxEventsVersion(q.model, q.query, q.headers) ?? -1));
}
exports.getMaxVersions = getMaxVersions;
async function fetchResultsForQuery(datastores, query, pageSize, queryIteration, opts) {
const isVersionOrdered = opts?.version_ordered ?? false;
const { data: results, headers } = await datastores
.get(query.datastore)
.walkNext(query.model, query.query, query.source, queryIteration.page, 2 * pageSize, {
cursor_last_id: queryIteration.cursor_last_id,
cursor_last_correlation_id: queryIteration.cursor_last_correlation_id,
current_version: queryIteration.version,
version_ordered: isVersionOrdered,
headers: query.headers,
});
query.correlationField = headers['correlation-field'];
if (results.length < pageSize && isVersionOrdered === false) {
queryIteration.is_exhausted = true;
queryIteration.results.push(...results);
return results;
}
const cursorLastId = headers?.['cursor-last-id'] ?? '';
const cursorLastCorrelationId = headers?.['cursor-last-correlation-id'] ?? '';
if (!!cursorLastId &&
!!queryIteration.cursor_last_id &&
cursorLastId === queryIteration.cursor_last_id &&
cursorLastCorrelationId === queryIteration.cursor_last_correlation_id) {
throw new Error('Same cursor last id after iteration');
}
queryIteration.results.push(...results);
queryIteration.page += 1;
queryIteration.cursor_last_id = cursorLastId;
queryIteration.cursor_last_correlation_id = cursorLastCorrelationId;
if (results.length < pageSize &&
queryIteration.version >= queryIteration.max_version) {
queryIteration.is_exhausted = true;
}
if (results.length < pageSize &&
queryIteration.version < queryIteration.max_version) {
const nextVersion = await datastores.get(query.datastore).minEventsVersion(query.model, {
...query.query,
version: {
$gt: queryIteration.version,
},
}, query.headers);
queryIteration.version = nextVersion;
queryIteration.cursor_last_id = '';
queryIteration.cursor_last_correlation_id = '';
queryIteration.page = 0;
}
return results;
}
exports.fetchResultsForQuery = fetchResultsForQuery;
async function handleResults(results, queries, iteration, handler, opts) {
let batch = [];
let batchId = 0;
let ids = new Set();
for (const [ri, r] of results.entries()) {
if (opts?.handle_in_order === true) {
await handler(r.result, queries[r.queryIndex], iteration.get(r.queryIndex), batchId, 0);
batchId += 1;
continue;
}
const query = queries[r.queryIndex];
const correlationId = r.result[query.correlationField];
const resultId = opts?.handle_in_parallel === true
? ri
: `${query.datastore}/${query.model}/${correlationId}`;
if (ids.has(resultId)) {
await Promise.all(batch.map(({ result, queryIndex }, index) => {
return handler(result, queries[queryIndex], iteration.get(queryIndex), batchId, index);
}));
batchId += 1;
batch = [];
ids = new Set();
}
ids.add(resultId);
batch.push(r);
}
await Promise.all(batch.map(({ result, queryIndex }, index) => {
return handler(result, queries[queryIndex], iteration.get(queryIndex), batchId, index);
}));
}
exports.handleResults = handleResults;
async function fetchResultsForQueries(datastores, iteration, queries, pageSize, opts) {
const batches = (0, chunk_1.default)(queries, opts?.chunk_size ?? 5);
for (const [i, batch] of batches.entries()) {
await Promise.all(batch.map((q, j) => {
const queryIteration = iteration.get(i * (opts?.chunk_size ?? 5) + j);
if (queryIteration.is_exhausted === true) {
return Promise.resolve({ data: [] });
}
if (queryIteration.results.length >= pageSize) {
return Promise.resolve(queryIteration.results);
}
return fetchResultsForQuery(datastores, q, pageSize, queryIteration, {
version_ordered: opts?.version_ordered,
});
}));
}
}
async function fetchSortedResults(datastores, iteration, queries, pageSize, sortHandler, opts) {
const results = [];
await fetchResultsForQueries(datastores, iteration, queries, pageSize, opts);
const sortableResults = [];
iteration.forEach((queryIteration, queryIndex) => {
sortResults(queryIteration.results, sortHandler);
sortableResults.push(...queryIteration.results.map((result, resultIndex) => ({
result,
created_at: result.created_at,
updated_at: result.updated_at,
queryIndex,
resultIndex,
})));
});
sortResults(sortableResults, sortHandler);
while (sortableResults.length > 0 && results.length < pageSize) {
const result = sortableResults.shift();
iteration.get(result.queryIndex).results.shift();
results.push(result);
}
return results;
}
async function handleIterationWithMutation(datastores, iteration, clonedIteration, queries, pageSize, opts) {
// Clean previous results
clonedIteration.forEach((queryIteration) => {
queryIteration.results = [];
});
await fetchResultsForQueries(datastores, clonedIteration, queries, pageSize, opts);
/**
* @todo check same state to avoid processing
*/
// if (!isEqual(resultsAfterMutation, results)) {
// }
iteration.forEach((queryIteration, index) => {
const clonedQueryIteration = clonedIteration.get(index);
const lastMatchingCorrelationIdIndex = getLastMatchingCorrelationIdIndex(queryIteration.results.map((r) => r[queries[index].correlationField]), clonedQueryIteration.results.map((r) => r[queries[index].correlationField]));
const resultsToPush = clonedQueryIteration.results.slice(lastMatchingCorrelationIdIndex + 1);
queryIteration.results.push(...resultsToPush);
queryIteration.cursor_last_correlation_id =
clonedQueryIteration.cursor_last_correlation_id;
queryIteration.cursor_last_id = clonedQueryIteration.cursor_last_id;
queryIteration.page = clonedQueryIteration.page;
queryIteration.version = clonedQueryIteration.version;
queryIteration.is_exhausted = clonedQueryIteration.is_exhausted;
});
}
exports.handleIterationWithMutation = handleIterationWithMutation;
function getLastMatchingCorrelationIdIndex(before, after) {
const lastCorrelationId = before[before.length - 1];
return after.lastIndexOf(lastCorrelationId);
}
exports.getLastMatchingCorrelationIdIndex = getLastMatchingCorrelationIdIndex;
function checkOptions(queries, opts) {
if (opts?.handle_in_order === true && opts?.handle_in_parallel === true) {
throw new Error(exports.ERRORS.INCOMPATIBLE_MULTIPLE_HANDLE_OPTIONS);
}
for (const query of queries) {
if (query.source === 'events' && opts?.is_mutating === true) {
throw new Error(exports.ERRORS.INCOMPATIBLE_MUTATION_ON_EVENTS);
}
}
}
exports.checkOptions = checkOptions;
async function walkMulti(datastores, queries, pageSize = 100, handler, opts, sortHandler = defaultWalkMultiSortHandler) {
checkOptions(queries, opts);
queries.forEach((q) => {
if ('_fields' in q.query) {
q.query._fields = {
...q.query._fields,
created_at: 1,
updated_at: 1,
};
}
return q;
});
const minVersions = opts?.version_ordered === true
? await getMinVersions(datastores, queries)
: queries.map(() => -1);
const maxVersions = opts?.version_ordered === true
? await getMaxVersions(datastores, queries)
: queries.map(() => -1);
const iteration = new Map(queries.map((_, i) => [
i,
{
query_index: i,
page: 0,
version: minVersions[i],
max_version: maxVersions[i],
is_exhausted: false,
cursor_last_id: '',
cursor_last_correlation_id: '',
results: [],
},
]));
const handledCorrelatoniIds = new Set();
let fullyExhausted;
let clonedIteration = new Map();
do {
fullyExhausted = true;
if (opts?.is_mutating === true) {
clonedIteration = (0, lodash_1.cloneDeep)(iteration);
}
let results = await fetchSortedResults(datastores, iteration, queries, pageSize, sortHandler, opts);
if (opts?.is_mutating === true) {
results = results.filter((result) => {
const query = queries[result.queryIndex];
const correlationId = `${query.datastore}:${query.model}:${result.result[query.correlationField]}`;
if (handledCorrelatoniIds.has(correlationId)) {
return false;
}
handledCorrelatoniIds.add(correlationId);
return true;
});
}
await handleResults(results, queries, iteration, handler, {
handle_in_order: opts?.handle_in_order,
handle_in_parallel: opts?.handle_in_parallel,
});
if (opts?.sleep && opts.sleep > 0) {
await new Promise((resolve) => setTimeout(resolve, opts?.sleep));
}
Array.from(iteration.values()).every((it) => {
if (it.is_exhausted === false || it.results.length > 0) {
fullyExhausted = false;
return false;
}
return true;
});
if (opts?.is_mutating === true) {
await handleIterationWithMutation(datastores, iteration, clonedIteration, queries, pageSize, opts);
}
} while (fullyExhausted === false);
}
exports.walkMulti = walkMulti;
//# sourceMappingURL=utils.js.map