@getanthill/datastore
Version:
Event-Sourced Datastore
343 lines • 13 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.aggregate = aggregate;
exports.importData = importData;
exports.replayEvents = replayEvents;
exports.exportData = exportData;
exports.validateData = validateData;
exports.default = register;
const fs_1 = __importDefault(require("fs"));
const commander_1 = require("commander");
const js_yaml_1 = __importDefault(require("js-yaml"));
const pick_1 = __importDefault(require("lodash/pick"));
const omit_1 = __importDefault(require("lodash/omit"));
const utils = __importStar(require("./utils"));
const sdk_1 = require("../sdk");
function filterFixtures(fixtures, ids) {
if (ids.length === 0) {
return [];
}
const toAdd = fixtures.filter((fixture) => ids.includes(fixture.id));
const dependencies = [];
for (const addition of toAdd) {
dependencies.push(...filterFixtures(fixtures, (addition.links || []).map((f) => f.id)));
}
const filteredFixtures = new Map();
dependencies.forEach((f) => {
filteredFixtures.set(f.id, f);
});
toAdd.forEach((f) => {
filteredFixtures.set(f.id, f);
});
return Array.from(filteredFixtures.values());
}
function aggregate(services) {
return async (filePath, cmd) => {
const aggregator = new sdk_1.Aggregator(services.datastores);
try {
const pipeline = js_yaml_1.default.load(fs_1.default.readFileSync(filePath, 'utf8'));
const data = await aggregator.aggregate(pipeline);
if (cmd.verbose === true) {
aggregator.logs.forEach((logLine) => {
utils.log(logLine, cmd.format);
});
}
utils.log(data, cmd.format);
}
catch (err) {
aggregator.logs.forEach((logLine) => {
utils.log(logLine, cmd.format);
});
if (err.response) {
utils.log(err.response.data, cmd.format);
return;
}
utils.log(err, cmd.format);
}
};
}
function importData(services) {
return async (filePath, cmd) => {
try {
const datastore = services.datastores.get(cmd.datastore);
if (!datastore) {
return;
}
let fixtures;
if (cmd.json === true) {
fixtures = require(filePath);
}
else {
fixtures = js_yaml_1.default.load(fs_1.default.readFileSync(filePath, 'utf8'));
}
/**
* @deprecated Backward compatibility for
* `fixtures` key
*/
if ('fixtures' in fixtures) {
fixtures = fixtures.fixtures;
}
if (cmd.ids) {
fixtures = filterFixtures(fixtures, cmd.ids);
}
const { data: modelConfigs } = await datastore.getModels();
const entities = await datastore.import(fixtures, modelConfigs, {
dryRun: cmd.dryRun,
});
let result = Array.from(entities.values());
if (cmd.diffOnly === true) {
result = result.filter((r) => r.__update__ === undefined || r.__update__.length > 0);
}
utils.log(result, cmd.format);
}
catch (err) {
if (err.response) {
utils.log(err.response.data, cmd.format);
return;
}
utils.log(err, cmd.format);
}
};
}
function replayEvents(services) {
return async (filePath, cmd) => {
try {
const datastore = services.datastores.get(cmd.datastore);
if (!datastore) {
return;
}
let fixtures;
if (cmd.json === true) {
fixtures = require(filePath);
}
else {
fixtures = js_yaml_1.default.load(fs_1.default.readFileSync(filePath, 'utf8'));
}
if (cmd.ids) {
fixtures = filterFixtures(fixtures, cmd.ids);
}
fixtures = fixtures.sort((a, b) => a.event.created_at.localeCompare(b.event.created_at));
const { data: modelConfigs } = await datastore.getModels();
const result = [];
for (const e of fixtures) {
const modelConfig = modelConfigs[e.model];
const event = e.event;
const { data: res } = await datastore?.apply(e.model, event[modelConfig.correlation_field], event.type, event.v, event, {
replay: 'true',
});
result.push(res);
}
utils.log(result, cmd.format);
}
catch (err) {
if (err.response) {
utils.log(err.response.data, cmd.format);
return;
}
utils.log(err, cmd.format);
}
};
}
function exportData(services) {
return async (model, filePath, cmd) => {
try {
const datastore = services.datastores.get(cmd.datastore);
if (!datastore) {
return;
}
const writeStream = fs_1.default.createWriteStream(filePath);
const { data: modelConfigs } = await datastore.getModel(model);
const modelConfig = modelConfigs[model];
if (cmd.format === 'json') {
writeStream.write('[');
}
const query = cmd.query || {};
const total = await datastore.count(model, query);
let count = 0;
await datastore.walk(model, query, async (entity) => {
const data = cmd.raw === true
? entity
: {
model,
id: `${entity[modelConfig.correlation_field]}${cmd.source === 'events' ? '/v' + entity.version : ''}`,
idempotency: (0, pick_1.default)(entity, cmd.idempotencyKeys ||
entity[modelConfig.correlation_field]),
event: cmd.source !== 'events' ? undefined : entity,
entity: cmd.source !== 'entities'
? undefined
: (0, omit_1.default)(entity, modelConfig.correlation_field, 'created_at', 'updated_at', 'version'),
};
writeStream.write(cmd.format === 'json'
? JSON.stringify(data, null, 2)
: js_yaml_1.default.dump([data]), 'utf-8');
count += 1;
if (cmd.format === 'json' && count < total) {
writeStream.write(', ');
}
}, 500, cmd.source, {}, {
sleep: 500,
});
if (cmd.format === 'json') {
writeStream.write(']');
}
writeStream.end();
utils.log({
msg: 'Export succeed',
count,
file_path: filePath,
}, cmd.format);
}
catch (err) {
if (err.response) {
utils.log(err.response.data, cmd.format);
return;
}
utils.log(err, cmd.format);
}
};
}
function validateData(services) {
return async (model, cmd) => {
try {
const datastore = services.datastores.get(cmd.datastore);
if (!datastore) {
return;
}
const query = cmd.query || {};
const total = await datastore.count(model, query);
utils.log({
msg: 'Starting the validation',
total,
}, cmd.format);
const stats = {
total,
processed: 0,
errors: 0,
};
try {
await datastore.walk(model, query, async () => {
stats.processed += 1;
}, 100, 'entities', {}, {
sleep: 500,
});
}
catch (err) {
stats.errors += 1;
console.error(err?.response?.data);
utils.log({
msg: 'Validation error',
}, cmd.format);
}
utils.log({
msg: 'Validation ended',
stats,
}, cmd.format);
}
catch (err) {
if (err.response) {
utils.log(err.response.data, cmd.format);
return;
}
utils.log(err, cmd.format);
}
};
}
function register(services, name = 'data') {
const program = new commander_1.Command(name);
program.summary('Data utilities commands');
// Aggregate
program
.command('aggregate <file_path>')
.addOption(new commander_1.Option('--format <format>', 'Response format').choices([
'json',
'yaml',
]))
.option('--verbose', 'Print the aggregation logs', false)
.description('Perform an aggregation on the datastore')
.action(aggregate(services));
// Import
let c = program
.command('import')
.argument('<file_path>', 'Path of the data file to import');
utils.addDatastoreOptions(c, services);
c.option('--ids <ids...>', 'IDs to import in the file')
.addOption(new commander_1.Option('--format <format>', 'Response format').choices([
'json',
'yaml',
]))
.option('--json', 'Input as JSON', false)
.option('--dry-run', 'If present, does not perform the import', false)
.option('--diff-only', 'Show changes only', false)
.description('Import entities into the Datastore')
.action(importData(services));
// Replay
c = program
.command('replay')
.argument('<file_path>', 'Path of the data file to import');
utils.addDatastoreOptions(c, services);
c.option('--ids <ids...>', 'IDs to import in the file')
.addOption(new commander_1.Option('--format <format>', 'Response format').choices([
'json',
'yaml',
]))
.option('--json', 'Input as JSON', false)
.option('--dry-run', 'If present, does not perform the import', false)
.description('Replay events into the Datastore')
.action(replayEvents(services));
// Export
c = program.command('export <model> <file_path>');
utils.addDatastoreOptions(c, services);
c.addOption(new commander_1.Option('-s, --source <source>', 'Count source: entities or events')
.default('entities')
.choices(['entities', 'events']));
c.option('-i, --idempotency-keys <idempotency_keys...>', 'Idempotency keys to use')
.option('-r, --raw', 'Export raw data without import logic')
.option('-f, --format <format>', 'Export data into the given format', 'yaml')
.option('-q, --query <query>', 'Export query to apply', JSON.parse)
.description('Export data from the Datastore')
.action(exportData(services));
// Validate
c = program.command('validate <model>');
utils.addDatastoreOptions(c, services);
c.option('-q, --query <query>', 'Validate query to apply', JSON.parse)
.description('Validate data from the Datastore')
.action(validateData(services));
return program;
}
//# sourceMappingURL=data.js.map