@grouparoo/core
Version:
The Grouparoo Core
404 lines (403 loc) • 17.2 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var Export_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Export = exports.ExportStates = void 0;
const sequelize_typescript_1 = require("sequelize-typescript");
const path_1 = __importDefault(require("path"));
const fs_extra_1 = require("fs-extra");
const actionhero_1 = require("actionhero");
const Destination_1 = require("./Destination");
const GrouparooRecord_1 = require("./GrouparooRecord");
const plugin_1 = require("../modules/plugin");
const moment_1 = __importDefault(require("moment"));
const sequelize_1 = require("sequelize");
const export_1 = require("../modules/ops/export");
const apiData_1 = require("../modules/apiData");
const stateMachine_1 = require("../modules/stateMachine");
const ExportProcessor_1 = require("./ExportProcessor");
const errors_1 = require("../modules/errors");
const commonModel_1 = require("../classes/commonModel");
const pluginDetails_1 = require("../modules/pluginDetails");
exports.ExportStates = [
"draft",
"pending",
"processing",
"canceled",
"failed",
"complete", // OK!
];
const STATE_TRANSITIONS = [
{ from: "draft", to: "pending", checks: [] },
{ from: "draft", to: "canceled", checks: [] },
{ from: "pending", to: "processing", checks: [] },
{ from: "pending", to: "canceled", checks: [] },
{ from: "pending", to: "failed", checks: [] },
{ from: "pending", to: "complete", checks: [] },
{ from: "processing", to: "canceled", checks: [] },
{ from: "processing", to: "failed", checks: [] },
{ from: "processing", to: "complete", checks: [] },
{ from: "processing", to: "pending", checks: [] },
];
let Export = Export_1 = class Export extends commonModel_1.CommonModel {
idPrefix() {
return "exp";
}
get oldRecordProperties() {
return export_1.ExportOps.deserializeExportRecordProperties(
//@ts-ignore
this.getDataValue("oldRecordProperties"));
}
set oldRecordProperties(value) {
//@ts-ignore
this.setDataValue("oldRecordProperties", JSON.stringify(value));
}
get newRecordProperties() {
return export_1.ExportOps.deserializeExportRecordProperties(
//@ts-ignore
this.getDataValue("newRecordProperties"));
}
set newRecordProperties(value) {
//@ts-ignore
this.setDataValue("newRecordProperties", JSON.stringify(value));
}
get oldGroups() {
//@ts-ignore
return JSON.parse(this.getDataValue("oldGroups") || "[]");
}
set oldGroups(value) {
//@ts-ignore
this.setDataValue("oldGroups", JSON.stringify(value));
}
get newGroups() {
//@ts-ignore
return JSON.parse(this.getDataValue("newGroups") || "[]");
}
set newGroups(value) {
//@ts-ignore
this.setDataValue("newGroups", JSON.stringify(value));
}
async setError(error, retryDelay) {
const maxExportAttempts = parseInt((await plugin_1.plugin.readSetting("core", "exports-max-retries-count")).value);
this.errorMessage = error.message || error.toString();
if (error["errorLevel"])
this.errorLevel = error["errorLevel"];
if (!retryDelay)
this.retryCount++;
if (this.retryCount >= maxExportAttempts) {
this.state = "failed";
this.sendAt = null;
}
else if (this.errorLevel === "info") {
this.state = "failed";
}
else {
this.state = "pending";
this.exportProcessorId = null;
this.sendAt = (0, moment_1.default)()
.add(retryDelay !== null && retryDelay !== void 0 ? retryDelay : actionhero_1.config.tasks.timeout, "ms")
.toDate();
this.startedAt = null;
}
return this.save();
}
async retry(retryDelay = actionhero_1.config.tasks.timeout, skipCount = false) {
const maxExportAttempts = parseInt((await plugin_1.plugin.readSetting("core", "exports-max-retries-count")).value);
if (!skipCount)
this.retryCount++;
if (this.retryCount >= maxExportAttempts) {
this.state = "failed";
this.sendAt = null;
}
else {
this.sendAt = (0, moment_1.default)().add(retryDelay, "ms").toDate();
this.startedAt = null;
}
return this.save();
}
async apiData(includeDestination = true) {
var _a, _b;
const destination = (_a = this.destination) !== null && _a !== void 0 ? _a : (await this.$get("destination", { scope: null }));
const record = (_b = this.record) !== null && _b !== void 0 ? _b : (await this.$get("record"));
return {
id: this.id,
destination: includeDestination && destination
? {
id: destination.id,
state: destination.state,
name: destination.name,
groupId: destination.groupId,
modelId: destination.modelId,
}
: undefined,
destinationName: destination ? destination.name : null,
recordId: this.recordId,
modelId: record === null || record === void 0 ? void 0 : record.modelId,
exportProcessorId: this.exportProcessorId,
state: this.state,
force: this.force,
createdAt: apiData_1.APIData.formatDate(this.createdAt),
sendAt: apiData_1.APIData.formatDate(this.sendAt),
startedAt: apiData_1.APIData.formatDate(this.startedAt),
completedAt: apiData_1.APIData.formatDate(this.completedAt),
retryCount: this.retryCount,
oldRecordProperties: this.oldRecordProperties,
newRecordProperties: this.newRecordProperties,
oldGroups: this.oldGroups,
newGroups: this.newGroups,
toDelete: this.toDelete,
hasChanges: this.hasChanges,
errorMessage: this.errorMessage,
errorLevel: this.errorLevel,
};
}
static async updateState(instance) {
await stateMachine_1.StateMachine.transition(instance, STATE_TRANSITIONS);
}
static ensureErrorLevel(instance) {
if (instance.errorMessage && !instance.errorLevel) {
instance.errorLevel = "error";
}
}
static async logExport(instance) {
if (!process.env.GROUPAROO_EXPORT_LOG)
return;
if (instance.changed("state") &&
["canceled", "failed", "complete"].includes(instance.state)) {
const exportData = {
...(await instance.apiData(false)),
timestamp: apiData_1.APIData.formatDate(new Date()),
};
const message = JSON.stringify(exportData);
if (process.env.GROUPAROO_EXPORT_LOG === "stdout") {
(0, actionhero_1.log)(`[ export ] ${message}`);
}
else {
const logPath = path_1.default.isAbsolute(process.env.GROUPAROO_EXPORT_LOG)
? process.env.GROUPAROO_EXPORT_LOG
: path_1.default.join((0, pluginDetails_1.getParentPath)(), process.env.GROUPAROO_EXPORT_LOG);
await (0, fs_extra_1.appendFile)(logPath, `${message}\n`);
}
}
}
static retryFailed(startDate, endDate, destination, saveExports = true) {
return export_1.ExportOps.retryFailedExports(startDate, endDate, destination, saveExports);
}
static retryById(exportId) {
return export_1.ExportOps.retryExportById(exportId);
}
static async sweep(limit) {
const days = 90; // keep all exports for at least 90 days
const whereDate = (0, moment_1.default)()
.subtract(days, "days")
.format("YYYY-MM-DD HH:mm:ss");
// for SQLite secondary changes queries
let responseCountWithCompleteExport;
let responseCountWithNoRecord;
let responseCountWithNoDestination;
// 1. Delete Complete Exports for the GrouparooRecord older than the newest complete Export for this Record+Destination
const rowsWithCompleteExport = await actionhero_1.api.sequelize.query(`
DELETE FROM exports
WHERE id IN (
SELECT id FROM exports
WHERE "state" IN ('complete', 'failed', 'canceled')
AND "createdAt" < '${whereDate}'
AND "createdAt" < (
SELECT MAX("createdAt")
FROM exports e2
WHERE
state = 'complete'
AND e2."recordId" = exports."recordId"
AND e2."destinationId" = exports."destinationId"
)
LIMIT ${limit}
)
${actionhero_1.config.sequelize.dialect === "postgres" ? "RETURNING id" : ""}
;`, { type: sequelize_1.QueryTypes.SELECT });
if (actionhero_1.config.sequelize.dialect === "sqlite") {
const changesRows = await actionhero_1.api.sequelize.query("SELECT changes() as count;", { type: sequelize_1.QueryTypes.SELECT });
responseCountWithCompleteExport = changesRows[0]["count"];
}
// 2. Delete exports older than the limit which no longer have a Record
const rowsWithNoRecord = await actionhero_1.api.sequelize.query(`
DELETE FROM exports
WHERE id IN (
SELECT id FROM exports
WHERE "createdAt" < '${whereDate}'
AND (
SELECT id
FROM records
WHERE "records"."id" = "exports"."recordId"
) IS NULL
LIMIT ${limit}
)
${actionhero_1.config.sequelize.dialect === "postgres" ? "RETURNING id" : ""}
;`, { type: sequelize_1.QueryTypes.SELECT });
if (actionhero_1.config.sequelize.dialect === "sqlite") {
const changesRows = await actionhero_1.api.sequelize.query("SELECT changes() as count;", { type: sequelize_1.QueryTypes.SELECT });
responseCountWithNoRecord = changesRows[0]["count"];
}
// 3. Delete exports older than the limit which no longer have a Destination
const rowsWithNoDestination = await actionhero_1.api.sequelize.query(`
DELETE FROM exports
WHERE id IN (
SELECT id FROM exports
WHERE "createdAt" < '${whereDate}'
AND (
SELECT id
FROM destinations
WHERE "destinations"."id" = "exports"."destinationId"
) IS NULL
LIMIT ${limit}
)
${actionhero_1.config.sequelize.dialect === "postgres" ? "RETURNING id" : ""}
;`, { type: sequelize_1.QueryTypes.SELECT });
if (actionhero_1.config.sequelize.dialect === "sqlite") {
const changesRows = await actionhero_1.api.sequelize.query("SELECT changes() as count;", { type: sequelize_1.QueryTypes.SELECT });
responseCountWithNoDestination = changesRows[0]["count"];
}
return {
count: actionhero_1.config.sequelize.dialect === "postgres"
? rowsWithCompleteExport.length +
rowsWithNoRecord.length +
rowsWithNoDestination.length
: responseCountWithCompleteExport +
responseCountWithNoRecord +
responseCountWithNoDestination,
days,
};
}
};
// --- Class Methods --- //
Export.defaultState = "pending";
__decorate([
(0, sequelize_typescript_1.ForeignKey)(() => Destination_1.Destination),
sequelize_typescript_1.Column,
__metadata("design:type", String)
], Export.prototype, "destinationId", void 0);
__decorate([
(0, sequelize_typescript_1.AllowNull)(false),
(0, sequelize_typescript_1.ForeignKey)(() => GrouparooRecord_1.GrouparooRecord),
sequelize_typescript_1.Column,
__metadata("design:type", String)
], Export.prototype, "recordId", void 0);
__decorate([
(0, sequelize_typescript_1.ForeignKey)(() => ExportProcessor_1.ExportProcessor),
sequelize_typescript_1.Column,
__metadata("design:type", String)
], Export.prototype, "exportProcessorId", void 0);
__decorate([
(0, sequelize_typescript_1.AllowNull)(false),
(0, sequelize_typescript_1.Default)(Export_1.defaultState),
(0, sequelize_typescript_1.Column)(sequelize_typescript_1.DataType.ENUM(...exports.ExportStates)),
__metadata("design:type", Object)
], Export.prototype, "state", void 0);
__decorate([
(0, sequelize_typescript_1.AllowNull)(false),
(0, sequelize_typescript_1.Default)(false),
sequelize_typescript_1.Column,
__metadata("design:type", Boolean)
], Export.prototype, "force", void 0);
__decorate([
sequelize_typescript_1.Column,
__metadata("design:type", Date)
], Export.prototype, "startedAt", void 0);
__decorate([
sequelize_typescript_1.Column,
__metadata("design:type", Date)
], Export.prototype, "sendAt", void 0);
__decorate([
(0, sequelize_typescript_1.Default)(0),
sequelize_typescript_1.Column,
__metadata("design:type", Number)
], Export.prototype, "retryCount", void 0);
__decorate([
sequelize_typescript_1.Column,
__metadata("design:type", Date)
], Export.prototype, "completedAt", void 0);
__decorate([
sequelize_typescript_1.Column,
__metadata("design:type", String)
], Export.prototype, "errorMessage", void 0);
__decorate([
(0, sequelize_typescript_1.Is)("ofValidErrorLevel", (value) => {
if (value && !errors_1.Errors.ERROR_LEVELS.includes(value)) {
throw new Error(`errorLevel must be one of: ${errors_1.Errors.ERROR_LEVELS.join(",")}`);
}
}),
(0, sequelize_typescript_1.Column)(sequelize_typescript_1.DataType.ENUM(...errors_1.Errors.ERROR_LEVELS)),
__metadata("design:type", String)
], Export.prototype, "errorLevel", void 0);
__decorate([
(0, sequelize_typescript_1.Column)(sequelize_typescript_1.DataType.TEXT),
__metadata("design:type", Object),
__metadata("design:paramtypes", [Object])
], Export.prototype, "oldRecordProperties", null);
__decorate([
(0, sequelize_typescript_1.Column)(sequelize_typescript_1.DataType.TEXT),
__metadata("design:type", Object),
__metadata("design:paramtypes", [Object])
], Export.prototype, "newRecordProperties", null);
__decorate([
(0, sequelize_typescript_1.Column)(sequelize_typescript_1.DataType.TEXT),
__metadata("design:type", Array),
__metadata("design:paramtypes", [Array])
], Export.prototype, "oldGroups", null);
__decorate([
(0, sequelize_typescript_1.Column)(sequelize_typescript_1.DataType.TEXT),
__metadata("design:type", Array),
__metadata("design:paramtypes", [Array])
], Export.prototype, "newGroups", null);
__decorate([
(0, sequelize_typescript_1.Default)(false),
(0, sequelize_typescript_1.AllowNull)(false),
sequelize_typescript_1.Column,
__metadata("design:type", Boolean)
], Export.prototype, "toDelete", void 0);
__decorate([
(0, sequelize_typescript_1.Default)(false),
(0, sequelize_typescript_1.AllowNull)(false),
sequelize_typescript_1.Column,
__metadata("design:type", Boolean)
], Export.prototype, "hasChanges", void 0);
__decorate([
(0, sequelize_typescript_1.BelongsTo)(() => Destination_1.Destination),
__metadata("design:type", Destination_1.Destination)
], Export.prototype, "destination", void 0);
__decorate([
(0, sequelize_typescript_1.BelongsTo)(() => GrouparooRecord_1.GrouparooRecord),
__metadata("design:type", GrouparooRecord_1.GrouparooRecord)
], Export.prototype, "record", void 0);
__decorate([
sequelize_typescript_1.BeforeSave,
__metadata("design:type", Function),
__metadata("design:paramtypes", [Export]),
__metadata("design:returntype", Promise)
], Export, "updateState", null);
__decorate([
sequelize_typescript_1.BeforeSave,
__metadata("design:type", Function),
__metadata("design:paramtypes", [Export]),
__metadata("design:returntype", void 0)
], Export, "ensureErrorLevel", null);
__decorate([
sequelize_typescript_1.AfterUpdate,
__metadata("design:type", Function),
__metadata("design:paramtypes", [Export]),
__metadata("design:returntype", Promise)
], Export, "logExport", null);
Export = Export_1 = __decorate([
(0, sequelize_typescript_1.Table)({ tableName: "exports", paranoid: false })
], Export);
exports.Export = Export;