@grouparoo/core
Version:
The Grouparoo Core
616 lines (615 loc) • 25 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.FinalSummaryReporters = exports.StatusReporters = void 0;
const os_1 = __importDefault(require("os"));
const actionhero_1 = require("actionhero");
const moment_1 = __importDefault(require("moment"));
const pluginDetails_1 = require("../modules/pluginDetails");
const sequelize_1 = require("sequelize");
const App_1 = require("../models/App");
const ApiKey_1 = require("../models/ApiKey");
const Source_1 = require("../models/Source");
const Schedule_1 = require("../models/Schedule");
const Destination_1 = require("../models/Destination");
const Import_1 = require("../models/Import");
const Group_1 = require("../models/Group");
const GroupRule_1 = require("../models/GroupRule");
const Export_1 = require("../models/Export");
const GrouparooRecord_1 = require("../models/GrouparooRecord");
const RecordProperty_1 = require("../models/RecordProperty");
const Property_1 = require("../models/Property");
const Run_1 = require("../models/Run");
const Team_1 = require("../models/Team");
const TeamMember_1 = require("../models/TeamMember");
const Notification_1 = require("../models/Notification");
const group_1 = require("../modules/ops/group");
const source_1 = require("../modules/ops/source");
const record_1 = require("./ops/record");
const GrouparooModel_1 = require("../models/GrouparooModel");
var StatusReporters;
(function (StatusReporters) {
let Cluster;
(function (Cluster) {
let Workers;
(function (Workers) {
async function countWorkers() {
return {
collection: "cluster",
topic: "workers",
aggregation: "count",
count: Object.keys(await actionhero_1.api.resque.queue.workers()).length,
};
}
Workers.countWorkers = countWorkers;
async function countErrors() {
return {
collection: "cluster",
topic: "resqueErrors",
aggregation: "count",
count: await actionhero_1.api.resque.queue.failedCount(),
};
}
Workers.countErrors = countErrors;
async function details() {
const resqueDetails = await actionhero_1.task.details();
return {
collection: "cluster",
topic: "resqueDetails",
aggregation: "exact",
value: resqueDetails.leader || "None",
metadata: JSON.stringify(resqueDetails),
};
}
Workers.details = details;
})(Workers = Cluster.Workers || (Cluster.Workers = {}));
let Process;
(function (Process) {
async function platform() {
return {
collection: "cluster",
topic: "os",
aggregation: "exact",
value: `${process.platform}/${os_1.default.release()}`,
};
}
Process.platform = platform;
async function env() {
var _a;
const metrics = [];
const envVars = [
["NODE_ENV", "development"],
["GROUPAROO_CLOUD", "false"],
["GROUPAROO_DISTRIBUTION", "unknown"],
["GROUPAROO_RUN_MODE", "unknown"],
["GROUPAROO_UI_EDITION", "unknown"],
];
for (const [k, defaultValue] of envVars) {
metrics.push({
collection: k,
topic: "env",
aggregation: "exact",
value: (_a = process.env[k]) !== null && _a !== void 0 ? _a : defaultValue,
});
}
return metrics;
}
Process.env = env;
async function sequelizeDialect() {
return {
collection: "sequelize",
topic: "dialect",
aggregation: "exact",
value: actionhero_1.config.sequelize.dialect,
};
}
Process.sequelizeDialect = sequelizeDialect;
})(Process = Cluster.Process || (Cluster.Process = {}));
let Notifications;
(function (Notifications) {
async function unread() {
return {
collection: "cluster",
topic: "unreadNotifications",
aggregation: "count",
count: await Notification_1.Notification.count({ where: { readAt: null } }),
};
}
Notifications.unread = unread;
})(Notifications = Cluster.Notifications || (Cluster.Notifications = {}));
})(Cluster = StatusReporters.Cluster || (StatusReporters.Cluster = {}));
let Plugins;
(function (Plugins) {
async function Versions() {
const metrics = [];
metrics.push({
collection: "cluster",
topic: "@grouparoo/core",
aggregation: "exact",
key: "version",
value: (0, pluginDetails_1.getCoreVersion)(),
});
const pluginManifest = (0, pluginDetails_1.getPluginManifest)();
pluginManifest.plugins.forEach((plugin) => {
metrics.push({
collection: "cluster",
topic: plugin.name,
aggregation: "exact",
key: "version",
value: plugin.version,
});
});
return mergeMetrics(metrics);
}
Plugins.Versions = Versions;
})(Plugins = StatusReporters.Plugins || (StatusReporters.Plugins = {}));
let Totals;
(function (Totals) {
async function Models(models = [
App_1.App,
ApiKey_1.ApiKey,
Source_1.Source,
Schedule_1.Schedule,
Destination_1.Destination,
Import_1.Import,
Group_1.Group,
GroupRule_1.GroupRule,
Export_1.Export,
GrouparooRecord_1.GrouparooRecord,
RecordProperty_1.RecordProperty,
Property_1.Property,
Run_1.Run,
Team_1.Team,
TeamMember_1.TeamMember,
GrouparooModel_1.GrouparooModel,
]) {
const metrics = [];
for (const i in models) {
//@ts-ignore
const model = models[i]; // pick one of the Models so that the types are the same. TODO: make this better
metrics.push({
collection: "totals",
topic: model.name,
aggregation: "count",
count: await model.count(),
});
}
return mergeMetrics(metrics);
}
Totals.Models = Models;
async function SourceTotals() {
const metrics = [];
const sources = await Source_1.Source.findAll();
for (const i in sources) {
const source = sources[i];
const schedule = await source.$get("schedule");
const { plugin } = await source.getPlugin();
metrics.push({
collection: "sourceTotals",
topic: plugin.name,
aggregation: "count",
imports: schedule
? await Import_1.Import.count({
include: [{ model: Run_1.Run, where: { creatorId: schedule.id } }],
})
: 0,
runs: schedule
? await Run_1.Run.count({ where: { creatorId: schedule.id } })
: 0,
});
}
return mergeMetrics(metrics);
}
Totals.SourceTotals = SourceTotals;
async function DestinationTotals() {
const metrics = [];
const destinations = await Destination_1.Destination.findAll();
for (const i in destinations) {
const destination = destinations[i];
const { plugin } = await destination.getPlugin();
metrics.push({
collection: "destinationTotals",
topic: plugin.name,
aggregation: "count",
exports: await Export_1.Export.count({
where: { destinationId: destination.id },
}),
});
}
return mergeMetrics(metrics);
}
Totals.DestinationTotals = DestinationTotals;
async function UniqueRecordsExported() {
const metrics = [];
const days = [1, 7, 30];
for (const dayCount of days) {
const count = await Export_1.Export.count({
distinct: true,
col: "recordId",
where: {
state: "complete",
createdAt: {
[sequelize_1.Op.gte]: (0, moment_1.default)().subtract(dayCount, "days").toDate(),
[sequelize_1.Op.lt]: new Date(),
},
},
});
metrics.push({
collection: `${dayCount}DayDistinct`,
topic: "Export",
aggregation: "count",
count,
});
}
return metrics;
}
Totals.UniqueRecordsExported = UniqueRecordsExported;
})(Totals = StatusReporters.Totals || (StatusReporters.Totals = {}));
let Pending;
(function (Pending) {
async function pendingRuns() {
const metrics = [];
const activeRuns = await Run_1.Run.findAll({ where: { state: "running" } });
metrics.push({
collection: "pending",
topic: "Run",
aggregation: "count",
count: activeRuns.length,
value: JSON.stringify(activeRuns.map((r) => r.id)),
});
for (const i in activeRuns) {
const run = activeRuns[i];
const creatorName = await run.getCreatorName();
const percentComplete = run.percentComplete;
const highWaterMark = run.highWaterMark
? Object.values(run.highWaterMark)[0]
: null;
metrics.push({
collection: "percentComplete",
topic: "Run",
aggregation: "exact",
key: run.id,
value: creatorName,
count: percentComplete,
metadata: highWaterMark === null || highWaterMark === void 0 ? void 0 : highWaterMark.toString(),
});
}
return metrics;
}
Pending.pendingRuns = pendingRuns;
async function pendingRecords() {
return {
collection: "pending",
topic: "GrouparooRecord",
aggregation: "count",
count: await GrouparooRecord_1.GrouparooRecord.count({
where: { state: { [sequelize_1.Op.ne]: "ready" } },
}),
};
}
Pending.pendingRecords = pendingRecords;
async function pendingImports() {
return {
collection: "pending",
topic: "Import",
aggregation: "count",
count: await Import_1.Import.count({
where: { state: { [sequelize_1.Op.notIn]: ["complete", "failed"] } },
}),
};
}
Pending.pendingImports = pendingImports;
async function pendingImportsBySource() {
const metrics = [];
const sources = await Source_1.Source.findAll();
const { counts } = await source_1.SourceOps.pendingImportsBySource();
for (const source of sources) {
metrics.push({
collection: "pendingBySource",
topic: "Import",
aggregation: "count",
key: source.id,
value: source.name,
count: counts[source.id] || 0,
});
}
return metrics;
}
Pending.pendingImportsBySource = pendingImportsBySource;
async function pendingExports() {
return {
collection: "pending",
topic: "Export",
aggregation: "count",
count: await Export_1.Export.count({
where: { state: "pending" },
}),
};
}
Pending.pendingExports = pendingExports;
async function pendingExportsByDestination() {
const metrics = [];
const destinations = await Destination_1.Destination.findAll();
for (const destination of destinations) {
const apiData = await destination.apiData();
metrics.push({
collection: "pendingByDestination",
topic: "Export",
aggregation: "count",
key: apiData.id,
value: apiData.name,
count: apiData.exportTotals.pending,
});
}
return metrics;
}
Pending.pendingExportsByDestination = pendingExportsByDestination;
})(Pending = StatusReporters.Pending || (StatusReporters.Pending = {}));
let Deleted;
(function (Deleted) {
async function deletedGroups() {
return {
collection: "deleted",
topic: "Group",
aggregation: "count",
count: await Group_1.Group.count({
where: { state: "deleted" },
}),
};
}
Deleted.deletedGroups = deletedGroups;
async function deletedDestinations() {
return {
collection: "deleted",
topic: "Destination",
aggregation: "count",
count: await Destination_1.Destination.count({
where: { state: "deleted" },
}),
};
}
Deleted.deletedDestinations = deletedDestinations;
async function deletedProperties() {
return {
collection: "deleted",
topic: "Property",
aggregation: "count",
count: await Property_1.Property.count({ where: { state: "deleted" } }),
};
}
Deleted.deletedProperties = deletedProperties;
async function deletedSources() {
return {
collection: "deleted",
topic: "Source",
aggregation: "count",
count: await Source_1.Source.count({ where: { state: "deleted" } }),
};
}
Deleted.deletedSources = deletedSources;
async function deletedRecords() {
const recordsToDestroy = await record_1.RecordOps.getRecordsToDestroy();
return {
collection: "deleted",
topic: "GrouparooRecord",
aggregation: "count",
count: recordsToDestroy.length,
};
}
Deleted.deletedRecords = deletedRecords;
async function deletedModels() {
return {
collection: "deleted",
topic: "Model",
aggregation: "count",
count: await GrouparooModel_1.GrouparooModel.count({ where: { state: "deleted" } }),
};
}
Deleted.deletedModels = deletedModels;
async function deletedApps() {
return {
collection: "deleted",
topic: "App",
aggregation: "count",
count: await App_1.App.count({ where: { state: "deleted" } }),
};
}
Deleted.deletedApps = deletedApps;
})(Deleted = StatusReporters.Deleted || (StatusReporters.Deleted = {}));
let Groups;
(function (Groups) {
async function byNewestMember() {
const metrics = [];
const { groups, newestMembersAdded } = await group_1.GroupOps.newestGroupMembers(25);
for (const group of groups) {
const apiData = await group.apiData();
metrics.push({
collection: "byNewestMember",
topic: "Group",
aggregation: "exact",
key: apiData.id,
value: apiData.name,
count: apiData.recordsCount,
metadata: newestMembersAdded[apiData.id]
? newestMembersAdded[apiData.id].toString()
: "No Group Members",
});
}
return metrics;
}
Groups.byNewestMember = byNewestMember;
})(Groups = StatusReporters.Groups || (StatusReporters.Groups = {}));
let Sources;
(function (Sources) {
async function nextRuns() {
const metrics = [];
const sources = await Source_1.Source.findAll();
for (const source of sources) {
const schedule = await source.$get("schedule");
const latestRun = schedule
? await Run_1.Run.findOne({
where: { creatorId: schedule.id },
order: [["updatedAt", "desc"]],
})
: null;
const nextRunAt = latestRun
? latestRun.updatedAt.getTime() + schedule.recurringFrequency
: -1;
metrics.push({
collection: "nextRun",
topic: "Source",
aggregation: "exact",
key: source.id,
value: source.name,
count: (schedule === null || schedule === void 0 ? void 0 : schedule.recurring) ? 1 : 0,
metadata: nextRunAt.toString(),
});
}
return metrics;
}
Sources.nextRuns = nextRuns;
})(Sources = StatusReporters.Sources || (StatusReporters.Sources = {}));
})(StatusReporters = exports.StatusReporters || (exports.StatusReporters = {}));
/*
* Merge the counts from metrics with matching collections, topics, and aggregations.
* This is useful for when you have 2 sources using the same app, and you want to report on the app's totals
*/
function mergeMetrics(metrics) {
var mergedMetrics = [];
metrics.forEach((item, idx) => {
const found = mergedMetrics.some((el, i) => {
if (i === idx)
return false;
return (el.collection === item.collection &&
el.topic === item.topic &&
el.aggregation === item.aggregation);
});
if (!found) {
mergedMetrics.push(item);
}
else if (idx !== null) {
for (const k in Object.keys(item)) {
if (item.hasOwnProperty(k)) {
mergedMetrics[idx].count = +item.count;
}
}
}
});
return mergedMetrics;
}
var FinalSummaryReporters;
(function (FinalSummaryReporters) {
const lastRunStart = new Date(actionhero_1.api.bootTime);
let Sources;
(function (Sources) {
async function getData() {
const runs = await Run_1.Run.findAll({
where: {
updatedAt: { [sequelize_1.Op.gte]: lastRunStart },
creatorType: "schedule",
},
});
const sources = {};
for (const run of runs) {
let source = null;
await run.updateTotals();
const schedule = await Schedule_1.Schedule.findByPk(run.creatorId);
if (schedule)
source = await schedule.$get("source", { scope: null });
const currentSource = sources[source.id] || {
name: source.name,
recordsCreated: 0,
recordsImported: 0,
importsCreated: 0,
error: null,
};
currentSource.recordsCreated += run.recordsCreated;
currentSource.recordsImported += run.recordsImported;
currentSource.importsCreated += run.importsCreated;
currentSource.error = currentSource.error || run.error;
sources[source.id] = currentSource;
}
return Object.values(sources);
}
Sources.getData = getData;
})(Sources = FinalSummaryReporters.Sources || (FinalSummaryReporters.Sources = {}));
let GrouparooRecords;
(function (GrouparooRecords) {
async function getData() {
const out = [];
const recordsUpdated = await GrouparooRecord_1.GrouparooRecord.count({
where: { updatedAt: { [sequelize_1.Op.gte]: lastRunStart } },
});
const recordsCreated = await GrouparooRecord_1.GrouparooRecord.count({
where: { createdAt: { [sequelize_1.Op.gte]: lastRunStart } },
});
const allRecords = await GrouparooRecord_1.GrouparooRecord.count();
const recordData = {
name: null,
recordsUpdated,
recordsCreated,
allRecords,
};
out.push(recordData);
return out;
}
GrouparooRecords.getData = getData;
})(GrouparooRecords = FinalSummaryReporters.GrouparooRecords || (FinalSummaryReporters.GrouparooRecords = {}));
let Destinations;
(function (Destinations) {
async function getData() {
const out = [];
const destinations = await Destination_1.Destination.scope(null).findAll();
for (const destination of destinations) {
const exportsCreated = await Export_1.Export.count({
where: {
createdAt: { [sequelize_1.Op.gte]: lastRunStart },
destinationId: destination.id,
},
});
const exportsFailed = await Export_1.Export.count({
where: {
state: "failed",
updatedAt: { [sequelize_1.Op.gte]: lastRunStart },
destinationId: destination.id,
},
});
const exportsComplete = await Export_1.Export.count({
where: {
state: "complete",
completedAt: { [sequelize_1.Op.gte]: lastRunStart },
destinationId: destination.id,
},
});
if (exportsCreated > 0 || exportsFailed > 0 || exportsComplete > 0) {
out.push({
name: destination.name,
exportsCreated,
exportsFailed,
exportsComplete,
});
}
}
return out;
}
Destinations.getData = getData;
})(Destinations = FinalSummaryReporters.Destinations || (FinalSummaryReporters.Destinations = {}));
let Warnings;
(function (Warnings) {
async function getWarnings() {
const out = [];
const schedules = await Schedule_1.Schedule.findAll();
if (schedules.length === 0) {
out.push({
name: "Schedules",
message: `No schedules found. The run command uses schedules to know what records to import.`,
link: `See this link for more info: https://www.grouparoo.com/docs/getting-started/product-concepts#schedule`,
});
}
return out;
}
Warnings.getWarnings = getWarnings;
})(Warnings = FinalSummaryReporters.Warnings || (FinalSummaryReporters.Warnings = {}));
})(FinalSummaryReporters = exports.FinalSummaryReporters || (exports.FinalSummaryReporters = {}));