@grouparoo/core
Version:
The Grouparoo Core
479 lines (478 loc) • 19.9 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SourceOps = void 0;
const sequelize_1 = __importDefault(require("sequelize"));
const RecordProperty_1 = require("../../models/RecordProperty");
const Property_1 = require("../../models/Property");
const GrouparooModel_1 = require("../../models/GrouparooModel");
const Option_1 = require("../../models/Option");
const actionhero_1 = require("actionhero");
const topologicalSort_1 = require("../topologicalSort");
const configWriter_1 = require("../configWriter");
const tableSpeculation_1 = require("../tableSpeculation");
const appsCache_1 = require("../../modules/caches/appsCache");
var SourceOps;
(function (SourceOps) {
/**
* Get the connection options for this source from the plugin
*/
async function sourceConnectionOptions(source, sourceOptions = {}) {
const { pluginConnection } = await source.getPlugin();
const app = await source.$get("app", { scope: null, include: [Option_1.Option] });
const connection = await app.getConnection();
const appOptions = await app.getOptions(true);
if (!pluginConnection.methods.sourceOptions)
return {};
return pluginConnection.methods.sourceOptions({
connection,
app,
appId: app.id,
appOptions,
sourceOptions,
});
}
SourceOps.sourceConnectionOptions = sourceConnectionOptions;
/**
* Load a preview of the data from this Source
*/
async function sourcePreview(source, sourceOptions) {
if (!sourceOptions)
sourceOptions = await source.getOptions(true);
try {
// if the options aren't set yet, return an empty array of rows
await source.validateOptions(sourceOptions);
}
catch {
return [];
}
const { pluginConnection } = await source.getPlugin();
const app = await source.$get("app", { scope: null, include: [Option_1.Option] });
const connection = await app.getConnection();
const appOptions = await app.getOptions(true);
if (!pluginConnection.methods.sourcePreview) {
throw new Error(`cannot return a source preview for ${source.type}`);
}
return pluginConnection.methods.sourcePreview({
connection,
app,
appId: app.id,
appOptions,
source,
sourceId: source.id,
sourceOptions,
});
}
SourceOps.sourcePreview = sourcePreview;
/**
* Import a record property for a GrouparooRecord from this source
*/
async function importRecordProperty(source, record, property, propertyOptionsOverride, propertyFiltersOverride) {
if (property.state !== "ready" && !propertyOptionsOverride)
return;
if (propertyOptionsOverride) {
await property.validateOptions(propertyOptionsOverride);
}
const { pluginConnection } = await source.getPlugin();
if (!pluginConnection) {
throw new Error(`cannot find connection for source ${source.type} (${source.id})`);
}
const method = pluginConnection.methods.recordProperty;
if (!method)
return;
const app = await appsCache_1.AppsCache.findOneWithCache(source.appId, undefined, "ready");
const connection = await app.getConnection();
const appOptions = await app.getOptions();
const sourceOptions = await source.getOptions();
const sourceMapping = await source.getMapping();
// we may not have the record property needed to make the mapping (ie: userId is not set on this anonymous record)
if (Object.values(sourceMapping).length > 0) {
const propertyMappingKey = Object.values(sourceMapping)[0];
const recordProperties = await record.getProperties();
if (!recordProperties[propertyMappingKey])
return;
}
while ((await app.checkAndUpdateParallelism("incr")) === false) {
(0, actionhero_1.log)(`parallelism limit reached for ${app.type}, sleeping...`);
actionhero_1.utils.sleep(100);
}
try {
const response = await method({
connection,
app,
appId: app.id,
appOptions,
source,
sourceId: source.id,
sourceOptions,
sourceMapping,
property,
propertyId: property.id,
propertyOptions: propertyOptionsOverride
? propertyOptionsOverride
: await property.getOptions(),
propertyFilters: propertyFiltersOverride
? propertyFiltersOverride
: await property.getFilters(),
record,
recordId: record.id,
});
return response;
}
catch (error) {
throw error;
}
finally {
await app.checkAndUpdateParallelism("decr");
}
}
SourceOps.importRecordProperty = importRecordProperty;
/**
* Import a record property for a GrouparooRecord from this source
*/
async function importRecordProperties(source, records, properties, propertyOptionsOverride, propertyFiltersOverride) {
if (properties.find((p) => p.state !== "ready") &&
!propertyOptionsOverride) {
return;
}
for (const key in propertyOptionsOverride) {
const property = properties.find((p) => (p.id = key));
await property.validateOptions(propertyOptionsOverride[key]);
}
const { pluginConnection } = await source.getPlugin();
if (!pluginConnection) {
throw new Error(`cannot find connection for source ${source.type} (${source.id})`);
}
const method = pluginConnection.methods.recordProperties;
if (!method)
return;
const app = await appsCache_1.AppsCache.findOneWithCache(source.appId, undefined, "ready");
const connection = await app.getConnection();
const appOptions = await app.getOptions();
const sourceOptions = await source.getOptions();
const sourceMapping = await source.getMapping();
while ((await app.checkAndUpdateParallelism("incr")) === false) {
(0, actionhero_1.log)(`parallelism limit reached for ${app.type}, sleeping...`);
actionhero_1.utils.sleep(100);
}
const propertyOptions = {};
const propertyFilters = {};
for (const property of properties) {
if (propertyOptionsOverride && propertyOptionsOverride[property.id]) {
propertyOptions[property.id] = propertyOptionsOverride[property.id];
}
else {
propertyOptions[property.id] = await property.getOptions();
}
if (propertyFiltersOverride && propertyFiltersOverride[property.id]) {
propertyFilters[property.id] = propertyFiltersOverride[property.id];
}
else {
propertyFilters[property.id] = await property.getFilters();
}
}
try {
const response = await method({
connection,
app,
appId: app.id,
appOptions,
source,
sourceId: source.id,
sourceOptions,
sourceMapping,
properties,
propertyIds: properties.map((p) => p.id),
propertyOptions,
propertyFilters,
records,
recordIds: records.map((p) => p.id),
});
await applyNonUniqueMappedResultsToAllRecords(response, {
records,
properties,
sourceMapping,
});
return response;
}
catch (error) {
throw error;
}
finally {
await app.checkAndUpdateParallelism("decr");
}
}
SourceOps.importRecordProperties = importRecordProperties;
// for non-unique mappings, we need to fan out the values we received back from the source
async function applyNonUniqueMappedResultsToAllRecords(response, { records, properties, sourceMapping, }) {
var _a, _b;
const mappedPropertyKey = Object.values(sourceMapping)[0];
const mappedProperty = await Property_1.Property.findOne({
where: { key: mappedPropertyKey },
});
if (!mappedProperty)
return;
if (mappedProperty.unique)
return;
const valueMap = {};
// load up the values
for (const recordId of Object.keys(response)) {
const record = records.find((p) => p.id === recordId);
const recordProperties = await record.getProperties();
for (const property of properties) {
if (!valueMap[property.id])
valueMap[property.id] = {};
if (recordProperties[mappedProperty.key].state !== "ready") {
throw new Error(`RecordProperty ${mappedProperty.key} for record ${record.id} is not ready`);
}
if (recordProperties[mappedProperty.key].values.length > 0 &&
recordProperties[mappedProperty.key].values[0] !== null &&
recordProperties[mappedProperty.key].values[0] !== undefined) {
valueMap[property.id][recordProperties[mappedProperty.key].values[0].toString()] = response[record.id][property.id];
}
}
}
// apply the values
for (const record of records) {
if (!response[record.id]) {
response[record.id] = {};
const recordProperties = await record.getProperties();
for (const propertyKey of Object.keys(valueMap)) {
const lookupValue = (_b = (_a = recordProperties[mappedProperty.key]) === null || _a === void 0 ? void 0 : _a.values[0]) === null || _b === void 0 ? void 0 : _b.toString();
if (lookupValue) {
response[record.id][propertyKey] =
valueMap[propertyKey][lookupValue];
}
}
}
}
}
SourceOps.applyNonUniqueMappedResultsToAllRecords = applyNonUniqueMappedResultsToAllRecords;
/**
* Import all record properties from a Source for a GrouparooRecord
*/
async function _import(source, record) {
const hash = {};
const properties = await source.$get("properties", {
where: { state: "ready" },
});
const { pluginConnection } = await source.getPlugin();
if (!pluginConnection) {
throw new Error(`cannot find connection for source ${source.type} (${source.id})`);
}
const canImport = pluginConnection.methods.recordProperty;
if (!canImport) {
return {
canImport: false,
properties: {},
};
}
for (const property of properties) {
hash[property.id] = await source.importRecordProperty(record, property, null, null);
}
// remove null and undefined as we cannot set that value
const hashKeys = Object.keys(hash);
for (const i in hashKeys) {
const id = hashKeys[i];
if (hash[id] === null || hash[id] === undefined) {
delete hash[id];
}
}
return {
canImport: true,
properties: hash,
};
}
SourceOps._import = _import;
/**
* Sorts an array of Sources by their dependencies.
* Be sure to eager-load Mappings and Properties
*/
function sortByDependencies(sources) {
const sortedSources = [];
const graph = {};
for (const source of sources) {
const provides = source.properties.map((p) => p.id);
const dependsOn = source.mappings.map((p) => p.propertyId);
for (const p of provides) {
graph[p] = dependsOn.filter((id) => id !== p);
}
}
const sortedPropertyIds = (0, topologicalSort_1.topologicalSort)(graph);
for (const propertyId of sortedPropertyIds) {
const source = sources.find((s) => s.properties.map((p) => p.id).includes(propertyId));
if (!sortedSources.map((s) => s.id).includes(source.id)) {
sortedSources.push(source);
}
}
return sortedSources;
}
SourceOps.sortByDependencies = sortByDependencies;
/**
* Get the default values of the options of a Property from this source
*/
async function defaultPropertyOptions(source) {
const { pluginConnection } = await source.getPlugin();
if (!pluginConnection) {
throw new Error(`cannot find a pluginConnection for type ${source.type}`);
}
if (!pluginConnection.methods.propertyOptions) {
throw new Error(`cannot find propertyOptions for type ${source.type}`);
}
const response = [];
const app = await source.$get("app", { include: [Option_1.Option], scope: null });
const appOptions = await app.getOptions(true);
const connection = await app.getConnection();
const sourceOptions = await source.getOptions(true);
const sourceMapping = await source.getMapping();
const propertyOptionOptions = await pluginConnection.methods.propertyOptions({
property: null,
propertyId: null,
propertyOptions: {},
});
for (const i in propertyOptionOptions) {
const opt = propertyOptionOptions[i];
const options = await opt.options({
connection,
app,
appId: app.id,
appOptions,
source,
sourceId: source.id,
sourceOptions,
sourceMapping,
property: null,
propertyId: null,
});
response.push({
key: opt.key,
displayName: opt.displayName,
description: opt.description,
required: opt.required,
type: opt.type,
primary: opt.primary,
options,
});
}
return response;
}
SourceOps.defaultPropertyOptions = defaultPropertyOptions;
/**
* This method is used to bootstrap a new source which requires a Property for a mapping, when the rule doesn't yet exist.
*/
async function bootstrapUniqueProperty(source, params) {
var _a;
const { mappedColumn, id, local = false, propertyOptions, sourceOptions, } = params;
let { key, type } = params;
const model = !key || !type ? await GrouparooModel_1.GrouparooModel.findById(source.modelId) : undefined;
let didGenerateKey = false;
const generateKey = (index) => `${configWriter_1.ConfigWriter.generateId(model.name)}_${configWriter_1.ConfigWriter.generateId(mappedColumn)}${!index ? "" : "_" + index}`;
if (!key) {
key = generateKey();
didGenerateKey = true;
}
if (!type) {
const preview = await source.sourcePreview(sourceOptions);
const samples = preview.map((row) => row[mappedColumn]);
type = tableSpeculation_1.TableSpeculation.columnType(mappedColumn, samples);
}
let retry;
let keyCount = 0;
do {
retry = false;
const property = Property_1.Property.build({
id,
key,
type,
state: "ready",
unique: true,
sourceId: source.id,
isArray: false,
});
try {
// manually run the hooks we want
Property_1.Property.generateId(property);
await Property_1.Property.ensureUnique(property);
await Property_1.Property.ensureNonArrayAndUnique(property);
// danger zone!
await property.save({ hooks: false });
// build the default options
const { pluginConnection } = await source.getPlugin();
if (!local) {
let ruleOptions = {};
if (typeof pluginConnection.methods.uniquePropertyBootstrapOptions ===
"function") {
const app = await source.$get("app", { include: [Option_1.Option] });
const connection = await app.getConnection();
const appOptions = await app.getOptions(true);
const options = await source.getOptions(true);
const defaultOptions = await pluginConnection.methods.uniquePropertyBootstrapOptions({
app,
appId: app.id,
connection,
appOptions,
source,
sourceId: source.id,
sourceOptions: options,
mappedColumn,
});
ruleOptions = defaultOptions;
}
if (propertyOptions) {
Object.assign(ruleOptions, propertyOptions);
}
await property.setOptions(ruleOptions, false, false);
}
return property;
}
catch (error) {
if (property) {
await property.destroy();
if (didGenerateKey &&
keyCount <= 10 &&
((_a = error === null || error === void 0 ? void 0 : error.message) === null || _a === void 0 ? void 0 : _a.match(/key ".+" is already in use/))) {
keyCount++;
key = generateKey(keyCount);
retry = true;
}
else {
throw error;
}
}
else {
throw error;
}
}
} while (retry);
}
SourceOps.bootstrapUniqueProperty = bootstrapUniqueProperty;
async function pendingImportsBySource() {
const countsBySource = await Property_1.Property.findAll({
attributes: [
"sourceId",
[
sequelize_1.default.fn("COUNT", sequelize_1.default.fn("DISTINCT", sequelize_1.default.col("recordId"))),
"count",
],
],
group: ["sourceId"],
include: [
{
model: RecordProperty_1.RecordProperty,
attributes: [],
where: { state: "pending" },
},
],
raw: true,
});
const counts = {};
countsBySource.forEach((record) => {
//@ts-ignore
counts[record.sourceId] = record["count"];
});
return { counts };
}
SourceOps.pendingImportsBySource = pendingImportsBySource;
})(SourceOps = exports.SourceOps || (exports.SourceOps = {}));