@golemio/pid
Version:
Golemio PID Module
246 lines • 11.3 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.RopidMetadataModel = exports.MetaDatasetInfoKeyEnum = exports.MetaStateEnum = exports.MetaTypeEnum = void 0;
const const_1 = require("../../schema-definitions/const");
const MetadataDto_1 = require("../../schema-definitions/shared/models/MetadataDto");
const connectors_1 = require("@golemio/core/dist/integration-engine/connectors");
const helpers_1 = require("@golemio/core/dist/integration-engine/helpers");
const models_1 = require("@golemio/core/dist/integration-engine/models");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
const sequelize_1 = __importDefault(require("@golemio/core/dist/shared/sequelize"));
var MetaTypeEnum;
(function (MetaTypeEnum) {
MetaTypeEnum["DATASET_INFO"] = "DATASET_INFO";
MetaTypeEnum["STATE"] = "STATE";
MetaTypeEnum["TABLE_TOTAL_COUNT"] = "TABLE_TOTAL_COUNT";
MetaTypeEnum["SAVED_ROWS"] = "SAVED_ROWS";
})(MetaTypeEnum || (exports.MetaTypeEnum = MetaTypeEnum = {}));
var MetaStateEnum;
(function (MetaStateEnum) {
MetaStateEnum["DOWNLOADED"] = "DOWNLOADED";
MetaStateEnum["SAVED"] = "SAVED";
})(MetaStateEnum || (exports.MetaStateEnum = MetaStateEnum = {}));
var MetaDatasetInfoKeyEnum;
(function (MetaDatasetInfoKeyEnum) {
MetaDatasetInfoKeyEnum["LAST_MODIFIED"] = "last_modified";
MetaDatasetInfoKeyEnum["FAILED"] = "failed";
MetaDatasetInfoKeyEnum["DEPLOYED"] = "deployed";
MetaDatasetInfoKeyEnum["NUMBER_OF_RETRIES"] = "number_of_retries";
MetaDatasetInfoKeyEnum["DIGEST"] = "digest";
})(MetaDatasetInfoKeyEnum || (exports.MetaDatasetInfoKeyEnum = MetaDatasetInfoKeyEnum = {}));
class RopidMetadataModel extends models_1.PostgresModel {
constructor(schemaDefinitions) {
super(schemaDefinitions.metadata.name + "Model", {
outputSequelizeAttributes: MetadataDto_1.MetadataDto.attributeModel,
pgTableName: schemaDefinitions.metadata.pgTableName,
pgSchema: const_1.PG_SCHEMA,
savingType: "insertOnly",
}, new golemio_validator_1.JSONSchemaValidator(schemaDefinitions.metadata.name + "ModelValidator", MetadataDto_1.MetadataDto.jsonSchema));
this.schemaDefinitions = schemaDefinitions;
this.getLastModified = async (dataset) => {
try {
const lastMod = await this.sequelizeModel.findOne({
order: [["version", "DESC"]],
where: {
dataset,
key: MetaDatasetInfoKeyEnum.LAST_MODIFIED,
type: MetaTypeEnum.DATASET_INFO,
},
});
return lastMod
? {
lastModified: lastMod.dataValues ? lastMod.dataValues.value : null,
version: lastMod.dataValues ? parseInt(lastMod.dataValues.version, 10) : 1,
}
: {
lastModified: null,
version: 0,
};
}
catch (err) {
helpers_1.log.warn(err);
return {
lastModified: null,
version: 0,
};
}
};
this.isDeployed = async (dataset) => {
try {
const metaRecord = await this.sequelizeModel.findOne({
attributes: ["key", "value"],
where: {
dataset,
type: MetaTypeEnum.DATASET_INFO,
},
order: [
["version", "DESC"],
["id", "DESC"],
],
raw: true,
});
return !!(metaRecord?.key === MetaDatasetInfoKeyEnum.DEPLOYED && metaRecord.value === "true");
}
catch (err) {
helpers_1.log.warn(err);
return false;
}
};
this.checkSavedRows = async (dataset, version) => {
const meta = await this.getTotalFromMeta(dataset, version);
const tables = await this.getTotalFromTables(dataset, version);
if (meta.totalRows !== tables.totalRows || meta.numOfTables !== tables.numOfTables) {
throw new golemio_errors_1.GeneralError(this.name + ": checkSavedRows() failed.");
}
};
this.replaceTmpTables = async (dataset, version) => {
const connection = connectors_1.PostgresConnector.getConnection();
const t = await connection.transaction();
try {
const tables = await this.sequelizeModel.findAll({
attributes: [["key", "tn"]],
transaction: t,
where: {
dataset,
type: MetaTypeEnum.STATE,
value: MetaStateEnum.SAVED,
version,
},
});
const promises = tables.map(async (table) => {
const pgTableName = this.schemaDefinitions[table.dataValues.tn].pgTableName;
helpers_1.log.debug(`Replacing tables: processing table ${pgTableName}`);
const tableName = `"${const_1.PG_SCHEMA}"."${pgTableName}"`;
const tmpTableName = `"tmp"."${pgTableName}"`;
// use DELETE (ROW EXCLUSIVE lock mode) instead of TRUNCATE (ACCESS EXCLUSIVE lock mode)
// as we don't want to have conflicts with SELECT queries (ACCESS SHARE lock mode) in output gateway
const rawReplaceTableQuery = `
DELETE FROM ${tableName};
INSERT INTO ${tableName} (SELECT * FROM ${tmpTableName});
DROP TABLE ${tmpTableName};
`;
return connection.query(rawReplaceTableQuery, {
type: sequelize_1.default.QueryTypes.RAW,
transaction: t,
});
});
await Promise.all(promises);
helpers_1.log.debug(`Replacing tables: processing finished.`);
// save meta
await this.sequelizeModel.create({
dataset,
key: MetaDatasetInfoKeyEnum.DEPLOYED,
type: MetaTypeEnum.DATASET_INFO,
value: "true",
version,
}, { transaction: t });
await this.sequelizeModel.destroy({
transaction: t,
where: {
dataset,
version: {
[sequelize_1.default.Op.and]: [
// delete all versions older than ten versions back
{ [sequelize_1.default.Op.lt]: version - 10 },
{ [sequelize_1.default.Op.ne]: -1 },
],
},
},
});
await t.commit();
return true;
}
catch (err) {
helpers_1.log.error(err);
await t.rollback();
throw new golemio_errors_1.GeneralError(this.name + ": replaceTmpTables() failed.", undefined, err);
}
};
this.rollbackFailedSaving = async (dataset, version) => {
await this.save({
dataset,
key: MetaDatasetInfoKeyEnum.FAILED,
type: MetaTypeEnum.DATASET_INFO,
value: new Date().toISOString(),
version,
});
const numberOfRetries = (await this.sequelizeModel.findOne({
attributes: [["value", "retries"]],
where: {
dataset,
key: MetaDatasetInfoKeyEnum.NUMBER_OF_RETRIES,
type: MetaTypeEnum.DATASET_INFO,
version: version - 1,
},
}))?.dataValues?.retries ?? 0;
await this.save({
dataset,
key: MetaDatasetInfoKeyEnum.NUMBER_OF_RETRIES,
type: MetaTypeEnum.DATASET_INFO,
value: +numberOfRetries + 1,
version,
});
};
this.getTotalFromMeta = async (dataset, version) => {
const tables = await this.sequelizeModel.findAll({
attributes: [["key", "tn"]],
where: {
dataset,
type: MetaTypeEnum.TABLE_TOTAL_COUNT,
version,
},
});
const result = await this.sequelizeModel.findAll({
attributes: [[sequelize_1.default.fn("SUM", sequelize_1.default.cast(sequelize_1.default.col("value"), "INTEGER")), "total"]],
where: {
dataset,
type: MetaTypeEnum.TABLE_TOTAL_COUNT,
version,
},
});
const res = {
numOfTables: tables.length,
totalRows: result[0].dataValues.total,
};
helpers_1.log.debug(this.name + " Total from metadata: " + JSON.stringify(res));
return res;
};
this.getTotalFromTables = async (dataset, version) => {
const connection = connectors_1.PostgresConnector.getConnection();
const tables = await this.sequelizeModel.findAll({
attributes: [["key", "tn"]],
where: {
dataset,
type: MetaTypeEnum.TABLE_TOTAL_COUNT,
version,
},
});
const tablesArray = [];
for (const table of tables) {
tablesArray.push("'" + this.schemaDefinitions[table.dataValues.tn].pgTableName + "'");
}
// TODO zbavit se raw query
const result = await connection.query("SELECT SUM(count_rows(table_schema, table_name)) as total " +
"FROM information_schema.tables " +
"WHERE " +
"table_schema NOT IN ('pg_catalog', 'information_schema') " +
"AND table_type = 'BASE TABLE' " +
"AND table_name in (" +
tablesArray.join(",") +
") " +
"AND table_schema = 'tmp'; ", { type: sequelize_1.default.QueryTypes.SELECT });
const res = {
numOfTables: tables.length,
totalRows: result[0].total,
};
helpers_1.log.debug(this.name + " Total from tables: " + JSON.stringify(res));
return res;
};
}
}
exports.RopidMetadataModel = RopidMetadataModel;
//# sourceMappingURL=RopidMetadataModel.js.map