@golemio/pid
Version:
Golemio PID Module
113 lines • 6.74 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.TransformAndSaveDataTask = void 0;
const Di_1 = require("../../../../ioc/Di");
const ropid_gtfs_1 = require("../../..");
const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
const Di_2 = require("../../../ioc/Di");
const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
const const_1 = require("../../../../../schema-definitions/const");
const ropid_gtfs_2 = require("../../../../../schema-definitions/ropid-gtfs");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const PostgresModel_1 = require("@golemio/core/dist/integration-engine/models/PostgresModel");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
const pg_copy_streams_1 = require("pg-copy-streams");
const HelperTypes_1 = require("./helpers/HelperTypes");
const CachedDatasetSchema_1 = require("./schema/CachedDatasetSchema");
class TransformAndSaveDataTask extends workers_1.AbstractTask {
constructor(queuePrefix) {
super(queuePrefix);
this.queueName = "transformAndSaveData";
this.queueTtl = 23 * 60 * 60 * 1000; // 23 hours
this.schema = CachedDatasetSchema_1.CachedDatasetSchema;
this.saveTransformedDataset = async (transformedData, dataset, modelName) => {
const saveMethod = this.ropidGtfsFactory.getSaveMethod(dataset, modelName);
if (saveMethod === HelperTypes_1.SaveMethod.STREAM) {
await this.streamDataToTmp(transformedData, modelName);
}
else {
await this.getTmpModelByName(modelName)?.save(transformedData, false);
this.logger.info(`Datasets: copying to tmp success (${modelName})`);
}
const dbLastModified = await this.metadataRepository.getLastModified(dataset);
await this.metadataRepository.updateState(dataset, modelName, RopidMetadataModel_1.MetaStateEnum.SAVED, dbLastModified.version);
};
this.streamDataToTmp = async (data, modelName) => {
// get connection and db client
const connection = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.PostgresConnector).getConnection();
const client = (await connection.connectionManager.getConnection({ type: "write" }));
const tmpTableName = ropid_gtfs_2.RopidGTFS[modelName].pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp;
// copy transformed data to tmp table by stream
await new Promise((resolve, reject) => {
const stream = client
.query((0, pg_copy_streams_1.from)(`COPY "${const_1.PG_SCHEMA}".${tmpTableName} FROM STDIN DELIMITER ',' CSV HEADER;`))
.on("error", async (err) => {
this.logger.error(`Datasets: copying to tmp error (${modelName}): ${err.toString()}`);
await connection.connectionManager.releaseConnection(client);
return reject(err);
})
.on("finish", async () => {
this.logger.info(`Datasets: copying to tmp success (${modelName})`);
await connection.connectionManager.releaseConnection(client);
return resolve();
});
data.pipe(stream);
});
};
this.getTmpModelByName = (name) => {
const modelDef = ropid_gtfs_2.RopidGTFS[name];
if (!modelDef || !modelDef.name || !modelDef.pgTableName || !modelDef.outputSequelizeAttributes) {
throw new Error("Model not found.");
}
const model = new PostgresModel_1.PostgresModel(modelDef.name + "Tmp" + "Model", {
outputSequelizeAttributes: modelDef.outputSequelizeAttributes,
pgSchema: const_1.PG_SCHEMA,
pgTableName: modelDef.pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp,
savingType: "insertOnly",
sequelizeAdditionalSettings: {
timestamps: false,
},
}, new golemio_validator_1.JSONSchemaValidator(modelDef.name + "Tmp" + "ModelValidator", modelDef.mongo));
return model;
};
this.logger = Di_2.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
this.ropidGtfsFactory = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
this.gtfsRedisChannel = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel);
this.metadataRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsMetadataRepository);
}
async execute(inputData, msgProperties) {
this.logger.info(`Datasets: transforming and saving (${inputData.name})`);
let modelName = (0, HelperTypes_1.datasetFileModelMap)(inputData.dataset, inputData.name);
if (!Array.isArray(modelName)) {
modelName = [modelName];
}
let errorMessage;
try {
const transformedData = await this.ropidGtfsFactory.loadAndTransformDataset(inputData, modelName[0]);
if (inputData.dataset === ropid_gtfs_1.DatasetEnum.CIS_STOPS) {
await this.saveTransformedDataset(transformedData.cis_stops, inputData.dataset, modelName[0]);
await this.saveTransformedDataset(transformedData.cis_stop_groups, inputData.dataset, modelName[1]);
}
else {
await this.saveTransformedDataset(transformedData, inputData.dataset, modelName[0]);
}
}
catch (err) {
this.logger.error(`Datasets: transforming and saving error (${modelName}): ${err.toString()}`);
errorMessage = err.message;
}
if (msgProperties?.headers[HelperTypes_1.ORIGIN_HOSTNAME_HEADER]) {
await this.gtfsRedisChannel.publishMessage(`${errorMessage ?? "OK"} (${modelName})`, {
channelSuffix: msgProperties.headers[HelperTypes_1.ORIGIN_HOSTNAME_HEADER],
});
}
if (errorMessage) {
throw new golemio_errors_1.GeneralError(errorMessage, this.constructor.name);
}
}
}
exports.TransformAndSaveDataTask = TransformAndSaveDataTask;
//# sourceMappingURL=TransformAndSaveDataTask.js.map