@golemio/pid
Version:
Golemio PID Module
140 lines • 8.57 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.TransformAndSaveDataTask = void 0;
const RopidGtfsMetadataRepository_1 = require("../../../RopidGtfsMetadataRepository");
const DatasetEnum_1 = require("../../../helpers/DatasetEnum");
const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
const const_1 = require("../../../../../schema-definitions/const");
const ropid_gtfs_1 = require("../../../../../schema-definitions/ropid-gtfs");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const pubsub_1 = require("@golemio/core/dist/integration-engine/data-access/pubsub");
const PostgresModel_1 = require("@golemio/core/dist/integration-engine/models/PostgresModel");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
const pg_copy_streams_1 = require("pg-copy-streams");
const constants_1 = require("../constants");
const HelperTypes_1 = require("./helpers/HelperTypes");
const RopidGtfsFactory_1 = require("./helpers/RopidGtfsFactory");
const CachedDatasetSchema_1 = require("./schema/CachedDatasetSchema");
let TransformAndSaveDataTask = class TransformAndSaveDataTask extends workers_1.AbstractTask {
constructor(metadataRepository, ropidGtfsFactory, gtfsRedisChannel, databaseConnector, logger) {
super(constants_1.TIMETABLE_WORKER_NAME);
this.metadataRepository = metadataRepository;
this.ropidGtfsFactory = ropidGtfsFactory;
this.gtfsRedisChannel = gtfsRedisChannel;
this.databaseConnector = databaseConnector;
this.logger = logger;
this.queueName = "transformAndSaveData";
this.queueTtl = 23 * 60 * 60 * 1000; // 23 hours
this.schema = CachedDatasetSchema_1.CachedDatasetSchema;
this.saveTransformedDataset = async (transformedData, dataset, modelName) => {
const saveMethod = this.ropidGtfsFactory.getSaveMethod(dataset, modelName);
if (saveMethod === HelperTypes_1.SaveMethod.STREAM) {
await this.streamDataToTmp(transformedData, modelName);
}
else {
await this.getTmpModelByName(modelName)?.save(transformedData, false);
this.logger.info(`Datasets: copying to tmp success (${modelName})`);
}
const dbLastModified = await this.metadataRepository.getLastModified(dataset);
await this.metadataRepository.updateState(dataset, modelName, RopidMetadataModel_1.MetaStateEnum.SAVED, dbLastModified.version);
};
this.streamDataToTmp = async (data, modelName) => {
// get connection and db client
const connection = this.databaseConnector.getConnection();
const client = (await connection.connectionManager.getConnection({ type: "write" }));
const tmpTableName = ropid_gtfs_1.RopidGTFS[modelName].pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp;
// copy transformed data to tmp table by stream
await new Promise((resolve, reject) => {
const stream = client
.query((0, pg_copy_streams_1.from)(`COPY "${const_1.PG_SCHEMA}".${tmpTableName} FROM STDIN DELIMITER ',' CSV HEADER;`))
.on("error", async (err) => {
this.logger.error(`Datasets: copying to tmp error (${modelName}): ${err.toString()}`);
await connection.connectionManager.releaseConnection(client);
return reject(err);
})
.on("finish", async () => {
this.logger.info(`Datasets: copying to tmp success (${modelName})`);
await connection.connectionManager.releaseConnection(client);
return resolve();
});
data.pipe(stream);
});
};
this.getTmpModelByName = (name) => {
const modelDef = ropid_gtfs_1.RopidGTFS[name];
if (!modelDef || !modelDef.name || !modelDef.pgTableName || !modelDef.outputSequelizeAttributes) {
throw new Error("Model not found.");
}
const model = new PostgresModel_1.PostgresModel(modelDef.name + "Tmp" + "Model", {
outputSequelizeAttributes: modelDef.outputSequelizeAttributes,
pgSchema: const_1.PG_SCHEMA,
pgTableName: modelDef.pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp,
savingType: "insertOnly",
sequelizeAdditionalSettings: {
timestamps: false,
},
}, new golemio_validator_1.JSONSchemaValidator(modelDef.name + "Tmp" + "ModelValidator", modelDef.mongo));
return model;
};
}
async execute(inputData, msgProperties) {
this.logger.info(`Datasets: transforming and saving (${inputData.name})`);
let modelName = (0, HelperTypes_1.datasetFileModelMap)(inputData.dataset, inputData.name);
if (!Array.isArray(modelName)) {
modelName = [modelName];
}
let errorMessage;
try {
const transformedData = await this.ropidGtfsFactory.loadAndTransformDataset(inputData, modelName[0]);
if (inputData.dataset === DatasetEnum_1.DatasetEnum.CIS_STOPS) {
await this.saveTransformedDataset(transformedData.cis_stops, inputData.dataset, modelName[0]);
await this.saveTransformedDataset(transformedData.cis_stop_groups, inputData.dataset, modelName[1]);
}
else {
await this.saveTransformedDataset(transformedData, inputData.dataset, modelName[0]);
}
}
catch (err) {
this.logger.error(`Datasets: transforming and saving error (${modelName}): ${err.toString()}`);
errorMessage = err.message;
}
if (msgProperties?.headers?.[HelperTypes_1.ORIGIN_HOSTNAME_HEADER]) {
await this.gtfsRedisChannel.publishMessage(`${errorMessage ?? "OK"} (${modelName})`, {
channelSuffix: msgProperties.headers[HelperTypes_1.ORIGIN_HOSTNAME_HEADER],
});
}
if (errorMessage) {
throw new golemio_errors_1.GeneralError(errorMessage, this.constructor.name);
}
}
};
exports.TransformAndSaveDataTask = TransformAndSaveDataTask;
exports.TransformAndSaveDataTask = TransformAndSaveDataTask = __decorate([
(0, tsyringe_1.injectable)(),
__param(0, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsMetadataRepository)),
__param(1, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory)),
__param(2, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel)),
__param(3, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.PostgresConnector)),
__param(4, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
__metadata("design:paramtypes", [RopidGtfsMetadataRepository_1.RopidGtfsMetadataRepository,
RopidGtfsFactory_1.RopidGtfsFactory,
pubsub_1.RedisPubSubChannel, Object, Object])
], TransformAndSaveDataTask);
//# sourceMappingURL=TransformAndSaveDataTask.js.map