@golemio/pid
Version:
Golemio PID Module
166 lines • 9.39 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DownloadDatasetsTask = void 0;
const RopidGtfsMetadataRepository_1 = require("../../../RopidGtfsMetadataRepository");
const RopidGtfsRepository_1 = require("../../../data-access/RopidGtfsRepository");
const StaticFileRedisRepository_1 = require("../../../data-access/cache/StaticFileRedisRepository");
const DatasetEnum_1 = require("../../../helpers/DatasetEnum");
const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
const pubsub_1 = require("@golemio/core/dist/integration-engine/data-access/pubsub");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
const node_os_1 = __importDefault(require("node:os"));
const HelperTypes_1 = require("./helpers/HelperTypes");
const RopidGtfsFactory_1 = require("./helpers/RopidGtfsFactory");
const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
const constants_1 = require("../constants");
let DownloadDatasetsTask = class DownloadDatasetsTask extends workers_1.AbstractTask {
constructor(metadataRepository, staticFileRedisRepository, ropidGtfsRepository, ropidGtfsFactory, gtfsRedisChannel, logger) {
super(constants_1.TIMETABLE_WORKER_NAME);
this.metadataRepository = metadataRepository;
this.staticFileRedisRepository = staticFileRedisRepository;
this.ropidGtfsRepository = ropidGtfsRepository;
this.ropidGtfsFactory = ropidGtfsFactory;
this.gtfsRedisChannel = gtfsRedisChannel;
this.logger = logger;
this.queueName = "downloadDatasets";
this.queueTtl = 59 * 60 * 1000; // 59 minutes
this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
/**
* Helper method - download and persist dataset
*/
this.downloadAndPersistDatasetFiles = async (dataset) => {
this.logger.info(`Datasets: processing ${dataset}`);
let files = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getAll();
if (!Array.isArray(files))
files = [files];
const dbLastModified = await this.metadataRepository.getLastModified(dataset);
const cachedDataset = [];
for (const file of files) {
this.logger.info(`Datasets: processing file '${file.name}'`);
const { data, ...meta } = file;
if (dataset === DatasetEnum_1.DatasetEnum.PID_GTFS || dataset === DatasetEnum_1.DatasetEnum.RUN_NUMBERS) {
await this.staticFileRedisRepository.pipeStream(file.filepath, data);
}
else {
await this.staticFileRedisRepository.set(file.filepath, typeof data === "string" ? data : JSON.stringify(data));
}
const key = (0, HelperTypes_1.datasetFileModelMap)(dataset, file.name);
const metaStates = (Array.isArray(key) ? key : [key]).map((name) => {
return {
dataset,
key: name,
type: RopidMetadataModel_1.MetaTypeEnum.STATE,
value: RopidMetadataModel_1.MetaStateEnum.DOWNLOADED,
version: dbLastModified.version + 1,
};
});
for (const metaState of metaStates) {
await this.metadataRepository.save(metaState);
}
cachedDataset.push({
...meta,
dataset,
});
}
const lastModified = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getLastModified();
await this.metadataRepository.save({
dataset,
key: RopidMetadataModel_1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
type: RopidMetadataModel_1.MetaTypeEnum.DATASET_INFO,
value: lastModified,
version: dbLastModified.version + 1,
});
this.logger.info(`Datasets: processing ${dataset} finished - ${JSON.stringify(cachedDataset)}`);
return cachedDataset;
};
}
async execute(data) {
this.logger.info(`Datasets: downloading ${data.datasets}`);
let cachedDatasets = [];
for (const dataset of data.datasets) {
try {
cachedDatasets = cachedDatasets.concat(await this.downloadAndPersistDatasetFiles(dataset));
}
catch (err) {
await integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
if (err instanceof golemio_errors_1.AbstractGolemioError) {
throw err;
}
throw new golemio_errors_1.GeneralError("Error while downloading GTFS datasets", this.constructor.name, err);
}
}
await this.ropidGtfsRepository.createTmpTables();
const subscriber = this.gtfsRedisChannel.createSubscriber({
channelSuffix: node_os_1.default.hostname(),
maxMessageCount: cachedDatasets.length,
});
await subscriber.subscribe();
for (const cachedDataset of cachedDatasets) {
integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "transformAndSaveData", cachedDataset, {
headers: {
[HelperTypes_1.ORIGIN_HOSTNAME_HEADER]: node_os_1.default.hostname(),
},
});
}
// Listen for messages from workers and wait for all of them to finish
// before continuing to the next step (replacing tables)
try {
await subscriber.listen(({ message, messageCount, isFinal }) => {
this.logger.debug(`Datasets: received message: ${message} (${messageCount}/${cachedDatasets.length})`);
if (!message.startsWith("OK")) {
this.logger.error(`Datasets: subscription error: ${message}`);
throw new golemio_errors_1.GeneralError(message, this.constructor.name);
}
if (isFinal) {
this.logger.info("Datasets: all messages received");
}
});
}
catch (err) {
if (err instanceof golemio_errors_1.AbstractGolemioError) {
throw err;
}
throw new golemio_errors_1.GeneralError("Error while listening for messages", this.constructor.name, err);
}
finally {
await subscriber.unsubscribe();
integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "checkSavedRowsAndReplaceTables", data);
}
}
};
exports.DownloadDatasetsTask = DownloadDatasetsTask;
exports.DownloadDatasetsTask = DownloadDatasetsTask = __decorate([
(0, tsyringe_1.injectable)(),
__param(0, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsMetadataRepository)),
__param(1, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository)),
__param(2, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsRepository)),
__param(3, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory)),
__param(4, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel)),
__param(5, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
__metadata("design:paramtypes", [RopidGtfsMetadataRepository_1.RopidGtfsMetadataRepository,
StaticFileRedisRepository_1.StaticFileRedisRepository,
RopidGtfsRepository_1.RopidGtfsRepository,
RopidGtfsFactory_1.RopidGtfsFactory,
pubsub_1.RedisPubSubChannel, Object])
], DownloadDatasetsTask);
//# sourceMappingURL=DownloadDatasetsTask.js.map