@golemio/pid
Version:
Golemio PID Module
137 lines • 7.37 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DownloadDatasetsTask = void 0;
const Di_1 = require("../../../../ioc/Di");
const RopidGtfsFacade_1 = require("../../../RopidGtfsFacade");
const Di_2 = require("../../../ioc/Di");
const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const os_1 = __importDefault(require("os"));
const HelperTypes_1 = require("./helpers/HelperTypes");
const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
class DownloadDatasetsTask extends workers_1.AbstractTask {
constructor(queuePrefix) {
super(queuePrefix);
this.queueName = "downloadDatasets";
this.queueTtl = 59 * 60 * 1000; // 59 minutes
this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
/**
* Helper method - download and persist dataset
*/
this.downloadAndPersistDatasetFiles = async (dataset) => {
this.logger.info(`Datasets: processing ${dataset}`);
let files = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getAll();
if (!Array.isArray(files))
files = [files];
const dbLastModified = await this.metadataRepository.getLastModified(dataset);
const cachedDataset = [];
for (const file of files) {
this.logger.info(`Datasets: processing file '${file.name}'`);
const { data, ...meta } = file;
if (dataset === RopidGtfsFacade_1.DatasetEnum.PID_GTFS || dataset === RopidGtfsFacade_1.DatasetEnum.RUN_NUMBERS) {
await this.staticFileRedisRepository.pipeStream(file.filepath, data);
}
else {
await this.staticFileRedisRepository.set(file.filepath, typeof data === "string" ? data : JSON.stringify(data));
}
const key = (0, HelperTypes_1.datasetFileModelMap)(dataset, file.name);
const metaStates = (Array.isArray(key) ? key : [key]).map((name) => {
return {
dataset,
key: name,
type: RopidMetadataModel_1.MetaTypeEnum.STATE,
value: RopidMetadataModel_1.MetaStateEnum.DOWNLOADED,
version: dbLastModified.version + 1,
};
});
for (const metaState of metaStates) {
await this.metadataRepository.save(metaState);
}
cachedDataset.push({
...meta,
dataset,
});
}
const lastModified = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getLastModified();
await this.metadataRepository.save({
dataset,
key: RopidMetadataModel_1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
type: RopidMetadataModel_1.MetaTypeEnum.DATASET_INFO,
value: lastModified,
version: dbLastModified.version + 1,
});
this.logger.info(`Datasets: processing ${dataset} finished - ${JSON.stringify(cachedDataset)}`);
return cachedDataset;
};
this.logger = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.Logger);
this.gtfsRedisChannel = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel);
this.staticFileRedisRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository);
// MetaData Model
this.metadataRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsMetadataRepository);
this.ropidGtfsRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsRepository);
}
async execute(data) {
this.logger.info(`Datasets: downloading ${data.datasets}`);
/** Ropid GTFS Dataset Map Dependencies **/
this.ropidGtfsFactory = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
let cachedDatasets = [];
for (const dataset of data.datasets) {
try {
cachedDatasets = cachedDatasets.concat(await this.downloadAndPersistDatasetFiles(dataset));
}
catch (err) {
await integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
if (err instanceof golemio_errors_1.AbstractGolemioError) {
throw err;
}
throw new golemio_errors_1.GeneralError("Error while downloading GTFS datasets", this.constructor.name, err);
}
}
await this.ropidGtfsRepository.createTmpTables();
const subscriber = this.gtfsRedisChannel.createSubscriber({
channelSuffix: os_1.default.hostname(),
maxMessageCount: cachedDatasets.length,
});
await subscriber.subscribe();
for (const cachedDataset of cachedDatasets) {
integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "transformAndSaveData", cachedDataset, {
headers: {
[HelperTypes_1.ORIGIN_HOSTNAME_HEADER]: os_1.default.hostname(),
},
});
}
// Listen for messages from workers and wait for all of them to finish
// before continuing to the next step (replacing tables)
try {
await subscriber.listen(({ message, messageCount, isFinal }) => {
this.logger.debug(`Datasets: received message: ${message} (${messageCount}/${cachedDatasets.length})`);
if (!message.startsWith("OK")) {
this.logger.error(`Datasets: subscription error: ${message}`);
throw new golemio_errors_1.GeneralError(message, this.constructor.name);
}
if (isFinal) {
this.logger.info("Datasets: all messages received");
}
});
}
catch (err) {
if (err instanceof golemio_errors_1.AbstractGolemioError) {
throw err;
}
throw new golemio_errors_1.GeneralError("Error while listening for messages", this.constructor.name, err);
}
finally {
await subscriber.unsubscribe();
integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "checkSavedRowsAndReplaceTables", data);
}
}
}
exports.DownloadDatasetsTask = DownloadDatasetsTask;
//# sourceMappingURL=DownloadDatasetsTask.js.map