@golemio/pid
Version:
Golemio PID Module
397 lines • 22.9 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.GenerateFilesTask = void 0;
const RouteTypeEnums_1 = require("../../../../../helpers/RouteTypeEnums");
const JISEventsRepository_1 = require("../../../../jis/repositories/JISEventsRepository");
const ropid_vymi_1 = require("../../../../ropid-vymi");
const Di_1 = require("../../../../ioc/Di");
const VPContainerToken_1 = require("../../../ioc/VPContainerToken");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const WebhookDecorators_1 = require("@golemio/core/dist/integration-engine/workers/helpers/WebhookDecorators");
const CloudflareCachePurgeWebhook_1 = require("@golemio/core/dist/integration-engine/workers/webhooks/CloudflareCachePurgeWebhook");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const connectors_1 = require("@golemio/core/dist/integration-engine/connectors");
const redis_semaphore_1 = require("@golemio/core/dist/shared/redis-semaphore");
const promises_1 = require("node:timers/promises");
const ovapi_gtfs_realtime_bindings_1 = require("@golemio/ovapi-gtfs-realtime-bindings");
const const_1 = require("../../../../../const");
const PublicStopTimeCacheRepository_1 = require("../../vehicle-positions/data-access/cache/PublicStopTimeCacheRepository");
const GtfsRtRedisRepository_1 = require("../data-access/GtfsRtRedisRepository");
const AlertsGenerator_1 = require("../helpers/AlertsGenerator");
const VehicleDescriptor_1 = require("../helpers/VehicleDescriptor");
const const_2 = require("../helpers/const");
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
const constants_1 = require("../constants");
let GenerateFilesTask = class GenerateFilesTask extends workers_1.AbstractEmptyTask {
constructor(config, logger, jisEventsRepository, stopTimeRepository, tripsRepository, gtfsRtRedisRepository) {
super(constants_1.WORKER_NAME);
this.config = config;
this.logger = logger;
this.stopTimeRepository = stopTimeRepository;
this.tripsRepository = tripsRepository;
this.gtfsRtRedisRepository = gtfsRtRedisRepository;
this.queueName = "generateFiles";
this.queueTtl = 20 * 1000; // 20 seconds
const vymiEventsRepository = new ropid_vymi_1.RopidVYMIEventsModel();
const vymiRoutesRepository = new ropid_vymi_1.RopidVYMIEventsRoutesModel();
this.alertsGenerator = new AlertsGenerator_1.AlertsGenerator(this.config, this.gtfsRtRedisRepository, jisEventsRepository, vymiEventsRepository, vymiRoutesRepository);
this.vehicleDescriptor = new VehicleDescriptor_1.VehicleDescriptor();
// should finish under 10 seconds
this.lockTimeout = this.config.getValue("module.pid.vehicle-positions.gtfsRealtime.mutexLockTimeout", 10_000);
this.refreshInterval = this.lockTimeout * const_2.LOCK_REFRESH_INTERVAL_RATIO;
}
async execute() {
const startTimeMs = Date.now();
const mutex = this.createMutex();
const lockAcquired = await mutex.tryAcquire();
if (!lockAcquired) {
this.logger.info(`${this.constructor.name}: mutex lock was not acquired, other task instance is running.`);
return;
}
try {
await this.generateFiles();
}
catch (err) {
if (err instanceof golemio_errors_1.AbstractGolemioError) {
throw err;
}
else {
throw new golemio_errors_1.GeneralError("Error while processing task to generate files for GTFS-RT.", this.constructor.name, err);
}
}
finally {
await mutex.release();
}
const ellapsedTimeInMs = Date.now() - startTimeMs;
this.logFinishTime(ellapsedTimeInMs);
if (ellapsedTimeInMs < this.refreshInterval) {
await (0, promises_1.setTimeout)(Math.abs(this.refreshInterval - ellapsedTimeInMs));
}
integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, this.queueName, {});
}
logFinishTime(ellapsedTimeInMs) {
const outrunLockTimeoutMsg = ellapsedTimeInMs > this.lockTimeout ? "Slow" : "OK";
const logLevel = ellapsedTimeInMs > this.lockTimeout ? "warn" : "info";
this.logger[logLevel](`${this.constructor.name}: finished in ${ellapsedTimeInMs}ms = ${outrunLockTimeoutMsg}.`);
}
readStopTimeIds(tripData) {
const stopTimesIds = [];
for (const tripRecord of tripData) {
const { id } = this.vehicleDescriptor.getVehicleDescriptor(tripRecord);
if (typeof id === "string") {
stopTimesIds.push(`${id}-${tripRecord.gtfs_trip_id}`);
}
}
return stopTimesIds;
}
speedMpsToKmh(speed) {
return speed ? parseFloat((speed / const_2.MPS_TO_KMH).toFixed(2)) : null;
}
feedEntityFromTripData(tripRecord, entityTimestamp, tripDescriptor, vehicleDescriptor) {
const feedEntity = {
id: tripRecord.id,
vehicle: {
currentStopSequence: tripRecord.last_position.last_stop_sequence,
position: {
bearing: tripRecord.last_position.bearing,
latitude: tripRecord.last_position.lat,
longitude: tripRecord.last_position.lng,
speed: this.speedMpsToKmh(tripRecord.last_position.speed),
},
timestamp: entityTimestamp,
trip: tripDescriptor,
vehicle: vehicleDescriptor,
},
};
return feedEntity;
}
stopTimeUpdatesFromVehicleTrip(stopTimes, vehicleDescriptor, tripRecord) {
if (typeof vehicleDescriptor.id === "string") {
const stopTimesForVehicle = stopTimes.get(`${vehicleDescriptor.id}-${tripRecord.gtfs_trip_id}`);
const stopTimeUpdates = this.generateStopTimeUpdate(tripRecord.gtfs_route_type, stopTimesForVehicle, tripRecord.last_position);
return stopTimeUpdates;
}
this.logger.error(new golemio_errors_1.GeneralError(`Cannot get stopTimeUpdates for trip with unknown vehicleId and gtfsTripId \
'${tripRecord.gtfs_trip_id}'`, this.constructor.name, undefined, undefined, "pid"));
return [];
}
async generateFiles() {
const currentDate = new Date();
const tripData = await this.tripsRepository.findAllForGTFSRt(currentDate);
const feedHeader = this.createFeedHeader();
const updatesMessage = ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedMessage.create({ header: feedHeader });
const positionsMessage = ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedMessage.create({ header: feedHeader });
const pidFeedMessage = ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedMessage.create({ header: feedHeader });
const stopTimesIds = this.readStopTimeIds(tripData);
if (stopTimesIds.length === 0) {
this.logger.warn("No stop time ids found for GTFS-RT generation. Most likely no vehicles are running.");
return;
}
const stopTimes = await this.stopTimeRepository.getPublicStopTimeCache(stopTimesIds);
const gtfsTripIdMap = {};
const entityTripStarts = [];
for (const tripRecord of tripData) {
let tripUpdateFeedEntity;
let feedMessageEntity;
const vehicleDescriptor = this.vehicleDescriptor.getVehicleDescriptor(tripRecord);
const entityTimestamp = Math.round(new Date(tripRecord.last_position.origin_timestamp).getTime() / 1000);
if (tripRecord.last_position.is_canceled && !gtfsTripIdMap[tripRecord.gtfs_trip_id]) {
tripUpdateFeedEntity = this.handleCanceledTrip(tripRecord, entityTimestamp, updatesMessage);
gtfsTripIdMap[tripRecord.gtfs_trip_id] = true;
// For canceled trips we don't want to create positions because they have lng 0 and lat 0
}
else {
const tripDescriptor = {
scheduleRelationship: ovapi_gtfs_realtime_bindings_1.transit_realtime.TripDescriptor.ScheduleRelationship.SCHEDULED,
startDate: this.parseUTCDateFromISO(tripRecord.start_timestamp),
startTime: tripRecord.start_time,
tripId: tripRecord.gtfs_trip_id,
routeId: tripRecord.gtfs_route_id,
};
const stopTimeUpdates = this.stopTimeUpdatesFromVehicleTrip(stopTimes, vehicleDescriptor, tripRecord);
const shouldSkipTripUpdate = stopTimeUpdates.every((el) => el.scheduleRelationship === ovapi_gtfs_realtime_bindings_1.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship.NO_DATA ||
gtfsTripIdMap[tripRecord.gtfs_trip_id]);
entityTripStarts.push(tripRecord.start_timestamp);
const positionEntity = this.feedEntityFromTripData(tripRecord, entityTimestamp, tripDescriptor, vehicleDescriptor);
const positionsMessageEntity = ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedEntity.fromObject(positionEntity);
positionsMessage.entity.push(positionsMessageEntity);
feedMessageEntity = structuredClone(positionsMessageEntity);
if (!shouldSkipTripUpdate) {
const tripUpdate = {
stopTimeUpdate: stopTimeUpdates,
timestamp: entityTimestamp,
trip: tripDescriptor,
vehicle: vehicleDescriptor,
};
const updateEntity = {
id: tripRecord.id,
tripUpdate,
};
const updatesMessageEntity = ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedEntity.fromObject(updateEntity);
updatesMessage.entity.push(updatesMessageEntity);
gtfsTripIdMap[tripRecord.gtfs_trip_id] = true;
feedMessageEntity.tripUpdate = tripUpdate;
}
}
if (feedMessageEntity) {
pidFeedMessage.entity.push(feedMessageEntity);
}
else if (tripUpdateFeedEntity) {
// If the trip is canceled we want to update just the trip without positions
pidFeedMessage.entity.push(tripUpdateFeedEntity);
}
}
this.removeDuplicateVehicleDescriptors(positionsMessage, entityTripStarts);
this.removeDuplicateVehicleDescriptors(pidFeedMessage, entityTripStarts, { shouldRemoveVehicleOnly: true });
const results = await Promise.allSettled([
this.validateAndSaveBuffer("trip_updates", updatesMessage),
this.validateAndSaveBuffer("vehicle_positions", positionsMessage),
this.validateAndSaveBuffer("pid_feed", pidFeedMessage),
this.alertsGenerator.generateAlerts(feedHeader),
]);
const rejectedPromise = results.find((result) => result.status === "rejected");
if (rejectedPromise) {
throw rejectedPromise.reason;
}
}
/**
* Get rid of vehicle descriptors with duplicate IDs from a given GTFS-RT message
*
* @param message The message to get rid of duplicate vehicle descriptors from
* @param tripData The trip data to get start timestamps from
* @param options Options for the duplicate removal process
* @param options.removeVehicleOnly Whether to only remove the duplicate vehicle descriptor from the message entity, instead
* of removing the whole entity. Defaults to `false`. Note that if an entity with a duplicate vehicle descriptor does not
* contain any other valuable data, it will be removed regardless of this option.
*/
removeDuplicateVehicleDescriptors(message, entityTripStarts, options = {
shouldRemoveVehicleOnly: false,
}) {
if (!message.entity) {
return;
}
const entitiesWithoutVehicleId = [];
const entitiesWithoutTheirDuplicateVehicle = [];
const entityWithTripStartByVehicleId = new Map();
for (let i = 0; i < message.entity.length; i++) {
const entity = message.entity[i];
const vehicleId = entity.vehicle?.vehicle?.id ?? null;
const entityTripStart = new Date(entityTripStarts[i]);
if (vehicleId === null) {
entitiesWithoutVehicleId.push(entity);
}
else if (!entityWithTripStartByVehicleId.has(vehicleId)) {
entityWithTripStartByVehicleId.set(vehicleId, { entity, entityTripStart });
}
else {
const unusedEntity = this.conditionallyUpdateEntityByVehicleId(entityWithTripStartByVehicleId, vehicleId, entity, entityTripStart);
if (options.shouldRemoveVehicleOnly) {
const { vehicle = null, ...entityWithoutVehicle } = unusedEntity;
if (Object.keys(entityWithoutVehicle).find((key) => key !== "id")) {
entitiesWithoutTheirDuplicateVehicle.push(entityWithoutVehicle);
}
}
}
}
message.entity = entitiesWithoutVehicleId.concat(entitiesWithoutTheirDuplicateVehicle);
for (const { entity } of entityWithTripStartByVehicleId.values())
message.entity.push(entity);
}
/**
* Update a given `EntityWithTripStartByVehicleId` Map with given values if the new entity has a lower trip start date/time
* and is not canceled. The entity that does not end up in the map (either the new entity if it does not meet the conditions
* for the update or the original entity if the new entity does) is returned.
*
* @param entityWithTripStartByVehicleId The Map to update
* @param vehicleId The vehicle ID to use as a key
* @param entity The entity to use as part of the new value, if the conditions are met
* @param entityTripStart The entity trip start date/time to use as part of the new value, if the conditions are met
*/
conditionallyUpdateEntityByVehicleId(entityWithTripStartByVehicleId, vehicleId, entity, entityTripStart) {
const mapItem = entityWithTripStartByVehicleId.get(vehicleId);
const entityTripScheduleRelationship = entity.vehicle?.trip?.scheduleRelationship ?? null;
if (entityTripScheduleRelationship !== ovapi_gtfs_realtime_bindings_1.transit_realtime.TripDescriptor.ScheduleRelationship.CANCELED &&
entityTripStart < mapItem.entityTripStart) {
entityWithTripStartByVehicleId.set(vehicleId, { entity, entityTripStart });
return mapItem.entity;
}
return entity;
}
generateStopTimeUpdate(routeType, stopTimes, vehiclePosition) {
const stopTimeUpdates = [];
// TODO this is a temporary solution, we need to fix the underlying issue in the source view
// v_public_vehiclepositions_combined_stop_times, the delay prediction is seemingly not working properly
// when the vehicle is before the track, see https://gitlab.com/operator-ict/golemio/code/modules/pid/-/issues/371
const shouldFallbackToScheduleTime = this.shouldFallbackToScheduleTime(vehiclePosition);
for (const stopTime of stopTimes) {
const scheduledTrack = stopTime.platform_code;
let actualTrack = stopTime.platform_code;
if (routeType === RouteTypeEnums_1.GTFSRouteTypeEnum.TRAIN) {
actualTrack = stopTime.cis_stop_platform_code;
}
const arrivalDelay = shouldFallbackToScheduleTime ? 0 : stopTime.arr_delay ?? stopTime.dep_delay;
const departureDelay = shouldFallbackToScheduleTime ? 0 : stopTime.dep_delay ?? stopTime.arr_delay;
stopTimeUpdates.push({
arrival: {
delay: arrivalDelay,
},
departure: {
delay: departureDelay,
},
...(arrivalDelay === null &&
departureDelay === null && {
scheduleRelationship: ovapi_gtfs_realtime_bindings_1.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship.NO_DATA,
}),
stopSequence: stopTime.sequence,
stopId: stopTime.stop_id,
".transit_realtime.ovapiStopTimeUpdate": {
...(scheduledTrack && { scheduledTrack }),
...(actualTrack && { actualTrack }),
},
});
}
return stopTimeUpdates;
}
createFeedHeader() {
const header = {
gtfsRealtimeVersion: "2.0",
incrementality: ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedHeader.Incrementality.FULL_DATASET,
timestamp: Math.round(Date.now() / 1000),
};
return ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedHeader.fromObject(header);
}
/**
* Parse UTC date from ISO timestamp
* @example 20230101
*/
parseUTCDateFromISO(timestamp) {
const startTimestamp = new Date(timestamp);
const formattedDate = startTimestamp.getUTCFullYear() +
("0" + (startTimestamp.getUTCMonth() + 1)).slice(-2) +
("0" + startTimestamp.getUTCDate()).slice(-2);
return formattedDate;
}
async validateAndSaveBuffer(feedName, message) {
const feedMessageVerificationResult = ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedMessage.verify(message);
if (feedMessageVerificationResult !== null) {
this.logger.error(new golemio_errors_1.GeneralError(`Invalid GTFS-RT message for feed '${feedName}'`, this.constructor.name, feedMessageVerificationResult, undefined, "pid"));
return;
}
const buffer = ovapi_gtfs_realtime_bindings_1.transit_realtime.FeedMessage.encode(message).finish();
await this.gtfsRtRedisRepository.set(feedName + "_timestamp", message.header.timestamp);
await this.gtfsRtRedisRepository.hset(feedName + ".pb", buffer.toString("binary"));
}
shouldFallbackToScheduleTime(vehiclePosition) {
return ((vehiclePosition.state_position === const_1.StatePositionEnum.BEFORE_TRACK ||
vehiclePosition.state_position === const_1.StatePositionEnum.BEFORE_TRACK_DELAYED) &&
(vehiclePosition.delay === 0 || vehiclePosition.delay === null));
}
handleCanceledTrip(tripRecord, timestamp, updatesMessage) {
const feedEntity = {
id: tripRecord.id,
tripUpdate: {
stopTimeUpdate: null,
timestamp,
trip: {
scheduleRelationship: ovapi_gtfs_realtime_bindings_1.transit_realtime.TripDescriptor.ScheduleRelationship.CANCELED,
startDate: this.parseUTCDateFromISO(tripRecord.start_timestamp),
startTime: tripRecord.start_time,
tripId: tripRecord.gtfs_trip_id,
routeId: tripRecord.gtfs_route_id,
},
vehicle: null,
},
};
updatesMessage.entity.push(feedEntity);
return feedEntity;
}
createMutex() {
const redisClient = connectors_1.RedisConnector.getConnection();
return new redis_semaphore_1.Mutex(redisClient, this.queueName, {
acquireAttemptsLimit: 1,
lockTimeout: this.lockTimeout,
refreshInterval: this.refreshInterval,
onLockLost: (err) => {
this.logger.info(`${this.constructor.name}: mutex lock was lost, will be generated in next task run.`);
},
});
}
};
exports.GenerateFilesTask = GenerateFilesTask;
// Used only for the class-level decorator below — NOT resolved inside the constructor
// TODO: Static fields cannot be injected via DI (tsyringe only injects constructor parameters). This is a
// pre-existing design limitation required by @WebhookDecorators, which runs at class definition time before
// any instance is created. Resolving directly from PidContainer here is intentional but bypasses the DI graph.
GenerateFilesTask.decoratorConfig = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.SimpleConfig);
__decorate([
WebhookDecorators_1.WebhookDecorators.after(new CloudflareCachePurgeWebhook_1.CloudflareCachePurgeWebhook(Object.values(GenerateFilesTask.decoratorConfig.getValue("module.pid.cloudflare.cachedApiUrlPaths", {})))),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], GenerateFilesTask.prototype, "execute", null);
exports.GenerateFilesTask = GenerateFilesTask = __decorate([
(0, tsyringe_1.injectable)(),
__param(0, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.SimpleConfig)),
__param(1, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
__param(2, (0, tsyringe_1.inject)(VPContainerToken_1.VPContainerToken.JISEventsRepository)),
__param(3, (0, tsyringe_1.inject)(VPContainerToken_1.VPContainerToken.PublicStopTimeCacheRepository)),
__param(4, (0, tsyringe_1.inject)(VPContainerToken_1.VPContainerToken.TripRepository)),
__param(5, (0, tsyringe_1.inject)(VPContainerToken_1.VPContainerToken.GtfsRtRedisRepository)),
__metadata("design:paramtypes", [Object, Object, JISEventsRepository_1.JISEventsRepository,
PublicStopTimeCacheRepository_1.PublicStopTimeCacheRepository, Function, GtfsRtRedisRepository_1.GtfsRtRedisRepository])
], GenerateFilesTask);
//# sourceMappingURL=GenerateFilesTask.js.map