minotor/src/gtfs/parser.ts

A lightweight client-side transit routing library.

import log from 'loglevel';
import { DateTime } from 'luxon';
import StreamZip from 'node-stream-zip';

import { StopId } from '../stops/stops.js';
import { StopsIndex } from '../stops/stopsIndex.js';
import { RouteType, Timetable } from '../timetable/timetable.js';
import { TransferGenerator } from '../transfers/generator.js';
import { FrequenciesMap, parseFrequencies } from './frequencies.js';
import { standardGtfsProfile } from './profiles/standard.js';
import { indexRoutes, parseRoutes } from './routes.js';
import { parseCalendar, parseCalendarDates, ServiceIds } from './services.js';
import { parseStops } from './stops.js';
import {
  addGeneratedTransfers,
  addMissingSiblingTransfers,
  buildTripTransfers,
  ForbiddenTransfersMap,
  GtfsTripTransfer,
  parseTransfers,
  TransfersMap,
} from './transfers.js';
import {
  buildStopsAdjacencyStructure,
  parseStopTimes,
  parseTrips,
} from './trips.js';
import { Maybe } from './utils.js';

const CALENDAR_FILE = 'calendar.txt';
const CALENDAR_DATES_FILE = 'calendar_dates.txt';
const FREQUENCIES_FILE = 'frequencies.txt';
const ROUTES_FILE = 'routes.txt';
const TRIPS_FILE = 'trips.txt';
const STOP_TIMES_FILE = 'stop_times.txt';
const STOPS_FILE = 'stops.txt';
const TRANSFERS_FILE = 'transfers.txt';

export type GtfsProfile = {
  routeTypeParser: (routeType: number) => Maybe<RouteType>;
  /**
   * Derive fallback transfers between active stops that share a parent station.
   *
   * @default true
   */
  deriveSiblingTransfers?: boolean;
};

export class GtfsParser {
  private path: string;
  private profile: GtfsProfile;
  private transferGenerator?: TransferGenerator;

  constructor(
    path: string,
    profile: GtfsProfile = standardGtfsProfile,
    transferGenerator?: TransferGenerator,
  ) {
    // TODO: support input from multiple sources
    this.path = path;
    this.profile = profile;
    this.transferGenerator = transferGenerator;
  }

  /**
   * Parses a GTFS feed to extract all the data relevant to a given day in a transit-planner friendly format.
   *
   * @param date The active date.
   * @returns The parsed timetable.
   */
  async parseTimetable(date: Date): Promise<Timetable> {
    log.setLevel('INFO');
    const zip = new StreamZip.async({ file: this.path });
    const entries = await zip.entries();
    const datetime = DateTime.fromJSDate(date);

    const activeServiceIds: ServiceIds = new Set();
    const activeStopIds = new Set<StopId>();

    log.info(`Parsing ${STOPS_FILE}`);
    const stopsStart = performance.now();
    const stopsStream = await zip.stream(STOPS_FILE);
    const parsedStops = await parseStops(stopsStream);
    const stopsEnd = performance.now();
    log.info(
      `${parsedStops.size} parsed stops. (${(stopsEnd - stopsStart).toFixed(2)}ms)`,
    );

    if (entries[CALENDAR_FILE]) {
      log.info(`Parsing ${CALENDAR_FILE}`);
      const calendarStart = performance.now();
      const calendarStream = await zip.stream(CALENDAR_FILE);
      await parseCalendar(calendarStream, activeServiceIds, datetime);
      const calendarEnd = performance.now();
      log.info(
        `${activeServiceIds.size} valid services. (${(calendarEnd - calendarStart).toFixed(2)}ms)`,
      );
    }

    if (entries[CALENDAR_DATES_FILE]) {
      log.info(`Parsing ${CALENDAR_DATES_FILE}`);
      const calendarDatesStart = performance.now();
      const calendarDatesStream = await zip.stream(CALENDAR_DATES_FILE);
      await parseCalendarDates(calendarDatesStream, activeServiceIds, datetime);
      const calendarDatesEnd = performance.now();
      log.info(
        `${activeServiceIds.size} valid services. (${(calendarDatesEnd - calendarDatesStart).toFixed(2)}ms)`,
      );
    }

    log.info(`Parsing ${ROUTES_FILE}`);
    const routesStart = performance.now();
    const routesStream = await zip.stream(ROUTES_FILE);
    const validGtfsRoutes = await parseRoutes(routesStream, this.profile);
    const routesEnd = performance.now();
    log.info(
      `${validGtfsRoutes.size} valid GTFS routes. (${(routesEnd - routesStart).toFixed(2)}ms)`,
    );

    log.info(`Parsing ${TRIPS_FILE}`);
    const tripsStart = performance.now();
    const tripsStream = await zip.stream(TRIPS_FILE);
    const trips = await parseTrips(
      tripsStream,
      activeServiceIds,
      validGtfsRoutes,
    );
    const tripsEnd = performance.now();
    log.info(
      `${trips.size} valid trips. (${(tripsEnd - tripsStart).toFixed(2)}ms)`,
    );

    let transfers: TransfersMap = new Map();
    let forbiddenTransfers: ForbiddenTransfersMap = new Map();
    let tripContinuationsList: GtfsTripTransfer[] = [];
    let guaranteedTripTransfersList: GtfsTripTransfer[] = [];
    if (entries[TRANSFERS_FILE]) {
      log.info(`Parsing ${TRANSFERS_FILE}`);
      const transfersStart = performance.now();
      const transfersStream = await zip.stream(TRANSFERS_FILE);
      const {
        transfers: parsedTransfers,
        forbiddenTransfers: parsedForbiddenTransfers,
        tripContinuations: parsedTripContinuations,
        guaranteedTripTransfers: parsedGuaranteedTripTransfers,
      } = await parseTransfers(transfersStream, parsedStops, activeServiceIds);
      transfers = parsedTransfers;
      forbiddenTransfers = parsedForbiddenTransfers;
      tripContinuationsList = parsedTripContinuations;
      guaranteedTripTransfersList = parsedGuaranteedTripTransfers;
      const transfersEnd = performance.now();
      log.info(
        `${transfers.size} valid transfers and ${tripContinuationsList.length} trip continuations and ${guaranteedTripTransfersList.length} guaranteed trip transfers. (${(transfersEnd - transfersStart).toFixed(2)}ms)`,
      );
    }

    let frequenciesMap: FrequenciesMap | undefined;
    if (entries[FREQUENCIES_FILE]) {
      log.info(`Parsing ${FREQUENCIES_FILE}`);
      const frequenciesStart = performance.now();
      const frequenciesStream = await zip.stream(FREQUENCIES_FILE);
      frequenciesMap = await parseFrequencies(
        frequenciesStream,
        new Set(trips.keys()),
      );
      const frequenciesEnd = performance.now();
      log.info(
        `${frequenciesMap.size} trips with frequency data. (${(frequenciesEnd - frequenciesStart).toFixed(2)}ms)`,
      );
    }

    log.info(`Parsing ${STOP_TIMES_FILE}`);
    const stopTimesStart = performance.now();
    const stopTimesStream = await zip.stream(STOP_TIMES_FILE);
    const { routes, serviceRoutesMap, tripsMapping } = await parseStopTimes(
      stopTimesStream,
      parsedStops,
      trips,
      activeStopIds,
      frequenciesMap,
    );
    const serviceRoutes = indexRoutes(validGtfsRoutes, serviceRoutesMap);
    const stopTimesEnd = performance.now();
    log.info(
      `${routes.length} valid unique routes. (${(stopTimesEnd - stopTimesStart).toFixed(2)}ms)`,
    );

    if (this.profile.deriveSiblingTransfers !== false) {
      const siblingTransfersStart = performance.now();
      const siblingTransfersAdded = addMissingSiblingTransfers(
        parsedStops,
        activeStopIds,
        transfers,
        forbiddenTransfers,
      );
      const siblingTransfersEnd = performance.now();
      log.info(
        `${siblingTransfersAdded} sibling transfers added. (${(siblingTransfersEnd - siblingTransfersStart).toFixed(2)}ms)`,
      );
    }

    if (this.transferGenerator) {
      log.info('Generating virtual transfers');
      const virtualTransfersStart = performance.now();
      // Only route-served stops with coordinates are worth connecting: a
      // transfer to a stop no route calls at is a dead end for the router.
      const originStops = Array.from(parsedStops.values()).filter(
        (stop) =>
          activeStopIds.has(stop.id) &&
          stop.lat !== undefined &&
          stop.lon !== undefined,
      );
      const stopsIndex = new StopsIndex(Array.from(parsedStops.values()));
      const generatedTransfers = await this.transferGenerator.generate(
        originStops,
        stopsIndex,
      );
      const addedTransfers = addGeneratedTransfers(
        generatedTransfers,
        activeStopIds,
        transfers,
        forbiddenTransfers,
      );
      const virtualTransfersEnd = performance.now();
      log.info(
        `${addedTransfers} virtual transfers added. (${(virtualTransfersEnd - virtualTransfersStart).toFixed(2)}ms)`,
      );
    }

    log.info('Building stops adjacency structure');
    const stopsAdjacencyStart = performance.now();
    const stopsAdjacency = buildStopsAdjacencyStructure(
      serviceRoutes,
      routes,
      transfers,
      parsedStops.size,
      activeStopIds,
    );

    const stopsAdjacencyEnd = performance.now();
    log.info(
      `${stopsAdjacency.length} valid stops in the structure. (${(stopsAdjacencyEnd - stopsAdjacencyStart).toFixed(2)}ms)`,
    );
    await zip.close();

    // temporary timetable for building continuations
    const timetable = new Timetable(stopsAdjacency, routes, serviceRoutes);

    log.info('Building in-seat trip continuations');
    const tripContinuationsStart = performance.now();
    const tripContinuations = buildTripTransfers(
      tripsMapping,
      tripContinuationsList,
      timetable,
      activeStopIds,
    );
    const tripContinuationsEnd = performance.now();
    log.info(
      `${tripContinuations.size} in-seat trip continuations origins created. (${(tripContinuationsEnd - tripContinuationsStart).toFixed(2)}ms)`,
    );
    log.info('Parsing complete.');
    log.info('Building guaranteed trip transfers');
    const guaranteedTripTransfersStart = performance.now();
    const guaranteedTripTransfers = buildTripTransfers(
      tripsMapping,
      guaranteedTripTransfersList,
      timetable,
      activeStopIds,
    );
    const guaranteedTripTransfersEnd = performance.now();
    log.info(
      `${guaranteedTripTransfers.size} guaranteed trip transfers origins created. (${(guaranteedTripTransfersEnd - guaranteedTripTransfersStart).toFixed(2)}ms)`,
    );
    log.info('Parsing complete.');

    return new Timetable(
      stopsAdjacency,
      routes,
      serviceRoutes,
      tripContinuations,
      guaranteedTripTransfers,
    );
  }

  /**
   * Parses a GTFS feed to extract all stops.
   *
   * @param activeStops The set of active stop IDs to include in the index.
   * @returns An index of stops.
   */
  async parseStops(): Promise<StopsIndex> {
    const zip = new StreamZip.async({ file: this.path });

    log.info(`Parsing ${STOPS_FILE}`);
    const stopsStart = performance.now();
    const stopsStream = await zip.stream(STOPS_FILE);
    const stops = await parseStops(stopsStream);
    const stopsEnd = performance.now();

    log.info(
      `${stops.size} parsed stops. (${(stopsEnd - stopsStart).toFixed(2)}ms)`,
    );

    await zip.close();

    return new StopsIndex(Array.from(stops.values()));
  }
}