gtfs
Version:
Import GTFS transit data into SQLite and query routes, stops, times, fares and more
1,008 lines (986 loc) • 26 kB
JavaScript
// src/bin/gtfsrealtime-update.ts
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import PrettyError from "pretty-error";
// src/lib/file-utils.ts
import path from "node:path";
import { existsSync } from "node:fs";
import { mkdir, readFile, rm } from "node:fs/promises";
import { omit, snakeCase } from "lodash-es";
import sanitize from "sanitize-filename";
import untildify from "untildify";
import StreamZip from "node-stream-zip";
// src/lib/log-utils.ts
import { clearLine, cursorTo } from "node:readline";
import { noop } from "lodash-es";
import * as colors from "yoctocolors";
function log(config) {
if (config.verbose === false) {
return noop;
}
if (config.logFunction) {
return config.logFunction;
}
return (text, overwrite = false) => {
if (overwrite && process.stdout.isTTY) {
clearLine(process.stdout, 0);
cursorTo(process.stdout, 0);
} else {
process.stdout.write("\n");
}
process.stdout.write(text);
};
}
function logWarning(config) {
if (config.logFunction) {
return config.logFunction;
}
return (text) => {
process.stdout.write(`
${formatWarning(text)}
`);
};
}
function logError(config) {
if (config.logFunction) {
return config.logFunction;
}
return (text) => {
process.stdout.write(`
${formatError(text)}
`);
};
}
function formatWarning(text) {
return colors.yellow(`${colors.underline("Warning")}: ${text}`);
}
function formatError(error) {
const messageText = error instanceof Error ? error.message : error;
const cleanMessage = messageText.replace(/^Error:\s*/i, "");
return colors.red(`${colors.underline("Error")}: ${cleanMessage}`);
}
// src/lib/file-utils.ts
async function getConfig(argv2) {
let config;
let data;
try {
if (argv2.configPath) {
const configPath = path.resolve(untildify(argv2.configPath));
data = await readFile(configPath, "utf8");
config = Object.assign(JSON.parse(data), argv2);
} else if (argv2.gtfsPath || argv2.gtfsUrl || argv2.sqlitePath) {
const agencies = [
...argv2.gtfsPath ? [{ path: argv2.gtfsPath }] : [],
...argv2.gtfsUrl ? [{ url: argv2.gtfsUrl }] : []
];
config = {
agencies,
...omit(argv2, ["path", "url"])
};
} else if (existsSync(path.resolve("./config.json"))) {
data = await readFile(path.resolve("./config.json"), "utf8");
config = Object.assign(JSON.parse(data), argv2);
log(config)("Using configuration from ./config.json");
} else {
throw new Error(
"Cannot find configuration file. Use config-sample.json as a starting point, pass --configPath option."
);
}
return config;
} catch (error) {
if (error instanceof SyntaxError) {
throw new Error(
`Cannot parse configuration file. Check to ensure that it is valid JSON. Error: ${error.message}`
);
}
throw error;
}
}
// src/lib/import-gtfs.ts
import { parse } from "csv-parse";
import pluralize2 from "pluralize";
import stripBomStream from "strip-bom-stream";
import { temporaryDirectory } from "tempy";
import Timer from "timer-machine";
import untildify3 from "untildify";
import mapSeries2 from "promise-map-series";
// src/models/gtfs-realtime/trip-updates.ts
var tripUpdates = {
filenameBase: "trip_updates",
extension: "gtfs-realtime",
schema: [
{
name: "id",
type: "text",
required: true,
primary: true,
index: true,
source: "id",
prefix: true
},
{
name: "vehicle_id",
type: "text",
index: true,
source: "tripUpdate.vehicle.id",
default: null,
prefix: true
},
{
name: "trip_id",
type: "text",
index: true,
source: "tripUpdate.trip.tripId",
default: null,
prefix: true
},
{
name: "trip_start_time",
type: "text",
source: "tripUpdate.trip.startTime",
default: null
},
{
name: "direction_id",
type: "integer",
source: "tripUpdate.trip.directionId",
default: null
},
{
name: "route_id",
type: "text",
index: true,
source: "tripUpdate.trip.routeId",
default: null,
prefix: true
},
{
name: "start_date",
type: "text",
source: "tripUpdate.trip.startDate",
default: null
},
{
name: "timestamp",
type: "text",
source: "tripUpdate.timestamp",
default: null
},
{
name: "schedule_relationship",
type: "text",
source: "tripUpdate.trip.scheduleRelationship",
default: null
},
{
name: "created_timestamp",
type: "integer",
required: true
},
{
name: "expiration_timestamp",
type: "integer",
required: true
}
]
};
// src/models/gtfs-realtime/stop-time-updates.ts
var stopTimeUpdates = {
filenameBase: "stop_time_updates",
extension: "gtfs-realtime",
schema: [
{
name: "trip_id",
type: "text",
index: true,
source: "parent.tripUpdate.trip.tripId",
default: null,
prefix: true
},
{
name: "trip_start_time",
type: "text",
source: "parent.tripUpdate.trip.startTime",
default: null
},
{
name: "direction_id",
type: "integer",
source: "parent.tripUpdate.trip.directionId",
default: null
},
{
name: "route_id",
type: "text",
index: true,
source: "parent.tripUpdate.trip.routeId",
default: null,
prefix: true
},
{
name: "stop_id",
type: "text",
index: true,
source: "stopId",
default: null,
prefix: true
},
{
name: "stop_sequence",
type: "integer",
source: "stopSequence",
default: null
},
{
name: "arrival_delay",
type: "integer",
source: "arrival.delay",
default: null
},
{
name: "departure_delay",
type: "integer",
source: "departure.delay",
default: null
},
{
name: "departure_timestamp",
type: "text",
source: "departure.time",
default: null
},
{
name: "arrival_timestamp",
type: "text",
source: "arrival.time",
default: null
},
{
name: "schedule_relationship",
type: "text",
source: "scheduleRelationship",
default: null
},
{
name: "created_timestamp",
type: "integer",
required: true
},
{
name: "expiration_timestamp",
type: "integer",
required: true
}
]
};
// src/models/gtfs-realtime/vehicle-positions.ts
var vehiclePositions = {
filenameBase: "vehicle_positions",
extension: "gtfs-realtime",
schema: [
{
name: "id",
type: "text",
required: true,
primary: true,
index: true,
source: "id",
prefix: true
},
{
name: "bearing",
type: "real",
source: "vehicle.position.bearing",
default: null
},
{
name: "latitude",
type: "real",
min: -90,
max: 90,
source: "vehicle.position.latitude",
default: null
},
{
name: "longitude",
type: "real",
source: "vehicle.position.longitude",
min: -180,
max: 180,
default: null
},
{
name: "speed",
type: "real",
min: 0,
source: "vehicle.position.speed",
default: null
},
{
name: "current_stop_sequence",
type: "integer",
source: "vehicle.currentStopSequence",
default: null
},
{
name: "trip_id",
type: "text",
index: true,
source: "vehicle.trip.tripId",
default: null,
prefix: true
},
{
name: "trip_start_date",
type: "text",
index: true,
source: "vehicle.trip.startDate",
default: null
},
{
name: "trip_start_time",
type: "text",
index: true,
source: "vehicle.trip.startTime",
default: null
},
{
name: "congestion_level",
type: "text",
source: "vehicle.congestionLevel",
default: null
},
{
name: "occupancy_status",
type: "text",
source: "vehicle.occupancyStatus",
default: null
},
{
name: "occupancy_percentage",
type: "integer",
source: "vehicle.occupancyPercentage",
default: null
},
{
name: "vehicle_stop_status",
type: "text",
source: "vehicle.vehicleStopStatus",
default: null
},
{
name: "vehicle_id",
type: "text",
index: true,
source: "vehicle.vehicle.id",
default: null,
prefix: true
},
{
name: "vehicle_label",
type: "text",
source: "vehicle.vehicle.label",
default: null
},
{
name: "vehicle_license_plate",
type: "text",
source: "vehicle.vehicle.licensePlate",
default: null
},
{
name: "vehicle_wheelchair_accessible",
type: "text",
source: "vehicle.vehicle.wheelchairAccessible",
default: null
},
{
name: "timestamp",
type: "text",
source: "vehicle.timestamp",
default: null
},
{
name: "created_timestamp",
type: "integer",
required: true
},
{
name: "expiration_timestamp",
type: "integer",
required: true
}
]
};
// src/models/gtfs-realtime/service-alerts.ts
var serviceAlerts = {
filenameBase: "service_alerts",
extension: "gtfs-realtime",
schema: [
{
name: "id",
type: "text",
required: true,
primary: true,
index: true,
source: "id",
prefix: true
},
{
name: "active_period",
type: "json",
source: "alert.activePeriod"
},
{
name: "cause",
type: "text",
source: "alert.cause"
},
{
name: "effect",
type: "text",
source: "alert.effect"
},
{
name: "url",
type: "text",
source: "alert.url.translation[0].text",
default: ""
},
{
name: "start_time",
type: "text",
required: true,
source: "alert.activePeriod[0].start",
default: ""
},
{
name: "end_time",
type: "text",
required: true,
source: "alert.activePeriod[0].end",
default: ""
},
{
name: "header_text",
type: "text",
required: true,
source: "alert.headerText.translation[0].text",
default: ""
},
{
name: "description_text",
type: "text",
required: true,
source: "alert.descriptionText.translation[0].text",
default: ""
},
{
name: "tts_header_text",
type: "text",
source: "alert.ttsHeaderText.translation[0].text"
},
{
name: "tts_description_text",
type: "text",
source: "alert.ttsDescriptionText.translation[0].text"
},
{
name: "severity_level",
type: "text",
source: "alert.severityLevel"
},
{
name: "created_timestamp",
type: "integer",
required: true
},
{
name: "expiration_timestamp",
type: "integer",
required: true
}
]
};
// src/models/gtfs-realtime/service-alert-informed_entities.ts
var serviceAlertInformedEntities = {
filenameBase: "service_alert_informed_entities",
extension: "gtfs-realtime",
schema: [
{
name: "alert_id",
type: "text",
required: true,
primary: true,
source: "parent.id",
prefix: true
},
{
name: "stop_id",
type: "text",
index: true,
source: "stopId",
default: null,
prefix: true
},
{
name: "route_id",
type: "text",
index: true,
source: "routeId",
default: null,
prefix: true
},
{
name: "route_type",
type: "integer",
index: true,
source: "routeType",
default: null
},
{
name: "trip_id",
type: "text",
index: true,
source: "trip.tripId",
default: null,
prefix: true
},
{
name: "direction_id",
type: "integer",
index: true,
source: "directionId",
default: null
},
{
name: "created_timestamp",
type: "integer",
required: true
},
{
name: "expiration_timestamp",
type: "integer",
required: true
}
]
};
// src/lib/db.ts
import Database from "better-sqlite3";
import untildify2 from "untildify";
var dbs = {};
function setupDb(sqlitePath) {
const db = new Database(untildify2(sqlitePath));
db.pragma("journal_mode = OFF");
db.pragma("synchronous = OFF");
db.pragma("temp_store = MEMORY");
dbs[sqlitePath] = db;
return db;
}
function openDb(config = null) {
if (config) {
const { sqlitePath = ":memory:", db } = config;
if (db) {
return db;
}
if (dbs[sqlitePath]) {
return dbs[sqlitePath];
}
return setupDb(sqlitePath);
}
if (Object.keys(dbs).length === 0) {
return setupDb(":memory:");
}
if (Object.keys(dbs).length === 1) {
const filename = Object.keys(dbs)[0];
return dbs[filename];
}
if (Object.keys(dbs).length > 1) {
throw new Error(
"Multiple databases open, please specify which one to use."
);
}
throw new Error("Unable to find database connection.");
}
// src/lib/geojson-utils.ts
import {
cloneDeep,
compact,
filter,
groupBy,
last,
omit as omit2,
sortBy,
omitBy
} from "lodash-es";
import { feature, featureCollection } from "@turf/helpers";
// src/lib/import-gtfs-realtime.ts
import pluralize from "pluralize";
import GtfsRealtimeBindings from "gtfs-realtime-bindings";
import mapSeries from "promise-map-series";
import { get } from "lodash-es";
// src/lib/utils.ts
import sqlString from "sqlstring-sqlite";
import Long from "long";
function validateConfigForImport(config) {
if (!config.agencies || config.agencies.length === 0) {
throw new Error("No `agencies` specified in config");
}
for (const [index, agency2] of config.agencies.entries()) {
if (!agency2.path && !agency2.url) {
throw new Error(
`No Agency \`url\` or \`path\` specified in config for agency index ${index}.`
);
}
}
return config;
}
function setDefaultConfig(initialConfig) {
const defaults = {
sqlitePath: ":memory:",
ignoreDuplicates: false,
ignoreErrors: false,
gtfsRealtimeExpirationSeconds: 0,
verbose: true
};
return {
...defaults,
...initialConfig
};
}
function convertLongTimeToDate(longDate) {
const { high, low, unsigned } = longDate;
return new Date(
Long.fromBits(low, high, unsigned).toNumber() * 1e3
).toISOString();
}
function applyPrefixToValue(value, columnShouldBePrefixed, prefix) {
if (!columnShouldBePrefixed || prefix === void 0 || value === null) {
return value;
}
return `${prefix}${value}`;
}
// src/lib/import-gtfs-realtime.ts
async function fetchGtfsRealtimeData(urlConfig, task) {
task.log(`Downloading GTFS-Realtime from ${urlConfig.url}`);
const response = await fetch(urlConfig.url, {
method: "GET",
headers: {
...urlConfig.headers ?? {},
"Accept-Encoding": "gzip"
},
signal: task.downloadTimeout ? AbortSignal.timeout(task.downloadTimeout) : void 0
});
if (response.status !== 200) {
task.logWarning(
`Unable to download GTFS-Realtime from ${urlConfig.url}. Got status ${response.status}.`
);
return null;
}
const buffer = await response.arrayBuffer();
const message = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(
new Uint8Array(buffer)
);
return GtfsRealtimeBindings.transit_realtime.FeedMessage.toObject(message, {
enums: String,
longs: String,
bytes: String,
defaults: false,
arrays: true,
objects: true,
oneofs: true
});
}
function removeExpiredRealtimeData(config) {
const db = openDb(config);
log(config)(`Removing expired GTFS-Realtime data`);
db.prepare(
`DELETE FROM vehicle_positions WHERE expiration_timestamp <= strftime('%s','now')`
).run();
db.prepare(
`DELETE FROM trip_updates WHERE expiration_timestamp <= strftime('%s','now')`
).run();
db.prepare(
`DELETE FROM stop_time_updates WHERE expiration_timestamp <= strftime('%s','now')`
).run();
db.prepare(
`DELETE FROM service_alerts WHERE expiration_timestamp <= strftime('%s','now')`
).run();
db.prepare(
`DELETE FROM service_alert_informed_entities WHERE expiration_timestamp <= strftime('%s','now')`
).run();
log(config)(`Removed expired GTFS-Realtime data\r`, true);
}
function prepareRealtimeFieldValue(entity, column, task) {
if (column.name === "created_timestamp") {
return task.currentTimestamp;
}
if (column.name === "expiration_timestamp") {
return task.currentTimestamp + task.gtfsRealtimeExpirationSeconds;
}
const baseValue = column.source === void 0 ? column.default : get(entity, column.source, column.default);
const timeAdjustedValue = baseValue?.__isLong__ ? convertLongTimeToDate(baseValue) : baseValue;
const prefixedValue = applyPrefixToValue(
timeAdjustedValue,
column.prefix,
task.prefix
);
return column.type === "json" ? JSON.stringify(prefixedValue) : prefixedValue;
}
async function processRealtimeAlerts(db, gtfsRealtimeData, task) {
const alertStmt = db.prepare(
`REPLACE INTO ${serviceAlerts.filenameBase} (${serviceAlerts.schema.map((column) => column.name).join(
", "
)}) VALUES (${serviceAlerts.schema.map(() => "?").join(", ")})`
);
const informedEntityStmt = db.prepare(
`REPLACE INTO ${serviceAlertInformedEntities.filenameBase} (${serviceAlertInformedEntities.schema.map((column) => column.name).join(
", "
)}) VALUES (${serviceAlertInformedEntities.schema.map(() => "?").join(", ")})`
);
let totalLineCount = 0;
db.transaction(() => {
for (const entity of gtfsRealtimeData.entity) {
const fieldValues = serviceAlerts.schema.map(
(column) => prepareRealtimeFieldValue(entity, column, task)
);
try {
alertStmt.run(fieldValues);
if (entity.alert.informedEntity?.length) {
const informedEntities = entity.alert.informedEntity.map(
(informedEntity) => {
informedEntity.parent = entity;
return serviceAlertInformedEntities.schema.map(
(column) => prepareRealtimeFieldValue(informedEntity, column, task)
);
}
);
for (const values of informedEntities) {
informedEntityStmt.run(values);
}
}
totalLineCount++;
} catch (error) {
task.logWarning(`Import error: ${error.message}`);
}
}
task.log(
`Importing - GTFS-Realtime service alerts - ${totalLineCount} entries imported\r`,
true
);
})();
}
async function processRealtimeTripUpdates(db, gtfsRealtimeData, task) {
let totalLineCount = 0;
const tripUpdateStmt = db.prepare(
`REPLACE INTO ${tripUpdates.filenameBase} (${tripUpdates.schema.map((column) => column.name).join(
", "
)}) VALUES (${tripUpdates.schema.map(() => "?").join(", ")})`
);
const stopTimeStmt = db.prepare(
`REPLACE INTO ${stopTimeUpdates.filenameBase} (${stopTimeUpdates.schema.map((column) => column.name).join(
", "
)}) VALUES (${stopTimeUpdates.schema.map(() => "?").join(", ")})`
);
db.transaction(() => {
for (const entity of gtfsRealtimeData.entity) {
try {
const fieldValues = tripUpdates.schema.map(
(column) => prepareRealtimeFieldValue(entity, column, task)
);
tripUpdateStmt.run(fieldValues);
for (const stopTimeUpdate of entity.tripUpdate.stopTimeUpdate) {
stopTimeUpdate.parent = entity;
const values = stopTimeUpdates.schema.map(
(column) => prepareRealtimeFieldValue(stopTimeUpdate, column, task)
);
stopTimeStmt.run(values);
}
totalLineCount++;
} catch (error) {
task.logWarning(`Import error: ${error.message}`);
}
}
task.log(
`Importing - GTFS-Realtime trip updates - ${totalLineCount} entries imported\r`,
true
);
})();
}
async function processRealtimeVehiclePositions(db, gtfsRealtimeData, task) {
let totalLineCount = 0;
const vehiclePositionStmt = db.prepare(
`REPLACE INTO ${vehiclePositions.filenameBase} (${vehiclePositions.schema.map((column) => column.name).join(
", "
)}) VALUES (${vehiclePositions.schema.map(() => "?").join(", ")})`
);
db.transaction(() => {
for (const entity of gtfsRealtimeData.entity) {
try {
const fieldValues = vehiclePositions.schema.map((column) => prepareRealtimeFieldValue(entity, column, task));
vehiclePositionStmt.run(fieldValues);
totalLineCount++;
} catch (error) {
task.logWarning(`Import error: ${error.message}`);
}
}
task.log(
`Importing - GTFS-Realtime vehicle positions - ${totalLineCount} entries imported\r`,
true
);
})();
}
async function updateGtfsRealtimeData(task) {
if (task.realtimeAlerts === void 0 && task.realtimeTripUpdates === void 0 && task.realtimeVehiclePositions === void 0) {
return;
}
const db = openDb({ sqlitePath: task.sqlitePath });
if (task.realtimeAlerts?.url) {
try {
const alertsData = await fetchGtfsRealtimeData(task.realtimeAlerts, task);
if (alertsData?.entity) {
await processRealtimeAlerts(db, alertsData, task);
}
} catch (error) {
if (task.ignoreErrors) {
task.logError(error.message);
} else {
throw error;
}
}
}
if (task.realtimeTripUpdates?.url) {
try {
const tripUpdatesData = await fetchGtfsRealtimeData(
task.realtimeTripUpdates,
task
);
if (tripUpdatesData?.entity) {
await processRealtimeTripUpdates(db, tripUpdatesData, task);
}
} catch (error) {
if (task.ignoreErrors) {
task.logError(error.message);
} else {
throw error;
}
}
}
if (task.realtimeVehiclePositions?.url) {
try {
const vehiclePositionsData = await fetchGtfsRealtimeData(
task.realtimeVehiclePositions,
task
);
if (vehiclePositionsData?.entity) {
await processRealtimeVehiclePositions(db, vehiclePositionsData, task);
}
} catch (error) {
if (task.ignoreErrors) {
task.logError(error.message);
} else {
throw error;
}
}
}
task.log(`GTFS-Realtime data import complete`);
}
async function updateGtfsRealtime(initialConfig) {
const config = setDefaultConfig(initialConfig);
validateConfigForImport(config);
try {
openDb(config);
const agencyCount = config.agencies.length;
log(config)(
`Starting GTFS-Realtime refresh for ${pluralize(
"agencies",
agencyCount,
true
)} using SQLite database at ${config.sqlitePath}`
);
removeExpiredRealtimeData(config);
await mapSeries(config.agencies, async (agency2) => {
try {
const task = {
realtimeAlerts: agency2.realtimeAlerts,
realtimeTripUpdates: agency2.realtimeTripUpdates,
realtimeVehiclePositions: agency2.realtimeVehiclePositions,
downloadTimeout: config.downloadTimeout,
gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
ignoreErrors: config.ignoreErrors,
sqlitePath: config.sqlitePath,
prefix: agency2.prefix,
currentTimestamp: Math.floor(Date.now() / 1e3),
log: log(config),
logWarning: logWarning(config),
logError: logError(config)
};
await updateGtfsRealtimeData(task);
} catch (error) {
if (config.ignoreErrors) {
logError(config)(error.message);
} else {
throw error;
}
}
});
log(config)(
`Completed GTFS-Realtime refresh for ${pluralize(
"agencies",
agencyCount,
true
)}
`
);
} catch (error) {
if (error?.code === "SQLITE_CANTOPEN") {
logError(config)(
`Unable to open sqlite database "${config.sqlitePath}" defined as \`sqlitePath\` config.json. Ensure the parent directory exists or remove \`sqlitePath\` from config.json.`
);
}
throw error;
}
}
// src/lib/export.ts
import { without, compact as compact2 } from "lodash-es";
import pluralize3 from "pluralize";
import { stringify } from "csv-stringify";
import sqlString2 from "sqlstring-sqlite";
import mapSeries3 from "promise-map-series";
import untildify4 from "untildify";
// src/lib/advancedQuery.ts
import sqlString3 from "sqlstring-sqlite";
// src/lib/gtfs/routes.ts
import { omit as omit3, pick } from "lodash-es";
// src/lib/gtfs/shapes.ts
import { compact as compact3, omit as omit4, pick as pick2 } from "lodash-es";
import { featureCollection as featureCollection2 } from "@turf/helpers";
// src/lib/gtfs/stops.ts
import { omit as omit5, orderBy, pick as pick3 } from "lodash-es";
// src/lib/gtfs/stop-times.ts
import { omit as omit6 } from "lodash-es";
import sqlString4 from "sqlstring-sqlite";
// src/bin/gtfsrealtime-update.ts
var pe = new PrettyError();
var argv = yargs(hideBin(process.argv)).usage("Usage: $0 --configPath ./config.json").help().option("c", {
alias: "configPath",
describe: "Path to config file",
type: "string"
}).default("configPath", void 0).parseSync();
var handleError = (error = "Unknown Error") => {
process.stdout.write(`
${formatError(error)}
`);
console.error(pe.render(error));
process.exit(1);
};
var setupImport = async () => {
const config = await getConfig({
configPath: argv.configPath
});
await updateGtfsRealtime(config);
process.exit();
};
setupImport().catch(handleError);
//# sourceMappingURL=gtfsrealtime-update.js.map