From 404618cc724cc077f0c4e920beed110d6ec2e06e Mon Sep 17 00:00:00 2001 From: rakow Date: Wed, 23 Oct 2024 14:23:02 +0200 Subject: [PATCH] Additional transit schedule validations (#3524) * add transit schedule validation for stops and travel times, update version of gtfs converter * fix duplicated argument * update gtfs converter * update gtfs version * improved validator, added class to fix departure time issues * fix departures that are the same at the end of a route * check for routes with empty departures --- contribs/application/pom.xml | 2 +- .../prepare/pt/AdjustSameDepartureTimes.java | 154 ++++++++++++++++++ .../pt/CreateTransitScheduleFromGtfs.java | 56 ++++++- .../pt/utils/TransitScheduleValidator.java | 152 +++++++++++++++-- 4 files changed, 342 insertions(+), 22 deletions(-) create mode 100644 contribs/application/src/main/java/org/matsim/application/prepare/pt/AdjustSameDepartureTimes.java diff --git a/contribs/application/pom.xml b/contribs/application/pom.xml index 0ac05d56158..83422007fee 100644 --- a/contribs/application/pom.xml +++ b/contribs/application/pom.xml @@ -87,7 +87,7 @@ com.github.matsim-org gtfs2matsim - fc8b13954d + 0bd5850fd6 diff --git a/contribs/application/src/main/java/org/matsim/application/prepare/pt/AdjustSameDepartureTimes.java b/contribs/application/src/main/java/org/matsim/application/prepare/pt/AdjustSameDepartureTimes.java new file mode 100644 index 00000000000..4d8b9378815 --- /dev/null +++ b/contribs/application/src/main/java/org/matsim/application/prepare/pt/AdjustSameDepartureTimes.java @@ -0,0 +1,154 @@ +package org.matsim.application.prepare.pt; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.matsim.core.utils.geometry.CoordUtils; +import org.matsim.core.utils.misc.OptionalTime; +import org.matsim.pt.transitSchedule.api.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Some providers set the same departure and arrival times for multiple stops. + * This usually happens with on-demand buses, but can also happen on night buses where multiple stops can have a departure at the same minute. + * These departure times will lead to artifacts when using a pseudo network. + * The purpose of this class is to spread these departures more evently. + * However, in the case of on demand buses, the travel times will likely still be too optimistic. + */ +public class AdjustSameDepartureTimes implements Consumer { + + private static Logger log = LogManager.getLogger(AdjustSameDepartureTimes.class); + + private static OptionalTime add(OptionalTime x, double t) { + if (!x.isDefined()) { + return x; + } + return OptionalTime.defined(x.seconds() + t); + } + + @Override + public void accept(TransitSchedule schedule) { + for (TransitLine line : schedule.getTransitLines().values()) { + + List routes = new ArrayList<>(line.getRoutes().values()); + for (TransitRoute route : routes) { + List newStops = adjustDepartures(schedule.getFactory(), route); + + if (newStops == null) { + continue; + } + + log.info("Adjusted departures for route {} in line {}", route.getId(), line.getId()); + + line.removeRoute(route); + + TransitRoute newRoute = schedule.getFactory().createTransitRoute(route.getId(), route.getRoute(), newStops, route.getTransportMode()); + newRoute.setDescription(route.getDescription()); + for (Map.Entry e : route.getAttributes().getAsMap().entrySet()) { + newRoute.getAttributes().putAttribute(e.getKey(), e.getValue()); + } + route.getDepartures().values().forEach(newRoute::addDeparture); + + line.addRoute(newRoute); + } + } + + } + + private List adjustDepartures(TransitScheduleFactory f, TransitRoute route) { + + List stops = new ArrayList<>(route.getStops()); + + boolean adjusted = false; + + // Check if the times at the end of the schedule are the same + // These need to be calculated and can not be interpolated + // The arrival at the last stop is shifted by small travel time + if (stops.size() > 1) { + TransitRouteStop last = stops.getLast(); + TransitRouteStop secondLast = stops.get(stops.size() - 2); + + OptionalTime lastDep = last.getDepartureOffset().or(last.getArrivalOffset()); + OptionalTime secondLastDep = secondLast.getDepartureOffset().or(secondLast.getArrivalOffset()); + + if (lastDep.isDefined() && secondLastDep.isDefined() && lastDep.equals(secondLastDep)) { + // Calculate the time between the last two stops + double dist = Math.max(20, CoordUtils.calcEuclideanDistance(last.getStopFacility().getCoord(), secondLast.getStopFacility().getCoord())); + double time = dist / 10; // 10 m/s + + // Calculate the time for the last stop + TransitRouteStop newStop = f.createTransitRouteStop( + last.getStopFacility(), + add(last.getArrivalOffset(), time), + add(last.getDepartureOffset(), time) + ); + + newStop.setAwaitDepartureTime(last.isAwaitDepartureTime()); + newStop.setAllowAlighting(last.isAllowAlighting()); + newStop.setAllowBoarding(last.isAllowBoarding()); + + stops.set(stops.size() - 1, newStop); + adjusted = true; + } + } + + for (int i = 0; i < stops.size() - 1; ) { + + TransitRouteStop stop = stops.get(i); + OptionalTime dep = stop.getDepartureOffset().or(stop.getArrivalOffset()); + + if (!dep.isDefined()) { + i++; + continue; + } + + OptionalTime arr = null; + int j = i + 1; + for (; j < stops.size(); j++) { + TransitRouteStop nextStop = stops.get(j); + arr = nextStop.getArrivalOffset().or(nextStop.getDepartureOffset()); + if (!dep.equals(arr)) { + break; + } + } + + if (arr == null) { + i++; + continue; + } + + if (j > i + 1) { + double time = dep.seconds(); + double diff = (arr.seconds() - time) / (j - i); + + for (int k = i + 1; k < j; k++) { + TransitRouteStop stopToAdjust = stops.get(k); + int add = (int) (diff * (k - i)); + + TransitRouteStop newStop = f.createTransitRouteStop( + stopToAdjust.getStopFacility(), + add(stopToAdjust.getArrivalOffset(), add), + add(stopToAdjust.getDepartureOffset(), add) + ); + + newStop.setAwaitDepartureTime(stopToAdjust.isAwaitDepartureTime()); + newStop.setAllowAlighting(stopToAdjust.isAllowAlighting()); + newStop.setAllowBoarding(stopToAdjust.isAllowBoarding()); + + stops.set(k, newStop); + adjusted = true; + } + } + + i = j; + } + + if (adjusted) + return stops; + + return null; + } +} diff --git a/contribs/application/src/main/java/org/matsim/application/prepare/pt/CreateTransitScheduleFromGtfs.java b/contribs/application/src/main/java/org/matsim/application/prepare/pt/CreateTransitScheduleFromGtfs.java index 5038d1f046c..ddbe0e7868a 100644 --- a/contribs/application/src/main/java/org/matsim/application/prepare/pt/CreateTransitScheduleFromGtfs.java +++ b/contribs/application/src/main/java/org/matsim/application/prepare/pt/CreateTransitScheduleFromGtfs.java @@ -1,5 +1,6 @@ package org.matsim.application.prepare.pt; +import com.conveyal.gtfs.model.Route; import com.conveyal.gtfs.model.Stop; import org.apache.commons.io.FilenameUtils; import org.apache.logging.log4j.LogManager; @@ -31,10 +32,8 @@ import java.io.File; import java.nio.file.Path; import java.time.LocalDate; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; +import java.util.function.Consumer; import java.util.function.Predicate; @@ -77,6 +76,15 @@ public class CreateTransitScheduleFromGtfs implements MATSimAppCommand { @CommandLine.Option(names = "--include-stops", description = "Fully qualified class name to a Predicate for filtering certain stops") private Class includeStops; + @CommandLine.Option(names = "--transform-stops", description = "Fully qualified class name to a Consumer for transforming stops before usage") + private Class transformStops; + + @CommandLine.Option(names = "--transform-routes", description = "Fully qualified class name to a Consumer for transforming routes before usage") + private Class transformRoutes; + + @CommandLine.Option(names = "--transform-schedule", description = "Fully qualified class name to a Consumer to be executed after the schedule was created", arity = "0..*", split = ",") + private List> transformSchedule; + @CommandLine.Option(names = "--merge-stops", description = "Whether stops should be merged by coordinate") private boolean mergeStops; @@ -134,6 +142,15 @@ public Integer call() throws Exception { log.info("Using prefix: {}", prefix); } + if (transformStops != null) { + converter.setTransformStop(createConsumer(transformStops, Stop.class)); + } + + if (transformRoutes != null) { + converter.setTransformRoute(createConsumer(transformRoutes, Route.class)); + } + + converter.build().convert(); i++; } @@ -144,6 +161,24 @@ public Integer call() throws Exception { TransitSchedulePostProcessTools.copyEarlyDeparturesToFollowingNight(scenario.getTransitSchedule(), 6 * 3600, "copied"); } + if (transformSchedule != null && !transformSchedule.isEmpty()) { + for (Class c : transformSchedule) { + Consumer f = createConsumer(c, TransitSchedule.class); + log.info("Applying {} to created schedule", c.getName()); + f.accept(scenario.getTransitSchedule()); + } + } + + for (TransitLine line : scenario.getTransitSchedule().getTransitLines().values()) { + List routes = new ArrayList<>(line.getRoutes().values()); + for (TransitRoute route : routes) { + if (route.getDepartures().isEmpty()) { + log.warn("Route {} in line {} with no departures removed.", route.getId(), line.getId()); + line.removeRoute(route); + } + } + } + Network network = NetworkUtils.readNetwork(networkFile); Scenario ptScenario = getScenarioWithPseudoPtNetworkAndTransitVehicles(network, scenario.getTransitSchedule(), "pt_"); @@ -151,11 +186,14 @@ public Integer call() throws Exception { if (validate) { //Check schedule and network TransitScheduleValidator.ValidationResult checkResult = TransitScheduleValidator.validateAll(ptScenario.getTransitSchedule(), ptScenario.getNetwork()); + List warnings = checkResult.getWarnings(); + if (!warnings.isEmpty()) + log.warn("TransitScheduleValidator warnings: {}", String.join("\n", warnings)); + if (checkResult.isValid()) { log.info("TransitSchedule and Network valid according to TransitScheduleValidator"); - log.warn("TransitScheduleValidator warnings: {}", checkResult.getWarnings()); } else { - log.error(checkResult.getErrors()); + log.error("TransitScheduleValidator errors: {}", String.join("\n", checkResult.getErrors())); throw new RuntimeException("TransitSchedule and/or Network invalid"); } } @@ -170,6 +208,12 @@ public Integer call() throws Exception { return 0; } + @SuppressWarnings({"unchecked", "unused"}) + private Consumer createConsumer(Class consumer, Class type) throws ReflectiveOperationException { + return (Consumer) consumer.getDeclaredConstructor().newInstance(); + } + + @SuppressWarnings("unchecked") private Predicate createFilter(int i) throws Exception { Predicate filter = (stop) -> true; diff --git a/matsim/src/main/java/org/matsim/pt/utils/TransitScheduleValidator.java b/matsim/src/main/java/org/matsim/pt/utils/TransitScheduleValidator.java index cfd96dbe88f..5f05825abc1 100644 --- a/matsim/src/main/java/org/matsim/pt/utils/TransitScheduleValidator.java +++ b/matsim/src/main/java/org/matsim/pt/utils/TransitScheduleValidator.java @@ -20,17 +20,12 @@ package org.matsim.pt.utils; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; +import java.util.*; import javax.xml.parsers.ParserConfigurationException; +import it.unimi.dsi.fastutil.doubles.DoubleArrayList; +import it.unimi.dsi.fastutil.doubles.DoubleList; import org.matsim.api.core.v01.Id; import org.matsim.api.core.v01.network.Link; import org.matsim.api.core.v01.network.Network; @@ -39,13 +34,9 @@ import org.matsim.core.population.routes.NetworkRoute; import org.matsim.core.scenario.MutableScenario; import org.matsim.core.scenario.ScenarioUtils; -import org.matsim.pt.transitSchedule.api.MinimalTransferTimes; -import org.matsim.pt.transitSchedule.api.TransitLine; -import org.matsim.pt.transitSchedule.api.TransitRoute; -import org.matsim.pt.transitSchedule.api.TransitRouteStop; -import org.matsim.pt.transitSchedule.api.TransitSchedule; -import org.matsim.pt.transitSchedule.api.TransitScheduleReader; -import org.matsim.pt.transitSchedule.api.TransitStopFacility; +import org.matsim.core.utils.geometry.CoordUtils; +import org.matsim.core.utils.geometry.GeometryUtils; +import org.matsim.pt.transitSchedule.api.*; import org.xml.sax.SAXException; /** @@ -248,6 +239,135 @@ public static ValidationResult validateTransfers(final TransitSchedule schedule) return result; } + public static ValidationResult validateDepartures(TransitSchedule schedule) { + ValidationResult result = new ValidationResult(); + for (TransitLine line : schedule.getTransitLines().values()) { + for (TransitRoute route : line.getRoutes().values()) { + if (route.getDepartures().isEmpty()) + result.addError("No departures defined for line %s, route %s".formatted(line.getId(), route.getId())); + + } + } + + return result; + } + + /** + * Validate if coordinates of stops and given travel times are plausible. + */ + public static ValidationResult validateStopCoordinates(final TransitSchedule schedule) { + + ValidationResult result = new ValidationResult(); + + // List of stops to the collected suspicious stops + Map suspiciousStops = new TreeMap<>(Comparator.comparing(TransitStopFacility::getName)); + + for (TransitLine line : schedule.getTransitLines().values()) { + + for (TransitRoute route : line.getRoutes().values()) { + + List routeStops = route.getStops(); + + // For too short routes, we can not detect outliers + if (routeStops.size() <= 4) + continue; + + double lastDepartureOffset = routeStops.getFirst().getDepartureOffset().or(routeStops.getFirst().getArrivalOffset()).seconds(); + + DoubleList speeds = new DoubleArrayList(); + DoubleList dists = new DoubleArrayList(); + + for (int i = 1; i < routeStops.size(); i++) { + TransitRouteStop routeStop = routeStops.get(i); + + if (routeStop.getStopFacility().getCoord() == null) + break; + + double departureOffset = routeStop.getArrivalOffset().or(routeStop.getDepartureOffset()).orElse(0); + double travelTime = departureOffset - lastDepartureOffset; + double length = CoordUtils.calcEuclideanDistance(routeStop.getStopFacility().getCoord(), + routeStops.get(i - 1).getStopFacility().getCoord()); + + dists.add(length); + + // Short distances are not checked, because here high speeds are not so problematic and arise from few seconds difference + if (length <= 20) { + speeds.add(-1); + continue; + } + + if (travelTime == 0) { + speeds.add(Double.POSITIVE_INFINITY); + continue; + } + + double speed = length / travelTime; + speeds.add(speed); + lastDepartureOffset = departureOffset; + } + + // If all speeds are valid, the stops and speeds can be checked + if (speeds.size() == routeStops.size() - 1) { + + // First check for suspicious stops + // These are stops with very high speed, and also high distance between stops + for (int i = 0; i < speeds.size() - 1; i++) { + TransitRouteStop stop = routeStops.get(i + 1); + double toStop = speeds.getDouble(i); + double fromStop = speeds.getDouble(i + 1); + + double both = (toStop + fromStop) / 2; + + double dist = (dists.getDouble(i) + dists.getDouble(i + 1)) / 2; + + // Only if the distance is large, we assume a mapping error might have occurred + if (dist < 5_000) + continue; + + // Remove the considered speeds from the calculation + DoubleList copy = new DoubleArrayList(speeds); + copy.removeDouble(i); + copy.removeDouble(i); + copy.removeIf(s -> s == -1 || s == Double.POSITIVE_INFINITY); + + double mean = copy.doubleStream().average().orElse(-1); + + // If no mean is known, use a high value to avoid false positives + if (mean == -1) { + mean = 70; + } + + // Some hard coded rules to detect suspicious stops, these are speed m/s, so quite high values + if (((toStop > 3 * mean && both > 50) || toStop > 120) && (((fromStop > 3 * mean && both > 50) || fromStop > 120))) { + DoubleList suspiciousSpeeds = suspiciousStops.computeIfAbsent(stop.getStopFacility(), (k) -> new DoubleArrayList()); + suspiciousSpeeds.add(toStop); + suspiciousSpeeds.add(fromStop); + } + } + + // Then check for implausible travel times + for (int i = 0; i < speeds.size(); i++) { + double speed = speeds.getDouble(i); + TransitStopFacility from = routeStops.get(i).getStopFacility(); + TransitStopFacility to = routeStops.get(i + 1).getStopFacility(); + if (speed > 230) { + result.addWarning("Suspicious high speed from stop %s (%s) to %s (%s) on line %s, route %s, index: %d: %.2f m/s, %.2fm" + .formatted(from.getName(), from.getId(), to.getName(), to.getId(), line.getId(), route.getId(), i, speed, dists.getDouble(i))); + } + } + } + } + } + + for (Map.Entry e : suspiciousStops.entrySet()) { + TransitStopFacility stop = e.getKey(); + double speed = e.getValue().doubleStream().average().orElse(-1); + result.addWarning("Suspicious location for stop %s (%s) at stop area %s: %s, avg. speed: %.2f m/s".formatted(stop.getName(), stop.getId(), stop.getStopAreaId(), stop.getCoord(), speed)); + } + + return result; + } + public static ValidationResult validateAll(final TransitSchedule schedule, final Network network) { ValidationResult v = validateUsedStopsHaveLinkId(schedule); v.add(validateNetworkRoutes(schedule, network)); @@ -259,6 +379,8 @@ public static ValidationResult validateAll(final TransitSchedule schedule, final v.add(validateAllStopsExist(schedule)); v.add(validateOffsets(schedule)); v.add(validateTransfers(schedule)); + v.add(validateStopCoordinates(schedule)); + v.add(validateDepartures(schedule)); return v; }