diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/ExtrapolateRidershipData.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/ExtrapolateRidershipData.java index aca49ac59..ffba0a7ce 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/ExtrapolateRidershipData.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/ExtrapolateRidershipData.java @@ -32,6 +32,8 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Set; +import java.util.HashSet; import java.util.HashMap; import java.util.Map.Entry; import java.util.List; @@ -155,6 +157,8 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) { riderships.add(ridership); ridershipMap.put(ridership.getTripId(), riderships); } + + } _log.error("Number of trips in ridership data {}", ridershipMap.size()); @@ -183,9 +187,86 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) { _log.error("Number of trips in GTFS: {}", daoTrips.size()); + //used for metrics, each ridership trip_id that is added to dao ridership.txt + Set ridershipIds = new HashSet(); + //Try to find trips that match by comparing the stops on that trip. We will get a lot of matches as identical trips run //on the same route each day - for (HashMap.Entry> trip : daoTrips.entrySet()) { + for (HashMap.Entry> trip : ridershipTrips.entrySet()) { + TreeMap ridershipStops = trip.getValue(); + //for each ridership trip, we now have a tree list of the stops on that trip. Compare to each dao trip + List ridershipStopList = new ArrayList<>(ridershipStops.values()); + for (HashMap.Entry> daoTrip : daoTrips.entrySet()) { + //daoStops is a treemap of stops on that trip, stop sequence: stop id + TreeMap daoStopsTree = daoTrip.getValue(); + //daoStopList is the stop ids on that trip in order, from the sequences being in order + List daoStopList = new ArrayList<>(daoStopsTree.values()); + boolean equal = ridershipStopList.equals(daoStopList); + //The trips are 'equal' in that they have the same stops, narrow it down by time + + //the Access trip id is: trip.getKey(), the GTFS trip id is: daoTrip.getKey().getId + if (equal) { + //get the first stop time for the trip + Trip GTFStrip = dao.getTripForId(daoTrip.getKey()); + StopTime firstStopTime = dao.getStopTimesForTrip(GTFStrip).get(0); + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("HH:mm:ss"); + + String GTFStime = StopTimeFieldMappingFactory.getSecondsAsString(firstStopTime.getArrivalTime()); + //a lot of stop times are 'invalid' hours of 24, 25, 26. For 24 hour clock, hour s/b 0-23 + GTFStime = ConvertToValidHours(GTFStime); + LocalTime GTFSArrival = LocalTime.parse(GTFStime, dtf); + + //We have the stop sequence and the stop id. From the stop id, get the stop time + Entry riderStop = ridershipStops.firstEntry(); + String firstStopId = riderStop.getValue(); + LocalTime passTime = null; + + //get the stop time for the first stop on this trip + ArrayList ridershipData = ridershipMap.get(trip.getKey()); + for (RidershipData rd : ridershipData) { + if (rd.getStopId() == firstStopId) { + passTime = rd.getPassingTime(); + } + } + long minutesDifference = 15; + + if (passTime != null) { + minutesDifference = Duration.between(GTFSArrival, passTime).toMinutes(); + } + if (minutesDifference < 4 && minutesDifference > -4) { + //_log.error("GTFS trip {}, time {}, Ridership trip {}, time {}, diff: {} ", trip.getKey().getId(), GTFSArrival, ridershipStops.getKey(),passTime, minutesDifference); + + //we have HashMap> ridershipMap = new HashMap<>(); + // and we need to set all of the ridershipData ids to the GTFS id, from the ridership id + + ArrayList rsList = ridershipMap.get(trip.getKey()); + //this is the list of data where the Trip ids are all the same, and each entry is a stop + //so, update the Trip id to go from Ridership_id to Trip_id + for (RidershipData rs : rsList) { + + //get trips added ids + //if its already been added, compare differences and use lowest? + rs.setRsTripId(trip.getKey()); + rs.setGtfsTripId(GTFStrip.getId().getId()); + rs.setTripId(GTFStrip.getId().getId()); + rs.setMinutesDifference(minutesDifference); + rs.setRouteId(dao.getTripForId(GTFStrip.getId()).getRoute().getId().getId()); + saveRidership(dao, rs); + ridershipIds.add(rs.getRsTripId()); + } + //Do we need to handle when two ridership stops match to one GTFS? Compare and take the one that is less? + //or what about the same ridership matching to multiple GTFS stops + } + } + } + } + + //the algorithm above takes each ridership trip and tries to find a GTFS trip to match it to. The algorithm below + //takes each GTFS trip and tries to find a ridership trip to match it to. The numbers of matched trips is the same either way. + + //Try to find trips that match by comparing the stops on that trip. We will get a lot of matches as identical trips run + //on the same route each day +/* for (HashMap.Entry> trip : daoTrips.entrySet()) { TreeMap daoStops = (TreeMap) trip.getValue(); //for each dao trip, we now have a tree list of the stops on that trip. Compare to each trip from ridership data List daoStopList = new ArrayList<>(daoStops.values()); @@ -243,21 +324,34 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) { rs.setMinutesDifference(minutesDifference); rs.setRouteId(dao.getTripForId(trip.getKey()).getRoute().getId().getId()); saveRidership(dao, rs); + ridershipIds.add(rs.getRsTripId()); } //Do we need to handle when two ridership stops match to one GTFS? Compare and take the one that is less? //or what about the same ridership matching to multiple GTFS stops - //TODO: what about the times? I am checking that A LOT and its inefficient - } } } - } + }*/ List riderships2 = new ArrayList<>(dao.getAllRiderships()); + _log.error("Ridership size: {}", dao.getAllRiderships().size()); _log.error("Ridership size: {}", riderships2.size()); + int in = 0; + int out = 0; + + //iterate over list of ridership data. For each ridershipData, is it in ridership.txt? + for (String ridership_trip_id : ridershipMap.keySet()) { + if (ridershipIds.contains(ridership_trip_id)) { + in++; + } + else { + out++; + } + } + _log.error("Trips ids in ridership.txt: {}, trip ids abandoned: {}", in, out); } diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/MTASubwayShuttleRouteStrategy.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/MTASubwayShuttleRouteStrategy.java new file mode 100644 index 000000000..d683b0108 --- /dev/null +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/MTASubwayShuttleRouteStrategy.java @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2018 Cambridge Systematics, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.onebusaway.gtfs_transformer.impl; + +import org.onebusaway.gtfs.model.AgencyAndId; +import org.onebusaway.gtfs.model.Route; +import org.onebusaway.gtfs.model.Stop; +import org.onebusaway.gtfs.model.Trip; +import org.onebusaway.gtfs.services.GtfsMutableRelationalDao; +import org.onebusaway.gtfs_transformer.services.GtfsTransformStrategy; +import org.onebusaway.gtfs_transformer.services.TransformContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +public class MTASubwayShuttleRouteStrategy implements GtfsTransformStrategy { + + private static final Logger _log = LoggerFactory.getLogger(MTASubwayShuttleRouteStrategy.class); + + private static final String SHUTTLE_HEADSIGN_SUFFIX = "SHUTTLE BUS"; + private static final String SHUTTLE_ID_SUFFIX = "-SS"; + private static final String SHUTTLE_NAME_SUFFIX = " Shuttle"; + private static final int SHUTTLE_ROUTE_TYPE = 714; + private static final String SHUTTLE_STOP_SUFFIX = "SHUTTLE BUS STOP"; + + @Override + public String getName() { + return getClass().getSimpleName(); + } + + @Override + public void run(TransformContext context, GtfsMutableRelationalDao dao) { + Map shuttleRoutes = new HashMap<>(); + + for (Trip trip : dao.getAllTrips()) { + if (trip.getTripHeadsign().endsWith(SHUTTLE_HEADSIGN_SUFFIX)) { + Route shuttleRoute = shuttleRoutes.computeIfAbsent(trip.getRoute(), r -> getShuttleRoute(dao, r)); + trip.setRoute(shuttleRoute); + dao.updateEntity(trip); + } + } + + // Shuttle stops share mta_stop_id with non-shuttle version + Map parentStopByMtaStopId = new HashMap<>(); + for (Stop stop : dao.getAllStops()) { + if (!stop.getName().endsWith(SHUTTLE_STOP_SUFFIX) && stop.getParentStation() != null) { + parentStopByMtaStopId.put(stop.getMtaStopId(), stop.getParentStation()); + } + } + for (Stop stop : dao.getAllStops()) { + if (stop.getName().endsWith(SHUTTLE_STOP_SUFFIX)) { + String parent = parentStopByMtaStopId.get(stop.getMtaStopId()); + if (parent == null) { + _log.info("No parent for shuttle stop {}", stop.getId()); + } + stop.setParentStation(parent); + dao.updateEntity(stop); + } + } + } + + private Route getShuttleRoute(GtfsMutableRelationalDao dao, Route orig) { + Route shuttleRoute = new Route(orig); + AgencyAndId id = new AgencyAndId(shuttleRoute.getId().getAgencyId(), + shuttleRoute.getId().getId() + SHUTTLE_ID_SUFFIX); + shuttleRoute.setId(id); + shuttleRoute.setShortName(shuttleRoute.getShortName() + SHUTTLE_ID_SUFFIX); + shuttleRoute.setLongName(shuttleRoute.getLongName() + SHUTTLE_NAME_SUFFIX); + shuttleRoute.setType(SHUTTLE_ROUTE_TYPE); + dao.saveEntity(shuttleRoute); + return shuttleRoute; + } +} diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/StationComplexStrategy.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/StationComplexStrategy.java index 6ade1e552..fa190ffaa 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/StationComplexStrategy.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/StationComplexStrategy.java @@ -15,6 +15,9 @@ */ package org.onebusaway.gtfs_transformer.impl; +import org.onebusaway.csv_entities.schema.EntitySchemaFactory; +import org.onebusaway.csv_entities.schema.FieldMapping; +import org.onebusaway.csv_entities.schema.FieldMappingFactory; import org.onebusaway.csv_entities.schema.annotations.CsvField; import org.onebusaway.gtfs.model.Pathway; import org.onebusaway.gtfs.model.Stop; @@ -35,11 +38,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.onebusaway.gtfs_transformer.util.PathwayUtil.PATHWAY_MODE_GENERIC; public class StationComplexStrategy implements GtfsTransformStrategy { + private enum Type { + PATHWAYS, + PARENT_STATION + } + private final Logger _log = LoggerFactory.getLogger(StationComplexStrategy.class); private static final String STOP_SEPARATOR = " "; @@ -50,26 +59,54 @@ public class StationComplexStrategy implements GtfsTransformStrategy { @CsvField(optional = true) private int genericPathwayTraversalTime = 60; + @CsvField(ignore = true) + private Type typeInternal = Type.PATHWAYS; + + @CsvField(optional = true) + private String type = null; + public String getName() { - return this.getClass().getName(); + return this.getClass().getName(); } // Create pathways between all stops in a station complex @Override public void run(TransformContext context, GtfsMutableRelationalDao dao) { File stComplexFile = new File(complexFile); - if(!stComplexFile.exists()) { + if (!stComplexFile.exists()) { throw new IllegalStateException( "Station Complex file does not exist: " + stComplexFile.getName()); } + Collection> complexes = getComplexList(dao); + if (typeInternal.equals(Type.PATHWAYS)) { + makePathways(complexes, dao); + } else if (typeInternal.equals(Type.PARENT_STATION)) { + setParents(complexes, dao); + } + } + + private void setParents(Collection> complexes, GtfsMutableRelationalDao dao) { + for (List complex : complexes) { + + Map> grouped = complex.stream() + .collect(Collectors.groupingBy(Stop::getName)); + for (List group : grouped.values()) { + String parent = group.get(0).getParentStation(); + for (Stop stop : group) { + stop.setParentStation(parent); + dao.updateEntity(stop); + } + } + } + } + private void makePathways(Collection> complexes, GtfsMutableRelationalDao dao) { String feedId = dao.getAllStops().iterator().next().getId().getAgencyId(); List newPathways = new ArrayList<>(); PathwayUtil util = new PathwayUtil(feedId, newPathways); - for (List complex : getComplexList(dao)) { + for (List complex : complexes) { for (Stop s : complex) { for (Stop t : complex) { - if (s != null && s.getParentStation() != null && t != null) { if (!s.equals(t)) { String id = String.format("complex-%s-%s", s.getId().getId(), t.getId().getId()); @@ -101,7 +138,7 @@ private Collection> getComplexList(GtfsDao dao) { } complexes.add(complex); } - } catch(IOException e) { + } catch (IOException e) { e.printStackTrace(); } return complexes; @@ -124,4 +161,9 @@ public void setComplexFile(String complexFile) { public void setGenericPathwayTraversalTime(int genericPathwayTraversalTime) { this.genericPathwayTraversalTime = genericPathwayTraversalTime; } -} + + public void setType(String type) { + this.type = type; + this.typeInternal = Type.valueOf(type); + } +} \ No newline at end of file