Merge branch 'master' of github.com:onebusaway/onebusaway-gtfs-modules

onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/ExtrapolateRidershipData.java

@@ -32,6 +32,8 @@
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
 import java.util.HashMap;
 import java.util.Map.Entry;
 import java.util.List;
@@ -155,6 +157,8 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) {
                 riderships.add(ridership);
                 ridershipMap.put(ridership.getTripId(), riderships);
             }
+
+
         }
 
         _log.error("Number of trips in ridership data {}", ridershipMap.size());
@@ -183,9 +187,86 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) {
 
         _log.error("Number of trips in GTFS: {}", daoTrips.size());
 
+        //used for metrics, each ridership trip_id that is added to dao ridership.txt
+        Set<String> ridershipIds = new HashSet<String>();
+
         //Try to find trips that match by comparing the stops on that trip.  We will get a lot of matches as identical trips run
         //on the same route each day
-        for (HashMap.Entry<AgencyAndId, TreeMap<Integer, String>> trip : daoTrips.entrySet()) {
+        for (HashMap.Entry<String, TreeMap<Integer, String>> trip : ridershipTrips.entrySet()) {
+            TreeMap<Integer, String> ridershipStops = trip.getValue();
+            //for each ridership trip, we now have a tree list of the stops on that trip.  Compare to each dao trip
+            List<String> ridershipStopList = new ArrayList<>(ridershipStops.values());
+            for (HashMap.Entry<AgencyAndId, TreeMap<Integer, String>> daoTrip : daoTrips.entrySet()) {
+                //daoStops is a treemap of stops on that trip, stop sequence: stop id
+                TreeMap<Integer, String> daoStopsTree = daoTrip.getValue();
+                //daoStopList is the stop ids on that trip in order, from the sequences being in order
+                List<String> daoStopList = new ArrayList<>(daoStopsTree.values());
+                boolean equal = ridershipStopList.equals(daoStopList);
+                //The trips are 'equal' in that they have the same stops, narrow it down by time
+
+                //the Access trip id is: trip.getKey(), the GTFS trip id is: daoTrip.getKey().getId
+                if (equal) {
+                    //get the first stop time for the trip
+                    Trip GTFStrip = dao.getTripForId(daoTrip.getKey());
+                    StopTime firstStopTime = dao.getStopTimesForTrip(GTFStrip).get(0);
+                    DateTimeFormatter dtf = DateTimeFormatter.ofPattern("HH:mm:ss");
+
+                    String GTFStime = StopTimeFieldMappingFactory.getSecondsAsString(firstStopTime.getArrivalTime());
+                    //a lot of stop times are 'invalid' hours of 24, 25, 26.  For 24 hour clock, hour s/b 0-23
+                    GTFStime = ConvertToValidHours(GTFStime);
+                    LocalTime GTFSArrival = LocalTime.parse(GTFStime, dtf);
+
+                    //We have the stop sequence and the stop id.  From the stop id, get the stop time
+                    Entry<Integer, String> riderStop = ridershipStops.firstEntry();
+                    String firstStopId = riderStop.getValue();
+                    LocalTime passTime = null;
+
+                    //get the stop time for the first stop on this trip
+                    ArrayList<RidershipData> ridershipData = ridershipMap.get(trip.getKey());
+                    for (RidershipData rd : ridershipData) {
+                        if (rd.getStopId() == firstStopId) {
+                            passTime = rd.getPassingTime();
+                        }
+                    }
+                    long minutesDifference = 15;
+
+                    if (passTime != null) {
+                        minutesDifference = Duration.between(GTFSArrival, passTime).toMinutes();
+                    }
+                    if (minutesDifference < 4 && minutesDifference > -4) {
+                        //_log.error("GTFS trip {}, time {}, Ridership trip {}, time {}, diff: {} ", trip.getKey().getId(), GTFSArrival, ridershipStops.getKey(),passTime, minutesDifference);
+
+                        //we have HashMap<String, ArrayList<RidershipData>> ridershipMap = new HashMap<>();
+                        // and we need to set all of the ridershipData ids to the GTFS id, from the ridership id
+
+                        ArrayList<RidershipData> rsList = ridershipMap.get(trip.getKey());
+                        //this is the list of data where the Trip ids are all the same, and each entry is a stop
+                        //so, update the Trip id to go from Ridership_id to Trip_id
+                        for (RidershipData rs : rsList) {
+
+                            //get trips added ids
+                            //if its already been added, compare differences and use lowest?
+                            rs.setRsTripId(trip.getKey());
+                            rs.setGtfsTripId(GTFStrip.getId().getId());
+                            rs.setTripId(GTFStrip.getId().getId());
+                            rs.setMinutesDifference(minutesDifference);
+                            rs.setRouteId(dao.getTripForId(GTFStrip.getId()).getRoute().getId().getId());
+                            saveRidership(dao, rs);
+                            ridershipIds.add(rs.getRsTripId());
+                        }
+                        //Do we need to handle when two ridership stops match to one GTFS? Compare and take the one that is less?
+                        //or what about the same ridership matching to multiple GTFS stops
+                    }
+                }
+            }
+        }
+
+        //the algorithm above takes each ridership trip and tries to find a GTFS trip to match it to.  The algorithm below
+        //takes each GTFS trip and tries to find a ridership trip to match it to.  The numbers of matched trips is the same either way.
+
+        //Try to find trips that match by comparing the stops on that trip.  We will get a lot of matches as identical trips run
+        //on the same route each day
+/*        for (HashMap.Entry<AgencyAndId, TreeMap<Integer, String>> trip : daoTrips.entrySet()) {
             TreeMap<Integer, String> daoStops = (TreeMap<Integer, String>) trip.getValue();
             //for each dao trip, we now have a tree list of the stops on that trip.  Compare to each trip from ridership data
             List<String> daoStopList = new ArrayList<>(daoStops.values());
@@ -243,21 +324,34 @@ public void run(TransformContext context, GtfsMutableRelationalDao dao) {
                             rs.setMinutesDifference(minutesDifference);
                             rs.setRouteId(dao.getTripForId(trip.getKey()).getRoute().getId().getId());
                             saveRidership(dao, rs);
+                            ridershipIds.add(rs.getRsTripId());
                         }
 
                         //Do we need to handle when two ridership stops match to one GTFS? Compare and take the one that is less?
                         //or what about the same ridership matching to multiple GTFS stops
 
-                        //TODO: what about the times?  I am checking that A LOT and its inefficient
-
                     }
                 }
             }
-        }
+        }*/
 
         List<Ridership> riderships2 = new ArrayList<>(dao.getAllRiderships());
+        _log.error("Ridership size: {}", dao.getAllRiderships().size());
         _log.error("Ridership size: {}", riderships2.size());
 
+        int in = 0;
+        int out = 0;
+
+        //iterate over list of ridership data. For each ridershipData, is it in ridership.txt?
+        for (String ridership_trip_id : ridershipMap.keySet()) {
+            if (ridershipIds.contains(ridership_trip_id)) {
+                in++;
+            }
+            else {
+                out++;
+            }
+        }
+        _log.error("Trips ids in ridership.txt: {}, trip ids abandoned: {}", in, out);
 
     }
 

onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/MTASubwayShuttleRouteStrategy.java

@@ -0,0 +1,88 @@
+/**
+ * Copyright (C) 2018 Cambridge Systematics, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onebusaway.gtfs_transformer.impl;
+
+import org.onebusaway.gtfs.model.AgencyAndId;
+import org.onebusaway.gtfs.model.Route;
+import org.onebusaway.gtfs.model.Stop;
+import org.onebusaway.gtfs.model.Trip;
+import org.onebusaway.gtfs.services.GtfsMutableRelationalDao;
+import org.onebusaway.gtfs_transformer.services.GtfsTransformStrategy;
+import org.onebusaway.gtfs_transformer.services.TransformContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MTASubwayShuttleRouteStrategy implements GtfsTransformStrategy {
+
+    private static final Logger _log = LoggerFactory.getLogger(MTASubwayShuttleRouteStrategy.class);
+
+    private static final String SHUTTLE_HEADSIGN_SUFFIX = "SHUTTLE BUS";
+    private static final String SHUTTLE_ID_SUFFIX = "-SS";
+    private static final String SHUTTLE_NAME_SUFFIX = " Shuttle";
+    private static final int SHUTTLE_ROUTE_TYPE = 714;
+    private static final String SHUTTLE_STOP_SUFFIX = "SHUTTLE BUS STOP";
+
+    @Override
+    public String getName() {
+        return getClass().getSimpleName();
+    }
+
+    @Override
+    public void run(TransformContext context, GtfsMutableRelationalDao dao) {
+        Map<Route, Route> shuttleRoutes = new HashMap<>();
+
+        for (Trip trip : dao.getAllTrips()) {
+            if (trip.getTripHeadsign().endsWith(SHUTTLE_HEADSIGN_SUFFIX)) {
+                Route shuttleRoute = shuttleRoutes.computeIfAbsent(trip.getRoute(), r -> getShuttleRoute(dao, r));
+                trip.setRoute(shuttleRoute);
+                dao.updateEntity(trip);
+            }
+        }
+
+        // Shuttle stops share mta_stop_id with non-shuttle version
+        Map<String, String> parentStopByMtaStopId = new HashMap<>();
+        for (Stop stop : dao.getAllStops()) {
+            if (!stop.getName().endsWith(SHUTTLE_STOP_SUFFIX) && stop.getParentStation() != null) {
+                parentStopByMtaStopId.put(stop.getMtaStopId(), stop.getParentStation());
+            }
+        }
+        for (Stop stop : dao.getAllStops()) {
+            if (stop.getName().endsWith(SHUTTLE_STOP_SUFFIX)) {
+                String parent = parentStopByMtaStopId.get(stop.getMtaStopId());
+                if (parent == null) {
+                   _log.info("No parent for shuttle stop {}", stop.getId());
+                }
+                stop.setParentStation(parent);
+                dao.updateEntity(stop);
+            }
+        }
+    }
+
+    private Route getShuttleRoute(GtfsMutableRelationalDao dao, Route orig) {
+        Route shuttleRoute = new Route(orig);
+        AgencyAndId id = new AgencyAndId(shuttleRoute.getId().getAgencyId(),
+                shuttleRoute.getId().getId() + SHUTTLE_ID_SUFFIX);
+        shuttleRoute.setId(id);
+        shuttleRoute.setShortName(shuttleRoute.getShortName() + SHUTTLE_ID_SUFFIX);
+        shuttleRoute.setLongName(shuttleRoute.getLongName() + SHUTTLE_NAME_SUFFIX);
+        shuttleRoute.setType(SHUTTLE_ROUTE_TYPE);
+        dao.saveEntity(shuttleRoute);
+        return shuttleRoute;
+    }
+}

onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/StationComplexStrategy.java

@@ -15,6 +15,9 @@
  */
 package org.onebusaway.gtfs_transformer.impl;
 
+import org.onebusaway.csv_entities.schema.EntitySchemaFactory;
+import org.onebusaway.csv_entities.schema.FieldMapping;
+import org.onebusaway.csv_entities.schema.FieldMappingFactory;
 import org.onebusaway.csv_entities.schema.annotations.CsvField;
 import org.onebusaway.gtfs.model.Pathway;
 import org.onebusaway.gtfs.model.Stop;
@@ -35,11 +38,17 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.onebusaway.gtfs_transformer.util.PathwayUtil.PATHWAY_MODE_GENERIC;
 
 public class StationComplexStrategy implements GtfsTransformStrategy {
 
+    private enum Type {
+        PATHWAYS,
+        PARENT_STATION
+    }
+
     private final Logger _log = LoggerFactory.getLogger(StationComplexStrategy.class);
 
     private static final String STOP_SEPARATOR = " ";
@@ -50,26 +59,54 @@ public class StationComplexStrategy implements GtfsTransformStrategy {
     @CsvField(optional = true)
     private int genericPathwayTraversalTime = 60;
 
+    @CsvField(ignore = true)
+    private Type typeInternal = Type.PATHWAYS;
+
+    @CsvField(optional = true)
+    private String type = null;
+
     public String getName() {
-      return this.getClass().getName();
+        return this.getClass().getName();
     }
 
     // Create pathways between all stops in a station complex
     @Override
     public void run(TransformContext context, GtfsMutableRelationalDao dao) {
         File stComplexFile = new File(complexFile);
-        if(!stComplexFile.exists()) {
+        if (!stComplexFile.exists()) {
             throw new IllegalStateException(
                     "Station Complex file does not exist: " + stComplexFile.getName());
         }
+        Collection<List<Stop>> complexes = getComplexList(dao);
+        if (typeInternal.equals(Type.PATHWAYS)) {
+            makePathways(complexes, dao);
+        } else if (typeInternal.equals(Type.PARENT_STATION)) {
+            setParents(complexes, dao);
+        }
+    }
+
+    private void setParents(Collection<List<Stop>> complexes, GtfsMutableRelationalDao dao) {
+        for (List<Stop> complex : complexes) {
+
+            Map<String, List<Stop>> grouped = complex.stream()
+                    .collect(Collectors.groupingBy(Stop::getName));
+            for (List<Stop> group : grouped.values()) {
+                String parent = group.get(0).getParentStation();
+                for (Stop stop : group) {
+                    stop.setParentStation(parent);
+                    dao.updateEntity(stop);
+                }
+            }
+        }
+    }
 
+    private void makePathways(Collection<List<Stop>> complexes, GtfsMutableRelationalDao dao) {
         String feedId = dao.getAllStops().iterator().next().getId().getAgencyId();
         List<Pathway> newPathways = new ArrayList<>();
         PathwayUtil util = new PathwayUtil(feedId, newPathways);
-        for (List<Stop> complex : getComplexList(dao)) {
+        for (List<Stop> complex : complexes) {
             for (Stop s : complex) {
                 for (Stop t : complex) {
-
                     if (s != null && s.getParentStation() != null && t != null) {
                         if (!s.equals(t)) {
                             String id = String.format("complex-%s-%s", s.getId().getId(), t.getId().getId());
@@ -101,7 +138,7 @@ private Collection<List<Stop>> getComplexList(GtfsDao dao) {
                 }
                 complexes.add(complex);
             }
-        } catch(IOException e) {
+        } catch (IOException e) {
             e.printStackTrace();
         }
         return complexes;
@@ -124,4 +161,9 @@ public void setComplexFile(String complexFile) {
     public void setGenericPathwayTraversalTime(int genericPathwayTraversalTime) {
         this.genericPathwayTraversalTime = genericPathwayTraversalTime;
     }
-}
+
+    public void setType(String type) {
+        this.type = type;
+        this.typeInternal = Type.valueOf(type);
+    }
+}