Merge pull request ibi-group#419 from ibi-group/add-pelias-update-job…

…-ltr

Add pelias update job ltr

src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java

@@ -11,6 +11,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.io.ByteStreams;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.eclipse.jetty.http.HttpStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -83,7 +84,10 @@ public static String formatJSON (String key, String value) {
      * supplied details about the exception encountered.
      */
     public static ObjectNode getObjectNode(String message, int code, Exception e) {
-        String detail = e != null ? e.getMessage() : null;
+        String detail = null;
+        if (e != null) {
+            detail = e.getMessage() != null ? e.getMessage() : ExceptionUtils.getStackTrace(e);
+        }
         return mapper.createObjectNode()
             .put("result", code >= 400 ? "ERR" : "OK")
             .put("message", message)

src/main/java/com/conveyal/datatools/manager/auth/Auth0UserProfile.java

@@ -387,7 +387,7 @@ public boolean canAdministerProject(Label label) {
     }
 
     public boolean canAdministerProject(Deployment deployment) {
-        return canAdministerProject(deployment.id, deployment.organizationId());
+        return canAdministerProject(deployment.projectId, deployment.organizationId());
     }
 
     public boolean canAdministerProject(OtpServer server) {

src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java

@@ -521,9 +521,8 @@ private static Deployment uploadToS3 (Request req, Response res) {
             removeDeletedCsvFiles(updatedCsvList, deployment, req);
             return Persistence.deployments.updateField(deployment.id, "peliasCsvFiles", updatedCsvList);
 
-        } catch (AmazonServiceException e) {
-            e.printStackTrace();
-            logMessageAndHalt(req, 500, "Failed to upload file to S3. Check server logs.");
+        } catch (Exception e) {
+            logMessageAndHalt(req, 500, "Failed to upload file to S3.", e);
             return null;
         }
     }

src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java

@@ -520,8 +520,9 @@ private int constructMergedTable(Table table, List<FeedToMerge> feedsToMerge,
                                 }
                                 LOG.info("total stops: {}", stopsCount);
                                 LOG.info("stops missing stop_code: {}", stopsMissingStopCodeCount);
-                                if (stopsMissingStopCodeCount == stopsCount) {
-                                    // If all stops are missing stop_code, we simply default to merging on stop_id.
+                                if (stopsMissingStopCodeCount + specialStopsCount == stopsCount) {
+                                    // If all stops are missing stop_code (taking into account the special stops that do
+                                    // not require stop_code), we simply default to merging on stop_id.
                                     LOG.warn(
                                         "stop_code is not present in file {}/{}. Reverting to stop_id",
                                         feedIndex + 1, feedsToMerge.size());

src/main/java/com/conveyal/datatools/manager/models/Deployment.java

@@ -76,7 +76,7 @@ public class Deployment extends Model implements Serializable {
     /* Pelias fields, used to determine where/if to send data to the Pelias webhook */
     public String peliasWebhookUrl;
     public boolean peliasUpdate;
-    public List<String> peliasCsvFiles;
+    public List<String> peliasCsvFiles = new ArrayList<>();
 
     /**
      * Get parent project for deployment. Note: at one point this was a JSON property of this class, but severe

src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java

@@ -8,6 +8,7 @@
 import com.conveyal.datatools.manager.models.Project;
 import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.gtfs.error.NewGTFSErrorType;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -26,6 +27,7 @@
 import static com.conveyal.datatools.TestUtils.zipFolderFiles;
 import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.MANUALLY_UPLOADED;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -45,6 +47,7 @@ public class MergeFeedsJobTest extends UnitTest {
     private static FeedVersion bothCalendarFilesVersion3;
     private static FeedVersion onlyCalendarVersion;
     private static FeedVersion onlyCalendarDatesVersion;
+    private static FeedSource bart;
 
     /**
      * Prepare and start a testing-specific web server
@@ -60,7 +63,7 @@ public static void setUp() throws IOException {
         Persistence.projects.create(project);
 
         // Bart
-        FeedSource bart = new FeedSource("BART", project.id, MANUALLY_UPLOADED);
+        bart = new FeedSource("BART", project.id, MANUALLY_UPLOADED);
         Persistence.feedSources.create(bart);
         bartVersion1 = createFeedVersionFromGtfsZip(bart, "bart_old.zip");
         bartVersion2 = createFeedVersionFromGtfsZip(bart, "bart_new.zip");
@@ -100,6 +103,13 @@ public static void setUp() throws IOException {
         );
     }
 
+    @AfterAll
+    public static void tearDown() {
+        if (project != null) {
+            project.delete();
+        }
+    }
+
     /**
      * Ensures that a regional feed merge will produce a feed that includes all entities from each feed.
      */
@@ -645,6 +655,31 @@ public void canMergeFeedsWithMTCForServiceIds4 () throws SQLException {
         );
     }
 
+    /**
+     * Verify that merging BART feeds that contain "special stops" (i.e., stops with location_type > 0, including
+     * entrances, generic nodes, etc.) handles missing stop codes correctly.
+     */
+    @Test
+    public void canMergeBARTFeedsWithSpecialStops() throws SQLException, IOException {
+        // Mini BART old/new feeds are pared down versions of the zips (bart_new.zip and bart_old.zip). They each have
+        // only one trip and its corresponding stop_times. They do contain a full set of routes and stops. The stops are
+        // from a recent (as of August 2021) GTFS file that includes a bunch of new stop records that act as entrances).
+        FeedVersion miniBartOld = createFeedVersion(bart, zipFolderFiles("mini-bart-old"));
+        FeedVersion miniBartNew = createFeedVersion(bart, zipFolderFiles("mini-bart-new"));
+        Set<FeedVersion> versions = new HashSet<>();
+        versions.add(miniBartOld);
+        versions.add(miniBartNew);
+        MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD);
+        mergeFeedsJob.run();
+        // Job should succeed.
+        assertFeedMergeSucceeded(mergeFeedsJob);
+        // Verify that the stop count is equal to the number of stops found in each of the input stops.txt files.
+        assertThatSqlCountQueryYieldsExpectedCount(
+            String.format("SELECT count(*) FROM %s.stops", mergeFeedsJob.mergedVersion.namespace),
+            182
+        );
+    }
+
     /**
      * Verifies that a completed merge feeds job did not fail.
      * @param mergeFeedsJob

src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/agency.txt

@@ -0,0 +1,2 @@
+agency_id,agency_name,agency_url,agency_timezone,agency_lang
+BART,Bay Area Rapid Transit,http://www.bart.gov,America/Los_Angeles,en

src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/calendar.txt

@@ -0,0 +1,4 @@
+service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date
+WKDY,1,1,1,1,1,0,0,20190211,20200210
+SAT,0,0,0,0,0,1,0,20190211,20200210
+SUN,0,0,0,0,0,0,1,20190211,20200210

src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/routes.txt

@@ -0,0 +1,10 @@
+route_id,agency_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color
+1,BART,Yellow,Antioch - SFO/Millbrae,,1,http://www.bart.gov/schedules/bylineresults?route=1,ffff33,
+3,BART,Orange,Warm Springs/South Fremont - Richmond,,1,http://www.bart.gov/schedules/,ff9933,
+5,BART,Green,Warm Springs/South Fremont - Daly City,,1,http://www.bart.gov/schedules/,339933,
+7,BART,Red,Richmond - Daly City/Millbrae,,1,http://www.bart.gov/schedules/,ff0000,
+9,BART,Blue-Sun,Dublin/Pleasanton - MacArthur,,1,http://www.bart.gov/schedules/,0099cc,
+11,BART,Blue-Wkd/Sat,Dublin/Pleasanton - Daly City,,1,http://www.bart.gov/schedules/,0099cc,
+13,BART,Purple,Millbrae - SFO,,1,http://www.bart.gov/schedules/,c463c5,
+19,BART,Beige,OAK - Coliseum,,1,http://www.bart.gov/schedules/,d5cfa3,
+

src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/stop_times.txt

@@ -0,0 +1,24 @@
+trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint
+3610458WKDY,04:58:00,04:58:00,CONC,1,San Francisco International Airport,,,,1
+3610458WKDY,05:03:00,05:03:00,PHIL,2,San Francisco International Airport,,,,1
+3610458WKDY,05:06:00,05:06:00,WCRK,3,San Francisco International Airport,,,,1
+3610458WKDY,05:11:00,05:11:00,LAFY,4,San Francisco International Airport,,,,1
+3610458WKDY,05:15:00,05:15:00,ORIN,5,San Francisco International Airport,,,,1
+3610458WKDY,05:21:00,05:21:00,ROCK,6,San Francisco International Airport,,,,1
+3610458WKDY,05:24:00,05:24:00,MCAR,7,San Francisco International Airport,,,,1
+3610458WKDY,05:28:00,05:28:00,19TH,8,San Francisco International Airport,,,,1
+3610458WKDY,05:30:00,05:30:00,12TH,9,San Francisco International Airport,,,,1
+3610458WKDY,05:35:00,05:35:00,WOAK,10,San Francisco International Airport,,,,1
+3610458WKDY,05:42:00,05:42:00,EMBR,11,San Francisco International Airport,,,,1
+3610458WKDY,05:43:00,05:43:00,MONT,12,San Francisco International Airport,,,,1
+3610458WKDY,05:45:00,05:45:00,POWL,13,San Francisco International Airport,,,,1
+3610458WKDY,05:46:00,05:46:00,CIVC,14,San Francisco International Airport,,,,1
+3610458WKDY,05:49:00,05:49:00,16TH,15,San Francisco International Airport,,,,1
+3610458WKDY,05:51:00,05:51:00,24TH,16,San Francisco International Airport,,,,1
+3610458WKDY,05:54:00,05:54:00,GLEN,17,San Francisco International Airport,,,,1
+3610458WKDY,05:56:00,05:56:00,BALB,18,San Francisco International Airport,,,,1
+3610458WKDY,06:00:00,06:00:00,DALY,19,San Francisco International Airport,,,,1
+3610458WKDY,06:04:00,06:04:00,COLM,20,San Francisco International Airport,,,,1
+3610458WKDY,06:07:00,06:07:00,SSAN,21,San Francisco International Airport,,,,1
+3610458WKDY,06:10:00,06:10:00,SBRN,22,San Francisco International Airport,,,,1
+3610458WKDY,06:15:00,06:15:00,SFIA,23,San Francisco International Airport,,,,1