From f802aa5650644172b2e649f07759e31d6b8786ac Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 29 Jul 2021 17:54:12 -0400 Subject: [PATCH 01/23] refactor: accomodate pelias update UI changes --- .../controllers/api/DeploymentController.java | 13 +++++++++++++ .../datatools/manager/models/Deployment.java | 4 ++++ .../conveyal/datatools/manager/models/Project.java | 3 +++ 3 files changed, 20 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 7b23b7765..aae6dc37b 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -218,6 +218,11 @@ private static Deployment createDeployment (Request req, Response res) { Project project = Persistence.projects.getById(projectId); Deployment newDeployment = new Deployment(project); + // Pre-populate the Pelias webhook URL from the project + if (project.lastUsedPeliasWebhookUrl != null) { + newDeployment.peliasWebhookUrl = project.lastUsedPeliasWebhookUrl; + } + // FIXME: Here we are creating a deployment and updating it with the JSON string (two db operations) // We do this because there is not currently apply JSON directly to an object (outside of Mongo codec // operations) @@ -311,6 +316,14 @@ private static Deployment updateDeployment (Request req, Response res) { List versionIds = versionsToInsert.stream().map(v -> v.id).collect(Collectors.toList()); Persistence.deployments.updateField(deploymentToUpdate.id, "feedVersionIds", versionIds); } + + // If Pelias Webhook URL is updated, set that to the project's as a helpful default + if (updateDocument.containsKey("peliasWebhookUrl")) { + Persistence.projects.updateField( + deploymentToUpdate.projectId, + "lastUsedPeliasWebhookUrl", + updateDocument.getString("peliasWebhookUrl")); + } Deployment updatedDeployment = Persistence.deployments.update(deploymentToUpdate.id, req.body()); // TODO: Should updates to the deployment's fields trigger a notification to subscribers? This could get // very noisy. diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 81e2bcb87..e207b798e 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -73,6 +73,10 @@ public class Deployment extends Model implements Serializable { private ObjectMapper otpConfigMapper = new ObjectMapper().setSerializationInclusion(Include.NON_NULL); + /* Pelias fields, used to determine where/if to send data to the Pelias webhook */ + public String peliasWebhookUrl; + public boolean peliasUpdate; + /** * Get parent project for deployment. Note: at one point this was a JSON property of this class, but severe * performance issues prevent this field from scaling to be fetched/assigned to a large collection of deployments. diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 64063fea3..886a7f8fc 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -45,6 +45,9 @@ public class Project extends Model { /** Last successful auto deploy. **/ public Date lastAutoDeploy; + /** The most recently entered Pelias webhook URL. Used when creating new Deployments */ + public String lastUsedPeliasWebhookUrl; + /** * A list of servers that are available to deploy project feeds/OSM to. This includes servers assigned to this * project as well as those that belong to no project. From 42b155a26304152cbcd83a8a4fd0a64976408c69 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 30 Jul 2021 14:53:31 -0400 Subject: [PATCH 02/23] feat(DeploymentController): add custom geocoder update job --- .../common/status/MonitorableJob.java | 3 +- .../controllers/api/DeploymentController.java | 28 +++++++++++++++++++ .../manager/jobs/PeliasUpdateJob.java | 19 +++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java diff --git a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java index 6eba99768..a3e487cbd 100644 --- a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java +++ b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java @@ -78,7 +78,8 @@ public enum JobType { VALIDATE_ALL_FEEDS, MONITOR_SERVER_STATUS, MERGE_FEED_VERSIONS, - RECREATE_BUILD_IMAGE + RECREATE_BUILD_IMAGE, + UPDATE_PELIAS } public MonitorableJob(Auth0UserProfile owner, String name, JobType type) { diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index aae6dc37b..efc0f8647 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -8,6 +8,7 @@ import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.DeployJob; +import com.conveyal.datatools.manager.jobs.PeliasUpdateJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.EC2InstanceSummary; import com.conveyal.datatools.manager.models.FeedSource; @@ -462,9 +463,36 @@ private static String deploy (Request req, Response res) { target); logMessageAndHalt(req, HttpStatus.ACCEPTED_202, message); } + + // If pelias update is requested, launch pelias update job + if (deployment.peliasUpdate) { + updatePelias(req, res); + } + return SparkUtils.formatJobMessage(job.jobId, "Deployment initiating."); } + /** + * Start an updatePelias job which will trigger the webhook, then check for status updates. + */ + private static String updatePelias (Request req, Response res) { + // Check parameters supplied in request for validity. + Auth0UserProfile userProfile = req.attribute("user"); + Deployment deployment = getDeploymentWithPermissions(req, res); + Project project = Persistence.projects.getById(deployment.projectId); + if (project == null) { + logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); + } + + // Check that permissions of user allow them to deploy to target. + boolean isProjectAdmin = userProfile.canAdministerProject(deployment.projectId, deployment.organizationId()); + + // Execute the pelias update job and keep track of it + PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "what is this"); + JobUtils.heavyExecutor.execute(peliasUpdateJob); + return SparkUtils.formatJobMessage(peliasUpdateJob.jobId, "Pelias update initiating."); + } + public static void register (String apiPrefix) { // Construct JSON managers which help serialize the response. Slim JSON is the generic JSON view. Full JSON // contains additional fields (at the moment just #ec2Instances) and should only be used when the controller diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java new file mode 100644 index 000000000..1499065b6 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -0,0 +1,19 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; + +public class PeliasUpdateJob extends MonitorableJob { + public PeliasUpdateJob(Auth0UserProfile owner, String name) { + super(owner, name, JobType.UPDATE_PELIAS); + } + /** + * This method must be overridden by subclasses to perform the core steps of the job. + */ + @Override + public void jobLogic() throws Exception { + status.update("Here we go!", 5.0); + Thread.sleep(4000); + status.completeSuccessfully("it's all done :)"); + } +} From b3efc80938d3e52bac117298edbe0b5938af4003 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 30 Jul 2021 17:16:50 -0400 Subject: [PATCH 03/23] refactor(PeliasUpdateJob): make request to webhook --- .../controllers/api/DeploymentController.java | 5 +- .../manager/jobs/PeliasUpdateJob.java | 120 +++++++++++++++++- 2 files changed, 118 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index efc0f8647..5a607b0ef 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -484,11 +484,8 @@ private static String updatePelias (Request req, Response res) { logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); } - // Check that permissions of user allow them to deploy to target. - boolean isProjectAdmin = userProfile.canAdministerProject(deployment.projectId, deployment.organizationId()); - // Execute the pelias update job and keep track of it - PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "what is this"); + PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "Updating Custom Geocoder Database", deployment); JobUtils.heavyExecutor.execute(peliasUpdateJob); return SparkUtils.formatJobMessage(peliasUpdateJob.jobId, "Pelias update initiating."); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 1499065b6..850752c1c 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -1,19 +1,133 @@ package com.conveyal.datatools.manager.jobs; import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.Deployment; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.json.JsonUtil; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; +import java.util.stream.Collectors; + +import static com.mongodb.client.model.Filters.in; public class PeliasUpdateJob extends MonitorableJob { - public PeliasUpdateJob(Auth0UserProfile owner, String name) { + /** + * The deployment to send to Pelias + */ + private Deployment deployment; + + public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deployment) { super(owner, name, JobType.UPDATE_PELIAS); + this.deployment = deployment; } + /** * This method must be overridden by subclasses to perform the core steps of the job. */ @Override public void jobLogic() throws Exception { status.update("Here we go!", 5.0); - Thread.sleep(4000); + String workerId = this.makeWebhookRequest(this.deployment); + // TODO: Check status endpoint every few seconds and update status + Thread.sleep(1000); + status.update(workerId, 55.0); + Thread.sleep(8000); status.completeSuccessfully("it's all done :)"); } -} + + /** + * Make a request to Pelias update webhook + * + * @return The workerID of the run created on the Pelias server + */ + private String makeWebhookRequest(Deployment deployment) throws IOException { + URL url; + try { + url = new URL(deployment.peliasWebhookUrl); + } catch (MalformedURLException ex) { + status.fail("Webhook URL was not a valid URL", ex); + return null; + } + + // Convert from feedVersionIds to Pelias Config objects + List gtfsFeeds = Persistence.feedVersions.getFiltered(in("_id", deployment.feedVersionIds)) + .stream() + .map(PeliasWebhookGTFSFeedFormat::new) + .collect(Collectors.toList()); + + PeliasWebhookRequestBody peliasWebhookRequestBody = new PeliasWebhookRequestBody(); + peliasWebhookRequestBody.gtfsFeeds = gtfsFeeds; + + String query = JsonUtil.toJson(peliasWebhookRequestBody); + + HttpResponse response; + + try { + CloseableHttpClient client = HttpClientBuilder.create().build(); + HttpPost request = new HttpPost(deployment.peliasWebhookUrl); + StringEntity queryEntity = new StringEntity(query); + request.setEntity(queryEntity); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + + response = client.execute(request); + } catch (IOException ex) { + status.fail("Couldn't connect to webhook URL given.", ex); + return null; + } + + String json = EntityUtils.toString(response.getEntity()); + JsonNode webhookResponse = null; + try { + webhookResponse = JsonUtil.objectMapper.readTree(json); + } catch (IOException ex) { + status.fail("Webhook server returned error:", ex); + return null; + } + + return webhookResponse.get("workerId").asText(); + } + + + /** + * The request body required by the Pelias webhook + */ + private class PeliasWebhookRequestBody { + public List gtfsFeeds; + public List csvFiles; + } + + /** + * The GTFS feed info format the Pelias webhook requires + */ + private class PeliasWebhookGTFSFeedFormat { + public String uri; + public String name; + public String filename; + + public PeliasWebhookGTFSFeedFormat(FeedVersion feedVersion) { + uri = S3Utils.getS3FeedUri(feedVersion.id); + name = Persistence.feedSources.getById(feedVersion.feedSourceId).name; + filename = feedVersion.id; + } + } + + private class PeliasWebhookErrorMessage { + public Boolean completed; + public String error; + public String message; + } +} \ No newline at end of file From ce49da00e3fb39d7021b562c4a30fab42d060937 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 2 Aug 2021 12:16:30 -0400 Subject: [PATCH 04/23] refactor(PeliasUpdateJob): display webhook status --- .../manager/jobs/PeliasUpdateJob.java | 154 ++++++++++++++---- .../datatools/manager/utils/HttpUtils.java | 25 ++- 2 files changed, 143 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 850752c1c..22e0b4bf1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -6,19 +6,21 @@ import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.json.JsonUtil; import com.fasterxml.jackson.databind.JsonNode; +import org.apache.http.Header; import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicHeader; import org.apache.http.util.EntityUtils; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; +import java.util.Timer; +import java.util.TimerTask; import java.util.stream.Collectors; import static com.mongodb.client.model.Filters.in; @@ -29,9 +31,21 @@ public class PeliasUpdateJob extends MonitorableJob { */ private Deployment deployment; + /** + * The workerId our request has on the webhook server. Used to get status updates + */ + private String workerId; + + + /** + * Timer used to poll the status endpoint + */ + Timer timer; + public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deployment) { super(owner, name, JobType.UPDATE_PELIAS); this.deployment = deployment; + this.timer = new Timer(); } /** @@ -39,13 +53,55 @@ public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deploymen */ @Override public void jobLogic() throws Exception { - status.update("Here we go!", 5.0); - String workerId = this.makeWebhookRequest(this.deployment); - // TODO: Check status endpoint every few seconds and update status + status.message = "Launching custom geocoder update request"; + workerId = this.makeWebhookRequest(); + status.percentComplete = 1.0; + + // Give server 1 second to create worker Thread.sleep(1000); - status.update(workerId, 55.0); - Thread.sleep(8000); - status.completeSuccessfully("it's all done :)"); + // Check status every 2 seconds + timer.schedule(new StatusChecker(), 0, 2000); + + } + + private void getWebhookStatus() { + URI url = getWebhookURI(deployment.peliasWebhookUrl + "/status/" + workerId); + HttpResponse response = HttpUtils.httpRequestRawResponse(url, 500, HttpUtils.REQUEST_METHOD.GET, null); + + // Convert raw body to JSON + String jsonResponse; + try { + jsonResponse = EntityUtils.toString(response.getEntity()); + } + catch (NullPointerException | IOException ex) { + status.fail("Webhook status did not provide a response!", ex); + return; + } + + // Parse JSON + PeliasWebhookStatusMessage statusResponse = null; + try { + statusResponse = JsonUtil.objectMapper.readValue(jsonResponse, PeliasWebhookStatusMessage.class); + } catch (IOException ex) { + status.fail("Status update wasn't in correct format:", ex); + return; + } + + if (!statusResponse.error.equals("false")) { + status.fail(statusResponse.error); + timer.cancel(); + return; + } + + if (statusResponse.completed) { + status.completeSuccessfully(statusResponse.message); + timer.cancel(); + return; + } + + status.message = statusResponse.message; + status.percentComplete = statusResponse.percentComplete; + status.completed = false; } /** @@ -53,14 +109,8 @@ public void jobLogic() throws Exception { * * @return The workerID of the run created on the Pelias server */ - private String makeWebhookRequest(Deployment deployment) throws IOException { - URL url; - try { - url = new URL(deployment.peliasWebhookUrl); - } catch (MalformedURLException ex) { - status.fail("Webhook URL was not a valid URL", ex); - return null; - } + private String makeWebhookRequest() { + URI url = getWebhookURI(deployment.peliasWebhookUrl); // Convert from feedVersionIds to Pelias Config objects List gtfsFeeds = Persistence.feedVersions.getFiltered(in("_id", deployment.feedVersionIds)) @@ -73,34 +123,63 @@ private String makeWebhookRequest(Deployment deployment) throws IOException { String query = JsonUtil.toJson(peliasWebhookRequestBody); - HttpResponse response; + // Create headers needed for Pelias webhook + List
headers = new ArrayList<>(); + headers.add(new BasicHeader("Accept", "application/json")); + headers.add(new BasicHeader("Content-type", "application/json")); + + // Get webhook response + HttpResponse response = HttpUtils.httpRequestRawResponse(url, 5000, HttpUtils.REQUEST_METHOD.POST, query, headers); + // Convert raw body to JSON + String jsonResponse; try { - CloseableHttpClient client = HttpClientBuilder.create().build(); - HttpPost request = new HttpPost(deployment.peliasWebhookUrl); - StringEntity queryEntity = new StringEntity(query); - request.setEntity(queryEntity); - request.setHeader("Accept", "application/json"); - request.setHeader("Content-type", "application/json"); - - response = client.execute(request); - } catch (IOException ex) { - status.fail("Couldn't connect to webhook URL given.", ex); + jsonResponse = EntityUtils.toString(response.getEntity()); + } + catch (NullPointerException | IOException ex) { + status.fail("Webhook server specified did not provide a response!", ex); return null; } - String json = EntityUtils.toString(response.getEntity()); + // Parse JSON JsonNode webhookResponse = null; try { - webhookResponse = JsonUtil.objectMapper.readTree(json); + webhookResponse = JsonUtil.objectMapper.readTree(jsonResponse); } catch (IOException ex) { status.fail("Webhook server returned error:", ex); return null; } + if (webhookResponse.get("error") != null) { + status.fail("Server returned an error: " + webhookResponse.get("error").asText()); + return null; + } + return webhookResponse.get("workerId").asText(); } + /** + * Helper function to convert Deployment webhook URL to URI object + * @param webhookUrlString String containing URL to parse + * @return URI object with webhook URL + */ + private URI getWebhookURI(String webhookUrlString) { + URI url; + try { + url = new URI(webhookUrlString); + } catch (URISyntaxException ex) { + status.fail("Webhook URL was not a valid URL", ex); + return null; + } + + return url; + } + + class StatusChecker extends TimerTask { + public void run() { + getWebhookStatus(); + } + } /** * The request body required by the Pelias webhook @@ -117,17 +196,24 @@ private class PeliasWebhookGTFSFeedFormat { public String uri; public String name; public String filename; + public String logUploadUrl; public PeliasWebhookGTFSFeedFormat(FeedVersion feedVersion) { uri = S3Utils.getS3FeedUri(feedVersion.id); name = Persistence.feedSources.getById(feedVersion.feedSourceId).name; filename = feedVersion.id; + // TODO: Where should the log be uploaded to? + logUploadUrl = ""; } } - private class PeliasWebhookErrorMessage { + /** + * The status object returned by the webhook/status endpoint + */ + public static class PeliasWebhookStatusMessage { public Boolean completed; public String error; public String message; + public Double percentComplete; } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java index 0c1cde3d5..83358e14d 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java @@ -1,5 +1,6 @@ package com.conveyal.datatools.manager.utils; +import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; @@ -17,6 +18,8 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; +import java.util.ArrayList; +import java.util.List; public class HttpUtils { private static final Logger LOG = LoggerFactory.getLogger(HttpUtils.class); @@ -25,12 +28,26 @@ public enum REQUEST_METHOD {GET, POST, DELETE, PUT} /** * Makes an http get/post request and returns the response. The request is based on the provided params. */ - //TODO: Replace with java.net.http once migrated to JDK 11. See HttpUtils under otp-middleware. + public static HttpResponse httpRequestRawResponse( + URI uri, + int connectionTimeout, + REQUEST_METHOD method, + String bodyContent + ) { + return httpRequestRawResponse(uri, connectionTimeout, method, bodyContent, new ArrayList<>()); + } + + /** + * Makes an http get/post request and returns the response, including custom headers. + * The request is based on the provided params, including headers. + */ public static HttpResponse httpRequestRawResponse( URI uri, int connectionTimeout, REQUEST_METHOD method, - String bodyContent) { + String bodyContent, + List
headers + ) { RequestConfig timeoutConfig = RequestConfig.custom() .setConnectionRequestTimeout(connectionTimeout) @@ -51,6 +68,10 @@ public static HttpResponse httpRequestRawResponse( try { HttpPost postRequest = new HttpPost(uri); if (bodyContent != null) postRequest.setEntity(new StringEntity(bodyContent)); + for (Header header : headers) { + postRequest.setHeader(header); + } + postRequest.setConfig(timeoutConfig); httpUriRequest = postRequest; } catch (UnsupportedEncodingException e) { From d9352a919147e1f7fe98e64730547b0013da7553 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 2 Aug 2021 15:49:02 -0400 Subject: [PATCH 05/23] refactor: support webhook http authentication --- .../manager/jobs/PeliasUpdateJob.java | 23 ++++++++++++++++--- .../datatools/manager/models/Deployment.java | 2 ++ .../datatools/manager/utils/HttpUtils.java | 3 +++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 22e0b4bf1..0884a09eb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -18,6 +18,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Timer; import java.util.TimerTask; @@ -42,10 +43,19 @@ public class PeliasUpdateJob extends MonitorableJob { */ Timer timer; + /** + * Webhook authorization from username and password + */ + Header webhookAuthorization; + public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deployment) { super(owner, name, JobType.UPDATE_PELIAS); this.deployment = deployment; this.timer = new Timer(); + + String authorizationString = deployment.peliasUsername + ":" + deployment.peliasPassword; + authorizationString = "Basic " + Base64.getEncoder().encodeToString(authorizationString.getBytes()); + this.webhookAuthorization = new BasicHeader("Authorization", authorizationString); } /** @@ -66,7 +76,11 @@ public void jobLogic() throws Exception { private void getWebhookStatus() { URI url = getWebhookURI(deployment.peliasWebhookUrl + "/status/" + workerId); - HttpResponse response = HttpUtils.httpRequestRawResponse(url, 500, HttpUtils.REQUEST_METHOD.GET, null); + + List
headers = new ArrayList<>(); + headers.add(this.webhookAuthorization); + + HttpResponse response = HttpUtils.httpRequestRawResponse(url, 1000, HttpUtils.REQUEST_METHOD.GET, null, headers); // Convert raw body to JSON String jsonResponse; @@ -75,6 +89,7 @@ private void getWebhookStatus() { } catch (NullPointerException | IOException ex) { status.fail("Webhook status did not provide a response!", ex); + timer.cancel(); return; } @@ -83,7 +98,8 @@ private void getWebhookStatus() { try { statusResponse = JsonUtil.objectMapper.readValue(jsonResponse, PeliasWebhookStatusMessage.class); } catch (IOException ex) { - status.fail("Status update wasn't in correct format:", ex); + status.fail("Server refused to provide a valid status update. Are the credentials correct?", ex); + timer.cancel(); return; } @@ -127,6 +143,7 @@ private String makeWebhookRequest() { List
headers = new ArrayList<>(); headers.add(new BasicHeader("Accept", "application/json")); headers.add(new BasicHeader("Content-type", "application/json")); + headers.add(this.webhookAuthorization); // Get webhook response HttpResponse response = HttpUtils.httpRequestRawResponse(url, 5000, HttpUtils.REQUEST_METHOD.POST, query, headers); @@ -146,7 +163,7 @@ private String makeWebhookRequest() { try { webhookResponse = JsonUtil.objectMapper.readTree(jsonResponse); } catch (IOException ex) { - status.fail("Webhook server returned error:", ex); + status.fail("The Webhook server's response was invalid! Are the credentials correct?", ex); return null; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index e207b798e..71352b241 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -75,6 +75,8 @@ public class Deployment extends Model implements Serializable { /* Pelias fields, used to determine where/if to send data to the Pelias webhook */ public String peliasWebhookUrl; + public String peliasUsername; + public String peliasPassword; public boolean peliasUpdate; /** diff --git a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java index 83358e14d..1df8a1fa6 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java @@ -61,6 +61,9 @@ public static HttpResponse httpRequestRawResponse( switch (method) { case GET: HttpGet getRequest = new HttpGet(uri); + for (Header header : headers) { + getRequest.setHeader(header); + } getRequest.setConfig(timeoutConfig); httpUriRequest = getRequest; break; From 7b6dad3c53ac11c33efa5b166176f5cdcb918cd8 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 2 Aug 2021 18:00:57 -0400 Subject: [PATCH 06/23] refactor(PeliasUpdateJob): pass log upload URI to webhook --- .../controllers/api/DeploymentController.java | 9 ++++--- .../manager/jobs/PeliasUpdateJob.java | 27 ++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 5a607b0ef..7c1707d67 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -466,7 +466,7 @@ private static String deploy (Request req, Response res) { // If pelias update is requested, launch pelias update job if (deployment.peliasUpdate) { - updatePelias(req, res); + updatePelias(req, res, job); } return SparkUtils.formatJobMessage(job.jobId, "Deployment initiating."); @@ -475,7 +475,7 @@ private static String deploy (Request req, Response res) { /** * Start an updatePelias job which will trigger the webhook, then check for status updates. */ - private static String updatePelias (Request req, Response res) { + private static String updatePelias (Request req, Response res, DeployJob deployJob) { // Check parameters supplied in request for validity. Auth0UserProfile userProfile = req.attribute("user"); Deployment deployment = getDeploymentWithPermissions(req, res); @@ -484,8 +484,11 @@ private static String updatePelias (Request req, Response res) { logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); } + // Get log upload URI from deploy job + AmazonS3URI logUploadS3URI = deployJob.getS3FolderURI(); + // Execute the pelias update job and keep track of it - PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "Updating Custom Geocoder Database", deployment); + PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "Updating Custom Geocoder Database", deployment, logUploadS3URI); JobUtils.heavyExecutor.execute(peliasUpdateJob); return SparkUtils.formatJobMessage(peliasUpdateJob.jobId, "Pelias update initiating."); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 0884a09eb..d83807457 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -1,5 +1,6 @@ package com.conveyal.datatools.manager.jobs; +import com.amazonaws.services.s3.AmazonS3URI; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; @@ -41,21 +42,29 @@ public class PeliasUpdateJob extends MonitorableJob { /** * Timer used to poll the status endpoint */ - Timer timer; + private Timer timer; /** * Webhook authorization from username and password */ - Header webhookAuthorization; + private Header webhookAuthorization; - public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deployment) { + /** + * S3 URI to upload logs to + */ + private AmazonS3URI logUploadS3URI; + + public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deployment, AmazonS3URI logUploadS3URI) { super(owner, name, JobType.UPDATE_PELIAS); this.deployment = deployment; this.timer = new Timer(); + this.logUploadS3URI = logUploadS3URI; - String authorizationString = deployment.peliasUsername + ":" + deployment.peliasPassword; - authorizationString = "Basic " + Base64.getEncoder().encodeToString(authorizationString.getBytes()); - this.webhookAuthorization = new BasicHeader("Authorization", authorizationString); + if (deployment.peliasUsername != "" && deployment.peliasPassword != "") { + String authorizationString = deployment.peliasUsername + ":" + deployment.peliasPassword; + authorizationString = "Basic " + Base64.getEncoder().encodeToString(authorizationString.getBytes()); + this.webhookAuthorization = new BasicHeader("Authorization", authorizationString); + } } /** @@ -136,9 +145,11 @@ private String makeWebhookRequest() { PeliasWebhookRequestBody peliasWebhookRequestBody = new PeliasWebhookRequestBody(); peliasWebhookRequestBody.gtfsFeeds = gtfsFeeds; + peliasWebhookRequestBody.logUploadUrl = this.logUploadS3URI.toString(); String query = JsonUtil.toJson(peliasWebhookRequestBody); + // Create headers needed for Pelias webhook List
headers = new ArrayList<>(); headers.add(new BasicHeader("Accept", "application/json")); @@ -204,6 +215,7 @@ public void run() { private class PeliasWebhookRequestBody { public List gtfsFeeds; public List csvFiles; + public String logUploadUrl; } /** @@ -213,14 +225,11 @@ private class PeliasWebhookGTFSFeedFormat { public String uri; public String name; public String filename; - public String logUploadUrl; public PeliasWebhookGTFSFeedFormat(FeedVersion feedVersion) { uri = S3Utils.getS3FeedUri(feedVersion.id); name = Persistence.feedSources.getById(feedVersion.feedSourceId).name; filename = feedVersion.id; - // TODO: Where should the log be uploaded to? - logUploadUrl = ""; } } From 307d3a91e6e4c7a742daebb484852f5192f8bcc4 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 3 Aug 2021 10:58:29 -0400 Subject: [PATCH 07/23] refactor: add pelias csv file upload --- .../controllers/api/DeploymentController.java | 65 ++++++++++++++++++- .../datatools/manager/jobs/DeployJob.java | 2 +- .../manager/jobs/PeliasUpdateJob.java | 7 ++ .../datatools/manager/models/Deployment.java | 1 + 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 7c1707d67..44e2c926a 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -1,9 +1,12 @@ package com.conveyal.datatools.manager.controllers.api; +import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.PutObjectRequest; import com.conveyal.datatools.common.status.MonitorableJob; -import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.common.utils.SparkUtils; +import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.common.utils.aws.EC2Utils; import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; @@ -19,6 +22,7 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.JobUtils; import com.conveyal.datatools.manager.utils.json.JsonManager; +import org.apache.commons.io.IOUtils; import org.bson.Document; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; @@ -26,9 +30,14 @@ import spark.Request; import spark.Response; +import javax.servlet.MultipartConfigElement; +import javax.servlet.ServletException; +import javax.servlet.http.Part; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -37,6 +46,7 @@ import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static com.conveyal.datatools.manager.DataManager.isExtensionEnabled; +import static com.conveyal.datatools.manager.jobs.DeployJob.bundlePrefix; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; import static spark.Spark.delete; @@ -493,6 +503,57 @@ private static String updatePelias (Request req, Response res, DeployJob deployJ return SparkUtils.formatJobMessage(peliasUpdateJob.jobId, "Pelias update initiating."); } + /** + * Uploads a file from Spark request object to the s3 bucket of the deployment the Pelias Update Job is associated with. + * Follows https://github.com/ibi-group/datatools-server/blob/dev/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java#L111 + * @return S3 URL the file has been uploaded to + */ + private static String uploadToS3 (Request req, Response res) throws IOException { + // Check parameters supplied in request for validity. + Auth0UserProfile userProfile = req.attribute("user"); + Deployment deployment = getDeploymentWithPermissions(req, res); + + String url; + + // Get file from request + if (req.raw().getAttribute("org.eclipse.jetty.multipartConfig") == null) { + MultipartConfigElement multipartConfigElement = new MultipartConfigElement(System.getProperty("java.io.tmpdir")); + req.raw().setAttribute("org.eclipse.jetty.multipartConfig", multipartConfigElement); + } + String extension = null; + File tempFile = null; + try { + Part part = req.raw().getPart("file"); + extension = "." + part.getContentType().split("/", 0)[1]; + tempFile = File.createTempFile(part.getName() + "_csv_upload", extension); + InputStream inputStream; + inputStream = part.getInputStream(); + FileOutputStream out = new FileOutputStream(tempFile); + IOUtils.copy(inputStream, out); + } catch (IOException | ServletException e) { + e.printStackTrace(); + } + + try { + String keyName = String.join("/", bundlePrefix, deployment.projectId, deployment.id, tempFile.getName()); + url = S3Utils.getDefaultBucketUrlForKey(keyName); + S3Utils.getDefaultS3Client().putObject(new PutObjectRequest( + S3Utils.DEFAULT_BUCKET, keyName, tempFile) + // Allow public read + // TODO: restrict? + .withCannedAcl(CannedAccessControlList.PublicRead)); + return url; + } catch (AmazonServiceException | CheckedAWSException e) { + e.printStackTrace(); + return null; + } finally { + boolean deleted = tempFile.delete(); + if (!deleted) { + throw new IOException("Failed to delete file temporarily stored on server"); + } + } + } + public static void register (String apiPrefix) { // Construct JSON managers which help serialize the response. Slim JSON is the generic JSON view. Full JSON // contains additional fields (at the moment just #ec2Instances) and should only be used when the controller @@ -518,5 +579,7 @@ public static void register (String apiPrefix) { post(apiPrefix + "secure/deployments", DeploymentController::createDeployment, fullJson::write); put(apiPrefix + "secure/deployments/:id", DeploymentController::updateDeployment, fullJson::write); post(apiPrefix + "secure/deployments/fromfeedsource/:id", DeploymentController::createDeploymentFromFeedSource, fullJson::write); + post(apiPrefix + "secure/deployments/:id/upload", DeploymentController::uploadToS3, slimJson::write); + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index bfc7f6259..dd505350f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -91,7 +91,7 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); - private static final String bundlePrefix = "bundles"; + public static final String bundlePrefix = "bundles"; // Indicates whether EC2 instances should be EBS optimized. private static final boolean EBS_OPTIMIZED = "true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ebs_optimized")); // Indicates the node.js version installed by nvm to set the PATH variable to point to diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index d83807457..0ebccb064 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -145,7 +145,10 @@ private String makeWebhookRequest() { PeliasWebhookRequestBody peliasWebhookRequestBody = new PeliasWebhookRequestBody(); peliasWebhookRequestBody.gtfsFeeds = gtfsFeeds; + peliasWebhookRequestBody.csvFiles = deployment.peliasCsvFiles; peliasWebhookRequestBody.logUploadUrl = this.logUploadS3URI.toString(); + peliasWebhookRequestBody.deploymentId = deployment.id; + String query = JsonUtil.toJson(peliasWebhookRequestBody); @@ -203,6 +206,9 @@ private URI getWebhookURI(String webhookUrlString) { return url; } + /** + * Class used to execute the status update + */ class StatusChecker extends TimerTask { public void run() { getWebhookStatus(); @@ -216,6 +222,7 @@ private class PeliasWebhookRequestBody { public List gtfsFeeds; public List csvFiles; public String logUploadUrl; + public String deploymentId; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 71352b241..7f0f97718 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -78,6 +78,7 @@ public class Deployment extends Model implements Serializable { public String peliasUsername; public String peliasPassword; public boolean peliasUpdate; + public List peliasCsvFiles; /** * Get parent project for deployment. Note: at one point this was a JSON property of this class, but severe From 12163e1851f28c1fba9403b6d86989dfcdb633ae Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 3 Aug 2021 11:42:29 -0400 Subject: [PATCH 08/23] refactor: adjust csv upload to match ui --- .../controllers/api/DeploymentController.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 44e2c926a..72b4b5bda 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -5,8 +5,8 @@ import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.PutObjectRequest; import com.conveyal.datatools.common.status.MonitorableJob; -import com.conveyal.datatools.common.utils.SparkUtils; import com.conveyal.datatools.common.utils.aws.CheckedAWSException; +import com.conveyal.datatools.common.utils.SparkUtils; import com.conveyal.datatools.common.utils.aws.EC2Utils; import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; @@ -42,6 +42,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; @@ -530,12 +531,14 @@ private static String uploadToS3 (Request req, Response res) throws IOException inputStream = part.getInputStream(); FileOutputStream out = new FileOutputStream(tempFile); IOUtils.copy(inputStream, out); - } catch (IOException | ServletException e) { - e.printStackTrace(); - } - try { - String keyName = String.join("/", bundlePrefix, deployment.projectId, deployment.id, tempFile.getName()); + String keyName = String.join( + "/", + bundlePrefix, + deployment.projectId, + deployment.id, + UUID.randomUUID() + "_" + part.getSubmittedFileName() + ); url = S3Utils.getDefaultBucketUrlForKey(keyName); S3Utils.getDefaultS3Client().putObject(new PutObjectRequest( S3Utils.DEFAULT_BUCKET, keyName, tempFile) @@ -543,7 +546,7 @@ private static String uploadToS3 (Request req, Response res) throws IOException // TODO: restrict? .withCannedAcl(CannedAccessControlList.PublicRead)); return url; - } catch (AmazonServiceException | CheckedAWSException e) { + } catch (IOException | ServletException | AmazonServiceException | CheckedAWSException e) { e.printStackTrace(); return null; } finally { From d7c91a0c178b3f4071edea2b9b629fd44010e02c Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 3 Aug 2021 14:49:36 -0400 Subject: [PATCH 09/23] refactor: cleanup and remove duplicated features --- .../controllers/api/DeploymentController.java | 8 +-- .../datatools/manager/jobs/DeployJob.java | 59 +++++++++---------- .../manager/jobs/PeliasUpdateJob.java | 6 +- .../datatools/manager/utils/HttpUtils.java | 6 ++ 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 72b4b5bda..3e73033d5 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -5,8 +5,8 @@ import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.PutObjectRequest; import com.conveyal.datatools.common.status.MonitorableJob; -import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.common.utils.SparkUtils; +import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.common.utils.aws.EC2Utils; import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; @@ -230,11 +230,6 @@ private static Deployment createDeployment (Request req, Response res) { Project project = Persistence.projects.getById(projectId); Deployment newDeployment = new Deployment(project); - // Pre-populate the Pelias webhook URL from the project - if (project.lastUsedPeliasWebhookUrl != null) { - newDeployment.peliasWebhookUrl = project.lastUsedPeliasWebhookUrl; - } - // FIXME: Here we are creating a deployment and updating it with the JSON string (two db operations) // We do this because there is not currently apply JSON directly to an object (outside of Mongo codec // operations) @@ -537,6 +532,7 @@ private static String uploadToS3 (Request req, Response res) throws IOException bundlePrefix, deployment.projectId, deployment.id, + // Where filenames are generated. Prepend random UUID to prevent overwriting UUID.randomUUID() + "_" + part.getSubmittedFileName() ); url = S3Utils.getDefaultBucketUrlForKey(keyName); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index dd505350f..09d36486b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -12,43 +12,14 @@ import com.amazonaws.services.ec2.model.InstanceType; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.Tag; -import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.ec2.model.TerminateInstancesResult; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.Serializable; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.WritableByteChannel; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Scanner; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - import com.amazonaws.waiters.Waiter; import com.amazonaws.waiters.WaiterParameters; import com.conveyal.datatools.common.status.MonitorableJob; @@ -81,6 +52,34 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Scanner; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_OTP_VERSION; /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 0ebccb064..4bc0acf26 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -87,7 +87,7 @@ private void getWebhookStatus() { URI url = getWebhookURI(deployment.peliasWebhookUrl + "/status/" + workerId); List
headers = new ArrayList<>(); - headers.add(this.webhookAuthorization); + headers.add(webhookAuthorization); HttpResponse response = HttpUtils.httpRequestRawResponse(url, 1000, HttpUtils.REQUEST_METHOD.GET, null, headers); @@ -146,7 +146,7 @@ private String makeWebhookRequest() { PeliasWebhookRequestBody peliasWebhookRequestBody = new PeliasWebhookRequestBody(); peliasWebhookRequestBody.gtfsFeeds = gtfsFeeds; peliasWebhookRequestBody.csvFiles = deployment.peliasCsvFiles; - peliasWebhookRequestBody.logUploadUrl = this.logUploadS3URI.toString(); + peliasWebhookRequestBody.logUploadUrl = logUploadS3URI.toString(); peliasWebhookRequestBody.deploymentId = deployment.id; @@ -157,7 +157,7 @@ private String makeWebhookRequest() { List
headers = new ArrayList<>(); headers.add(new BasicHeader("Accept", "application/json")); headers.add(new BasicHeader("Content-type", "application/json")); - headers.add(this.webhookAuthorization); + headers.add(webhookAuthorization); // Get webhook response HttpResponse response = HttpUtils.httpRequestRawResponse(url, 5000, HttpUtils.REQUEST_METHOD.POST, query, headers); diff --git a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java index 1df8a1fa6..a56784b71 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java @@ -85,6 +85,9 @@ public static HttpResponse httpRequestRawResponse( case PUT: try { HttpPut putRequest = new HttpPut(uri); + for (Header header : headers) { + putRequest.setHeader(header); + } putRequest.setEntity(new StringEntity(bodyContent)); putRequest.setConfig(timeoutConfig); httpUriRequest = putRequest; @@ -95,6 +98,9 @@ public static HttpResponse httpRequestRawResponse( break; case DELETE: HttpDelete deleteRequest = new HttpDelete(uri); + for (Header header : headers) { + deleteRequest.setHeader(header); + } deleteRequest.setConfig(timeoutConfig); httpUriRequest = deleteRequest; break; From f1161b34caf6da36319ce713f5f40308be7e73cf Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 3 Aug 2021 15:49:42 -0400 Subject: [PATCH 10/23] revert: remove username/password functionality --- .../conveyal/datatools/manager/jobs/PeliasUpdateJob.java | 6 ------ .../com/conveyal/datatools/manager/models/Deployment.java | 2 -- 2 files changed, 8 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 4bc0acf26..3769dcacf 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -59,12 +59,6 @@ public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deploymen this.deployment = deployment; this.timer = new Timer(); this.logUploadS3URI = logUploadS3URI; - - if (deployment.peliasUsername != "" && deployment.peliasPassword != "") { - String authorizationString = deployment.peliasUsername + ":" + deployment.peliasPassword; - authorizationString = "Basic " + Base64.getEncoder().encodeToString(authorizationString.getBytes()); - this.webhookAuthorization = new BasicHeader("Authorization", authorizationString); - } } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 7f0f97718..33be6d766 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -75,8 +75,6 @@ public class Deployment extends Model implements Serializable { /* Pelias fields, used to determine where/if to send data to the Pelias webhook */ public String peliasWebhookUrl; - public String peliasUsername; - public String peliasPassword; public boolean peliasUpdate; public List peliasCsvFiles; From ff4070cb2ba88c83030b129baf0caeca4117e05d Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 4 Aug 2021 08:30:19 -0400 Subject: [PATCH 11/23] refactor(DeploymentController): delete csv url from s3 when it is deleted --- .../controllers/api/DeploymentController.java | 32 +++++++++++++++++-- .../manager/jobs/PeliasUpdateJob.java | 3 -- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 3e73033d5..e10740ad9 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -3,6 +3,7 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.PutObjectRequest; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.common.utils.SparkUtils; @@ -276,7 +277,7 @@ private static Deployment createDeploymentFromFeedSource (Request req, Response * Update a single deployment. If the deployment's feed versions are updated, checks to ensure that each * version exists and is a part of the same parent project are performed before updating. */ - private static Deployment updateDeployment (Request req, Response res) { + private static Deployment updateDeployment (Request req, Response res) throws CheckedAWSException { Deployment deploymentToUpdate = getDeploymentWithPermissions(req, res); Document updateDocument = Document.parse(req.body()); // FIXME use generic update hook, also feedVersions is getting serialized into MongoDB (which is undesirable) @@ -331,6 +332,26 @@ private static Deployment updateDeployment (Request req, Response res) { "lastUsedPeliasWebhookUrl", updateDocument.getString("peliasWebhookUrl")); } + + // If updatedDocument has deleted a CSV file, also delete that CSV file from S3 + if (updateDocument.containsKey("peliasCsvFiles")) { + List csvUrls = (List) updateDocument.get("peliasCsvFiles"); + // Only delete if the array differs + if (!csvUrls.equals(deploymentToUpdate.peliasCsvFiles)) { + for (String existingCsvUrl : deploymentToUpdate.peliasCsvFiles) { + // Only delete if the file does not exist in the deployment + if (!csvUrls.contains(existingCsvUrl)) { + try { + AmazonS3URI s3URIToDelete = new AmazonS3URI(existingCsvUrl); + S3Utils.getDefaultS3Client().deleteObject(new DeleteObjectRequest(s3URIToDelete.getBucket(), s3URIToDelete.getKey())); + } catch(Exception e) { + logMessageAndHalt(req, 500, "Failed to delete file from S3.", e); + } + } + } + + } + } Deployment updatedDeployment = Persistence.deployments.update(deploymentToUpdate.id, req.body()); // TODO: Should updates to the deployment's fields trigger a notification to subscribers? This could get // very noisy. @@ -510,6 +531,7 @@ private static String uploadToS3 (Request req, Response res) throws IOException Deployment deployment = getDeploymentWithPermissions(req, res); String url; + Exception failure = null; // Get file from request if (req.raw().getAttribute("org.eclipse.jetty.multipartConfig") == null) { @@ -544,11 +566,15 @@ private static String uploadToS3 (Request req, Response res) throws IOException return url; } catch (IOException | ServletException | AmazonServiceException | CheckedAWSException e) { e.printStackTrace(); + failure = e; return null; } finally { boolean deleted = tempFile.delete(); if (!deleted) { - throw new IOException("Failed to delete file temporarily stored on server"); + logMessageAndHalt(req, 500, "Failed to delete file temporarily stored on server"); + } + if (failure != null) { + logMessageAndHalt(req, 500, "Failed to upload file. Please try again"); } } } @@ -578,7 +604,7 @@ public static void register (String apiPrefix) { post(apiPrefix + "secure/deployments", DeploymentController::createDeployment, fullJson::write); put(apiPrefix + "secure/deployments/:id", DeploymentController::updateDeployment, fullJson::write); post(apiPrefix + "secure/deployments/fromfeedsource/:id", DeploymentController::createDeploymentFromFeedSource, fullJson::write); - post(apiPrefix + "secure/deployments/:id/upload", DeploymentController::uploadToS3, slimJson::write); + post(apiPrefix + "secure/deployments/:id/upload", DeploymentController::uploadToS3); } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 3769dcacf..70f2c88b1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -19,7 +19,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Base64; import java.util.List; import java.util.Timer; import java.util.TimerTask; @@ -143,10 +142,8 @@ private String makeWebhookRequest() { peliasWebhookRequestBody.logUploadUrl = logUploadS3URI.toString(); peliasWebhookRequestBody.deploymentId = deployment.id; - String query = JsonUtil.toJson(peliasWebhookRequestBody); - // Create headers needed for Pelias webhook List
headers = new ArrayList<>(); headers.add(new BasicHeader("Accept", "application/json")); From 7aab4a64041e70370cce800d09b930135cadc2d2 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 6 Aug 2021 08:16:38 -0400 Subject: [PATCH 12/23] revert: remove project webhook assistance feature --- .../manager/controllers/api/DeploymentController.java | 8 -------- .../com/conveyal/datatools/manager/models/Project.java | 3 --- 2 files changed, 11 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index e10740ad9..686ca85a7 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -325,14 +325,6 @@ private static Deployment updateDeployment (Request req, Response res) throws Ch Persistence.deployments.updateField(deploymentToUpdate.id, "feedVersionIds", versionIds); } - // If Pelias Webhook URL is updated, set that to the project's as a helpful default - if (updateDocument.containsKey("peliasWebhookUrl")) { - Persistence.projects.updateField( - deploymentToUpdate.projectId, - "lastUsedPeliasWebhookUrl", - updateDocument.getString("peliasWebhookUrl")); - } - // If updatedDocument has deleted a CSV file, also delete that CSV file from S3 if (updateDocument.containsKey("peliasCsvFiles")) { List csvUrls = (List) updateDocument.get("peliasCsvFiles"); diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 886a7f8fc..64063fea3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -45,9 +45,6 @@ public class Project extends Model { /** Last successful auto deploy. **/ public Date lastAutoDeploy; - /** The most recently entered Pelias webhook URL. Used when creating new Deployments */ - public String lastUsedPeliasWebhookUrl; - /** * A list of servers that are available to deploy project feeds/OSM to. This includes servers assigned to this * project as well as those that belong to no project. From 3ed943abf53df3baa41486c2d6efee5f3ca257b3 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 6 Aug 2021 08:32:01 -0400 Subject: [PATCH 13/23] refactor: remove unused class fields, more lenient webhook status parsing --- .../datatools/manager/jobs/PeliasUpdateJob.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 70f2c88b1..c2cf65380 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -44,9 +44,9 @@ public class PeliasUpdateJob extends MonitorableJob { private Timer timer; /** - * Webhook authorization from username and password + * The number of webhook status requests allowed to fail before considering the server down */ - private Header webhookAuthorization; + private int webhookStatusFailuresAllowed = 3; /** * S3 URI to upload logs to @@ -79,10 +79,7 @@ public void jobLogic() throws Exception { private void getWebhookStatus() { URI url = getWebhookURI(deployment.peliasWebhookUrl + "/status/" + workerId); - List
headers = new ArrayList<>(); - headers.add(webhookAuthorization); - - HttpResponse response = HttpUtils.httpRequestRawResponse(url, 1000, HttpUtils.REQUEST_METHOD.GET, null, headers); + HttpResponse response = HttpUtils.httpRequestRawResponse(url, 1000, HttpUtils.REQUEST_METHOD.GET, null); // Convert raw body to JSON String jsonResponse; @@ -90,8 +87,10 @@ private void getWebhookStatus() { jsonResponse = EntityUtils.toString(response.getEntity()); } catch (NullPointerException | IOException ex) { - status.fail("Webhook status did not provide a response!", ex); - timer.cancel(); + if (--webhookStatusFailuresAllowed == 0) { + status.fail("Webhook status did not provide a response!", ex); + timer.cancel(); + } return; } @@ -148,7 +147,6 @@ private String makeWebhookRequest() { List
headers = new ArrayList<>(); headers.add(new BasicHeader("Accept", "application/json")); headers.add(new BasicHeader("Content-type", "application/json")); - headers.add(webhookAuthorization); // Get webhook response HttpResponse response = HttpUtils.httpRequestRawResponse(url, 5000, HttpUtils.REQUEST_METHOD.POST, query, headers); From 8d25ce26ad545a9a1c00258c8ca44dc152651f34 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 6 Aug 2021 08:42:27 -0400 Subject: [PATCH 14/23] refactor: cleanup --- .../controllers/api/FeedSourceController.java | 5 +---- .../datatools/manager/jobs/PeliasUpdateJob.java | 14 +++++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index faba03200..c8420cbb6 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -2,8 +2,8 @@ import com.conveyal.datatools.common.utils.Scheduler; import com.conveyal.datatools.manager.DataManager; -import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.auth.Actions; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.extensions.ExternalFeedResource; import com.conveyal.datatools.manager.jobs.FetchSingleFeedJob; import com.conveyal.datatools.manager.jobs.NotifyUsersForSubscriptionJob; @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.StringUtils; -import org.bson.conversions.Bson; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,8 +43,6 @@ import static com.conveyal.datatools.manager.models.ExternalFeedSourceProperty.constructId; import static com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation.getInvalidSubstitutionMessage; import static com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation.getInvalidSubstitutionPatterns; -import static com.mongodb.client.model.Filters.and; -import static com.mongodb.client.model.Filters.eq; import static com.mongodb.client.model.Filters.in; import static spark.Spark.delete; import static spark.Spark.get; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 8f3dae42e..af9a057da 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -29,7 +29,7 @@ public class PeliasUpdateJob extends MonitorableJob { /** * The deployment to send to Pelias */ - private Deployment deployment; + private final Deployment deployment; /** * The workerId our request has on the webhook server. Used to get status updates @@ -40,7 +40,7 @@ public class PeliasUpdateJob extends MonitorableJob { /** * Timer used to poll the status endpoint */ - private Timer timer; + private final Timer timer; /** * The number of webhook status requests allowed to fail before considering the server down @@ -50,7 +50,7 @@ public class PeliasUpdateJob extends MonitorableJob { /** * S3 URI to upload logs to */ - private AmazonS3URI logUploadS3URI; + private final AmazonS3URI logUploadS3URI; public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deployment, AmazonS3URI logUploadS3URI) { super(owner, name, JobType.UPDATE_PELIAS); @@ -94,7 +94,7 @@ private void getWebhookStatus() { } // Parse JSON - PeliasWebhookStatusMessage statusResponse = null; + PeliasWebhookStatusMessage statusResponse; try { statusResponse = JsonUtil.objectMapper.readValue(jsonResponse, PeliasWebhookStatusMessage.class); } catch (IOException ex) { @@ -161,7 +161,7 @@ private String makeWebhookRequest() { } // Parse JSON - JsonNode webhookResponse = null; + JsonNode webhookResponse; try { webhookResponse = JsonUtil.objectMapper.readTree(jsonResponse); } catch (IOException ex) { @@ -206,7 +206,7 @@ public void run() { /** * The request body required by the Pelias webhook */ - private class PeliasWebhookRequestBody { + private static class PeliasWebhookRequestBody { public List gtfsFeeds; public List csvFiles; public String logUploadUrl; @@ -216,7 +216,7 @@ private class PeliasWebhookRequestBody { /** * The GTFS feed info format the Pelias webhook requires */ - private class PeliasWebhookGTFSFeedFormat { + private static class PeliasWebhookGTFSFeedFormat { public String uri; public String name; public String filename; From 0ddf61f8d6e3a96db61d37545bebed5dfc25445f Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 9 Aug 2021 11:01:37 +0200 Subject: [PATCH 15/23] refactor(PeliasUpdateJob): catch non-checked exceptions --- .../manager/jobs/PeliasUpdateJob.java | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index af9a057da..29c2da0f9 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -78,31 +78,23 @@ public void jobLogic() throws Exception { private void getWebhookStatus() { URI url = getWebhookURI(deployment.peliasWebhookUrl + "/status/" + workerId); - SimpleHttpResponse response = HttpUtils.httpRequestRawResponse(url, 1000, HttpUtils.REQUEST_METHOD.GET, null); - // Convert raw body to JSON - String jsonResponse; + PeliasWebhookStatusMessage statusResponse; + try { - jsonResponse = response.body; - } - catch (NullPointerException ex) { + SimpleHttpResponse response = HttpUtils.httpRequestRawResponse(url, 1000, HttpUtils.REQUEST_METHOD.GET, null); + // Convert raw body to PeliasWebhookStatusMessage + String jsonResponse = response.body; + statusResponse = JsonUtil.objectMapper.readValue(jsonResponse, PeliasWebhookStatusMessage.class); + } catch (Exception ex) { + // Allow a set number of failed requests before showing the user failure message if (--webhookStatusFailuresAllowed == 0) { - status.fail("Webhook status did not provide a response!", ex); + status.fail("Webhook status did not provide a valid response!", ex); timer.cancel(); } return; } - // Parse JSON - PeliasWebhookStatusMessage statusResponse; - try { - statusResponse = JsonUtil.objectMapper.readValue(jsonResponse, PeliasWebhookStatusMessage.class); - } catch (IOException ex) { - status.fail("Server refused to provide a valid status update. Are the credentials correct?", ex); - timer.cancel(); - return; - } - if (!statusResponse.error.equals("false")) { status.fail(statusResponse.error); timer.cancel(); From 2c0f44d8258a6c6bf93526031b7ef88249cdcf2a Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 10 Aug 2021 20:33:37 +0200 Subject: [PATCH 16/23] refactor: remove reference to removed credentials feature --- .../com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 29c2da0f9..61e3d85bf 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -157,7 +157,7 @@ private String makeWebhookRequest() { try { webhookResponse = JsonUtil.objectMapper.readTree(jsonResponse); } catch (IOException ex) { - status.fail("The Webhook server's response was invalid! Are the credentials correct?", ex); + status.fail("The Webhook server's response was invalid! Is the server URL correct?", ex); return null; } From ce48cea206b53840bdf605967275005230499af7 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 24 Aug 2021 09:22:24 +0100 Subject: [PATCH 17/23] refactor(DeploymentController): move csv upload replace code to server --- .../controllers/api/DeploymentController.java | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 1d647c3b3..fb4d1b20f 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -328,21 +328,7 @@ private static Deployment updateDeployment (Request req, Response res) throws Ch // If updatedDocument has deleted a CSV file, also delete that CSV file from S3 if (updateDocument.containsKey("peliasCsvFiles")) { List csvUrls = (List) updateDocument.get("peliasCsvFiles"); - // Only delete if the array differs - if (!csvUrls.equals(deploymentToUpdate.peliasCsvFiles)) { - for (String existingCsvUrl : deploymentToUpdate.peliasCsvFiles) { - // Only delete if the file does not exist in the deployment - if (!csvUrls.contains(existingCsvUrl)) { - try { - AmazonS3URI s3URIToDelete = new AmazonS3URI(existingCsvUrl); - S3Utils.getDefaultS3Client().deleteObject(new DeleteObjectRequest(s3URIToDelete.getBucket(), s3URIToDelete.getKey())); - } catch(Exception e) { - logMessageAndHalt(req, 500, "Failed to delete file from S3.", e); - } - } - } - - } + removeDeletedCsvFiles(csvUrls, deploymentToUpdate, req); } Deployment updatedDeployment = Persistence.deployments.update(deploymentToUpdate.id, req.body()); // TODO: Should updates to the deployment's fields trigger a notification to subscribers? This could get @@ -370,6 +356,29 @@ private static Deployment updateDeployment (Request req, Response res) throws Ch // // } + /** + * Helper method for update steps which removes all removed csv files from s3. + * @param csvUrls The new list of csv files + * @param deploymentToUpdate An existing deployment, which contains csv files to check changes against + * @param req A request object used to report failure + */ + private static void removeDeletedCsvFiles(List csvUrls, Deployment deploymentToUpdate, Request req) { + // Only delete if the array differs + if (!csvUrls.equals(deploymentToUpdate.peliasCsvFiles)) { + for (String existingCsvUrl : deploymentToUpdate.peliasCsvFiles) { + // Only delete if the file does not exist in the deployment + if (!csvUrls.contains(existingCsvUrl)) { + try { + AmazonS3URI s3URIToDelete = new AmazonS3URI(existingCsvUrl); + S3Utils.getDefaultS3Client().deleteObject(new DeleteObjectRequest(s3URIToDelete.getBucket(), s3URIToDelete.getKey())); + } catch(Exception e) { + logMessageAndHalt(req, 500, "Failed to delete file from S3.", e); + } + } + } + } + } + /** * HTTP endpoint to deregister and terminate a set of instance IDs that are associated with a particular deployment. * The intent here is to give the user a device by which they can terminate an EC2 instance that has started up, but @@ -517,7 +526,7 @@ private static String updatePelias (Request req, Response res, DeployJob deployJ * Follows https://github.com/ibi-group/datatools-server/blob/dev/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java#L111 * @return S3 URL the file has been uploaded to */ - private static String uploadToS3 (Request req, Response res) throws IOException { + private static Deployment uploadToS3 (Request req, Response res) { // Check parameters supplied in request for validity. Auth0UserProfile userProfile = req.attribute("user"); Deployment deployment = getDeploymentWithPermissions(req, res); @@ -530,6 +539,7 @@ private static String uploadToS3 (Request req, Response res) throws IOException MultipartConfigElement multipartConfigElement = new MultipartConfigElement(System.getProperty("java.io.tmpdir")); req.raw().setAttribute("org.eclipse.jetty.multipartConfig", multipartConfigElement); } + String extension = null; File tempFile = null; try { @@ -555,7 +565,21 @@ private static String uploadToS3 (Request req, Response res) throws IOException // Allow public read // TODO: restrict? .withCannedAcl(CannedAccessControlList.PublicRead)); - return url; + + // Update deployment csvs + List updatedCsvList = new ArrayList<>(deployment.peliasCsvFiles); + updatedCsvList.add(url); + + // If this is set, a file is being replaced + String s3FileToRemove = req.raw().getHeader("urlToDelete"); + if (s3FileToRemove != null) { + updatedCsvList.remove(s3FileToRemove); + } + + // Persist changes after removing deleted csv files from s3 + removeDeletedCsvFiles(updatedCsvList, deployment, req); + return Persistence.deployments.updateField(deployment.id, "peliasCsvFiles", updatedCsvList); + } catch (IOException | ServletException | AmazonServiceException | CheckedAWSException e) { e.printStackTrace(); failure = e; @@ -596,7 +620,7 @@ public static void register (String apiPrefix) { post(apiPrefix + "secure/deployments", DeploymentController::createDeployment, fullJson::write); put(apiPrefix + "secure/deployments/:id", DeploymentController::updateDeployment, fullJson::write); post(apiPrefix + "secure/deployments/fromfeedsource/:id", DeploymentController::createDeploymentFromFeedSource, fullJson::write); - post(apiPrefix + "secure/deployments/:id/upload", DeploymentController::uploadToS3); + post(apiPrefix + "secure/deployments/:id/upload", DeploymentController::uploadToS3, fullJson::write); } } From 1f5673d390acd5a8af7ad68b32260b14e5afe526 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 24 Aug 2021 09:32:53 +0100 Subject: [PATCH 18/23] refactor(DeploymentController): cleanup --- .../manager/controllers/api/DeploymentController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index fb4d1b20f..289355b2e 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -277,7 +277,7 @@ private static Deployment createDeploymentFromFeedSource (Request req, Response * Update a single deployment. If the deployment's feed versions are updated, checks to ensure that each * version exists and is a part of the same parent project are performed before updating. */ - private static Deployment updateDeployment (Request req, Response res) throws CheckedAWSException { + private static Deployment updateDeployment (Request req, Response res) { Deployment deploymentToUpdate = getDeploymentWithPermissions(req, res); Document updateDocument = Document.parse(req.body()); // FIXME use generic update hook, also feedVersions is getting serialized into MongoDB (which is undesirable) @@ -364,7 +364,7 @@ private static Deployment updateDeployment (Request req, Response res) throws Ch */ private static void removeDeletedCsvFiles(List csvUrls, Deployment deploymentToUpdate, Request req) { // Only delete if the array differs - if (!csvUrls.equals(deploymentToUpdate.peliasCsvFiles)) { + if (deploymentToUpdate.peliasCsvFiles != null && !csvUrls.equals(deploymentToUpdate.peliasCsvFiles)) { for (String existingCsvUrl : deploymentToUpdate.peliasCsvFiles) { // Only delete if the file does not exist in the deployment if (!csvUrls.contains(existingCsvUrl)) { From b17de71f83ac9c47b560d98d89b56e63205537a7 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 24 Aug 2021 09:38:45 +0100 Subject: [PATCH 19/23] refactor: only start pelias update job once OTP deployment completes --- .../controllers/api/DeploymentController.java | 28 ------------------- .../datatools/manager/jobs/DeployJob.java | 11 ++++++++ 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 289355b2e..5290fc79b 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -12,7 +12,6 @@ import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.DeployJob; -import com.conveyal.datatools.manager.jobs.PeliasUpdateJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.EC2InstanceSummary; import com.conveyal.datatools.manager.models.FeedSource; @@ -491,36 +490,9 @@ private static String deploy (Request req, Response res) { target); logMessageAndHalt(req, HttpStatus.ACCEPTED_202, message); } - - // If pelias update is requested, launch pelias update job - if (deployment.peliasUpdate) { - updatePelias(req, res, job); - } - return SparkUtils.formatJobMessage(job.jobId, "Deployment initiating."); } - /** - * Start an updatePelias job which will trigger the webhook, then check for status updates. - */ - private static String updatePelias (Request req, Response res, DeployJob deployJob) { - // Check parameters supplied in request for validity. - Auth0UserProfile userProfile = req.attribute("user"); - Deployment deployment = getDeploymentWithPermissions(req, res); - Project project = Persistence.projects.getById(deployment.projectId); - if (project == null) { - logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); - } - - // Get log upload URI from deploy job - AmazonS3URI logUploadS3URI = deployJob.getS3FolderURI(); - - // Execute the pelias update job and keep track of it - PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "Updating Custom Geocoder Database", deployment, logUploadS3URI); - JobUtils.heavyExecutor.execute(peliasUpdateJob); - return SparkUtils.formatJobMessage(peliasUpdateJob.jobId, "Pelias update initiating."); - } - /** * Uploads a file from Spark request object to the s3 bucket of the deployment the Pelias Update Job is associated with. * Follows https://github.com/ibi-group/datatools-server/blob/dev/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java#L111 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 09d36486b..a54883b7a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -340,6 +340,17 @@ public void jobLogic () { // Set baseUrl after success. status.baseUrl = otpServer.publicUrl; } + + // Now that the main instance is updated successfully, update Pelias + if (deployment.peliasUpdate) { + // Get log upload URI from deploy job + AmazonS3URI logUploadS3URI = getS3FolderURI(); + + // Execute the pelias update job and keep track of it + PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(owner, "Updating Custom Geocoder Database", deployment, logUploadS3URI); + JobUtils.heavyExecutor.execute(peliasUpdateJob); + } + status.completed = true; } From 2abcd28674fc5e4185a71587c261094ec53700bd Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 24 Aug 2021 10:17:02 +0100 Subject: [PATCH 20/23] refactor: unify shared s3 upload code --- .../datatools/common/utils/SparkUtils.java | 52 +++++++++++++++++ .../datatools/common/utils/aws/S3Utils.java | 18 ++++++ .../controllers/api/EditorController.java | 57 +------------------ .../controllers/api/DeploymentController.java | 46 +-------------- 4 files changed, 74 insertions(+), 99 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index 9dd8622d4..a60b4cb43 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -1,5 +1,8 @@ package com.conveyal.datatools.common.utils; +import com.amazonaws.AmazonServiceException; +import com.conveyal.datatools.common.utils.aws.CheckedAWSException; +import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.utils.ErrorUtils; import com.fasterxml.jackson.core.JsonProcessingException; @@ -7,6 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.io.ByteStreams; +import org.apache.commons.io.IOUtils; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,14 +18,18 @@ import spark.Request; import spark.Response; +import javax.servlet.MultipartConfigElement; +import javax.servlet.ServletException; import javax.servlet.ServletInputStream; import javax.servlet.ServletOutputStream; import javax.servlet.ServletRequestWrapper; import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.Part; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; @@ -265,6 +273,50 @@ public static void copyRequestStreamIntoFile(Request req, File file) { } } + /** + * Copies a multi-part file upload to disk, attempts to upload it to S3, then deletes the local file. + * @param req Request object containing file to upload + * @param uploadType A string to include in the uploaded filename. Will also be added to the temporary file + * which makes debugging easier should the upload fail. + * @param key The S3 key to upload the file to + * @return An HTTP S3 url containing the uploaded file + */ + public static String uploadMultipartRequestBodyToS3(Request req, String uploadType, String key) { + // Get file from request + if (req.raw().getAttribute("org.eclipse.jetty.multipartConfig") == null) { + MultipartConfigElement multipartConfigElement = new MultipartConfigElement(System.getProperty("java.io.tmpdir")); + req.raw().setAttribute("org.eclipse.jetty.multipartConfig", multipartConfigElement); + } + String extension = null; + File tempFile = null; + String uploadedFileName = null; + try { + Part part = req.raw().getPart("file"); + uploadedFileName = part.getSubmittedFileName(); + + extension = "." + part.getContentType().split("/", 0)[1]; + tempFile = File.createTempFile(part.getName() + "_" + uploadType, extension); + InputStream inputStream; + inputStream = part.getInputStream(); + FileOutputStream out = new FileOutputStream(tempFile); + IOUtils.copy(inputStream, out); + } catch (IOException | ServletException e) { + e.printStackTrace(); + logMessageAndHalt(req, 400, "Unable to read uploaded file"); + } + try { + return S3Utils.uploadObject(key + "_" + uploadedFileName, tempFile); + } catch (AmazonServiceException | CheckedAWSException e) { + logMessageAndHalt(req, 500, "Error uploading file to S3", e); + return null; + } finally { + boolean deleted = tempFile.delete(); + if (!deleted) { + LOG.error("Could not delete s3 temporary upload file"); + } + } + } + private static String trimLines(String str) { if (str == null) return ""; String[] lines = str.split("\n"); diff --git a/src/main/java/com/conveyal/datatools/common/utils/aws/S3Utils.java b/src/main/java/com/conveyal/datatools/common/utils/aws/S3Utils.java index a5ee96bf9..078781132 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/aws/S3Utils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/aws/S3Utils.java @@ -7,7 +7,9 @@ import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.PutObjectRequest; import com.conveyal.datatools.common.utils.SparkUtils; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.models.OtpServer; @@ -192,6 +194,22 @@ public static String downloadObject( } } + /** + * Uploads a file to S3 using a given key + * @param keyName The s3 key to uplaod the file to + * @param fileToUpload The file to upload to S3 + * @return A URL where the file is publicly accessible + */ + public static String uploadObject(String keyName, File fileToUpload) throws AmazonServiceException, CheckedAWSException { + String url = S3Utils.getDefaultBucketUrlForKey(keyName); + // FIXME: This may need to change during feed store refactor + getDefaultS3Client().putObject(new PutObjectRequest( + S3Utils.DEFAULT_BUCKET, keyName, fileToUpload) + // grant public read + .withCannedAcl(CannedAccessControlList.PublicRead)); + return url; + } + public static AmazonS3 getDefaultS3Client() throws CheckedAWSException { return getS3Client (null, null); } diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java index 7400fbeaf..ea73c0cdf 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java @@ -1,11 +1,6 @@ package com.conveyal.datatools.editor.controllers.api; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.model.CannedAccessControlList; -import com.amazonaws.services.s3.model.PutObjectRequest; import com.conveyal.datatools.common.utils.SparkUtils; -import com.conveyal.datatools.common.utils.aws.CheckedAWSException; -import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.editor.controllers.EditorLockController; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.FeedSource; @@ -23,7 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.dbutils.DbUtils; -import org.apache.commons.io.IOUtils; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,14 +25,8 @@ import spark.Request; import spark.Response; -import javax.servlet.MultipartConfigElement; -import javax.servlet.ServletException; -import javax.servlet.http.Part; import javax.sql.DataSource; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -108,49 +96,6 @@ public abstract class EditorController { registerRoutes(); } - public static String uploadBranding(Request req, String key) { - String url; - - // Get file from request - if (req.raw().getAttribute("org.eclipse.jetty.multipartConfig") == null) { - MultipartConfigElement multipartConfigElement = new MultipartConfigElement(System.getProperty("java.io.tmpdir")); - req.raw().setAttribute("org.eclipse.jetty.multipartConfig", multipartConfigElement); - } - String extension = null; - File tempFile = null; - try { - Part part = req.raw().getPart("file"); - extension = "." + part.getContentType().split("/", 0)[1]; - tempFile = File.createTempFile(key + "_branding", extension); - InputStream inputStream; - inputStream = part.getInputStream(); - FileOutputStream out = new FileOutputStream(tempFile); - IOUtils.copy(inputStream, out); - } catch (IOException | ServletException e) { - e.printStackTrace(); - logMessageAndHalt(req, 400, "Unable to read uploaded file"); - } - - try { - String keyName = "branding/" + key + extension; - url = S3Utils.getDefaultBucketUrlForKey(keyName); - // FIXME: This may need to change during feed store refactor - S3Utils.getDefaultS3Client().putObject(new PutObjectRequest( - S3Utils.DEFAULT_BUCKET, keyName, tempFile) - // grant public read - .withCannedAcl(CannedAccessControlList.PublicRead)); - return url; - } catch (AmazonServiceException | CheckedAWSException e) { - logMessageAndHalt(req, 500, "Error uploading file to S3", e); - return null; - } finally { - boolean deleted = tempFile.delete(); - if (!deleted) { - LOG.error("Could not delete s3 upload file."); - } - } - } - /** * Add static HTTP endpoints to Spark static instance. */ @@ -399,7 +344,7 @@ private String uploadEntityBranding (Request req, Response res) { int id = getIdFromRequest(req); String url; try { - url = uploadBranding(req, String.format("%s_%d", classToLowercase, id)); + url = SparkUtils.uploadMultipartRequestBodyToS3(req, "branding", String.format("%s_%d", classToLowercase, id)); } catch (HaltException e) { // Do not re-catch halts thrown for exceptions that have already been caught. throw e; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 5290fc79b..781ae565d 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -2,9 +2,7 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3URI; -import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.PutObjectRequest; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.common.utils.SparkUtils; import com.conveyal.datatools.common.utils.aws.CheckedAWSException; @@ -22,7 +20,6 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.JobUtils; import com.conveyal.datatools.manager.utils.json.JsonManager; -import org.apache.commons.io.IOUtils; import org.bson.Document; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; @@ -30,14 +27,9 @@ import spark.Request; import spark.Response; -import javax.servlet.MultipartConfigElement; -import javax.servlet.ServletException; -import javax.servlet.http.Part; import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -500,28 +492,10 @@ private static String deploy (Request req, Response res) { */ private static Deployment uploadToS3 (Request req, Response res) { // Check parameters supplied in request for validity. - Auth0UserProfile userProfile = req.attribute("user"); Deployment deployment = getDeploymentWithPermissions(req, res); String url; - Exception failure = null; - - // Get file from request - if (req.raw().getAttribute("org.eclipse.jetty.multipartConfig") == null) { - MultipartConfigElement multipartConfigElement = new MultipartConfigElement(System.getProperty("java.io.tmpdir")); - req.raw().setAttribute("org.eclipse.jetty.multipartConfig", multipartConfigElement); - } - - String extension = null; - File tempFile = null; try { - Part part = req.raw().getPart("file"); - extension = "." + part.getContentType().split("/", 0)[1]; - tempFile = File.createTempFile(part.getName() + "_csv_upload", extension); - InputStream inputStream; - inputStream = part.getInputStream(); - FileOutputStream out = new FileOutputStream(tempFile); - IOUtils.copy(inputStream, out); String keyName = String.join( "/", @@ -529,14 +503,9 @@ private static Deployment uploadToS3 (Request req, Response res) { deployment.projectId, deployment.id, // Where filenames are generated. Prepend random UUID to prevent overwriting - UUID.randomUUID() + "_" + part.getSubmittedFileName() + UUID.randomUUID() + "" ); - url = S3Utils.getDefaultBucketUrlForKey(keyName); - S3Utils.getDefaultS3Client().putObject(new PutObjectRequest( - S3Utils.DEFAULT_BUCKET, keyName, tempFile) - // Allow public read - // TODO: restrict? - .withCannedAcl(CannedAccessControlList.PublicRead)); + url = SparkUtils.uploadMultipartRequestBodyToS3(req, "csvUpload", keyName); // Update deployment csvs List updatedCsvList = new ArrayList<>(deployment.peliasCsvFiles); @@ -552,18 +521,9 @@ private static Deployment uploadToS3 (Request req, Response res) { removeDeletedCsvFiles(updatedCsvList, deployment, req); return Persistence.deployments.updateField(deployment.id, "peliasCsvFiles", updatedCsvList); - } catch (IOException | ServletException | AmazonServiceException | CheckedAWSException e) { + } catch (AmazonServiceException e) { e.printStackTrace(); - failure = e; return null; - } finally { - boolean deleted = tempFile.delete(); - if (!deleted) { - logMessageAndHalt(req, 500, "Failed to delete file temporarily stored on server"); - } - if (failure != null) { - logMessageAndHalt(req, 500, "Failed to upload file. Please try again"); - } } } From 164284c72332e390961ece9bf91e2bf7ff660eb3 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 1 Sep 2021 11:31:45 +0100 Subject: [PATCH 21/23] refactor: fix upload path regression in introducing s3 upload method --- .../java/com/conveyal/datatools/common/utils/SparkUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index a60b4cb43..fa0a3f289 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -305,7 +305,7 @@ public static String uploadMultipartRequestBodyToS3(Request req, String uploadTy logMessageAndHalt(req, 400, "Unable to read uploaded file"); } try { - return S3Utils.uploadObject(key + "_" + uploadedFileName, tempFile); + return S3Utils.uploadObject(uploadType + "/" + key + "_" + uploadedFileName, tempFile); } catch (AmazonServiceException | CheckedAWSException e) { logMessageAndHalt(req, 500, "Error uploading file to S3", e); return null; From 4b73157ca781655923e944dad2ed1ccb0c32ce05 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 1 Sep 2021 11:35:42 +0100 Subject: [PATCH 22/23] refactor: address PR feedback --- .../manager/controllers/api/DeploymentController.java | 3 ++- .../java/com/conveyal/datatools/manager/jobs/DeployJob.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 781ae565d..1760908b4 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -503,7 +503,7 @@ private static Deployment uploadToS3 (Request req, Response res) { deployment.projectId, deployment.id, // Where filenames are generated. Prepend random UUID to prevent overwriting - UUID.randomUUID() + "" + UUID.randomUUID().toString() ); url = SparkUtils.uploadMultipartRequestBodyToS3(req, "csvUpload", keyName); @@ -523,6 +523,7 @@ private static Deployment uploadToS3 (Request req, Response res) { } catch (AmazonServiceException e) { e.printStackTrace(); + logMessageAndHalt(req, 500, "Failed to upload file to S3. Check server logs."); return null; } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index a54883b7a..ddcb2c09b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -348,7 +348,7 @@ public void jobLogic () { // Execute the pelias update job and keep track of it PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(owner, "Updating Custom Geocoder Database", deployment, logUploadS3URI); - JobUtils.heavyExecutor.execute(peliasUpdateJob); + this.addNextJob(peliasUpdateJob); } status.completed = true; From 1a1cc856fb5ff2e183e176465a5fb4c499af0067 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Sat, 11 Sep 2021 09:44:01 -0400 Subject: [PATCH 23/23] refactor: tweaks for #408 --- .../com/conveyal/datatools/common/utils/SparkUtils.java | 6 +++++- .../manager/controllers/api/DeploymentController.java | 5 ++--- .../com/conveyal/datatools/manager/models/Deployment.java | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index fa0a3f289..d93681227 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.io.ByteStreams; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +84,10 @@ public static String formatJSON (String key, String value) { * supplied details about the exception encountered. */ public static ObjectNode getObjectNode(String message, int code, Exception e) { - String detail = e != null ? e.getMessage() : null; + String detail = null; + if (e != null) { + detail = e.getMessage() != null ? e.getMessage() : ExceptionUtils.getStackTrace(e); + } return mapper.createObjectNode() .put("result", code >= 400 ? "ERR" : "OK") .put("message", message) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 1760908b4..7ef59d772 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -521,9 +521,8 @@ private static Deployment uploadToS3 (Request req, Response res) { removeDeletedCsvFiles(updatedCsvList, deployment, req); return Persistence.deployments.updateField(deployment.id, "peliasCsvFiles", updatedCsvList); - } catch (AmazonServiceException e) { - e.printStackTrace(); - logMessageAndHalt(req, 500, "Failed to upload file to S3. Check server logs."); + } catch (Exception e) { + logMessageAndHalt(req, 500, "Failed to upload file to S3.", e); return null; } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 33be6d766..8541d5ddc 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -76,7 +76,7 @@ public class Deployment extends Model implements Serializable { /* Pelias fields, used to determine where/if to send data to the Pelias webhook */ public String peliasWebhookUrl; public boolean peliasUpdate; - public List peliasCsvFiles; + public List peliasCsvFiles = new ArrayList<>(); /** * Get parent project for deployment. Note: at one point this was a JSON property of this class, but severe