From 99105ecb052929dcc475a5cbddfa9c4238359ec4 Mon Sep 17 00:00:00 2001 From: Evan Siroky Date: Tue, 25 Feb 2020 10:23:29 -0800 Subject: [PATCH 1/7] feat(deploy): add addition ec2 customizations Add graph building ami/instance type Add custom region --- .../controllers/api/ServerController.java | 50 ++++++++++++++----- .../datatools/manager/models/EC2Info.java | 9 ++++ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 5c50787e9..7c37f6e60 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -3,6 +3,7 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.AmazonEC2Exception; import com.amazonaws.services.ec2.model.DescribeImagesRequest; import com.amazonaws.services.ec2.model.DescribeImagesResult; @@ -19,6 +20,7 @@ import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClientBuilder; import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException; import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersRequest; @@ -62,6 +64,7 @@ import static com.conveyal.datatools.common.utils.SparkUtils.getPOJOFromRequestBody; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE; +import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds; import static spark.Spark.delete; import static spark.Spark.get; import static spark.Spark.options; @@ -250,10 +253,24 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc AWSStaticCredentialsProvider credentials = AWSUtils.getCredentialsForRole(server.role, "validate"); // If alternative credentials exist, override the default AWS clients. if (credentials != null) { + // build ec2 client ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build(); iamClient = AmazonIdentityManagementClientBuilder.standard().withCredentials(credentials).build(); s3Client = AWSUtils.getS3ClientForRole(server.role, null); } + if (server.ec2Info.region != null) { + AmazonEC2ClientBuilder builder = AmazonEC2Client.builder(); + if (credentials != null) { + builder.withCredentials(credentials); + } + builder.withRegion(server.ec2Info.region); + ec2Client = builder.build(); + if (credentials != null) { + s3Client = AWSUtils.getS3ClientForRole(server.role, server.ec2Info.region); + } else { + s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), server.ec2Info.region); + } + } // Check that projectId is valid. if (server.projectId != null) { Project project = Persistence.projects.getById(server.projectId); @@ -265,7 +282,7 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc if (server.ec2Info != null) { validateInstanceType(server.ec2Info.instanceType, req); // Validate target group and get load balancer to validate subnetId and security group ID. - LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info.targetGroupArn, req, credentials); + LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info, req, credentials); validateSubnetId(loadBalancer, server.ec2Info, req, ec2Client); validateSecurityGroupId(loadBalancer, server.ec2Info, req); // Validate remaining AWS values. @@ -304,7 +321,7 @@ private static boolean verifyS3WritePermissions(AmazonS3 s3Client, String s3Buck s3Client.putObject(s3Bucket, key, File.createTempFile("test", ".zip")); s3Client.deleteObject(s3Bucket, key); } catch (IOException | AmazonS3Exception e) { - LOG.warn("S3 client cannot write to bucket" + s3Bucket, e); + LOG.warn("S3 client cannot write to bucket: " + s3Bucket, e); return false; } return true; @@ -448,16 +465,21 @@ private static void validateInstanceType(String instanceType, Request req) { * - https://serverfault.com/a/865422 * - https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html */ - private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn, AWSStaticCredentialsProvider credentials) { + private static LoadBalancer getLoadBalancerForTargetGroup (EC2Info ec2Info, AWSStaticCredentialsProvider credentials) { // If alternative credentials exist, use them to assume the role. Otherwise, use default ELB client. - AmazonElasticLoadBalancing elbClient = credentials != null - ? AmazonElasticLoadBalancingClient.builder() - .withCredentials(credentials) - .build() - : elb; + AmazonElasticLoadBalancingClientBuilder builder = AmazonElasticLoadBalancingClient.builder(); + if (credentials != null) { + builder.withCredentials(credentials); + } + + if (ec2Info.region != null) { + builder.withRegion(ec2Info.region); + } + + AmazonElasticLoadBalancing elbClient = builder.build(); try { DescribeTargetGroupsRequest targetGroupsRequest = new DescribeTargetGroupsRequest() - .withTargetGroupArns(targetGroupArn); + .withTargetGroupArns(ec2Info.targetGroupArn); List targetGroups = elbClient.describeTargetGroups(targetGroupsRequest).getTargetGroups(); for (TargetGroup tg : targetGroups) { DescribeLoadBalancersRequest request = new DescribeLoadBalancersRequest() @@ -467,7 +489,7 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn return result.getLoadBalancers().iterator().next(); } } catch (AmazonElasticLoadBalancingException e) { - LOG.warn("Invalid value for Target Group ARN: {}", targetGroupArn); + LOG.warn("Invalid value for Target Group ARN: {}", ec2Info.targetGroupArn); } // If no target group/load balancer found, return null. return null; @@ -477,11 +499,13 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn * Validate that ELB target group exists and is not empty and return associated load balancer for validating related * fields. */ - private static LoadBalancer validateTargetGroupAndGetLoadBalancer(String targetGroupArn, Request req, AWSStaticCredentialsProvider credentials) { - if (isEmpty(targetGroupArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); + private static LoadBalancer validateTargetGroupAndGetLoadBalancer(EC2Info ec2Info, Request req, AWSStaticCredentialsProvider credentials) { + if (isEmpty(ec2Info.targetGroupArn)) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); + } // Get load balancer for target group. This essentially checks that the target group exists and is assigned // to a load balancer. - LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(targetGroupArn, credentials); + LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(ec2Info, credentials); if (loadBalancer == null) { logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group or Load Balancer."); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java index fcbd37aa7..f4ebfadd5 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -27,10 +27,19 @@ public EC2Info () {} public String securityGroupId; /** The Amazon machine image (AMI) to be used for the OTP EC2 machines. */ public String amiId; + /** + * The AWS-style instance type (e.g., t2.medium) to use for new EC2 machines used specifically for graph building. + * Defaults to {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} if null during deployment. + */ + public String buildInstanceType; + /** The Amazon machine image (AMI) to be used for the OTP EC2 machine used specifically for graph building. */ + public String buildAmiId; /** The IAM instance profile ARN that the OTP EC2 server should assume. For example, arn:aws:iam::123456789012:instance-profile/otp-ec2-role */ public String iamInstanceProfileArn; /** The AWS key file (.pem) that should be used to set up OTP EC2 servers (gives a way for admins to SSH into machine). */ public String keyName; /** The target group to deploy new EC2 instances to. */ public String targetGroupArn; + /** An optional custom AWS region */ + public String region; } \ No newline at end of file From fdc04a23e928f18a2f5f1a4869122f9fdb3c6951 Mon Sep 17 00:00:00 2001 From: Evan Siroky Date: Tue, 25 Feb 2020 17:47:10 -0800 Subject: [PATCH 2/7] refactor(deploy): move ec2 region checks in block that check for ec2info --- .../controllers/api/ServerController.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 7c37f6e60..793232c56 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -258,19 +258,6 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc iamClient = AmazonIdentityManagementClientBuilder.standard().withCredentials(credentials).build(); s3Client = AWSUtils.getS3ClientForRole(server.role, null); } - if (server.ec2Info.region != null) { - AmazonEC2ClientBuilder builder = AmazonEC2Client.builder(); - if (credentials != null) { - builder.withCredentials(credentials); - } - builder.withRegion(server.ec2Info.region); - ec2Client = builder.build(); - if (credentials != null) { - s3Client = AWSUtils.getS3ClientForRole(server.role, server.ec2Info.region); - } else { - s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), server.ec2Info.region); - } - } // Check that projectId is valid. if (server.projectId != null) { Project project = Persistence.projects.getById(server.projectId); @@ -280,6 +267,20 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc // If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to // AWS. (e.g., target group ARN and instance type). if (server.ec2Info != null) { + // do some custom items if a custom region should be used + if (server.ec2Info.region != null) { + AmazonEC2ClientBuilder builder = AmazonEC2Client.builder(); + if (credentials != null) { + builder.withCredentials(credentials); + } + builder.withRegion(server.ec2Info.region); + ec2Client = builder.build(); + if (credentials != null) { + s3Client = AWSUtils.getS3ClientForRole(server.role, server.ec2Info.region); + } else { + s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), server.ec2Info.region); + } + } validateInstanceType(server.ec2Info.instanceType, req); // Validate target group and get load balancer to validate subnetId and security group ID. LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info, req, credentials); From 077de675f25f1ebf0f15095f40c007f3587dff0b Mon Sep 17 00:00:00 2001 From: Evan Siroky Date: Wed, 26 Feb 2020 21:29:28 -0800 Subject: [PATCH 3/7] feat(deploy): allow region and graph elb deployment customization --- .../common/status/MonitorableJob.java | 3 + .../datatools/common/utils/AWSUtils.java | 14 +- .../datatools/common/utils/SparkUtils.java | 2 + .../controllers/api/DeploymentController.java | 18 ++- .../controllers/api/ServerController.java | 29 +++- .../datatools/manager/jobs/DeployJob.java | 148 ++++++++++++++---- .../manager/jobs/MonitorServerStatusJob.java | 50 ++++-- .../datatools/manager/models/Deployment.java | 13 +- .../datatools/manager/models/EC2Info.java | 11 ++ .../datatools/manager/models/OtpServer.java | 15 +- .../datatools/manager/jobs/DeployJobTest.java | 5 +- 11 files changed, 243 insertions(+), 65 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java index 7f66d1dc8..e9f76c093 100644 --- a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java +++ b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java @@ -76,6 +76,7 @@ public MonitorableJob(Auth0UserProfile owner, String name, JobType type) { } this.owner = owner; this.name = name; + status.name = name; this.type = type; registerJob(); } @@ -279,6 +280,7 @@ public void update (boolean isError, String message, double percentComplete, boo } public void fail (String message, Exception e) { + LOG.warn("Job `{}` has been failed with message: `{}` and Exception: `{}`", name, message, e); this.error = true; this.percentComplete = 100; this.completed = true; @@ -288,6 +290,7 @@ public void fail (String message, Exception e) { } public void fail (String message) { + LOG.warn("Job `{}` has been failed with message: `{}`", name, message); this.error = true; this.percentComplete = 100; this.completed = true; diff --git a/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java b/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java index f887bac81..16a81efcc 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java @@ -148,9 +148,11 @@ public static AWSStaticCredentialsProvider getCredentialsForRole(String role, St * Shorthand method to obtain an EC2 client for the provided role ARN. If role is null, the default EC2 credentials * will be used. */ - public static AmazonEC2 getEC2ClientForRole (String role) { + public static AmazonEC2 getEC2ClientForRole (String role, String region) { AWSStaticCredentialsProvider credentials = getCredentialsForRole(role, "ec2-client"); - return getEC2ClientForCredentials(credentials); + return region == null + ? getEC2ClientForCredentials(credentials) + : getEC2ClientForCredentials(credentials, region); } /** @@ -161,6 +163,14 @@ public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider crede return AmazonEC2Client.builder().withCredentials(credentials).build(); } + /** + * Shorthand method to obtain an EC2 client for the provided credentials and region. If credentials are null, the + * default EC2 credentials will be used. + */ + public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider credentials, String region) { + return AmazonEC2Client.builder().withCredentials(credentials).withRegion(region).build(); + } + /** * Shorthand method to obtain an S3 client for the provided credentials. If credentials are null, the default EC2 * credentials will be used. diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index 9eaa8eb98..ba7fa0c47 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -234,6 +234,8 @@ public static void logRequestOrResponse( LOG.warn("Request object is null. Cannot log."); return; } + // don't log job status requests/responses, they clutter things up + if (request.pathInfo().contains("status/jobs")) return; Auth0UserProfile userProfile = request.attribute("user"); String userEmail = userProfile != null ? userProfile.getEmail() : "no-auth"; String queryString = request.queryParams().size() > 0 ? "?" + request.queryString() : ""; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 437abc675..02fc71b32 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -9,6 +9,7 @@ import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3URI; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.common.utils.AWSUtils; @@ -47,6 +48,7 @@ import static com.conveyal.datatools.common.utils.AWSUtils.downloadFromS3; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; +import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds; import static spark.Spark.delete; import static spark.Spark.get; import static spark.Spark.options; @@ -101,6 +103,7 @@ private static String downloadBuildArtifact (Request req, Response res) { // Default client to use if no role was used during the deployment. AmazonS3 s3Client = FeedStore.s3Client; String role = null; + String region = null; String uriString; String filename = req.queryParams("filename"); if (filename == null) { @@ -131,6 +134,7 @@ private static String downloadBuildArtifact (Request req, Response res) { logMessageAndHalt(req, 400, "The deployment does not have job history or associated server information to construct URI for build artifact. " + uriString); return null; } + region = server.ec2Info == null ? null : server.ec2Info.region; uriString = String.format("s3://%s/bundles/%s/%s/%s", server.s3Bucket, deployment.projectId, deployment.id, jobId); LOG.warn("Could not find deploy summary for job. Attempting to use {}", uriString); } @@ -138,10 +142,15 @@ private static String downloadBuildArtifact (Request req, Response res) { // If summary is readily available, just use the ready-to-use build artifacts field. uriString = summaryToDownload.buildArtifactsFolder; role = summaryToDownload.role; + region = summaryToDownload.ec2Info == null ? null : summaryToDownload.ec2Info.region; } AmazonS3URI uri = new AmazonS3URI(uriString); // Assume the alternative role if needed to download the deploy artifact. - if (role != null) s3Client = AWSUtils.getS3ClientForRole(role); + if (role != null) { + s3Client = AWSUtils.getS3ClientForRole(role, region); + } else if (region != null) { + s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), region); + } return downloadFromS3(s3Client, uri.getBucket(), String.join("/", uri.getKey(), filename), false, res); } @@ -373,7 +382,12 @@ private static boolean terminateEC2InstanceForDeployment(Request req, Response r } } // If checks are ok, terminate instances. - boolean success = ServerController.deRegisterAndTerminateInstances(credentials, targetGroupArn, idsToTerminate); + boolean success = ServerController.deRegisterAndTerminateInstances( + credentials, + targetGroupArn, + latest.ec2Info.region, + idsToTerminate + ); if (!success) { logMessageAndHalt(req, 400, "Could not complete termination request"); return false; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 793232c56..d3f81379f 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -120,7 +120,10 @@ private static OtpServer terminateEC2InstancesForServer(Request req, Response re OtpServer server = getServerWithPermissions(req, res); List instances = server.retrieveEC2Instances(); List ids = getIds(instances); - AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole(server.role); + AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole( + server.role, + server.ec2Info == null ? null : server.ec2Info.region + ); terminateInstances(ec2Client, ids); for (Deployment deployment : Deployment.retrieveDeploymentForServerAndRouterId(server.id, null)) { Persistence.deployments.updateField(deployment.id, "deployedTo", null); @@ -160,7 +163,12 @@ public static TerminateInstancesResult terminateInstances(AmazonEC2 ec2Client, L * De-register instances from the specified target group/load balancer and terminate the instances. * */ - public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvider credentials, String targetGroupArn, List instanceIds) { + public static boolean deRegisterAndTerminateInstances( + AWSStaticCredentialsProvider credentials, + String targetGroupArn, + String region, + List instanceIds + ) { LOG.info("De-registering instances from load balancer {}", instanceIds); TargetDescription[] targetDescriptions = instanceIds.stream() .map(id -> new TargetDescription().withId(id)) @@ -172,12 +180,21 @@ public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvid AmazonElasticLoadBalancing elbClient = elb; AmazonEC2 ec2Client = ec2; // If OTP Server has role defined/alt credentials, override default AWS clients. - if (credentials != null) { - elbClient = AmazonElasticLoadBalancingClient.builder().withCredentials(credentials).build(); - ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build(); + if (credentials != null || region != null) { + AmazonElasticLoadBalancingClientBuilder elbBuilder = AmazonElasticLoadBalancingClient.builder(); + AmazonEC2ClientBuilder ec2Builder = AmazonEC2Client.builder(); + if (credentials != null) { + elbBuilder.withCredentials(credentials); + ec2Builder.withCredentials(credentials); + } + if (region != null) { + elbBuilder.withRegion(region); + ec2Builder.withRegion(region); + } + elbClient = elbBuilder.build(); + ec2Client = ec2Builder.build(); } elbClient.deregisterTargets(request); - // FIXME default to regular ec2 client ServerController.terminateInstances(ec2Client, instanceIds); } catch (AmazonEC2Exception | AmazonElasticLoadBalancingException e) { LOG.warn("Could not terminate EC2 instances: " + String.join(",", instanceIds), e); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 039572b73..77b7ef755 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -104,6 +104,8 @@ public class DeployJob extends MonitorableJob { private final int targetCount; private final DeployType deployType; private final AWSStaticCredentialsProvider credentials; + private final String customRegion; + private int tasksCompleted = 0; private int totalTasks; @@ -161,13 +163,18 @@ public DeployJob(Deployment deployment, Auth0UserProfile owner, OtpServer otpSer } public DeployJob(Deployment deployment, Auth0UserProfile owner, OtpServer otpServer, String bundlePath, DeployType deployType) { + this("Deploying " + deployment.name, deployment, owner, otpServer, bundlePath, deployType); + } + + public DeployJob(String jobName, Deployment deployment, Auth0UserProfile owner, OtpServer otpServer, String bundlePath, DeployType deployType) { // TODO add new job type or get rid of enum in favor of just using class names - super(owner, "Deploying " + deployment.name, JobType.DEPLOY_TO_OTP); + super(owner, jobName, JobType.DEPLOY_TO_OTP); this.deployment = deployment; this.otpServer = otpServer; this.s3Bucket = otpServer.s3Bucket != null ? otpServer.s3Bucket : DataManager.feedBucket; // Use a special subclass of status here that has additional fields this.status = new DeployStatus(); + this.status.name = jobName; this.targetCount = otpServer.internalUrl != null ? otpServer.internalUrl.size() : 0; this.totalTasks = 1 + targetCount; status.message = "Initializing..."; @@ -177,19 +184,22 @@ public DeployJob(Deployment deployment, Auth0UserProfile owner, OtpServer otpSer this.deployType = deployType; if (bundlePath == null) { // Use standard path for bundle. - setJobRelativePath(String.join("/", bundlePrefix, deployment.projectId, deployment.id, this.jobId)); + this.jobRelativePath = String.join("/", bundlePrefix, deployment.projectId, deployment.id, this.jobId); } else { // Override job relative path so that bundle can be downloaded directly. Note: this is currently only used // for testing (DeployJobTest), but the uses may be expanded in order to perhaps add a server to an existing // deployment using either a specified bundle or Graph.obj. - setJobRelativePath(bundlePath); + this.jobRelativePath = bundlePath; } // CONNECT TO EC2/S3 - // FIXME Should this ec2 client be longlived? credentials = AWSUtils.getCredentialsForRole(otpServer.role, this.jobId); - ec2 = AWSUtils.getEC2ClientForCredentials(credentials); - s3Client = AWSUtils.getS3ClientForCredentials(credentials, null); - + this.customRegion = otpServer.ec2Info != null && otpServer.ec2Info.region != null + ? otpServer.ec2Info.region + : null; + ec2 = customRegion == null + ? AWSUtils.getEC2ClientForCredentials(credentials) + : AWSUtils.getEC2ClientForCredentials(credentials, customRegion); + s3Client = AWSUtils.getS3ClientForCredentials(credentials, customRegion); } public void jobLogic () { @@ -449,6 +459,10 @@ private String getS3BundleURI() { return joinToS3FolderURI("bundle.zip"); } + public String getCustomRegion() { + return customRegion; + } + private String getLatestS3BundleKey() { String name = StringUtils.getCleanName(deployment.parentProject().name.toLowerCase()); return String.format("%s/%s/%s-latest.zip", bundlePrefix, deployment.projectId, name); @@ -500,14 +514,19 @@ private void replaceEC2Servers() { // First start graph-building instance and wait for graph to successfully build. if (!deployType.equals(DeployType.USE_PREBUILT_GRAPH)) { status.message = "Starting up graph building EC2 instance"; - instances.addAll(startEC2Instances(1, false)); + List graphBuildingInstances = startEC2Instances(1, false); // Exit if an error was encountered. - if (status.error || instances.size() == 0) { - ServerController.terminateInstances(ec2, instances); + if (status.error || graphBuildingInstances.size() == 0) { + ServerController.terminateInstances(ec2, graphBuildingInstances); return; } status.message = "Waiting for graph build to complete..."; - MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob(owner, this, instances.get(0), false); + MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob( + owner, + this, + graphBuildingInstances.get(0), + false + ); monitorInitialServerJob.run(); status.update("Graph build is complete!", 50); @@ -526,12 +545,25 @@ private void replaceEC2Servers() { statusMessage = "Error encountered while building graph. Inspect build logs."; LOG.error(statusMessage); status.fail(statusMessage); - ServerController.terminateInstances(ec2, instances); + ServerController.terminateInstances(ec2, graphBuildingInstances); return; } + // Check whether the graph build instance type or AMI ID is different from the non-graph building type. + // If so, terminate the graph building instance. If not, add the graph building instance to the list + // of started instances. + if (otpServer.ec2Info.hasSeparateGraphBuildConfig()) { + // different instance type and/or ami exists for graph building. Terminate graph building instance + ServerController.terminateInstances(ec2, graphBuildingInstances); + status.numServersRemaining = Math.max(otpServer.ec2Info.instanceCount, 0); + } else { + // same configuration exists, so keep instance on and add to list of running instances + instances.addAll(graphBuildingInstances); + status.numServersRemaining = otpServer.ec2Info.instanceCount <= 0 + ? 0 + : otpServer.ec2Info.instanceCount - 1; + } } // Spin up remaining servers which will download the graph from S3. - status.numServersRemaining = otpServer.ec2Info.instanceCount <= 0 ? 0 : otpServer.ec2Info.instanceCount - 1; List remainingServerMonitorJobs = new ArrayList<>(); List remainingInstances = new ArrayList<>(); if (status.numServersRemaining > 0) { @@ -577,6 +609,7 @@ private void replaceEC2Servers() { boolean success = ServerController.deRegisterAndTerminateInstances( credentials, otpServer.ec2Info.targetGroupArn, + customRegion, previousInstanceIds ); // If there was a problem during de-registration/termination, notify via status message. @@ -601,7 +634,6 @@ private void replaceEC2Servers() { * TODO: Booting up R5 servers has not been fully tested. */ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { - String instanceType = otpServer.ec2Info.instanceType == null ? DEFAULT_INSTANCE_TYPE : otpServer.ec2Info.instanceType; // User data should contain info about: // 1. Downloading GTFS/OSM info (s3) // 2. Time to live until shutdown/termination (for test servers) @@ -621,19 +653,32 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withAssociatePublicIpAddress(true) .withGroups(otpServer.ec2Info.securityGroupId) .withDeviceIndex(0); - // If AMI not defined, use the default AMI ID. - String amiId = otpServer.ec2Info.amiId; - if (amiId == null) { + // Pick proper ami depending on whether graph is being built and what is defined. + String amiId; + if (!graphAlreadyBuilt && otpServer.ec2Info.buildAmiId != null) { + amiId = otpServer.ec2Info.buildAmiId; + } else if (otpServer.ec2Info.amiId != null) { + amiId = otpServer.ec2Info.amiId; + } else { amiId = DEFAULT_AMI_ID; - // Verify that AMI is correctly defined. - if (amiId == null || !ServerController.amiExists(amiId, ec2)) { - statusMessage = String.format( - "Default AMI ID (%s) is missing or bad. Should be provided in config at %s", - amiId, - AMI_CONFIG_PATH); - LOG.error(statusMessage); - status.fail(statusMessage); - } + } + // Verify that AMI is correctly defined. + if (amiId == null || !ServerController.amiExists(amiId, ec2)) { + statusMessage = String.format( + "Default AMI ID (%s) is missing or bad. Should be provided in config at %s", + amiId, + AMI_CONFIG_PATH); + LOG.error(statusMessage); + status.fail(statusMessage); + } + // Pick proper instance type depending on whether graph is being built and what is defined. + String instanceType; + if (!graphAlreadyBuilt && otpServer.ec2Info.buildInstanceType != null) { + instanceType = otpServer.ec2Info.buildInstanceType; + } else if (otpServer.ec2Info.instanceType != null) { + instanceType = otpServer.ec2Info.instanceType; + } else { + instanceType = DEFAULT_INSTANCE_TYPE; } RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) @@ -790,7 +835,12 @@ private String constructUserData(boolean graphAlreadyBuilt) { lines.add("aws configure set default.s3.multipart_chunksize 32MB"); // Get region from config or default to us-east-1 - String region = DataManager.getConfigPropertyAsText("application.data.s3_region"); + String region; + if (customRegion != null) { + region = customRegion; + } else { + region = DataManager.getConfigPropertyAsText("application.data.s3_region"); + } if (region == null) region = "us-east-1"; lines.add(String.format("aws configure set default.region %s", region)); // Create the directory for the graph inputs. @@ -826,26 +876,60 @@ private String constructUserData(boolean graphAlreadyBuilt) { lines.add(String.format("sudo echo $BUNDLE_STATUS > $WEB_DIR/%s", BUNDLE_DOWNLOAD_COMPLETE_FILE)); // Put unzipped bundle data into router directory. lines.add(String.format("unzip /tmp/bundle.zip -d %s", routerDir)); + // copy cached_elevations if it exists into routerDir + String cachedElevationFile = String.format( + "/var/%s/cache/cached_elevations.obj", + getTripPlannerString() + ); + String builtCachedElevationFile = String.format( + "%s/cached_elevations.obj", + routerDir + ); + lines.add(String.format( + "[ -f %s ] && cp %s %s", + cachedElevationFile, + cachedElevationFile, + builtCachedElevationFile + )); lines.add("echo 'starting graph build'"); // Build the graph. if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu java -Xmx${MEM}k -jar %s/%s.jar point --build %s", jarDir, jarName, routerDir)); else lines.add(String.format("sudo -H -u ubuntu java -jar -Xmx${MEM}k %s/%s.jar --build %s > $BUILDLOGFILE 2>&1", jarDir, jarName, routerDir)); + // copy resulting cached elevations to cache folder. This is useful if creating an AMI image is desired + // after graph build. + lines.add(String.format( + "[ -f %s ] && cp %s %s", + builtCachedElevationFile, + builtCachedElevationFile, + cachedElevationFile + )); // Re-upload user data log after build command. lines.add(uploadUserDataLogCommand); - // Upload the build log file and graph to S3. + // Upload the build log file, build report and graph to S3. if (!deployment.r5) { String s3BuildLogPath = joinToS3FolderURI(getBuildLogFilename()); + // upload log file lines.add(String.format("aws s3 cp $BUILDLOGFILE %s ", s3BuildLogPath)); + // upload report if it was generated + String reportPath = String.format("%s/report", routerDir); + lines.add(String.format( + "[ -e %s ] && cd %s && zip -r report.zip report && cd - && aws s3 cp %s.zip %s", + reportPath, + routerDir, + reportPath, + joinToS3FolderURI("graph-build-report.zip") + )); + // upload graph lines.add(String.format("aws s3 cp %s %s ", graphPath, getS3GraphURI())); } } // Determine if graph build/download was successful (and that Graph.obj is not zero bytes). lines.add(String.format("FILESIZE=$(wc -c <%s)", graphPath)); lines.add(String.format("[ -f %s ] && (($FILESIZE > 0)) && GRAPH_STATUS='SUCCESS' || GRAPH_STATUS='FAILURE'", graphPath)); + // Re-upload user data log before indicating that graph build/download is complete. + lines.add(uploadUserDataLogCommand); // Create file with bundle status in web dir to notify Data Tools that download is complete. lines.add(String.format("sudo echo $GRAPH_STATUS > $WEB_DIR/%s", GRAPH_STATUS_FILE)); - // Re-upload user data log before final command (and before optional shutdown statement). - lines.add(uploadUserDataLogCommand); if (deployment.buildGraphOnly) { // If building graph only, tell the instance to shut itself down after the graph build (and log upload) is // complete. @@ -874,10 +958,6 @@ public String getJobRelativePath() { return jobRelativePath; } - public void setJobRelativePath(String jobRelativePath) { - this.jobRelativePath = jobRelativePath; - } - @JsonIgnore public AmazonS3URI getS3FolderURI() { return new AmazonS3URI(String.format("s3://%s/%s", otpServer.s3Bucket, getJobRelativePath())); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 4fcd5e0ed..b2347d8e5 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -8,6 +8,7 @@ import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClientBuilder; import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.amazonaws.services.s3.AmazonS3URI; @@ -69,7 +70,9 @@ public MonitorServerStatusJob(Auth0UserProfile owner, DeployJob deployJob, Insta status.message = "Checking server status..."; startTime = System.currentTimeMillis(); credentials = AWSUtils.getCredentialsForRole(otpServer.role, "monitor-" + instance.getInstanceId()); - ec2 = AWSUtils.getEC2ClientForCredentials(credentials); + ec2 = deployJob.getCustomRegion() == null + ? AWSUtils.getEC2ClientForCredentials(credentials) + : AWSUtils.getEC2ClientForCredentials(credentials, deployJob.getCustomRegion()); } @JsonProperty @@ -106,14 +109,14 @@ public void jobLogic() { wait("bundle download check:" + bundleUrl); bundleIsDownloaded = checkForSuccessfulRequest(bundleUrl); if (jobHasTimedOut()) { - status.fail(String.format("Job timed out while checking for server bundle download status (%s)", instance.getInstanceId())); + failJob("Job timed out while checking for server bundle download status."); return; } } // Check status of bundle download and fail job if there was a failure. String bundleStatus = getUrlAsString(bundleUrl); if (bundleStatus == null || !bundleStatus.contains("SUCCESS")) { - status.fail("Failure encountered while downloading transit bundle."); + failJob("Failure encountered while downloading transit bundle."); return; } long bundleDownloadSeconds = (System.currentTimeMillis() - bundleDownloadStartTime) / 1000; @@ -130,26 +133,27 @@ public void jobLogic() { wait("graph build/download check: " + graphStatusUrl); graphIsAvailable = checkForSuccessfulRequest(graphStatusUrl); if (jobHasTimedOut()) { - message = String.format("Job timed out while waiting for graph build/download (%s). If this was a graph building machine, it may have run out of memory.", instance.getInstanceId()); + message = "Job timed out while waiting for graph build/download. If this was a graph building machine, it may have run out of memory."; LOG.error(message); - status.fail(message); + failJob(message); return; } } // Check status of bundle download and fail job if there was a failure. String graphStatus = getUrlAsString(graphStatusUrl); if (graphStatus == null || !graphStatus.contains("SUCCESS")) { - message = String.format("Failure encountered while building/downloading graph (%s).", instance.getInstanceId()); + message = "Failure encountered while building/downloading graph."; LOG.error(message); - status.fail(message); + failJob(message); return; } graphBuildSeconds = (System.currentTimeMillis() - graphBuildStartTime) / 1000; message = String.format("Graph build/download completed in %d seconds!", graphBuildSeconds); LOG.info(message); // If only task is to build graph, this machine's job is complete and we can consider this job done. - if (deployment.buildGraphOnly) { + if (deployment.buildGraphOnly || (!graphAlreadyBuilt && otpServer.ec2Info.hasSeparateGraphBuildConfig())) { status.update(false, message, 100); + LOG.info("View logs at {}", getUserDataLogS3Path()); return; } status.update("Loading graph...", 70); @@ -162,8 +166,8 @@ public void jobLogic() { wait("router to become available: " + routerUrl); routerIsAvailable = checkForSuccessfulRequest(routerUrl); if (jobHasTimedOut()) { - message = String.format("Job timed out while waiting for trip planner to start up (%s)", instance.getInstanceId()); - status.fail(message); + message = "Job timed out while waiting for trip planner to start up."; + failJob(message); LOG.error(message); return; } @@ -173,9 +177,10 @@ public void jobLogic() { // After the router is available, the EC2 instance can be registered with the load balancer. // REGISTER INSTANCE WITH LOAD BALANCER // Use alternative credentials if they exist. - AmazonElasticLoadBalancing elbClient = AmazonElasticLoadBalancingClient.builder() - .withCredentials(credentials) - .build(); + AmazonElasticLoadBalancingClientBuilder builder = AmazonElasticLoadBalancingClient.builder() + .withCredentials(credentials); + if (deployJob.getCustomRegion() != null) builder.withRegion(deployJob.getCustomRegion()); + AmazonElasticLoadBalancing elbClient = builder.build(); RegisterTargetsRequest registerTargetsRequest = new RegisterTargetsRequest() .withTargetGroupArn(otpServer.ec2Info.targetGroupArn) .withTargets(new TargetDescription().withId(instance.getInstanceId())); @@ -184,14 +189,29 @@ public void jobLogic() { message = String.format("Server successfully registered with load balancer %s. OTP running at %s", otpServer.ec2Info.targetGroupArn, routerUrl); LOG.info(message); status.update(false, message, 100, true); + LOG.info("View logs at {}", getUserDataLogS3Path()); deployJob.incrementCompletedServers(); } else { - message = String.format("There is no load balancer under which to register ec2 instance %s.", instance.getInstanceId()); + message = "There is no load balancer under which to register ec2 instance."; LOG.error(message); - status.fail(message); + failJob(message); } } + /** + * Gets the expected path to the user data logs that get uploaded to s3 + */ + private String getUserDataLogS3Path() { + return String.format("%s/%s.log", deployJob.getS3FolderURI(), instance.getInstanceId()); + } + + /** + * Helper that fails with a helpful message about where to find uploaded logs. + */ + private void failJob(String message) { + status.fail(String.format("%s Check logs at: %s", message, getUserDataLogS3Path())); + } + /** Determine if job has passed time limit for its run time. */ private boolean jobHasTimedOut() { long runTime = System.currentTimeMillis() - startTime; diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index cd0426926..391151fce 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -122,11 +122,20 @@ public List retrieveEC2Instances() { Filter deploymentFilter = new Filter("tag:deploymentId", Collections.singletonList(id)); // Check if the latest deployment used alternative credentials/AWS role. String role = null; + String region = null; if (this.latest() != null) { OtpServer server = Persistence.servers.getById(this.latest().serverId); - if (server != null) role = server.role; + if (server != null) { + role = server.role; + if (server.ec2Info != null) { + region = server.ec2Info.region; + } + } } - return DeploymentController.fetchEC2InstanceSummaries(AWSUtils.getEC2ClientForRole(role), deploymentFilter); + return DeploymentController.fetchEC2InstanceSummaries( + AWSUtils.getEC2ClientForRole(role, region), + deploymentFilter + ); } public void storeFeedVersions(Collection versions) { diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java index f4ebfadd5..9d66e32bf 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -42,4 +42,15 @@ public EC2Info () {} public String targetGroupArn; /** An optional custom AWS region */ public String region; + + /** + * Returns true if the instance type or ami ids are set and are different for a graph build. + */ + public boolean hasSeparateGraphBuildConfig() { + return ( + buildInstanceType != null && !buildInstanceType.equals(instanceType) + ) || ( + buildAmiId != null && !buildAmiId.equals(amiId) + ); + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java index c86bdabf4..77e84b64a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java @@ -59,13 +59,22 @@ public List retrieveEC2InstanceSummaries() { // Prevent calling EC2 method on servers that do not have EC2 info defined because this is a JSON property. if (ec2Info == null) return Collections.EMPTY_LIST; Filter serverFilter = new Filter("tag:serverId", Collections.singletonList(id)); - return DeploymentController.fetchEC2InstanceSummaries(AWSUtils.getEC2ClientForRole(this.role), serverFilter); + return DeploymentController.fetchEC2InstanceSummaries( + AWSUtils.getEC2ClientForRole(this.role, ec2Info.region), + serverFilter + ); } public List retrieveEC2Instances() { - if (!"true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled"))) return Collections.EMPTY_LIST; + if ( + !"true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled")) || + ec2Info == null + ) return Collections.EMPTY_LIST; Filter serverFilter = new Filter("tag:serverId", Collections.singletonList(id)); - return DeploymentController.fetchEC2Instances(AWSUtils.getEC2ClientForRole(this.role), serverFilter); + return DeploymentController.fetchEC2Instances( + AWSUtils.getEC2ClientForRole(this.role, ec2Info.region), + serverFilter + ); } @JsonProperty("organizationId") diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java index 151efdb52..df8f47cf0 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java @@ -97,7 +97,10 @@ public void canDeployFromPrebuiltGraph () { public static void cleanUp() { List instances = server.retrieveEC2Instances(); List ids = getIds(instances); - terminateInstances(AWSUtils.getEC2ClientForRole(server.role), ids); + terminateInstances( + AWSUtils.getEC2ClientForRole(server.role, server.ec2Info == null ? null : server.ec2Info.region), + ids + ); } } From 2289407b0012524871dc09e816b612eb52d431a3 Mon Sep 17 00:00:00 2001 From: Evan Siroky Date: Thu, 27 Feb 2020 12:00:39 -0800 Subject: [PATCH 4/7] refactor(deploy): remove copying of cached_elevations.obj --- .../datatools/manager/jobs/DeployJob.java | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 77b7ef755..de8cfe5df 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -876,33 +876,10 @@ private String constructUserData(boolean graphAlreadyBuilt) { lines.add(String.format("sudo echo $BUNDLE_STATUS > $WEB_DIR/%s", BUNDLE_DOWNLOAD_COMPLETE_FILE)); // Put unzipped bundle data into router directory. lines.add(String.format("unzip /tmp/bundle.zip -d %s", routerDir)); - // copy cached_elevations if it exists into routerDir - String cachedElevationFile = String.format( - "/var/%s/cache/cached_elevations.obj", - getTripPlannerString() - ); - String builtCachedElevationFile = String.format( - "%s/cached_elevations.obj", - routerDir - ); - lines.add(String.format( - "[ -f %s ] && cp %s %s", - cachedElevationFile, - cachedElevationFile, - builtCachedElevationFile - )); lines.add("echo 'starting graph build'"); // Build the graph. if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu java -Xmx${MEM}k -jar %s/%s.jar point --build %s", jarDir, jarName, routerDir)); else lines.add(String.format("sudo -H -u ubuntu java -jar -Xmx${MEM}k %s/%s.jar --build %s > $BUILDLOGFILE 2>&1", jarDir, jarName, routerDir)); - // copy resulting cached elevations to cache folder. This is useful if creating an AMI image is desired - // after graph build. - lines.add(String.format( - "[ -f %s ] && cp %s %s", - builtCachedElevationFile, - builtCachedElevationFile, - cachedElevationFile - )); // Re-upload user data log after build command. lines.add(uploadUserDataLogCommand); // Upload the build log file, build report and graph to S3. From a0224ee0e1fc4ab15953bffe7ea0bfadf77178ad Mon Sep 17 00:00:00 2001 From: Evan Siroky Date: Thu, 27 Feb 2020 12:09:55 -0800 Subject: [PATCH 5/7] refactor: correct JavaDoc --- .../java/com/conveyal/datatools/manager/models/EC2Info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java index 9d66e32bf..69a5b2e28 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -29,7 +29,7 @@ public EC2Info () {} public String amiId; /** * The AWS-style instance type (e.g., t2.medium) to use for new EC2 machines used specifically for graph building. - * Defaults to {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} if null during deployment. + * Defaults to {@link com.conveyal.datatools.manager.models.EC2Info#instanceType} if null during deployment. */ public String buildInstanceType; /** The Amazon machine image (AMI) to be used for the OTP EC2 machine used specifically for graph building. */ From ce7c8262f1fe002b31ce377f8532f475e4e70cd0 Mon Sep 17 00:00:00 2001 From: Evan Siroky Date: Tue, 3 Mar 2020 15:28:08 -0800 Subject: [PATCH 6/7] refactor(deploy): consolidate some code --- .../datatools/manager/jobs/DeployJob.java | 21 ++--------- .../manager/jobs/MonitorServerStatusJob.java | 17 +++------ .../datatools/manager/models/EC2Info.java | 36 +++++++++++++++++++ 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index de8cfe5df..59f6aea01 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -80,8 +80,7 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); private static final String bundlePrefix = "bundles"; public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; - private static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; - private static final String DEFAULT_AMI_ID = DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); + public static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; // Indicates whether EC2 instances should be EBS optimized. private static final boolean EBS_OPTIMIZED = "true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ebs_optimized")); private static final String OTP_GRAPH_FILENAME = "Graph.obj"; @@ -654,14 +653,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withGroups(otpServer.ec2Info.securityGroupId) .withDeviceIndex(0); // Pick proper ami depending on whether graph is being built and what is defined. - String amiId; - if (!graphAlreadyBuilt && otpServer.ec2Info.buildAmiId != null) { - amiId = otpServer.ec2Info.buildAmiId; - } else if (otpServer.ec2Info.amiId != null) { - amiId = otpServer.ec2Info.amiId; - } else { - amiId = DEFAULT_AMI_ID; - } + String amiId = otpServer.ec2Info.getAmiId(graphAlreadyBuilt); // Verify that AMI is correctly defined. if (amiId == null || !ServerController.amiExists(amiId, ec2)) { statusMessage = String.format( @@ -672,14 +664,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { status.fail(statusMessage); } // Pick proper instance type depending on whether graph is being built and what is defined. - String instanceType; - if (!graphAlreadyBuilt && otpServer.ec2Info.buildInstanceType != null) { - instanceType = otpServer.ec2Info.buildInstanceType; - } else if (otpServer.ec2Info.instanceType != null) { - instanceType = otpServer.ec2Info.instanceType; - } else { - instanceType = DEFAULT_INSTANCE_TYPE; - } + String instanceType = otpServer.ec2Info.getInstanceType(graphAlreadyBuilt); RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) .withInstanceType(instanceType) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index b2347d8e5..1640897fb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -133,18 +133,14 @@ public void jobLogic() { wait("graph build/download check: " + graphStatusUrl); graphIsAvailable = checkForSuccessfulRequest(graphStatusUrl); if (jobHasTimedOut()) { - message = "Job timed out while waiting for graph build/download. If this was a graph building machine, it may have run out of memory."; - LOG.error(message); - failJob(message); + failJob("Job timed out while waiting for graph build/download. If this was a graph building machine, it may have run out of memory."); return; } } // Check status of bundle download and fail job if there was a failure. String graphStatus = getUrlAsString(graphStatusUrl); if (graphStatus == null || !graphStatus.contains("SUCCESS")) { - message = "Failure encountered while building/downloading graph."; - LOG.error(message); - failJob(message); + failJob("Failure encountered while building/downloading graph."); return; } graphBuildSeconds = (System.currentTimeMillis() - graphBuildStartTime) / 1000; @@ -166,9 +162,7 @@ public void jobLogic() { wait("router to become available: " + routerUrl); routerIsAvailable = checkForSuccessfulRequest(routerUrl); if (jobHasTimedOut()) { - message = "Job timed out while waiting for trip planner to start up."; - failJob(message); - LOG.error(message); + failJob("Job timed out while waiting for trip planner to start up."); return; } } @@ -192,9 +186,7 @@ public void jobLogic() { LOG.info("View logs at {}", getUserDataLogS3Path()); deployJob.incrementCompletedServers(); } else { - message = "There is no load balancer under which to register ec2 instance."; - LOG.error(message); - failJob(message); + failJob("There is no load balancer under which to register ec2 instance."); } } @@ -209,6 +201,7 @@ private String getUserDataLogS3Path() { * Helper that fails with a helpful message about where to find uploaded logs. */ private void failJob(String message) { + LOG.error(message); status.fail(String.format("%s Check logs at: %s", message, getUserDataLogS3Path())); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java index 69a5b2e28..1ec29976c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -1,10 +1,14 @@ package com.conveyal.datatools.manager.models; +import com.conveyal.datatools.manager.DataManager; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; import java.util.List; +import static com.conveyal.datatools.manager.jobs.DeployJob.AMI_CONFIG_PATH; +import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE; + /** * Contains the fields specific to starting up new EC2 servers for an ELB target group. If null, at least one internal * URLs must be provided. @@ -53,4 +57,36 @@ public boolean hasSeparateGraphBuildConfig() { buildAmiId != null && !buildAmiId.equals(amiId) ); } + + /** + * Returns the appropriate ami ID to use when creating a new ec2 instance during a deploy job. + * + * @param graphAlreadyBuilt whether or not a graph has already been built. If false, this means a build ami should + * be used if available. + */ + public String getAmiId(boolean graphAlreadyBuilt) { + if (!graphAlreadyBuilt && buildAmiId != null) { + return buildAmiId; + } else if (amiId != null) { + return amiId; + } else { + return DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); + } + } + + /** + * Returns the appropriate instance type to use when creating a new ec2 instance during a deploy job. + * + * @param graphAlreadyBuilt whether or not a graph has already been built. If false, this means a build instance + * type should be used if available. + */ + public String getInstanceType(boolean graphAlreadyBuilt) { + if (!graphAlreadyBuilt && buildInstanceType != null) { + return buildInstanceType; + } else if (instanceType != null) { + return instanceType; + } else { + return DEFAULT_INSTANCE_TYPE; + } + } } \ No newline at end of file From 3d77359ee9be5492040681fd44ab5e54e4fad3cc Mon Sep 17 00:00:00 2001 From: Evan Siroky Date: Thu, 5 Mar 2020 16:07:26 -0800 Subject: [PATCH 7/7] A few small improvements for PR review --- .../controllers/api/ServerController.java | 11 +++++--- .../datatools/manager/jobs/DeployJob.java | 27 +++++++++++++++---- .../datatools/manager/models/EC2Info.java | 27 ++++++++++++------- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index d3f81379f..7098cea50 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -63,7 +63,7 @@ import static com.conveyal.datatools.common.utils.SparkUtils.getPOJOFromRequestBody; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; -import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE; +import static com.conveyal.datatools.manager.models.EC2Info.DEFAULT_INSTANCE_TYPE; import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds; import static spark.Spark.delete; import static spark.Spark.get; @@ -284,7 +284,7 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc // If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to // AWS. (e.g., target group ARN and instance type). if (server.ec2Info != null) { - // do some custom items if a custom region should be used + // create custom clients if credentials and or a custom region exist if (server.ec2Info.region != null) { AmazonEC2ClientBuilder builder = AmazonEC2Client.builder(); if (credentials != null) { @@ -464,14 +464,17 @@ private static void validateSubnetId(LoadBalancer loadBalancer, EC2Info ec2Info, /** * Validate that EC2 instance type (e.g., t2-medium) exists. This value can be empty and will default to - * {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} at deploy time. + * {@link com.conveyal.datatools.manager.models.EC2Info#DEFAULT_INSTANCE_TYPE} at deploy time. */ private static void validateInstanceType(String instanceType, Request req) { if (instanceType == null) return; try { InstanceType.fromValue(instanceType); } catch (IllegalArgumentException e) { - String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE); + String message = String.format( + "Must provide valid instance type (if none provided, defaults to %s).", + DEFAULT_INSTANCE_TYPE + ); logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e); } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 59f6aea01..d197ee192 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -6,10 +6,12 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.model.CreateTagsRequest; import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.Filter; import com.amazonaws.services.ec2.model.IamInstanceProfileSpecification; import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.InstanceNetworkInterfaceSpecification; +import com.amazonaws.services.ec2.model.InstanceType; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.Tag; import com.amazonaws.services.s3.AmazonS3; @@ -69,6 +71,8 @@ import static com.conveyal.datatools.manager.controllers.api.ServerController.getIds; import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_OTP_VERSION; import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_R5_VERSION; +import static com.conveyal.datatools.manager.models.EC2Info.AMI_CONFIG_PATH; +import static com.conveyal.datatools.manager.models.EC2Info.DEFAULT_INSTANCE_TYPE; /** * Deploy the given deployment to the OTP servers specified by targets. @@ -79,8 +83,6 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); private static final String bundlePrefix = "bundles"; - public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; - public static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; // Indicates whether EC2 instances should be EBS optimized. private static final boolean EBS_OPTIMIZED = "true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ebs_optimized")); private static final String OTP_GRAPH_FILENAME = "Graph.obj"; @@ -553,7 +555,7 @@ private void replaceEC2Servers() { if (otpServer.ec2Info.hasSeparateGraphBuildConfig()) { // different instance type and/or ami exists for graph building. Terminate graph building instance ServerController.terminateInstances(ec2, graphBuildingInstances); - status.numServersRemaining = Math.max(otpServer.ec2Info.instanceCount, 0); + status.numServersRemaining = Math.max(otpServer.ec2Info.instanceCount, 1); } else { // same configuration exists, so keep instance on and add to list of running instances instances.addAll(graphBuildingInstances); @@ -657,14 +659,29 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { // Verify that AMI is correctly defined. if (amiId == null || !ServerController.amiExists(amiId, ec2)) { statusMessage = String.format( - "Default AMI ID (%s) is missing or bad. Should be provided in config at %s", + "AMI ID (%s) is missing or bad. Check the deployment settings or the default value in the app config at %s", amiId, - AMI_CONFIG_PATH); + AMI_CONFIG_PATH + ); LOG.error(statusMessage); status.fail(statusMessage); + return Collections.EMPTY_LIST; } // Pick proper instance type depending on whether graph is being built and what is defined. String instanceType = otpServer.ec2Info.getInstanceType(graphAlreadyBuilt); + // Verify that instance type is correctly defined. + try { + InstanceType.fromValue(instanceType); + } catch (IllegalArgumentException e) { + statusMessage = String.format( + "Instance type (%s) is bad. Check the deployment settings. The default value is %s", + instanceType, + DEFAULT_INSTANCE_TYPE + ); + LOG.error(statusMessage); + status.fail(statusMessage); + return Collections.EMPTY_LIST; + } RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) .withInstanceType(instanceType) diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java index 1ec29976c..bae33f20a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -4,10 +4,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; -import java.util.List; - -import static com.conveyal.datatools.manager.jobs.DeployJob.AMI_CONFIG_PATH; -import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE; /** * Contains the fields specific to starting up new EC2 servers for an ELB target group. If null, at least one internal @@ -16,11 +12,15 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class EC2Info implements Serializable { private static final long serialVersionUID = 1L; + + public static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; + public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; + /** Empty constructor for serialization. */ public EC2Info () {} /** * The AWS-style instance type (e.g., t2.medium) to use for new EC2 machines. Defaults to - * {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} if null during deployment. + * {@link com.conveyal.datatools.manager.models.EC2Info#DEFAULT_INSTANCE_TYPE} if null during deployment. */ public String instanceType; /** Number of instances to spin up and add to target group. If zero, defaults to 1. */ @@ -29,22 +29,31 @@ public EC2Info () {} public String subnetId; /** The security group ID associated with the target group. */ public String securityGroupId; - /** The Amazon machine image (AMI) to be used for the OTP EC2 machines. */ + /** + * The Amazon machine image (AMI) to be used for the OTP EC2 machines. Defaults to the app config value at + * {@link com.conveyal.datatools.manager.models.EC2Info#AMI_CONFIG_PATH} if null during deployment. + */ public String amiId; /** * The AWS-style instance type (e.g., t2.medium) to use for new EC2 machines used specifically for graph building. * Defaults to {@link com.conveyal.datatools.manager.models.EC2Info#instanceType} if null during deployment. */ public String buildInstanceType; - /** The Amazon machine image (AMI) to be used for the OTP EC2 machine used specifically for graph building. */ + /** + * The Amazon machine image (AMI) (e.g. ami-12345678) to be used for the OTP EC2 machine used specifically for + * graph building. Defaults to {@link com.conveyal.datatools.manager.models.EC2Info#amiId} if null during deployment. + */ public String buildAmiId; - /** The IAM instance profile ARN that the OTP EC2 server should assume. For example, arn:aws:iam::123456789012:instance-profile/otp-ec2-role */ + /** + * The IAM instance profile ARN that the OTP EC2 server should assume. For example, + * arn:aws:iam::123456789012:instance-profile/otp-ec2-role + */ public String iamInstanceProfileArn; /** The AWS key file (.pem) that should be used to set up OTP EC2 servers (gives a way for admins to SSH into machine). */ public String keyName; /** The target group to deploy new EC2 instances to. */ public String targetGroupArn; - /** An optional custom AWS region */ + /** An optional custom AWS region (e.g., us-east-1) */ public String region; /**