diff --git a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java index 7f66d1dc8..e9f76c093 100644 --- a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java +++ b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java @@ -76,6 +76,7 @@ public MonitorableJob(Auth0UserProfile owner, String name, JobType type) { } this.owner = owner; this.name = name; + status.name = name; this.type = type; registerJob(); } @@ -279,6 +280,7 @@ public void update (boolean isError, String message, double percentComplete, boo } public void fail (String message, Exception e) { + LOG.warn("Job `{}` has been failed with message: `{}` and Exception: `{}`", name, message, e); this.error = true; this.percentComplete = 100; this.completed = true; @@ -288,6 +290,7 @@ public void fail (String message, Exception e) { } public void fail (String message) { + LOG.warn("Job `{}` has been failed with message: `{}`", name, message); this.error = true; this.percentComplete = 100; this.completed = true; diff --git a/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java b/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java index f887bac81..16a81efcc 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java @@ -148,9 +148,11 @@ public static AWSStaticCredentialsProvider getCredentialsForRole(String role, St * Shorthand method to obtain an EC2 client for the provided role ARN. If role is null, the default EC2 credentials * will be used. */ - public static AmazonEC2 getEC2ClientForRole (String role) { + public static AmazonEC2 getEC2ClientForRole (String role, String region) { AWSStaticCredentialsProvider credentials = getCredentialsForRole(role, "ec2-client"); - return getEC2ClientForCredentials(credentials); + return region == null + ? getEC2ClientForCredentials(credentials) + : getEC2ClientForCredentials(credentials, region); } /** @@ -161,6 +163,14 @@ public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider crede return AmazonEC2Client.builder().withCredentials(credentials).build(); } + /** + * Shorthand method to obtain an EC2 client for the provided credentials and region. If credentials are null, the + * default EC2 credentials will be used. + */ + public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider credentials, String region) { + return AmazonEC2Client.builder().withCredentials(credentials).withRegion(region).build(); + } + /** * Shorthand method to obtain an S3 client for the provided credentials. If credentials are null, the default EC2 * credentials will be used. diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index 9eaa8eb98..ba7fa0c47 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -234,6 +234,8 @@ public static void logRequestOrResponse( LOG.warn("Request object is null. Cannot log."); return; } + // don't log job status requests/responses, they clutter things up + if (request.pathInfo().contains("status/jobs")) return; Auth0UserProfile userProfile = request.attribute("user"); String userEmail = userProfile != null ? userProfile.getEmail() : "no-auth"; String queryString = request.queryParams().size() > 0 ? "?" + request.queryString() : ""; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 437abc675..02fc71b32 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -9,6 +9,7 @@ import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3URI; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.common.utils.AWSUtils; @@ -47,6 +48,7 @@ import static com.conveyal.datatools.common.utils.AWSUtils.downloadFromS3; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; +import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds; import static spark.Spark.delete; import static spark.Spark.get; import static spark.Spark.options; @@ -101,6 +103,7 @@ private static String downloadBuildArtifact (Request req, Response res) { // Default client to use if no role was used during the deployment. AmazonS3 s3Client = FeedStore.s3Client; String role = null; + String region = null; String uriString; String filename = req.queryParams("filename"); if (filename == null) { @@ -131,6 +134,7 @@ private static String downloadBuildArtifact (Request req, Response res) { logMessageAndHalt(req, 400, "The deployment does not have job history or associated server information to construct URI for build artifact. " + uriString); return null; } + region = server.ec2Info == null ? null : server.ec2Info.region; uriString = String.format("s3://%s/bundles/%s/%s/%s", server.s3Bucket, deployment.projectId, deployment.id, jobId); LOG.warn("Could not find deploy summary for job. Attempting to use {}", uriString); } @@ -138,10 +142,15 @@ private static String downloadBuildArtifact (Request req, Response res) { // If summary is readily available, just use the ready-to-use build artifacts field. uriString = summaryToDownload.buildArtifactsFolder; role = summaryToDownload.role; + region = summaryToDownload.ec2Info == null ? null : summaryToDownload.ec2Info.region; } AmazonS3URI uri = new AmazonS3URI(uriString); // Assume the alternative role if needed to download the deploy artifact. - if (role != null) s3Client = AWSUtils.getS3ClientForRole(role); + if (role != null) { + s3Client = AWSUtils.getS3ClientForRole(role, region); + } else if (region != null) { + s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), region); + } return downloadFromS3(s3Client, uri.getBucket(), String.join("/", uri.getKey(), filename), false, res); } @@ -373,7 +382,12 @@ private static boolean terminateEC2InstanceForDeployment(Request req, Response r } } // If checks are ok, terminate instances. - boolean success = ServerController.deRegisterAndTerminateInstances(credentials, targetGroupArn, idsToTerminate); + boolean success = ServerController.deRegisterAndTerminateInstances( + credentials, + targetGroupArn, + latest.ec2Info.region, + idsToTerminate + ); if (!success) { logMessageAndHalt(req, 400, "Could not complete termination request"); return false; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 5c50787e9..7098cea50 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -3,6 +3,7 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.AmazonEC2Exception; import com.amazonaws.services.ec2.model.DescribeImagesRequest; import com.amazonaws.services.ec2.model.DescribeImagesResult; @@ -19,6 +20,7 @@ import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClientBuilder; import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException; import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersRequest; @@ -61,7 +63,8 @@ import static com.conveyal.datatools.common.utils.SparkUtils.getPOJOFromRequestBody; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; -import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE; +import static com.conveyal.datatools.manager.models.EC2Info.DEFAULT_INSTANCE_TYPE; +import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds; import static spark.Spark.delete; import static spark.Spark.get; import static spark.Spark.options; @@ -117,7 +120,10 @@ private static OtpServer terminateEC2InstancesForServer(Request req, Response re OtpServer server = getServerWithPermissions(req, res); List instances = server.retrieveEC2Instances(); List ids = getIds(instances); - AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole(server.role); + AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole( + server.role, + server.ec2Info == null ? null : server.ec2Info.region + ); terminateInstances(ec2Client, ids); for (Deployment deployment : Deployment.retrieveDeploymentForServerAndRouterId(server.id, null)) { Persistence.deployments.updateField(deployment.id, "deployedTo", null); @@ -157,7 +163,12 @@ public static TerminateInstancesResult terminateInstances(AmazonEC2 ec2Client, L * De-register instances from the specified target group/load balancer and terminate the instances. * */ - public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvider credentials, String targetGroupArn, List instanceIds) { + public static boolean deRegisterAndTerminateInstances( + AWSStaticCredentialsProvider credentials, + String targetGroupArn, + String region, + List instanceIds + ) { LOG.info("De-registering instances from load balancer {}", instanceIds); TargetDescription[] targetDescriptions = instanceIds.stream() .map(id -> new TargetDescription().withId(id)) @@ -169,12 +180,21 @@ public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvid AmazonElasticLoadBalancing elbClient = elb; AmazonEC2 ec2Client = ec2; // If OTP Server has role defined/alt credentials, override default AWS clients. - if (credentials != null) { - elbClient = AmazonElasticLoadBalancingClient.builder().withCredentials(credentials).build(); - ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build(); + if (credentials != null || region != null) { + AmazonElasticLoadBalancingClientBuilder elbBuilder = AmazonElasticLoadBalancingClient.builder(); + AmazonEC2ClientBuilder ec2Builder = AmazonEC2Client.builder(); + if (credentials != null) { + elbBuilder.withCredentials(credentials); + ec2Builder.withCredentials(credentials); + } + if (region != null) { + elbBuilder.withRegion(region); + ec2Builder.withRegion(region); + } + elbClient = elbBuilder.build(); + ec2Client = ec2Builder.build(); } elbClient.deregisterTargets(request); - // FIXME default to regular ec2 client ServerController.terminateInstances(ec2Client, instanceIds); } catch (AmazonEC2Exception | AmazonElasticLoadBalancingException e) { LOG.warn("Could not terminate EC2 instances: " + String.join(",", instanceIds), e); @@ -250,6 +270,7 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc AWSStaticCredentialsProvider credentials = AWSUtils.getCredentialsForRole(server.role, "validate"); // If alternative credentials exist, override the default AWS clients. if (credentials != null) { + // build ec2 client ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build(); iamClient = AmazonIdentityManagementClientBuilder.standard().withCredentials(credentials).build(); s3Client = AWSUtils.getS3ClientForRole(server.role, null); @@ -263,9 +284,23 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc // If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to // AWS. (e.g., target group ARN and instance type). if (server.ec2Info != null) { + // create custom clients if credentials and or a custom region exist + if (server.ec2Info.region != null) { + AmazonEC2ClientBuilder builder = AmazonEC2Client.builder(); + if (credentials != null) { + builder.withCredentials(credentials); + } + builder.withRegion(server.ec2Info.region); + ec2Client = builder.build(); + if (credentials != null) { + s3Client = AWSUtils.getS3ClientForRole(server.role, server.ec2Info.region); + } else { + s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), server.ec2Info.region); + } + } validateInstanceType(server.ec2Info.instanceType, req); // Validate target group and get load balancer to validate subnetId and security group ID. - LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info.targetGroupArn, req, credentials); + LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info, req, credentials); validateSubnetId(loadBalancer, server.ec2Info, req, ec2Client); validateSecurityGroupId(loadBalancer, server.ec2Info, req); // Validate remaining AWS values. @@ -304,7 +339,7 @@ private static boolean verifyS3WritePermissions(AmazonS3 s3Client, String s3Buck s3Client.putObject(s3Bucket, key, File.createTempFile("test", ".zip")); s3Client.deleteObject(s3Bucket, key); } catch (IOException | AmazonS3Exception e) { - LOG.warn("S3 client cannot write to bucket" + s3Bucket, e); + LOG.warn("S3 client cannot write to bucket: " + s3Bucket, e); return false; } return true; @@ -429,14 +464,17 @@ private static void validateSubnetId(LoadBalancer loadBalancer, EC2Info ec2Info, /** * Validate that EC2 instance type (e.g., t2-medium) exists. This value can be empty and will default to - * {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} at deploy time. + * {@link com.conveyal.datatools.manager.models.EC2Info#DEFAULT_INSTANCE_TYPE} at deploy time. */ private static void validateInstanceType(String instanceType, Request req) { if (instanceType == null) return; try { InstanceType.fromValue(instanceType); } catch (IllegalArgumentException e) { - String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE); + String message = String.format( + "Must provide valid instance type (if none provided, defaults to %s).", + DEFAULT_INSTANCE_TYPE + ); logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e); } } @@ -448,16 +486,21 @@ private static void validateInstanceType(String instanceType, Request req) { * - https://serverfault.com/a/865422 * - https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html */ - private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn, AWSStaticCredentialsProvider credentials) { + private static LoadBalancer getLoadBalancerForTargetGroup (EC2Info ec2Info, AWSStaticCredentialsProvider credentials) { // If alternative credentials exist, use them to assume the role. Otherwise, use default ELB client. - AmazonElasticLoadBalancing elbClient = credentials != null - ? AmazonElasticLoadBalancingClient.builder() - .withCredentials(credentials) - .build() - : elb; + AmazonElasticLoadBalancingClientBuilder builder = AmazonElasticLoadBalancingClient.builder(); + if (credentials != null) { + builder.withCredentials(credentials); + } + + if (ec2Info.region != null) { + builder.withRegion(ec2Info.region); + } + + AmazonElasticLoadBalancing elbClient = builder.build(); try { DescribeTargetGroupsRequest targetGroupsRequest = new DescribeTargetGroupsRequest() - .withTargetGroupArns(targetGroupArn); + .withTargetGroupArns(ec2Info.targetGroupArn); List targetGroups = elbClient.describeTargetGroups(targetGroupsRequest).getTargetGroups(); for (TargetGroup tg : targetGroups) { DescribeLoadBalancersRequest request = new DescribeLoadBalancersRequest() @@ -467,7 +510,7 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn return result.getLoadBalancers().iterator().next(); } } catch (AmazonElasticLoadBalancingException e) { - LOG.warn("Invalid value for Target Group ARN: {}", targetGroupArn); + LOG.warn("Invalid value for Target Group ARN: {}", ec2Info.targetGroupArn); } // If no target group/load balancer found, return null. return null; @@ -477,11 +520,13 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn * Validate that ELB target group exists and is not empty and return associated load balancer for validating related * fields. */ - private static LoadBalancer validateTargetGroupAndGetLoadBalancer(String targetGroupArn, Request req, AWSStaticCredentialsProvider credentials) { - if (isEmpty(targetGroupArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); + private static LoadBalancer validateTargetGroupAndGetLoadBalancer(EC2Info ec2Info, Request req, AWSStaticCredentialsProvider credentials) { + if (isEmpty(ec2Info.targetGroupArn)) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); + } // Get load balancer for target group. This essentially checks that the target group exists and is assigned // to a load balancer. - LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(targetGroupArn, credentials); + LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(ec2Info, credentials); if (loadBalancer == null) { logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group or Load Balancer."); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 039572b73..d197ee192 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -6,10 +6,12 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.model.CreateTagsRequest; import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.Filter; import com.amazonaws.services.ec2.model.IamInstanceProfileSpecification; import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.InstanceNetworkInterfaceSpecification; +import com.amazonaws.services.ec2.model.InstanceType; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.Tag; import com.amazonaws.services.s3.AmazonS3; @@ -69,6 +71,8 @@ import static com.conveyal.datatools.manager.controllers.api.ServerController.getIds; import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_OTP_VERSION; import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_R5_VERSION; +import static com.conveyal.datatools.manager.models.EC2Info.AMI_CONFIG_PATH; +import static com.conveyal.datatools.manager.models.EC2Info.DEFAULT_INSTANCE_TYPE; /** * Deploy the given deployment to the OTP servers specified by targets. @@ -79,9 +83,6 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); private static final String bundlePrefix = "bundles"; - public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; - private static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; - private static final String DEFAULT_AMI_ID = DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); // Indicates whether EC2 instances should be EBS optimized. private static final boolean EBS_OPTIMIZED = "true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ebs_optimized")); private static final String OTP_GRAPH_FILENAME = "Graph.obj"; @@ -104,6 +105,8 @@ public class DeployJob extends MonitorableJob { private final int targetCount; private final DeployType deployType; private final AWSStaticCredentialsProvider credentials; + private final String customRegion; + private int tasksCompleted = 0; private int totalTasks; @@ -161,13 +164,18 @@ public DeployJob(Deployment deployment, Auth0UserProfile owner, OtpServer otpSer } public DeployJob(Deployment deployment, Auth0UserProfile owner, OtpServer otpServer, String bundlePath, DeployType deployType) { + this("Deploying " + deployment.name, deployment, owner, otpServer, bundlePath, deployType); + } + + public DeployJob(String jobName, Deployment deployment, Auth0UserProfile owner, OtpServer otpServer, String bundlePath, DeployType deployType) { // TODO add new job type or get rid of enum in favor of just using class names - super(owner, "Deploying " + deployment.name, JobType.DEPLOY_TO_OTP); + super(owner, jobName, JobType.DEPLOY_TO_OTP); this.deployment = deployment; this.otpServer = otpServer; this.s3Bucket = otpServer.s3Bucket != null ? otpServer.s3Bucket : DataManager.feedBucket; // Use a special subclass of status here that has additional fields this.status = new DeployStatus(); + this.status.name = jobName; this.targetCount = otpServer.internalUrl != null ? otpServer.internalUrl.size() : 0; this.totalTasks = 1 + targetCount; status.message = "Initializing..."; @@ -177,19 +185,22 @@ public DeployJob(Deployment deployment, Auth0UserProfile owner, OtpServer otpSer this.deployType = deployType; if (bundlePath == null) { // Use standard path for bundle. - setJobRelativePath(String.join("/", bundlePrefix, deployment.projectId, deployment.id, this.jobId)); + this.jobRelativePath = String.join("/", bundlePrefix, deployment.projectId, deployment.id, this.jobId); } else { // Override job relative path so that bundle can be downloaded directly. Note: this is currently only used // for testing (DeployJobTest), but the uses may be expanded in order to perhaps add a server to an existing // deployment using either a specified bundle or Graph.obj. - setJobRelativePath(bundlePath); + this.jobRelativePath = bundlePath; } // CONNECT TO EC2/S3 - // FIXME Should this ec2 client be longlived? credentials = AWSUtils.getCredentialsForRole(otpServer.role, this.jobId); - ec2 = AWSUtils.getEC2ClientForCredentials(credentials); - s3Client = AWSUtils.getS3ClientForCredentials(credentials, null); - + this.customRegion = otpServer.ec2Info != null && otpServer.ec2Info.region != null + ? otpServer.ec2Info.region + : null; + ec2 = customRegion == null + ? AWSUtils.getEC2ClientForCredentials(credentials) + : AWSUtils.getEC2ClientForCredentials(credentials, customRegion); + s3Client = AWSUtils.getS3ClientForCredentials(credentials, customRegion); } public void jobLogic () { @@ -449,6 +460,10 @@ private String getS3BundleURI() { return joinToS3FolderURI("bundle.zip"); } + public String getCustomRegion() { + return customRegion; + } + private String getLatestS3BundleKey() { String name = StringUtils.getCleanName(deployment.parentProject().name.toLowerCase()); return String.format("%s/%s/%s-latest.zip", bundlePrefix, deployment.projectId, name); @@ -500,14 +515,19 @@ private void replaceEC2Servers() { // First start graph-building instance and wait for graph to successfully build. if (!deployType.equals(DeployType.USE_PREBUILT_GRAPH)) { status.message = "Starting up graph building EC2 instance"; - instances.addAll(startEC2Instances(1, false)); + List graphBuildingInstances = startEC2Instances(1, false); // Exit if an error was encountered. - if (status.error || instances.size() == 0) { - ServerController.terminateInstances(ec2, instances); + if (status.error || graphBuildingInstances.size() == 0) { + ServerController.terminateInstances(ec2, graphBuildingInstances); return; } status.message = "Waiting for graph build to complete..."; - MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob(owner, this, instances.get(0), false); + MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob( + owner, + this, + graphBuildingInstances.get(0), + false + ); monitorInitialServerJob.run(); status.update("Graph build is complete!", 50); @@ -526,12 +546,25 @@ private void replaceEC2Servers() { statusMessage = "Error encountered while building graph. Inspect build logs."; LOG.error(statusMessage); status.fail(statusMessage); - ServerController.terminateInstances(ec2, instances); + ServerController.terminateInstances(ec2, graphBuildingInstances); return; } + // Check whether the graph build instance type or AMI ID is different from the non-graph building type. + // If so, terminate the graph building instance. If not, add the graph building instance to the list + // of started instances. + if (otpServer.ec2Info.hasSeparateGraphBuildConfig()) { + // different instance type and/or ami exists for graph building. Terminate graph building instance + ServerController.terminateInstances(ec2, graphBuildingInstances); + status.numServersRemaining = Math.max(otpServer.ec2Info.instanceCount, 1); + } else { + // same configuration exists, so keep instance on and add to list of running instances + instances.addAll(graphBuildingInstances); + status.numServersRemaining = otpServer.ec2Info.instanceCount <= 0 + ? 0 + : otpServer.ec2Info.instanceCount - 1; + } } // Spin up remaining servers which will download the graph from S3. - status.numServersRemaining = otpServer.ec2Info.instanceCount <= 0 ? 0 : otpServer.ec2Info.instanceCount - 1; List remainingServerMonitorJobs = new ArrayList<>(); List remainingInstances = new ArrayList<>(); if (status.numServersRemaining > 0) { @@ -577,6 +610,7 @@ private void replaceEC2Servers() { boolean success = ServerController.deRegisterAndTerminateInstances( credentials, otpServer.ec2Info.targetGroupArn, + customRegion, previousInstanceIds ); // If there was a problem during de-registration/termination, notify via status message. @@ -601,7 +635,6 @@ private void replaceEC2Servers() { * TODO: Booting up R5 servers has not been fully tested. */ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { - String instanceType = otpServer.ec2Info.instanceType == null ? DEFAULT_INSTANCE_TYPE : otpServer.ec2Info.instanceType; // User data should contain info about: // 1. Downloading GTFS/OSM info (s3) // 2. Time to live until shutdown/termination (for test servers) @@ -621,19 +654,33 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withAssociatePublicIpAddress(true) .withGroups(otpServer.ec2Info.securityGroupId) .withDeviceIndex(0); - // If AMI not defined, use the default AMI ID. - String amiId = otpServer.ec2Info.amiId; - if (amiId == null) { - amiId = DEFAULT_AMI_ID; - // Verify that AMI is correctly defined. - if (amiId == null || !ServerController.amiExists(amiId, ec2)) { - statusMessage = String.format( - "Default AMI ID (%s) is missing or bad. Should be provided in config at %s", - amiId, - AMI_CONFIG_PATH); - LOG.error(statusMessage); - status.fail(statusMessage); - } + // Pick proper ami depending on whether graph is being built and what is defined. + String amiId = otpServer.ec2Info.getAmiId(graphAlreadyBuilt); + // Verify that AMI is correctly defined. + if (amiId == null || !ServerController.amiExists(amiId, ec2)) { + statusMessage = String.format( + "AMI ID (%s) is missing or bad. Check the deployment settings or the default value in the app config at %s", + amiId, + AMI_CONFIG_PATH + ); + LOG.error(statusMessage); + status.fail(statusMessage); + return Collections.EMPTY_LIST; + } + // Pick proper instance type depending on whether graph is being built and what is defined. + String instanceType = otpServer.ec2Info.getInstanceType(graphAlreadyBuilt); + // Verify that instance type is correctly defined. + try { + InstanceType.fromValue(instanceType); + } catch (IllegalArgumentException e) { + statusMessage = String.format( + "Instance type (%s) is bad. Check the deployment settings. The default value is %s", + instanceType, + DEFAULT_INSTANCE_TYPE + ); + LOG.error(statusMessage); + status.fail(statusMessage); + return Collections.EMPTY_LIST; } RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) @@ -790,7 +837,12 @@ private String constructUserData(boolean graphAlreadyBuilt) { lines.add("aws configure set default.s3.multipart_chunksize 32MB"); // Get region from config or default to us-east-1 - String region = DataManager.getConfigPropertyAsText("application.data.s3_region"); + String region; + if (customRegion != null) { + region = customRegion; + } else { + region = DataManager.getConfigPropertyAsText("application.data.s3_region"); + } if (region == null) region = "us-east-1"; lines.add(String.format("aws configure set default.region %s", region)); // Create the directory for the graph inputs. @@ -832,20 +884,31 @@ private String constructUserData(boolean graphAlreadyBuilt) { else lines.add(String.format("sudo -H -u ubuntu java -jar -Xmx${MEM}k %s/%s.jar --build %s > $BUILDLOGFILE 2>&1", jarDir, jarName, routerDir)); // Re-upload user data log after build command. lines.add(uploadUserDataLogCommand); - // Upload the build log file and graph to S3. + // Upload the build log file, build report and graph to S3. if (!deployment.r5) { String s3BuildLogPath = joinToS3FolderURI(getBuildLogFilename()); + // upload log file lines.add(String.format("aws s3 cp $BUILDLOGFILE %s ", s3BuildLogPath)); + // upload report if it was generated + String reportPath = String.format("%s/report", routerDir); + lines.add(String.format( + "[ -e %s ] && cd %s && zip -r report.zip report && cd - && aws s3 cp %s.zip %s", + reportPath, + routerDir, + reportPath, + joinToS3FolderURI("graph-build-report.zip") + )); + // upload graph lines.add(String.format("aws s3 cp %s %s ", graphPath, getS3GraphURI())); } } // Determine if graph build/download was successful (and that Graph.obj is not zero bytes). lines.add(String.format("FILESIZE=$(wc -c <%s)", graphPath)); lines.add(String.format("[ -f %s ] && (($FILESIZE > 0)) && GRAPH_STATUS='SUCCESS' || GRAPH_STATUS='FAILURE'", graphPath)); + // Re-upload user data log before indicating that graph build/download is complete. + lines.add(uploadUserDataLogCommand); // Create file with bundle status in web dir to notify Data Tools that download is complete. lines.add(String.format("sudo echo $GRAPH_STATUS > $WEB_DIR/%s", GRAPH_STATUS_FILE)); - // Re-upload user data log before final command (and before optional shutdown statement). - lines.add(uploadUserDataLogCommand); if (deployment.buildGraphOnly) { // If building graph only, tell the instance to shut itself down after the graph build (and log upload) is // complete. @@ -874,10 +937,6 @@ public String getJobRelativePath() { return jobRelativePath; } - public void setJobRelativePath(String jobRelativePath) { - this.jobRelativePath = jobRelativePath; - } - @JsonIgnore public AmazonS3URI getS3FolderURI() { return new AmazonS3URI(String.format("s3://%s/%s", otpServer.s3Bucket, getJobRelativePath())); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 4fcd5e0ed..1640897fb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -8,6 +8,7 @@ import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClientBuilder; import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.amazonaws.services.s3.AmazonS3URI; @@ -69,7 +70,9 @@ public MonitorServerStatusJob(Auth0UserProfile owner, DeployJob deployJob, Insta status.message = "Checking server status..."; startTime = System.currentTimeMillis(); credentials = AWSUtils.getCredentialsForRole(otpServer.role, "monitor-" + instance.getInstanceId()); - ec2 = AWSUtils.getEC2ClientForCredentials(credentials); + ec2 = deployJob.getCustomRegion() == null + ? AWSUtils.getEC2ClientForCredentials(credentials) + : AWSUtils.getEC2ClientForCredentials(credentials, deployJob.getCustomRegion()); } @JsonProperty @@ -106,14 +109,14 @@ public void jobLogic() { wait("bundle download check:" + bundleUrl); bundleIsDownloaded = checkForSuccessfulRequest(bundleUrl); if (jobHasTimedOut()) { - status.fail(String.format("Job timed out while checking for server bundle download status (%s)", instance.getInstanceId())); + failJob("Job timed out while checking for server bundle download status."); return; } } // Check status of bundle download and fail job if there was a failure. String bundleStatus = getUrlAsString(bundleUrl); if (bundleStatus == null || !bundleStatus.contains("SUCCESS")) { - status.fail("Failure encountered while downloading transit bundle."); + failJob("Failure encountered while downloading transit bundle."); return; } long bundleDownloadSeconds = (System.currentTimeMillis() - bundleDownloadStartTime) / 1000; @@ -130,26 +133,23 @@ public void jobLogic() { wait("graph build/download check: " + graphStatusUrl); graphIsAvailable = checkForSuccessfulRequest(graphStatusUrl); if (jobHasTimedOut()) { - message = String.format("Job timed out while waiting for graph build/download (%s). If this was a graph building machine, it may have run out of memory.", instance.getInstanceId()); - LOG.error(message); - status.fail(message); + failJob("Job timed out while waiting for graph build/download. If this was a graph building machine, it may have run out of memory."); return; } } // Check status of bundle download and fail job if there was a failure. String graphStatus = getUrlAsString(graphStatusUrl); if (graphStatus == null || !graphStatus.contains("SUCCESS")) { - message = String.format("Failure encountered while building/downloading graph (%s).", instance.getInstanceId()); - LOG.error(message); - status.fail(message); + failJob("Failure encountered while building/downloading graph."); return; } graphBuildSeconds = (System.currentTimeMillis() - graphBuildStartTime) / 1000; message = String.format("Graph build/download completed in %d seconds!", graphBuildSeconds); LOG.info(message); // If only task is to build graph, this machine's job is complete and we can consider this job done. - if (deployment.buildGraphOnly) { + if (deployment.buildGraphOnly || (!graphAlreadyBuilt && otpServer.ec2Info.hasSeparateGraphBuildConfig())) { status.update(false, message, 100); + LOG.info("View logs at {}", getUserDataLogS3Path()); return; } status.update("Loading graph...", 70); @@ -162,9 +162,7 @@ public void jobLogic() { wait("router to become available: " + routerUrl); routerIsAvailable = checkForSuccessfulRequest(routerUrl); if (jobHasTimedOut()) { - message = String.format("Job timed out while waiting for trip planner to start up (%s)", instance.getInstanceId()); - status.fail(message); - LOG.error(message); + failJob("Job timed out while waiting for trip planner to start up."); return; } } @@ -173,9 +171,10 @@ public void jobLogic() { // After the router is available, the EC2 instance can be registered with the load balancer. // REGISTER INSTANCE WITH LOAD BALANCER // Use alternative credentials if they exist. - AmazonElasticLoadBalancing elbClient = AmazonElasticLoadBalancingClient.builder() - .withCredentials(credentials) - .build(); + AmazonElasticLoadBalancingClientBuilder builder = AmazonElasticLoadBalancingClient.builder() + .withCredentials(credentials); + if (deployJob.getCustomRegion() != null) builder.withRegion(deployJob.getCustomRegion()); + AmazonElasticLoadBalancing elbClient = builder.build(); RegisterTargetsRequest registerTargetsRequest = new RegisterTargetsRequest() .withTargetGroupArn(otpServer.ec2Info.targetGroupArn) .withTargets(new TargetDescription().withId(instance.getInstanceId())); @@ -184,14 +183,28 @@ public void jobLogic() { message = String.format("Server successfully registered with load balancer %s. OTP running at %s", otpServer.ec2Info.targetGroupArn, routerUrl); LOG.info(message); status.update(false, message, 100, true); + LOG.info("View logs at {}", getUserDataLogS3Path()); deployJob.incrementCompletedServers(); } else { - message = String.format("There is no load balancer under which to register ec2 instance %s.", instance.getInstanceId()); - LOG.error(message); - status.fail(message); + failJob("There is no load balancer under which to register ec2 instance."); } } + /** + * Gets the expected path to the user data logs that get uploaded to s3 + */ + private String getUserDataLogS3Path() { + return String.format("%s/%s.log", deployJob.getS3FolderURI(), instance.getInstanceId()); + } + + /** + * Helper that fails with a helpful message about where to find uploaded logs. + */ + private void failJob(String message) { + LOG.error(message); + status.fail(String.format("%s Check logs at: %s", message, getUserDataLogS3Path())); + } + /** Determine if job has passed time limit for its run time. */ private boolean jobHasTimedOut() { long runTime = System.currentTimeMillis() - startTime; diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index cd0426926..391151fce 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -122,11 +122,20 @@ public List retrieveEC2Instances() { Filter deploymentFilter = new Filter("tag:deploymentId", Collections.singletonList(id)); // Check if the latest deployment used alternative credentials/AWS role. String role = null; + String region = null; if (this.latest() != null) { OtpServer server = Persistence.servers.getById(this.latest().serverId); - if (server != null) role = server.role; + if (server != null) { + role = server.role; + if (server.ec2Info != null) { + region = server.ec2Info.region; + } + } } - return DeploymentController.fetchEC2InstanceSummaries(AWSUtils.getEC2ClientForRole(role), deploymentFilter); + return DeploymentController.fetchEC2InstanceSummaries( + AWSUtils.getEC2ClientForRole(role, region), + deploymentFilter + ); } public void storeFeedVersions(Collection versions) { diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java index fcbd37aa7..bae33f20a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -1,9 +1,9 @@ package com.conveyal.datatools.manager.models; +import com.conveyal.datatools.manager.DataManager; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; -import java.util.List; /** * Contains the fields specific to starting up new EC2 servers for an ELB target group. If null, at least one internal @@ -12,11 +12,15 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class EC2Info implements Serializable { private static final long serialVersionUID = 1L; + + public static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; + public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; + /** Empty constructor for serialization. */ public EC2Info () {} /** * The AWS-style instance type (e.g., t2.medium) to use for new EC2 machines. Defaults to - * {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} if null during deployment. + * {@link com.conveyal.datatools.manager.models.EC2Info#DEFAULT_INSTANCE_TYPE} if null during deployment. */ public String instanceType; /** Number of instances to spin up and add to target group. If zero, defaults to 1. */ @@ -25,12 +29,73 @@ public EC2Info () {} public String subnetId; /** The security group ID associated with the target group. */ public String securityGroupId; - /** The Amazon machine image (AMI) to be used for the OTP EC2 machines. */ + /** + * The Amazon machine image (AMI) to be used for the OTP EC2 machines. Defaults to the app config value at + * {@link com.conveyal.datatools.manager.models.EC2Info#AMI_CONFIG_PATH} if null during deployment. + */ public String amiId; - /** The IAM instance profile ARN that the OTP EC2 server should assume. For example, arn:aws:iam::123456789012:instance-profile/otp-ec2-role */ + /** + * The AWS-style instance type (e.g., t2.medium) to use for new EC2 machines used specifically for graph building. + * Defaults to {@link com.conveyal.datatools.manager.models.EC2Info#instanceType} if null during deployment. + */ + public String buildInstanceType; + /** + * The Amazon machine image (AMI) (e.g. ami-12345678) to be used for the OTP EC2 machine used specifically for + * graph building. Defaults to {@link com.conveyal.datatools.manager.models.EC2Info#amiId} if null during deployment. + */ + public String buildAmiId; + /** + * The IAM instance profile ARN that the OTP EC2 server should assume. For example, + * arn:aws:iam::123456789012:instance-profile/otp-ec2-role + */ public String iamInstanceProfileArn; /** The AWS key file (.pem) that should be used to set up OTP EC2 servers (gives a way for admins to SSH into machine). */ public String keyName; /** The target group to deploy new EC2 instances to. */ public String targetGroupArn; + /** An optional custom AWS region (e.g., us-east-1) */ + public String region; + + /** + * Returns true if the instance type or ami ids are set and are different for a graph build. + */ + public boolean hasSeparateGraphBuildConfig() { + return ( + buildInstanceType != null && !buildInstanceType.equals(instanceType) + ) || ( + buildAmiId != null && !buildAmiId.equals(amiId) + ); + } + + /** + * Returns the appropriate ami ID to use when creating a new ec2 instance during a deploy job. + * + * @param graphAlreadyBuilt whether or not a graph has already been built. If false, this means a build ami should + * be used if available. + */ + public String getAmiId(boolean graphAlreadyBuilt) { + if (!graphAlreadyBuilt && buildAmiId != null) { + return buildAmiId; + } else if (amiId != null) { + return amiId; + } else { + return DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); + } + } + + /** + * Returns the appropriate instance type to use when creating a new ec2 instance during a deploy job. + * + * @param graphAlreadyBuilt whether or not a graph has already been built. If false, this means a build instance + * type should be used if available. + */ + public String getInstanceType(boolean graphAlreadyBuilt) { + if (!graphAlreadyBuilt && buildInstanceType != null) { + return buildInstanceType; + } else if (instanceType != null) { + return instanceType; + } else { + return DEFAULT_INSTANCE_TYPE; + } + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java index c86bdabf4..77e84b64a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java @@ -59,13 +59,22 @@ public List retrieveEC2InstanceSummaries() { // Prevent calling EC2 method on servers that do not have EC2 info defined because this is a JSON property. if (ec2Info == null) return Collections.EMPTY_LIST; Filter serverFilter = new Filter("tag:serverId", Collections.singletonList(id)); - return DeploymentController.fetchEC2InstanceSummaries(AWSUtils.getEC2ClientForRole(this.role), serverFilter); + return DeploymentController.fetchEC2InstanceSummaries( + AWSUtils.getEC2ClientForRole(this.role, ec2Info.region), + serverFilter + ); } public List retrieveEC2Instances() { - if (!"true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled"))) return Collections.EMPTY_LIST; + if ( + !"true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled")) || + ec2Info == null + ) return Collections.EMPTY_LIST; Filter serverFilter = new Filter("tag:serverId", Collections.singletonList(id)); - return DeploymentController.fetchEC2Instances(AWSUtils.getEC2ClientForRole(this.role), serverFilter); + return DeploymentController.fetchEC2Instances( + AWSUtils.getEC2ClientForRole(this.role, ec2Info.region), + serverFilter + ); } @JsonProperty("organizationId") diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java index 151efdb52..df8f47cf0 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java @@ -97,7 +97,10 @@ public void canDeployFromPrebuiltGraph () { public static void cleanUp() { List instances = server.retrieveEC2Instances(); List ids = getIds(instances); - terminateInstances(AWSUtils.getEC2ClientForRole(server.role), ids); + terminateInstances( + AWSUtils.getEC2ClientForRole(server.role, server.ec2Info == null ? null : server.ec2Info.region), + ids + ); } }