Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Custom ec2 ami, instance type and region #282

Merged
merged 7 commits into from
Mar 10, 2020
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public MonitorableJob(Auth0UserProfile owner, String name, JobType type) {
}
this.owner = owner;
this.name = name;
status.name = name;
this.type = type;
registerJob();
}
Expand Down Expand Up @@ -279,6 +280,7 @@ public void update (boolean isError, String message, double percentComplete, boo
}

public void fail (String message, Exception e) {
LOG.warn("Job `{}` has been failed with message: `{}` and Exception: `{}`", name, message, e);
this.error = true;
this.percentComplete = 100;
this.completed = true;
Expand All @@ -288,6 +290,7 @@ public void fail (String message, Exception e) {
}

public void fail (String message) {
LOG.warn("Job `{}` has been failed with message: `{}`", name, message);
this.error = true;
this.percentComplete = 100;
this.completed = true;
Expand Down
14 changes: 12 additions & 2 deletions src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,11 @@ public static AWSStaticCredentialsProvider getCredentialsForRole(String role, St
* Shorthand method to obtain an EC2 client for the provided role ARN. If role is null, the default EC2 credentials
* will be used.
*/
public static AmazonEC2 getEC2ClientForRole (String role) {
public static AmazonEC2 getEC2ClientForRole (String role, String region) {
AWSStaticCredentialsProvider credentials = getCredentialsForRole(role, "ec2-client");
return getEC2ClientForCredentials(credentials);
return region == null
? getEC2ClientForCredentials(credentials)
: getEC2ClientForCredentials(credentials, region);
}

/**
Expand All @@ -161,6 +163,14 @@ public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider crede
return AmazonEC2Client.builder().withCredentials(credentials).build();
}

/**
* Shorthand method to obtain an EC2 client for the provided credentials and region. If credentials are null, the
* default EC2 credentials will be used.
*/
public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider credentials, String region) {
return AmazonEC2Client.builder().withCredentials(credentials).withRegion(region).build();
}

/**
* Shorthand method to obtain an S3 client for the provided credentials. If credentials are null, the default EC2
* credentials will be used.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,8 @@ public static void logRequestOrResponse(
LOG.warn("Request object is null. Cannot log.");
return;
}
// don't log job status requests/responses, they clutter things up
if (request.pathInfo().contains("status/jobs")) return;
evansiroky marked this conversation as resolved.
Show resolved Hide resolved
Auth0UserProfile userProfile = request.attribute("user");
String userEmail = userProfile != null ? userProfile.getEmail() : "no-auth";
String queryString = request.queryParams().size() > 0 ? "?" + request.queryString() : "";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.Reservation;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.AmazonS3URI;
import com.conveyal.datatools.common.status.MonitorableJob;
import com.conveyal.datatools.common.utils.AWSUtils;
Expand Down Expand Up @@ -47,6 +48,7 @@

import static com.conveyal.datatools.common.utils.AWSUtils.downloadFromS3;
import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds;
import static spark.Spark.delete;
import static spark.Spark.get;
import static spark.Spark.options;
Expand Down Expand Up @@ -101,6 +103,7 @@ private static String downloadBuildArtifact (Request req, Response res) {
// Default client to use if no role was used during the deployment.
AmazonS3 s3Client = FeedStore.s3Client;
String role = null;
String region = null;
String uriString;
String filename = req.queryParams("filename");
if (filename == null) {
Expand Down Expand Up @@ -131,17 +134,23 @@ private static String downloadBuildArtifact (Request req, Response res) {
logMessageAndHalt(req, 400, "The deployment does not have job history or associated server information to construct URI for build artifact. " + uriString);
return null;
}
region = server.ec2Info == null ? null : server.ec2Info.region;
uriString = String.format("s3://%s/bundles/%s/%s/%s", server.s3Bucket, deployment.projectId, deployment.id, jobId);
LOG.warn("Could not find deploy summary for job. Attempting to use {}", uriString);
}
} else {
// If summary is readily available, just use the ready-to-use build artifacts field.
uriString = summaryToDownload.buildArtifactsFolder;
role = summaryToDownload.role;
region = summaryToDownload.ec2Info == null ? null : summaryToDownload.ec2Info.region;
}
AmazonS3URI uri = new AmazonS3URI(uriString);
// Assume the alternative role if needed to download the deploy artifact.
if (role != null) s3Client = AWSUtils.getS3ClientForRole(role);
if (role != null) {
s3Client = AWSUtils.getS3ClientForRole(role, region);
} else if (region != null) {
s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), region);
}
return downloadFromS3(s3Client, uri.getBucket(), String.join("/", uri.getKey(), filename), false, res);
}

Expand Down Expand Up @@ -373,7 +382,12 @@ private static boolean terminateEC2InstanceForDeployment(Request req, Response r
}
}
// If checks are ok, terminate instances.
boolean success = ServerController.deRegisterAndTerminateInstances(credentials, targetGroupArn, idsToTerminate);
boolean success = ServerController.deRegisterAndTerminateInstances(
credentials,
targetGroupArn,
latest.ec2Info.region,
idsToTerminate
);
if (!success) {
logMessageAndHalt(req, 400, "Could not complete termination request");
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
import com.amazonaws.services.ec2.model.AmazonEC2Exception;
import com.amazonaws.services.ec2.model.DescribeImagesRequest;
import com.amazonaws.services.ec2.model.DescribeImagesResult;
Expand All @@ -19,6 +20,7 @@
import com.amazonaws.services.ec2.model.TerminateInstancesResult;
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing;
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient;
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClientBuilder;
import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException;
import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest;
import com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersRequest;
Expand Down Expand Up @@ -62,6 +64,7 @@
import static com.conveyal.datatools.common.utils.SparkUtils.getPOJOFromRequestBody;
import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE;
import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds;
import static spark.Spark.delete;
import static spark.Spark.get;
import static spark.Spark.options;
Expand Down Expand Up @@ -117,7 +120,10 @@ private static OtpServer terminateEC2InstancesForServer(Request req, Response re
OtpServer server = getServerWithPermissions(req, res);
List<Instance> instances = server.retrieveEC2Instances();
List<String> ids = getIds(instances);
AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole(server.role);
AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole(
server.role,
server.ec2Info == null ? null : server.ec2Info.region
);
terminateInstances(ec2Client, ids);
for (Deployment deployment : Deployment.retrieveDeploymentForServerAndRouterId(server.id, null)) {
Persistence.deployments.updateField(deployment.id, "deployedTo", null);
Expand Down Expand Up @@ -157,7 +163,12 @@ public static TerminateInstancesResult terminateInstances(AmazonEC2 ec2Client, L
* De-register instances from the specified target group/load balancer and terminate the instances.
*
*/
public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvider credentials, String targetGroupArn, List<String> instanceIds) {
public static boolean deRegisterAndTerminateInstances(
AWSStaticCredentialsProvider credentials,
String targetGroupArn,
String region,
List<String> instanceIds
) {
LOG.info("De-registering instances from load balancer {}", instanceIds);
TargetDescription[] targetDescriptions = instanceIds.stream()
.map(id -> new TargetDescription().withId(id))
Expand All @@ -169,12 +180,21 @@ public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvid
AmazonElasticLoadBalancing elbClient = elb;
AmazonEC2 ec2Client = ec2;
// If OTP Server has role defined/alt credentials, override default AWS clients.
if (credentials != null) {
elbClient = AmazonElasticLoadBalancingClient.builder().withCredentials(credentials).build();
ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build();
if (credentials != null || region != null) {
AmazonElasticLoadBalancingClientBuilder elbBuilder = AmazonElasticLoadBalancingClient.builder();
AmazonEC2ClientBuilder ec2Builder = AmazonEC2Client.builder();
if (credentials != null) {
elbBuilder.withCredentials(credentials);
ec2Builder.withCredentials(credentials);
}
if (region != null) {
elbBuilder.withRegion(region);
ec2Builder.withRegion(region);
}
elbClient = elbBuilder.build();
ec2Client = ec2Builder.build();
}
elbClient.deregisterTargets(request);
// FIXME default to regular ec2 client
ServerController.terminateInstances(ec2Client, instanceIds);
} catch (AmazonEC2Exception | AmazonElasticLoadBalancingException e) {
LOG.warn("Could not terminate EC2 instances: " + String.join(",", instanceIds), e);
Expand Down Expand Up @@ -250,6 +270,7 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc
AWSStaticCredentialsProvider credentials = AWSUtils.getCredentialsForRole(server.role, "validate");
// If alternative credentials exist, override the default AWS clients.
if (credentials != null) {
// build ec2 client
ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build();
iamClient = AmazonIdentityManagementClientBuilder.standard().withCredentials(credentials).build();
s3Client = AWSUtils.getS3ClientForRole(server.role, null);
Expand All @@ -263,9 +284,23 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc
// If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to
// AWS. (e.g., target group ARN and instance type).
if (server.ec2Info != null) {
// do some custom items if a custom region should be used
evansiroky marked this conversation as resolved.
Show resolved Hide resolved
if (server.ec2Info.region != null) {
AmazonEC2ClientBuilder builder = AmazonEC2Client.builder();
if (credentials != null) {
builder.withCredentials(credentials);
}
builder.withRegion(server.ec2Info.region);
ec2Client = builder.build();
if (credentials != null) {
s3Client = AWSUtils.getS3ClientForRole(server.role, server.ec2Info.region);
} else {
s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), server.ec2Info.region);
}
}
validateInstanceType(server.ec2Info.instanceType, req);
// Validate target group and get load balancer to validate subnetId and security group ID.
LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info.targetGroupArn, req, credentials);
LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info, req, credentials);
validateSubnetId(loadBalancer, server.ec2Info, req, ec2Client);
validateSecurityGroupId(loadBalancer, server.ec2Info, req);
// Validate remaining AWS values.
Expand Down Expand Up @@ -304,7 +339,7 @@ private static boolean verifyS3WritePermissions(AmazonS3 s3Client, String s3Buck
s3Client.putObject(s3Bucket, key, File.createTempFile("test", ".zip"));
s3Client.deleteObject(s3Bucket, key);
} catch (IOException | AmazonS3Exception e) {
LOG.warn("S3 client cannot write to bucket" + s3Bucket, e);
LOG.warn("S3 client cannot write to bucket: " + s3Bucket, e);
return false;
}
return true;
Expand Down Expand Up @@ -448,16 +483,21 @@ private static void validateInstanceType(String instanceType, Request req) {
* - https://serverfault.com/a/865422
* - https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html
*/
private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn, AWSStaticCredentialsProvider credentials) {
private static LoadBalancer getLoadBalancerForTargetGroup (EC2Info ec2Info, AWSStaticCredentialsProvider credentials) {
// If alternative credentials exist, use them to assume the role. Otherwise, use default ELB client.
AmazonElasticLoadBalancing elbClient = credentials != null
? AmazonElasticLoadBalancingClient.builder()
.withCredentials(credentials)
.build()
: elb;
AmazonElasticLoadBalancingClientBuilder builder = AmazonElasticLoadBalancingClient.builder();
if (credentials != null) {
builder.withCredentials(credentials);
}

if (ec2Info.region != null) {
builder.withRegion(ec2Info.region);
}

AmazonElasticLoadBalancing elbClient = builder.build();
try {
DescribeTargetGroupsRequest targetGroupsRequest = new DescribeTargetGroupsRequest()
.withTargetGroupArns(targetGroupArn);
.withTargetGroupArns(ec2Info.targetGroupArn);
List<TargetGroup> targetGroups = elbClient.describeTargetGroups(targetGroupsRequest).getTargetGroups();
for (TargetGroup tg : targetGroups) {
DescribeLoadBalancersRequest request = new DescribeLoadBalancersRequest()
Expand All @@ -467,7 +507,7 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn
return result.getLoadBalancers().iterator().next();
}
} catch (AmazonElasticLoadBalancingException e) {
LOG.warn("Invalid value for Target Group ARN: {}", targetGroupArn);
LOG.warn("Invalid value for Target Group ARN: {}", ec2Info.targetGroupArn);
}
// If no target group/load balancer found, return null.
return null;
Expand All @@ -477,11 +517,13 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn
* Validate that ELB target group exists and is not empty and return associated load balancer for validating related
* fields.
*/
private static LoadBalancer validateTargetGroupAndGetLoadBalancer(String targetGroupArn, Request req, AWSStaticCredentialsProvider credentials) {
if (isEmpty(targetGroupArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN.");
private static LoadBalancer validateTargetGroupAndGetLoadBalancer(EC2Info ec2Info, Request req, AWSStaticCredentialsProvider credentials) {
if (isEmpty(ec2Info.targetGroupArn)) {
logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN.");
}
// Get load balancer for target group. This essentially checks that the target group exists and is assigned
// to a load balancer.
LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(targetGroupArn, credentials);
LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(ec2Info, credentials);
if (loadBalancer == null) {
logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group or Load Balancer.");
}
Expand Down
Loading