Skip to content

Commit

Permalink
Merge pull request #282 from ibi-group/custom-ec2-ami-and-region
Browse files Browse the repository at this point in the history 
Custom ec2 ami, instance type and region
  • Loading branch information
@landonreed
landonreed authored Mar 10, 2020
2 parents 134a323 + 3d77359 commit 13fbbe2
Show file tree
Hide file tree
Showing 11 changed files with 325 additions and 93 deletions.
3 changes: 3 additions & 0 deletions src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public MonitorableJob(Auth0UserProfile owner, String name, JobType type) {
}
this.owner = owner;
this.name = name;
status.name = name;
this.type = type;
registerJob();
}
Expand Down Expand Up @@ -279,6 +280,7 @@ public void update (boolean isError, String message, double percentComplete, boo
}

public void fail (String message, Exception e) {
LOG.warn("Job `{}` has been failed with message: `{}` and Exception: `{}`", name, message, e);
this.error = true;
this.percentComplete = 100;
this.completed = true;
Expand All @@ -288,6 +290,7 @@ public void fail (String message, Exception e) {
}

public void fail (String message) {
LOG.warn("Job `{}` has been failed with message: `{}`", name, message);
this.error = true;
this.percentComplete = 100;
this.completed = true;
Expand Down
14 changes: 12 additions & 2 deletions src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,11 @@ public static AWSStaticCredentialsProvider getCredentialsForRole(String role, St
* Shorthand method to obtain an EC2 client for the provided role ARN. If role is null, the default EC2 credentials
* will be used.
*/
public static AmazonEC2 getEC2ClientForRole (String role) {
public static AmazonEC2 getEC2ClientForRole (String role, String region) {
AWSStaticCredentialsProvider credentials = getCredentialsForRole(role, "ec2-client");
return getEC2ClientForCredentials(credentials);
return region == null
? getEC2ClientForCredentials(credentials)
: getEC2ClientForCredentials(credentials, region);
}

/**
Expand All @@ -161,6 +163,14 @@ public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider crede
return AmazonEC2Client.builder().withCredentials(credentials).build();
}

/**
* Shorthand method to obtain an EC2 client for the provided credentials and region. If credentials are null, the
* default EC2 credentials will be used.
*/
public static AmazonEC2 getEC2ClientForCredentials (AWSCredentialsProvider credentials, String region) {
return AmazonEC2Client.builder().withCredentials(credentials).withRegion(region).build();
}

/**
* Shorthand method to obtain an S3 client for the provided credentials. If credentials are null, the default EC2
* credentials will be used.
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,8 @@ public static void logRequestOrResponse(
LOG.warn("Request object is null. Cannot log.");
return;
}
// don't log job status requests/responses, they clutter things up
if (request.pathInfo().contains("status/jobs")) return;
Auth0UserProfile userProfile = request.attribute("user");
String userEmail = userProfile != null ? userProfile.getEmail() : "no-auth";
String queryString = request.queryParams().size() > 0 ? "?" + request.queryString() : "";
Expand Down
18 changes: 16 additions & 2 deletions src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.Reservation;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.AmazonS3URI;
import com.conveyal.datatools.common.status.MonitorableJob;
import com.conveyal.datatools.common.utils.AWSUtils;
Expand Down Expand Up @@ -47,6 +48,7 @@

import static com.conveyal.datatools.common.utils.AWSUtils.downloadFromS3;
import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds;
import static spark.Spark.delete;
import static spark.Spark.get;
import static spark.Spark.options;
Expand Down Expand Up @@ -101,6 +103,7 @@ private static String downloadBuildArtifact (Request req, Response res) {
// Default client to use if no role was used during the deployment.
AmazonS3 s3Client = FeedStore.s3Client;
String role = null;
String region = null;
String uriString;
String filename = req.queryParams("filename");
if (filename == null) {
Expand Down Expand Up @@ -131,17 +134,23 @@ private static String downloadBuildArtifact (Request req, Response res) {
logMessageAndHalt(req, 400, "The deployment does not have job history or associated server information to construct URI for build artifact. " + uriString);
return null;
}
region = server.ec2Info == null ? null : server.ec2Info.region;
uriString = String.format("s3://%s/bundles/%s/%s/%s", server.s3Bucket, deployment.projectId, deployment.id, jobId);
LOG.warn("Could not find deploy summary for job. Attempting to use {}", uriString);
}
} else {
// If summary is readily available, just use the ready-to-use build artifacts field.
uriString = summaryToDownload.buildArtifactsFolder;
role = summaryToDownload.role;
region = summaryToDownload.ec2Info == null ? null : summaryToDownload.ec2Info.region;
}
AmazonS3URI uri = new AmazonS3URI(uriString);
// Assume the alternative role if needed to download the deploy artifact.
if (role != null) s3Client = AWSUtils.getS3ClientForRole(role);
if (role != null) {
s3Client = AWSUtils.getS3ClientForRole(role, region);
} else if (region != null) {
s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), region);
}
return downloadFromS3(s3Client, uri.getBucket(), String.join("/", uri.getKey(), filename), false, res);
}

Expand Down Expand Up @@ -373,7 +382,12 @@ private static boolean terminateEC2InstanceForDeployment(Request req, Response r
}
}
// If checks are ok, terminate instances.
boolean success = ServerController.deRegisterAndTerminateInstances(credentials, targetGroupArn, idsToTerminate);
boolean success = ServerController.deRegisterAndTerminateInstances(
credentials,
targetGroupArn,
latest.ec2Info.region,
idsToTerminate
);
if (!success) {
logMessageAndHalt(req, 400, "Could not complete termination request");
return false;
Expand Down
89 changes: 67 additions & 22 deletions src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
import com.amazonaws.services.ec2.model.AmazonEC2Exception;
import com.amazonaws.services.ec2.model.DescribeImagesRequest;
import com.amazonaws.services.ec2.model.DescribeImagesResult;
Expand All @@ -19,6 +20,7 @@
import com.amazonaws.services.ec2.model.TerminateInstancesResult;
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing;
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient;
import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClientBuilder;
import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException;
import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest;
import com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersRequest;
Expand Down Expand Up @@ -61,7 +63,8 @@

import static com.conveyal.datatools.common.utils.SparkUtils.getPOJOFromRequestBody;
import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE;
import static com.conveyal.datatools.manager.models.EC2Info.DEFAULT_INSTANCE_TYPE;
import static com.conveyal.datatools.manager.persistence.FeedStore.getAWSCreds;
import static spark.Spark.delete;
import static spark.Spark.get;
import static spark.Spark.options;
Expand Down Expand Up @@ -117,7 +120,10 @@ private static OtpServer terminateEC2InstancesForServer(Request req, Response re
OtpServer server = getServerWithPermissions(req, res);
List<Instance> instances = server.retrieveEC2Instances();
List<String> ids = getIds(instances);
AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole(server.role);
AmazonEC2 ec2Client = AWSUtils.getEC2ClientForRole(
server.role,
server.ec2Info == null ? null : server.ec2Info.region
);
terminateInstances(ec2Client, ids);
for (Deployment deployment : Deployment.retrieveDeploymentForServerAndRouterId(server.id, null)) {
Persistence.deployments.updateField(deployment.id, "deployedTo", null);
Expand Down Expand Up @@ -157,7 +163,12 @@ public static TerminateInstancesResult terminateInstances(AmazonEC2 ec2Client, L
* De-register instances from the specified target group/load balancer and terminate the instances.
*
*/
public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvider credentials, String targetGroupArn, List<String> instanceIds) {
public static boolean deRegisterAndTerminateInstances(
AWSStaticCredentialsProvider credentials,
String targetGroupArn,
String region,
List<String> instanceIds
) {
LOG.info("De-registering instances from load balancer {}", instanceIds);
TargetDescription[] targetDescriptions = instanceIds.stream()
.map(id -> new TargetDescription().withId(id))
Expand All @@ -169,12 +180,21 @@ public static boolean deRegisterAndTerminateInstances(AWSStaticCredentialsProvid
AmazonElasticLoadBalancing elbClient = elb;
AmazonEC2 ec2Client = ec2;
// If OTP Server has role defined/alt credentials, override default AWS clients.
if (credentials != null) {
elbClient = AmazonElasticLoadBalancingClient.builder().withCredentials(credentials).build();
ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build();
if (credentials != null || region != null) {
AmazonElasticLoadBalancingClientBuilder elbBuilder = AmazonElasticLoadBalancingClient.builder();
AmazonEC2ClientBuilder ec2Builder = AmazonEC2Client.builder();
if (credentials != null) {
elbBuilder.withCredentials(credentials);
ec2Builder.withCredentials(credentials);
}
if (region != null) {
elbBuilder.withRegion(region);
ec2Builder.withRegion(region);
}
elbClient = elbBuilder.build();
ec2Client = ec2Builder.build();
}
elbClient.deregisterTargets(request);
// FIXME default to regular ec2 client
ServerController.terminateInstances(ec2Client, instanceIds);
} catch (AmazonEC2Exception | AmazonElasticLoadBalancingException e) {
LOG.warn("Could not terminate EC2 instances: " + String.join(",", instanceIds), e);
Expand Down Expand Up @@ -250,6 +270,7 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc
AWSStaticCredentialsProvider credentials = AWSUtils.getCredentialsForRole(server.role, "validate");
// If alternative credentials exist, override the default AWS clients.
if (credentials != null) {
// build ec2 client
ec2Client = AmazonEC2Client.builder().withCredentials(credentials).build();
iamClient = AmazonIdentityManagementClientBuilder.standard().withCredentials(credentials).build();
s3Client = AWSUtils.getS3ClientForRole(server.role, null);
Expand All @@ -263,9 +284,23 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc
// If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to
// AWS. (e.g., target group ARN and instance type).
if (server.ec2Info != null) {
// create custom clients if credentials and or a custom region exist
if (server.ec2Info.region != null) {
AmazonEC2ClientBuilder builder = AmazonEC2Client.builder();
if (credentials != null) {
builder.withCredentials(credentials);
}
builder.withRegion(server.ec2Info.region);
ec2Client = builder.build();
if (credentials != null) {
s3Client = AWSUtils.getS3ClientForRole(server.role, server.ec2Info.region);
} else {
s3Client = AWSUtils.getS3ClientForCredentials(getAWSCreds(), server.ec2Info.region);
}
}
validateInstanceType(server.ec2Info.instanceType, req);
// Validate target group and get load balancer to validate subnetId and security group ID.
LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info.targetGroupArn, req, credentials);
LoadBalancer loadBalancer = validateTargetGroupAndGetLoadBalancer(server.ec2Info, req, credentials);
validateSubnetId(loadBalancer, server.ec2Info, req, ec2Client);
validateSecurityGroupId(loadBalancer, server.ec2Info, req);
// Validate remaining AWS values.
Expand Down Expand Up @@ -304,7 +339,7 @@ private static boolean verifyS3WritePermissions(AmazonS3 s3Client, String s3Buck
s3Client.putObject(s3Bucket, key, File.createTempFile("test", ".zip"));
s3Client.deleteObject(s3Bucket, key);
} catch (IOException | AmazonS3Exception e) {
LOG.warn("S3 client cannot write to bucket" + s3Bucket, e);
LOG.warn("S3 client cannot write to bucket: " + s3Bucket, e);
return false;
}
return true;
Expand Down Expand Up @@ -429,14 +464,17 @@ private static void validateSubnetId(LoadBalancer loadBalancer, EC2Info ec2Info,

/**
* Validate that EC2 instance type (e.g., t2-medium) exists. This value can be empty and will default to
* {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} at deploy time.
* {@link com.conveyal.datatools.manager.models.EC2Info#DEFAULT_INSTANCE_TYPE} at deploy time.
*/
private static void validateInstanceType(String instanceType, Request req) {
if (instanceType == null) return;
try {
InstanceType.fromValue(instanceType);
} catch (IllegalArgumentException e) {
String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE);
String message = String.format(
"Must provide valid instance type (if none provided, defaults to %s).",
DEFAULT_INSTANCE_TYPE
);
logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e);
}
}
Expand All @@ -448,16 +486,21 @@ private static void validateInstanceType(String instanceType, Request req) {
* - https://serverfault.com/a/865422
* - https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-limits.html
*/
private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn, AWSStaticCredentialsProvider credentials) {
private static LoadBalancer getLoadBalancerForTargetGroup (EC2Info ec2Info, AWSStaticCredentialsProvider credentials) {
// If alternative credentials exist, use them to assume the role. Otherwise, use default ELB client.
AmazonElasticLoadBalancing elbClient = credentials != null
? AmazonElasticLoadBalancingClient.builder()
.withCredentials(credentials)
.build()
: elb;
AmazonElasticLoadBalancingClientBuilder builder = AmazonElasticLoadBalancingClient.builder();
if (credentials != null) {
builder.withCredentials(credentials);
}

if (ec2Info.region != null) {
builder.withRegion(ec2Info.region);
}

AmazonElasticLoadBalancing elbClient = builder.build();
try {
DescribeTargetGroupsRequest targetGroupsRequest = new DescribeTargetGroupsRequest()
.withTargetGroupArns(targetGroupArn);
.withTargetGroupArns(ec2Info.targetGroupArn);
List<TargetGroup> targetGroups = elbClient.describeTargetGroups(targetGroupsRequest).getTargetGroups();
for (TargetGroup tg : targetGroups) {
DescribeLoadBalancersRequest request = new DescribeLoadBalancersRequest()
Expand All @@ -467,7 +510,7 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn
return result.getLoadBalancers().iterator().next();
}
} catch (AmazonElasticLoadBalancingException e) {
LOG.warn("Invalid value for Target Group ARN: {}", targetGroupArn);
LOG.warn("Invalid value for Target Group ARN: {}", ec2Info.targetGroupArn);
}
// If no target group/load balancer found, return null.
return null;
Expand All @@ -477,11 +520,13 @@ private static LoadBalancer getLoadBalancerForTargetGroup (String targetGroupArn
* Validate that ELB target group exists and is not empty and return associated load balancer for validating related
* fields.
*/
private static LoadBalancer validateTargetGroupAndGetLoadBalancer(String targetGroupArn, Request req, AWSStaticCredentialsProvider credentials) {
if (isEmpty(targetGroupArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN.");
private static LoadBalancer validateTargetGroupAndGetLoadBalancer(EC2Info ec2Info, Request req, AWSStaticCredentialsProvider credentials) {
if (isEmpty(ec2Info.targetGroupArn)) {
logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN.");
}
// Get load balancer for target group. This essentially checks that the target group exists and is assigned
// to a load balancer.
LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(targetGroupArn, credentials);
LoadBalancer loadBalancer = getLoadBalancerForTargetGroup(ec2Info, credentials);
if (loadBalancer == null) {
logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group or Load Balancer.");
}
Expand Down
Loading

0 comments on commit 13fbbe2

Please sign in to comment.