From c0d8fdca3052f95e41b97b8bf31a7eeb54739db9 Mon Sep 17 00:00:00 2001 From: "Moellering, Sascha" Date: Sun, 27 May 2018 19:10:18 +0200 Subject: [PATCH] - Added deployment logic in bootstrap verticle - Ported AWS Lambda functions to Golang - Added Service Auto Scaling for Fargate --- NOTICE | 2 +- README.md | 2 +- infrastructure/lambda.yaml | 17 ++-- master.yaml | 2 +- .../reactive-vertx/Dockerfile | 1 - .../tracking-service/reactive-vertx/pom.xml | 6 -- .../java/com/amazon/BootStrapVerticle.java | 43 ++++++++-- services/tracking-service/service.yaml | 82 ++++++++++++++++++- 8 files changed, 127 insertions(+), 28 deletions(-) diff --git a/NOTICE b/NOTICE index a307967..7cf5a41 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,2 @@ reactive-refarch-cloudformation -Copyright 2011-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2011-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/README.md b/README.md index 801fafa..e9dc9f2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Reactive Microservices Architecture with Amazon ECS, AWS Lambda, Amazon Kinesis Streams, Amazon ElastiCache, and Amazon DynamoDB +# Reactive Microservices Architectures with Amazon ECS, AWS Lambda, Amazon Kinesis Streams, Amazon ElastiCache, and Amazon DynamoDB This reference architecture provides a set of YAML templates for deploying a reactive microservices architecture based on [Amazon Elastic Container Service (Amazon ECS)](http://docs.aws.amazon.com/AmazonECS/latest/developerguide/Welcome.html), [AWS Lambda](https://aws.amazon.com/lambda/), [Amazon Kinesis Streams](https://aws.amazon.com/kinesis/streams/), [Amazon ElastiCache](https://aws.amazon.com/elasticache/), and [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) using [Amazon CloudFormation](https://aws.amazon.com/cloudformation/). diff --git a/infrastructure/lambda.yaml b/infrastructure/lambda.yaml index d479775..07bfd92 100644 --- a/infrastructure/lambda.yaml +++ b/infrastructure/lambda.yaml @@ -104,13 +104,13 @@ Resources: Type: "AWS::Lambda::Function" Properties: FunctionName: !Sub ${EnvironmentName}-KinesisConsumerFunction-${AWS::Region} - Handler: "com.amazon.KinesisConsumerHandler" + Handler: "kinesis-consumer" Role: !GetAtt LambdaExecutionRoleDynamoDB.Arn Code: S3Bucket: "reactive-refarch-cloudformation-us-east-1" - S3Key: "lambda/kinesis-consumer-1.1.jar" - Runtime: "java8" - MemorySize: 256 + S3Key: "lambda/kinesis-consumer-2.1.zip" + Runtime: "go1.x" + MemorySize: 128 Timeout: "30" Environment: Variables: @@ -120,13 +120,13 @@ Resources: Type: "AWS::Lambda::Function" Properties: FunctionName: !Sub ${EnvironmentName}-RedisUpdateFunction-${AWS::Region} - Handler: "com.amazon.RedisUpdaterHandler" + Handler: "redis-updater" Role: !GetAtt LambdaExecutionRoleRedis.Arn Code: S3Bucket: "reactive-refarch-cloudformation-us-east-1" - S3Key: "lambda/redis-updater-1.1.jar" - Runtime: "java8" - MemorySize: 256 + S3Key: "lambda/redis-updater-2.1.zip" + Runtime: "go1.x" + MemorySize: 128 Timeout: "30" VpcConfig: SecurityGroupIds: @@ -136,6 +136,7 @@ Resources: Variables: REDIS_HOST: !Ref RedisHost REDIS_PORT: 6379 + REDIS_CHANNEL: "channel1" EventSourceMappingConsumer: Type: "AWS::Lambda::EventSourceMapping" diff --git a/master.yaml b/master.yaml index 638e002..43f1a5b 100644 --- a/master.yaml +++ b/master.yaml @@ -9,7 +9,7 @@ Description: > Amazon EC2 Container Registry (Amazon ECR). In addition, a DynamoDB table, Kinesis Data Streams, an ElastiCache cluster and AWS Lambda-functions are created. - Last Modified: 29. March 2018 + Last Modified: 19. May 2018 Author: Sascha Möllering Resources: diff --git a/services/tracking-service/reactive-vertx/Dockerfile b/services/tracking-service/reactive-vertx/Dockerfile index df98a2f..e611ec6 100644 --- a/services/tracking-service/reactive-vertx/Dockerfile +++ b/services/tracking-service/reactive-vertx/Dockerfile @@ -97,7 +97,6 @@ ADD target/reactive-vertx-1.1-fat.jar srv/vertx/ CMD ["java", "-server", "-XX:+DoEscapeAnalysis", "-XX:+UseStringDeduplication", \ "-XX:+UseCompressedOops", "-Xms100M", "-Xmx200M", "-XX:+UseG1GC", \ - "-Dcom.amazonaws.sdk.disableCbor", \ "-jar", "srv/vertx/reactive-vertx-1.1-fat.jar"] EXPOSE 8080 \ No newline at end of file diff --git a/services/tracking-service/reactive-vertx/pom.xml b/services/tracking-service/reactive-vertx/pom.xml index 917767d..b9f2c59 100644 --- a/services/tracking-service/reactive-vertx/pom.xml +++ b/services/tracking-service/reactive-vertx/pom.xml @@ -61,12 +61,6 @@ test - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - 2.9.3 - - diff --git a/services/tracking-service/reactive-vertx/src/main/java/com/amazon/BootStrapVerticle.java b/services/tracking-service/reactive-vertx/src/main/java/com/amazon/BootStrapVerticle.java index 174c594..f517a7c 100644 --- a/services/tracking-service/reactive-vertx/src/main/java/com/amazon/BootStrapVerticle.java +++ b/services/tracking-service/reactive-vertx/src/main/java/com/amazon/BootStrapVerticle.java @@ -20,10 +20,7 @@ import com.amazon.verticles.HttpVerticle; import com.amazon.verticles.KinesisVerticle; import com.amazon.verticles.RedisVerticle; -import io.vertx.core.AbstractVerticle; -import io.vertx.core.DeploymentOptions; -import io.vertx.core.Future; -import io.vertx.core.Vertx; +import io.vertx.core.*; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; @@ -31,6 +28,10 @@ import java.io.InputStreamReader; import java.net.URL; import java.net.URLConnection; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class BootStrapVerticle extends AbstractVerticle { @@ -46,18 +47,42 @@ public static void main (String ... args) { } @Override - public void start(Future startFuture) throws Exception { + public void start(Future startFuture) { + + List futures = Stream.generate(Future::future).limit(4) + .collect(Collectors.toList()); LOGGER.info("Deploying " + RedisVerticle.class.getCanonicalName()); - vertx.deployVerticle(RedisVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(1)); + this.deployVerticle(RedisVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(1), futures.get(0)); LOGGER.info("Deploying " + CacheVerticle.class.getCanonicalName()); - vertx.deployVerticle(CacheVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(1)); + this.deployVerticle(CacheVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(1), futures.get(1)); LOGGER.info("Deploying " + HttpVerticle.class.getCanonicalName()); - vertx.deployVerticle(HttpVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(5)); + this.deployVerticle(HttpVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(5), futures.get(2)); LOGGER.info("Deploying " + KinesisVerticle.class.getCanonicalName()); - vertx.deployVerticle(KinesisVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(5)); + this.deployVerticle(KinesisVerticle.class.getCanonicalName(), new DeploymentOptions().setInstances(5), futures.get(3)); + + CompositeFuture.all(futures).setHandler(ar -> { + if (ar.succeeded()) { + startFuture.complete(); + } else { + startFuture.fail(ar.cause()); + } + }); + } + + private void deployVerticle(final String verticleName, final DeploymentOptions deploymentOptions, + Future future) { + vertx.deployVerticle(verticleName, deploymentOptions, deployment -> + { + if (!deployment.succeeded()) { + LOGGER.error(deployment.cause()); + future.fail(deployment.cause()); + } else { + future.complete(); + } + }); } } diff --git a/services/tracking-service/service.yaml b/services/tracking-service/service.yaml index 3e3b8df..8eb377b 100644 --- a/services/tracking-service/service.yaml +++ b/services/tracking-service/service.yaml @@ -103,7 +103,7 @@ Resources: ContainerDefinitions: - Name: reactive-service Essential: true - Image: 275396840892.dkr.ecr.us-east-1.amazonaws.com/reactive-refarch:2.1 + Image: 275396840892.dkr.ecr.us-east-1.amazonaws.com/reactive-refarch:2.2 Memory: 2048 Cpu: 1024 HealthCheck: @@ -133,3 +133,83 @@ Resources: Properties: LogGroupName: !Ref AWS::StackName RetentionInDays: 7 + + ServiceScalingTarget: + Type: AWS::ApplicationAutoScaling::ScalableTarget + DependsOn: Service + Properties: + MaxCapacity: 4 + MinCapacity: 2 + ResourceId: !Join ['', [service/, !Ref 'Cluster', /, !GetAtt [Service, Name]]] + RoleARN: !GetAtt [AutoscalingRole, Arn] + ScalableDimension: ecs:service:DesiredCount + ServiceNamespace: ecs + + ServiceScalingPolicy: + Type: AWS::ApplicationAutoScaling::ScalingPolicy + Properties: + PolicyName: AStepPolicy + PolicyType: StepScaling + ScalingTargetId: !Ref 'ServiceScalingTarget' + StepScalingPolicyConfiguration: + AdjustmentType: PercentChangeInCapacity + Cooldown: 60 + MetricAggregationType: Average + StepAdjustments: + - MetricIntervalLowerBound: 0 + ScalingAdjustment: 200 + + CPUPAlarmScaleUp: + Type: AWS::CloudWatch::Alarm + Properties: + EvaluationPeriods: '1' + Statistic: Average + Threshold: '75' + AlarmDescription: Alarm if CPU is > 75% + Period: '60' + AlarmActions: [!Ref 'ServiceScalingPolicy'] + Namespace: AWS/ECS + Dimensions: + - Name: ECSService + Value: !Ref 'Service' + ComparisonOperator: GreaterThanThreshold + MetricName: CPUUtilization + + CPUPAlarmScaleDown: + Type: AWS::CloudWatch::Alarm + Properties: + EvaluationPeriods: '1' + Statistic: Average + Threshold: '10' + AlarmDescription: Alarm if CPU is < 10% + Period: '60' + AlarmActions: [!Ref 'ServiceScalingPolicy'] + Namespace: AWS/ECS + Dimensions: + - Name: ECSService + Value: !Ref 'Service' + ComparisonOperator: LessThanThreshold + MetricName: CPUUtilization + + # This IAM Role grants the service access to register/unregister with the + # Application Load Balancer (ALB). It is based on the default documented here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/service_IAM_role.html + + AutoscalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: [application-autoscaling.amazonaws.com] + Action: ['sts:AssumeRole'] + Path: / + Policies: + - PolicyName: service-autoscaling + PolicyDocument: + Statement: + - Effect: Allow + Action: ['application-autoscaling:*', 'cloudwatch:DescribeAlarms', 'cloudwatch:PutMetricAlarm', + 'ecs:DescribeServices', 'ecs:UpdateService'] + Resource: '*' \ No newline at end of file