Merge pull request #11 from TimBuss/master

feat: Reduce alarm noise

autoscaling.tf

@@ -15,7 +15,7 @@ resource "aws_cloudwatch_metric_alarm" "memory_or_cpu_high" {
   count               = var.enable_autoscale ? 1 : 0
   alarm_name          = "${aws_ecs_service.main.name}-Memory-${var.task_scale_out_memory_threshold_percent}-OR-CPU-${var.task_scale_out_cpu_threshold_percent}-High"
   comparison_operator = "GreaterThanOrEqualToThreshold"
-  evaluation_periods  = "1"
+  evaluation_periods  = var.task_alarm_evaluation_periods
   threshold           = "1"
   alarm_description   = "Scale out ${aws_ecs_service.main.name} tasks"
 
@@ -32,7 +32,7 @@ resource "aws_cloudwatch_metric_alarm" "memory_or_cpu_high" {
     metric {
       metric_name = "CPUUtilization"
       namespace   = "AWS/ECS"
-      period      = var.task_scale_out_alarm_evaluation_period
+      period      = var.task_alarm_period
       stat        = "Average"
       unit        = "Percent"
 
@@ -49,7 +49,7 @@ resource "aws_cloudwatch_metric_alarm" "memory_or_cpu_high" {
     metric {
       metric_name = "MemoryUtilization"
       namespace   = "AWS/ECS"
-      period      = var.task_scale_out_alarm_evaluation_period
+      period      = var.task_alarm_period
       stat        = "Average"
       unit        = "Percent"
 
@@ -71,13 +71,13 @@ resource "aws_cloudwatch_metric_alarm" "memory_and_cpu_low" {
   count              = var.enable_autoscale ? 1 : 0
   alarm_name          = "${aws_ecs_service.main.name}-Memory-${var.task_scale_in_memory_threshold_percent}-AND-CPU-${var.task_scale_in_cpu_threshold_percent}-Low"
   comparison_operator = "LessThanThreshold"
-  evaluation_periods  = "1"
+  evaluation_periods  = var.task_alarm_evaluation_periods
   threshold           = "1"
   alarm_description   = "Scale in ${aws_ecs_service.main.name} tasks"
 
   metric_query {
     id          = "e1"
-    expression  = "CEIL((cpu-${var.task_scale_in_cpu_threshold_percent})/(100))+CEIL((memory-${var.task_scale_in_memory_threshold_percent})/(100))"
+    expression  = "CEIL((cpu-${var.task_scale_in_cpu_threshold_percent})/(100))+CEIL((memory-${var.task_scale_in_memory_threshold_percent})/(100))+((${var.task_count}+1)-healthyhosts)"
     label       = "CPU and Memory Utilization low"
     return_data = "true"
   }
@@ -88,7 +88,7 @@ resource "aws_cloudwatch_metric_alarm" "memory_and_cpu_low" {
     metric {
       metric_name = "CPUUtilization"
       namespace   = "AWS/ECS"
-      period      = var.task_scale_in_alarm_evaluation_period
+      period      = var.task_alarm_period
       stat        = "Average"
       unit        = "Percent"
 
@@ -105,7 +105,7 @@ resource "aws_cloudwatch_metric_alarm" "memory_and_cpu_low" {
     metric {
       metric_name = "MemoryUtilization"
       namespace   = "AWS/ECS"
-      period      = var.task_scale_in_alarm_evaluation_period
+      period      = var.task_alarm_period
       stat        = "Average"
       unit        = "Percent"
 
@@ -116,6 +116,22 @@ resource "aws_cloudwatch_metric_alarm" "memory_and_cpu_low" {
     }
   }
 
+  metric_query {
+    id = "healthyhosts"
+
+    metric {
+      metric_name = "HealthyHostCount"
+      namespace   = "AWS/ApplicationELB"
+      period      = var.task_alarm_period
+      stat        = "Average"
+
+      dimensions = {
+        TargetGroup = aws_lb_target_group.app.arn_suffix
+        LoadBalancer = aws_lb.main.arn_suffix
+      }
+    }
+  }
+
     alarm_actions = var.enable_monitoring ? [aws_appautoscaling_policy.app_in[0].arn, module.notify-slack.this_slack_topic_arn] : [aws_appautoscaling_policy.app_in[0].arn]
 
   depends_on = [
@@ -167,4 +183,4 @@ resource "aws_appautoscaling_policy" "app_in" {
   depends_on = [
     aws_appautoscaling_target.app_scale_target[0],
   ]
-}
+}

main.tf

@@ -42,6 +42,7 @@ resource "aws_lb_target_group" "app" {
   protocol    = "HTTPS"
   vpc_id      = var.vpc_id
   target_type = "ip"
+  slow_start  = var.target_group_slow_start
 
   health_check {
     path     = var.app_healthcheck_endpoint

monitoring.tf

@@ -56,7 +56,6 @@ resource "aws_cloudwatch_metric_alarm" "httpcode_target_5xx_count" {
   alarm_description         = "${var.resource_prefix}-${terraform.workspace}-5XX-target-group-errors"
   alarm_actions             = [module.notify-slack.this_slack_topic_arn]
   ok_actions                = [module.notify-slack.this_slack_topic_arn]
-  insufficient_data_actions = [module.notify-slack.this_slack_topic_arn]
 
   dimensions = {
     "TargetGroup"  = aws_lb_target_group.app.arn_suffix
@@ -74,10 +73,10 @@ resource "aws_cloudwatch_metric_alarm" "unhealthy_host_count" {
   period                    = var.monitoring_period
   statistic                 = "Average"
   threshold                 = 0
+  treat_missing_data        = "notBreaching"
   alarm_description         = "${var.resource_prefix}-${terraform.workspace}-unhealthy-hosts"
   alarm_actions             = [module.notify-slack.this_slack_topic_arn]
   ok_actions                = [module.notify-slack.this_slack_topic_arn]
-  insufficient_data_actions = [module.notify-slack.this_slack_topic_arn]
 
   dimensions = {
     "TargetGroup"  = aws_lb_target_group.app.arn_suffix

variables.tf

@@ -189,35 +189,41 @@ variable "task_scale_out_cpu_threshold_percent" {
 variable "task_scale_in_memory_threshold_percent" {
   type = number
   default = 25
-  description= "[Optional] Autoscale - When both cpu and memory are below scale in thresholds decrease tasks by 1"
+  description= "[Optional] Autoscale - When both cpu and memory are below scale in thresholds decrease tasks by 1. Only applies when the total number of tasks exceed the minimum task count. number of running tasks determined by HealthyHostCount."
 }
 
 variable "task_scale_in_cpu_threshold_percent" {
-  type = number
-  default = 30
-  description= "[Optional] Autoscale - When both cpu and memory are below scale in thresholds decrease tasks by 1"
+  type        = number
+  default     = 30
+  description = "[Optional] Autoscale - When both cpu and memory are below scale in thresholds decrease tasks by 1. Only applies when the total number of tasks exceed the minimum task count. number of running tasks determined by HealthyHostCount."
 }
 
 variable "task_scale_in_cooldown_period" {
-  type = number
-  default = 300
+  type        = number
+  default     = 300
   description = "[Optional] Autoscale - scale in cooldown period, in seconds"
 }
 
 variable "task_scale_out_cooldown_period" {
-  type = number
-  default = 300
+  type        = number
+  default     = 300
   description = "[Optional] Autoscale - scale out cooldown period, in seconds"
 }
 
-variable "task_scale_in_alarm_evaluation_period" {
-  type = number
-  default = 60
-  description = "[Optional] Autoscale - scale in alarm evaluation period, in seconds"
+variable "task_alarm_period" {
+  type        = number
+  default     = 60
+  description = "[Optional] Autoscale - period of time to evaluate, in seconds"
 }
 
-variable "task_scale_out_alarm_evaluation_period" {
-  type = number
-  default = 60
-  description = "[Optional] Autoscale - scale out alarm evaluation period, in seconds"
-}
+variable "task_alarm_evaluation_periods" {
+  type        = number
+  default     = 3
+  description = "[Optional] Autoscale - number of data points to use for evaluation"
+}
+
+variable "target_group_slow_start" {
+  type        = number
+  default     = 300
+  description = "[Optional] Load balancer - time period to wait before forwarding requests to the target group, time in seconds."
+}