feat(loadbalancing): add fail-open monitoring (#537)

Fixes #532

This pull request adds fail-open monitoring to our Load Balancing
monitors.

To give context about fail open
([ref](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/target-group-health-checks.html)):
> If a target group contains only unhealthy registered targets, the load
balancer routes requests to all those targets, regardless of their
health status. This means that if all targets fail health checks at the
same time in all enabled Availability Zones, the load balancer fails
open. The effect of the fail-open is to allow traffic to all targets in
all enabled Availability Zones, regardless of their health status, based
on the load balancing algorithm

Adding this metric will give better visibility into whether the Load
Balancer's fail open routing was used during incidents. The metrics
added as part of this pull request are in line with the AWS
documentation:
* `UnhealthyRoutingRequestCount` for ApplicationLoadBalancer with
`LoadBalancer` and `TargetGroup` dimensions
([ref](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-cloudwatch-metrics.html))
* `UnhealthyRoutingFlowCount` for NetworkLoadBalancer with
`LoadBalancer` dimension
([ref](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-cloudwatch-metrics.html))

These metrics are reported conditionally, only when they have nonzero
values. Thus, I added a `FILL(metric, 0)` metric math to correctly
represent the values on the dashboards.

Tested with NetworkLoadBalancer, as that's the setup I have on my
account - the ApplicationLoadBalancer values are based only on the
documentation.

---

_By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license_

Co-authored-by: Milosz Watroba <[email protected]>

lib/monitoring/aws-ecs-patterns/Ec2ServiceMonitoring.ts

@@ -181,6 +181,7 @@ export class Ec2ServiceMonitoring extends Monitoring {
   readonly ephemeralStorageUsageMetric: MetricWithAlarmSupport;
   readonly activeTcpFlowCountMetric?: MetricWithAlarmSupport;
   readonly newTcpFlowCountMetric?: MetricWithAlarmSupport;
+  readonly unhealthyRoutingFlowCountMetric?: MetricWithAlarmSupport;
   readonly processedBytesMetric?: MetricWithAlarmSupport;
 
   private hasLoadBalancer: boolean;
@@ -219,6 +220,8 @@ export class Ec2ServiceMonitoring extends Monitoring {
         this.loadBalancerMetricFactory.metricActiveConnectionCount();
       this.newTcpFlowCountMetric =
         this.loadBalancerMetricFactory.metricNewConnectionCount();
+      this.unhealthyRoutingFlowCountMetric =
+        this.loadBalancerMetricFactory.metricUnhealthyRoutingCount();
       this.processedBytesMetric =
         this.loadBalancerMetricFactory.metricProcessedBytesMin();
     }
@@ -451,6 +454,10 @@ export class Ec2ServiceMonitoring extends Monitoring {
       left.push(this.newTcpFlowCountMetric);
     }
 
+    if (this.unhealthyRoutingFlowCountMetric) {
+      left.push(this.unhealthyRoutingFlowCountMetric);
+    }
+
     if (this.processedBytesMetric) {
       right.push(this.processedBytesMetric);
     }

lib/monitoring/aws-ecs-patterns/FargateServiceMonitoring.ts

@@ -181,6 +181,7 @@ export class FargateServiceMonitoring extends Monitoring {
   readonly ephemeralStorageUsageMetric: MetricWithAlarmSupport;
   readonly activeTcpFlowCountMetric?: MetricWithAlarmSupport;
   readonly newTcpFlowCountMetric?: MetricWithAlarmSupport;
+  readonly unhealthyRoutingFlowCountMetric?: MetricWithAlarmSupport;
   readonly processedBytesMetric?: MetricWithAlarmSupport;
 
   private hasLoadBalancer: boolean;
@@ -222,6 +223,8 @@ export class FargateServiceMonitoring extends Monitoring {
         this.loadBalancerMetricFactory.metricActiveConnectionCount();
       this.newTcpFlowCountMetric =
         this.loadBalancerMetricFactory.metricNewConnectionCount();
+      this.unhealthyRoutingFlowCountMetric =
+        this.loadBalancerMetricFactory.metricUnhealthyRoutingCount();
       this.processedBytesMetric =
         this.loadBalancerMetricFactory.metricProcessedBytesMin();
     }
@@ -455,6 +458,10 @@ export class FargateServiceMonitoring extends Monitoring {
       left.push(this.newTcpFlowCountMetric);
     }
 
+    if (this.unhealthyRoutingFlowCountMetric) {
+      left.push(this.unhealthyRoutingFlowCountMetric);
+    }
+
     if (this.processedBytesMetric) {
       right.push(this.processedBytesMetric);
     }

lib/monitoring/aws-loadbalancing/ApplicationLoadBalancerMetricFactory.ts

@@ -107,4 +107,21 @@ export class ApplicationLoadBalancerMetricFactory
       })
     );
   }
+
+  metricUnhealthyRoutingCount() {
+    const unhealthyRoutingRequestCount = this.metricFactory.adaptMetric(
+      this.applicationTargetGroup.metrics.custom(
+        "UnhealthyRoutingRequestCount",
+        {
+          statistic: MetricStatistic.SUM,
+        }
+      )
+    );
+
+    return this.metricFactory.createMetricMath(
+      "FILL(unhealthyRoutingRequestCount, 0)",
+      { unhealthyRoutingRequestCount },
+      "Unhealthy routing (fail open)"
+    );
+  }
 }

lib/monitoring/aws-loadbalancing/LoadBalancerMetricFactory.ts

@@ -104,5 +104,7 @@ export interface ILoadBalancerMetricFactory {
 
   metricNewConnectionCount(): MetricWithAlarmSupport;
 
+  metricUnhealthyRoutingCount(): MetricWithAlarmSupport;
+
   metricProcessedBytesMin(): MetricWithAlarmSupport;
 }

lib/monitoring/aws-loadbalancing/NetworkLoadBalancerMetricFactory.ts

@@ -107,4 +107,18 @@ export class NetworkLoadBalancerMetricFactory
       })
     );
   }
+
+  metricUnhealthyRoutingCount() {
+    const unhealthyRoutingFlowCount = this.metricFactory.adaptMetric(
+      this.networkLoadBalancer.metrics.custom("UnhealthyRoutingFlowCount", {
+        statistic: MetricStatistic.SUM,
+      })
+    );
+
+    return this.metricFactory.createMetricMath(
+      "FILL(unhealthyRoutingFlowCount, 0)",
+      { unhealthyRoutingFlowCount },
+      "Unhealthy routing (fail open)"
+    );
+  }
 }

lib/monitoring/aws-loadbalancing/NetworkLoadBalancerMonitoring.ts

@@ -62,6 +62,7 @@ export class NetworkLoadBalancerMonitoring extends Monitoring {
   protected readonly healthyTaskPercentMetric: MetricWithAlarmSupport;
   protected readonly activeTcpFlowCountMetric: MetricWithAlarmSupport;
   protected readonly newTcpFlowCountMetric: MetricWithAlarmSupport;
+  protected readonly unhealthyRoutingFlowCountMetric: MetricWithAlarmSupport;
   protected readonly processedBytesMetric: MetricWithAlarmSupport;
 
   constructor(
@@ -89,6 +90,8 @@ export class NetworkLoadBalancerMonitoring extends Monitoring {
     this.activeTcpFlowCountMetric =
       this.metricFactory.metricActiveConnectionCount();
     this.newTcpFlowCountMetric = this.metricFactory.metricNewConnectionCount();
+    this.unhealthyRoutingFlowCountMetric =
+      this.metricFactory.metricUnhealthyRoutingCount();
     this.processedBytesMetric = this.metricFactory.metricProcessedBytesMin();
 
     const alarmFactory = this.createAlarmFactory(
@@ -184,7 +187,11 @@ export class NetworkLoadBalancerMonitoring extends Monitoring {
       width,
       height,
       title: "TCP Flows",
-      left: [this.activeTcpFlowCountMetric, this.newTcpFlowCountMetric],
+      left: [
+        this.activeTcpFlowCountMetric,
+        this.newTcpFlowCountMetric,
+        this.unhealthyRoutingFlowCountMetric,
+      ],
       leftYAxis: CountAxisFromZero,
       right: [this.processedBytesMetric],
       rightYAxis: SizeAxisBytesFromZero,