From 6aa1cdb6d91afabc631c1220e9144f32c6c4ca8b Mon Sep 17 00:00:00 2001
From: ConjunctiveNormalForm <zhiyuan.zach.yang@gmail.com>
Date: Wed, 13 Mar 2024 14:33:11 -0400
Subject: [PATCH] add success rate alarms for log processor lambdas

---
 bin/stacks/analytics-stack.ts | 58 ++++++++++++++++++++++++++++++++++-
 bin/stacks/api-stack.ts       |  7 +++--
 2 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/bin/stacks/analytics-stack.ts b/bin/stacks/analytics-stack.ts
index 5208418e..be2683b1 100644
--- a/bin/stacks/analytics-stack.ts
+++ b/bin/stacks/analytics-stack.ts
@@ -41,6 +41,7 @@ export interface AnalyticsStackProps extends cdk.NestedStackProps {
   envVars: Record<string, string>;
   analyticsStreamArn: string;
   stage: string;
+  chatbotSNSArn?: string;
 }
 
 /**
@@ -59,7 +60,7 @@ export class AnalyticsStack extends cdk.NestedStack {
 
   constructor(scope: Construct, id: string, props: AnalyticsStackProps) {
     super(scope, id, props);
-    const { quoteLambda, analyticsStreamArn, stage } = props;
+    const { quoteLambda, analyticsStreamArn, stage, chatbotSNSArn } = props;
 
     /* S3 Initialization */
     const rfqRequestBucket = new aws_s3.Bucket(this, 'RfqRequestBucket');
@@ -470,6 +471,61 @@ export class AnalyticsStack extends cdk.NestedStack {
         ],
       })
     );
+    let chatBotTopic: cdk.aws_sns.ITopic;
+    if (chatbotSNSArn) {
+      chatBotTopic = cdk.aws_sns.Topic.fromTopicArn(this, 'ChatbotTopic', chatbotSNSArn);
+    }
+
+    /* log processor alarms */
+    [quoteProcessorLambda, fillEventProcessorLambda, postOrderProcessorLambda, botOrderEventsProcessorLambda].forEach(
+      (lambda) => {
+        const successRateSev2Name = `${lambda.node.id}-SEV2-SuccessRate`;
+        const successRateSev3Name = `${lambda.node.id}-SEV3-SuccessRate`;
+
+        const errors = lambda.metricErrors({
+          period: cdk.Duration.minutes(5),
+          statistic: cdk.aws_cloudwatch.Stats.SUM,
+          label: `${lambda.node.id} Errors`,
+        });
+
+        const invocations = lambda.metricInvocations({
+          period: cdk.Duration.minutes(5),
+          statistic: cdk.aws_cloudwatch.Stats.SUM,
+          label: `${lambda.node.id} Invocations`,
+        });
+
+        const successRate = new cdk.aws_cloudwatch.MathExpression({
+          expression: '100 - 100 * errors / MAX([errors, invocations])',
+          usingMetrics: {
+            errors,
+            invocations,
+          },
+          label: `${lambda.node.id} Success Rate`,
+        });
+
+        const successRateSev2 = new cdk.aws_cloudwatch.Alarm(this, successRateSev2Name, {
+          metric: successRate,
+          threshold: 60,
+          evaluationPeriods: 1,
+          comparisonOperator: cdk.aws_cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
+          actionsEnabled: true,
+        });
+
+        const successRateSev3 = new cdk.aws_cloudwatch.Alarm(this, successRateSev3Name, {
+          metric: successRate,
+          threshold: 90,
+          evaluationPeriods: 1,
+          comparisonOperator: cdk.aws_cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
+          actionsEnabled: true,
+        });
+
+        if (chatBotTopic) {
+          successRateSev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
+          successRateSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
+        }
+      }
+    );
+
     // CDK doesn't have this implemented yet, so have to use the CloudFormation resource (lower level of abstraction)
     // https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-kinesisfirehose-deliverystream.html
     const uraRequestStream = new aws_firehose.CfnDeliveryStream(this, 'uraRequestStream', {
diff --git a/bin/stacks/api-stack.ts b/bin/stacks/api-stack.ts
index 955184c9..651439b5 100644
--- a/bin/stacks/api-stack.ts
+++ b/bin/stacks/api-stack.ts
@@ -10,9 +10,9 @@ import * as aws_lambda from 'aws-cdk-lib/aws-lambda';
 import * as aws_lambda_nodejs from 'aws-cdk-lib/aws-lambda-nodejs';
 import * as aws_logs from 'aws-cdk-lib/aws-logs';
 import * as aws_waf from 'aws-cdk-lib/aws-wafv2';
-import { KmsStack } from './kms-stack'
 import { Construct } from 'constructs';
 import * as path from 'path';
+import { KmsStack } from './kms-stack';
 
 import { Metric } from '../../lib/entities';
 import { STAGE } from '../../lib/util/stage';
@@ -156,7 +156,7 @@ export class APIStack extends cdk.Stack {
     });
 
     // KMS initialization
-    const kmsStack = new KmsStack(this, `${SERVICE_NAME}HardQuoteCosignerKey-1`)
+    const kmsStack = new KmsStack(this, `${SERVICE_NAME}HardQuoteCosignerKey-1`);
 
     /*
      * Firehose Initialization
@@ -184,7 +184,7 @@ export class APIStack extends cdk.Stack {
         actions: ['kms:GetPublicKey', 'kms:Sign'],
         effect: aws_iam.Effect.ALLOW,
       })
-    )
+    );
 
     lambdaRole.addToPolicy(
       new aws_iam.PolicyStatement({
@@ -461,6 +461,7 @@ export class APIStack extends cdk.Stack {
       envVars: props.envVars,
       analyticsStreamArn: firehoseStack.analyticsStreamArn,
       stage,
+      chatbotSNSArn,
     });
 
     const cronStack = new CronStack(this, 'CronStack', {