Skip to content

Commit

Permalink
feat(lambda): allow alarming on max latency (#486)
Browse files Browse the repository at this point in the history
Closes #448

This also aligns how we create/present latency metrics with how we do it
for other things like API Gateway.

---

_By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license_
  • Loading branch information
echeung-amzn authored Jan 17, 2024
1 parent 01e865f commit 3ce4dbd
Show file tree
Hide file tree
Showing 9 changed files with 330 additions and 86 deletions.
58 changes: 55 additions & 3 deletions API.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 8 additions & 1 deletion lib/common/monitoring/alarms/LatencyAlarmFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export enum LatencyType {
TM999_TOP = "TM(99.9%:100%)",
TM9999_TOP = "TM(99.99%:100%)",
AVERAGE = "Average",
MAX = "Maximum",
}

export function getLatencyTypeStatistic(latencyType: LatencyType) {
Expand Down Expand Up @@ -72,6 +73,8 @@ export function getLatencyTypeStatistic(latencyType: LatencyType) {
return MetricStatistic.TM9999_TOP;
case LatencyType.AVERAGE:
return MetricStatistic.AVERAGE;
case LatencyType.MAX:
return MetricStatistic.MAX;
default:
throw new Error(
"Unsupported latency type (unknown statistic): " + latencyType
Expand All @@ -90,10 +93,12 @@ export function getLatencyTypeExpressionId(latencyType: LatencyType) {
case LatencyType.P9999:
case LatencyType.P100:
// remove the P prefix
return latencyType.substr(1);
return latencyType.substring(1);
case LatencyType.AVERAGE:
// making it shorter for backwards compatibility
return "Avg";
case LatencyType.MAX:
return "Max";
default:
// use as-is
return latencyType;
Expand All @@ -119,6 +124,8 @@ export function getLatencyTypeLabel(latencyType: LatencyType) {
case LatencyType.AVERAGE:
// no suffix here, since we already have average
return "Average";
case LatencyType.MAX:
return "Maximum";
default:
// use as-is
return latencyType + averageSuffix;
Expand Down
6 changes: 3 additions & 3 deletions lib/monitoring/aws-apigateway/ApiGatewayMetricFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,21 +138,21 @@ export class ApiGatewayMetricFactory {
}

/**
* @deprecated use metricLatencyInMillis instead
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP99InMillis() {
return this.metricLatencyInMillis(LatencyType.P99);
}

/**
* @deprecated use metricLatencyInMillis instead
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP90InMillis() {
return this.metricLatencyInMillis(LatencyType.P90);
}

/**
* @deprecated use metricLatencyInMillis instead
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP50InMillis() {
return this.metricLatencyInMillis(LatencyType.P50);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,42 +133,42 @@ export class ApiGatewayV2HttpApiMetricFactory {
}

/**
* @deprecated use metricLatencyInMillis instead
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP50InMillis() {
return this.metricLatencyInMillis(LatencyType.P50);
}

/**
* @deprecated use metricLatencyInMillis instead
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP90InMillis() {
return this.metricLatencyInMillis(LatencyType.P90);
}

/**
* @deprecated use metricLatencyInMillis instead
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP99InMillis() {
return this.metricLatencyInMillis(LatencyType.P99);
}

/**
* @deprecated use metricIntegrationLatencyInMillis instead
* @deprecated Use {@link metricIntegrationLatencyInMillis} instead.
*/
metricIntegrationLatencyP50InMillis() {
return this.metricIntegrationLatencyInMillis(LatencyType.P50);
}

/**
* @deprecated use metricIntegrationLatencyInMillis instead
* @deprecated Use {@link metricIntegrationLatencyInMillis} instead.
*/
metricIntegrationLatencyP90InMillis() {
return this.metricIntegrationLatencyInMillis(LatencyType.P90);
}

/**
* @deprecated use metricIntegrationLatencyInMillis instead
* @deprecated Use {@link metricIntegrationLatencyInMillis} instead.
*/
metricIntegrationLatencyP99InMillis() {
return this.metricIntegrationLatencyInMillis(LatencyType.P99);
Expand Down
38 changes: 22 additions & 16 deletions lib/monitoring/aws-lambda/LambdaFunctionMetricFactory.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import { IFunction } from "aws-cdk-lib/aws-lambda";

import {
LatencyType,
MetricFactory,
MetricStatistic,
RateComputationMethod,
getLatencyTypeLabel,
getLatencyTypeStatistic,
} from "../../common";

export interface LambdaFunctionMetricFactoryProps {
Expand Down Expand Up @@ -45,7 +48,7 @@ export class LambdaFunctionMetricFactory {
}

/**
* @deprecated use metricInvocationRate
* @deprecated Use {@link metricInvocationRate} instead.
*/
metricTps() {
return this.metricFactory.toRate(
Expand Down Expand Up @@ -111,31 +114,34 @@ export class LambdaFunctionMetricFactory {
);
}

metricLatencyP99InMillis() {
metricLatencyInMillis(latencyType: LatencyType) {
return this.metricFactory.adaptMetric(
this.lambdaFunction.metricDuration({
statistic: MetricStatistic.P99,
label: "P99",
statistic: getLatencyTypeStatistic(latencyType),
label: getLatencyTypeLabel(latencyType),
})
);
}

/**
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP99InMillis() {
return this.metricLatencyInMillis(LatencyType.P99);
}

/**
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP90InMillis() {
return this.metricFactory.adaptMetric(
this.lambdaFunction.metricDuration({
statistic: MetricStatistic.P90,
label: "P90",
})
);
return this.metricLatencyInMillis(LatencyType.P90);
}

/**
* @deprecated Use {@link metricLatencyInMillis} instead.
*/
metricLatencyP50InMillis() {
return this.metricFactory.adaptMetric(
this.lambdaFunction.metricDuration({
statistic: MetricStatistic.P50,
label: "P50",
})
);
return this.metricLatencyInMillis(LatencyType.P50);
}

metricConcurrentExecutions() {
Expand Down
29 changes: 26 additions & 3 deletions lib/monitoring/aws-lambda/LambdaFunctionMonitoring.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ export interface LambdaFunctionMonitoringOptions extends BaseMonitoringProps {
readonly addLatencyP50Alarm?: Record<string, LatencyThreshold>;
readonly addLatencyP90Alarm?: Record<string, LatencyThreshold>;
readonly addLatencyP99Alarm?: Record<string, LatencyThreshold>;
readonly addMaxLatencyAlarm?: Record<string, LatencyThreshold>;

readonly addFaultCountAlarm?: Record<string, ErrorCountThreshold>;
readonly addFaultRateAlarm?: Record<string, ErrorRateThreshold>;
Expand Down Expand Up @@ -143,6 +144,7 @@ export class LambdaFunctionMonitoring extends Monitoring {
readonly p50LatencyMetric: MetricWithAlarmSupport;
readonly p90LatencyMetric: MetricWithAlarmSupport;
readonly p99LatencyMetric: MetricWithAlarmSupport;
readonly maxLatencyMetric: MetricWithAlarmSupport;
readonly faultCountMetric: MetricWithAlarmSupport;
readonly faultRateMetric: MetricWithAlarmSupport;
readonly invocationCountMetric: MetricWithAlarmSupport;
Expand Down Expand Up @@ -202,9 +204,18 @@ export class LambdaFunctionMonitoring extends Monitoring {
props
);
this.tpsMetric = this.metricFactory.metricTps();
this.p50LatencyMetric = this.metricFactory.metricLatencyP50InMillis();
this.p90LatencyMetric = this.metricFactory.metricLatencyP90InMillis();
this.p99LatencyMetric = this.metricFactory.metricLatencyP99InMillis();
this.p50LatencyMetric = this.metricFactory.metricLatencyInMillis(
LatencyType.P50
);
this.p90LatencyMetric = this.metricFactory.metricLatencyInMillis(
LatencyType.P90
);
this.p99LatencyMetric = this.metricFactory.metricLatencyInMillis(
LatencyType.P99
);
this.maxLatencyMetric = this.metricFactory.metricLatencyInMillis(
LatencyType.MAX
);
this.faultCountMetric = this.metricFactory.metricFaultCount();
this.faultRateMetric = this.metricFactory.metricFaultRate();
this.invocationCountMetric = this.metricFactory.metricInvocationCount();
Expand Down Expand Up @@ -359,6 +370,18 @@ export class LambdaFunctionMonitoring extends Monitoring {
this.latencyAnnotations.push(createdAlarm.annotation);
this.addAlarm(createdAlarm);
}
for (const disambiguator in props.addMaxLatencyAlarm) {
const alarmProps = props.addMaxLatencyAlarm[disambiguator];
const createdAlarm = this.latencyAlarmFactory.addLatencyAlarm(
this.maxLatencyMetric,
LatencyType.MAX,
alarmProps,
disambiguator
);
this.latencyAnnotations.push(createdAlarm.annotation);
this.addAlarm(createdAlarm);
}

for (const disambiguator in props.addFaultCountAlarm) {
const alarmProps = props.addFaultCountAlarm[disambiguator];
const createdAlarm = this.errorAlarmFactory.addErrorCountAlarm(
Expand Down
Loading

0 comments on commit 3ce4dbd

Please sign in to comment.