Skip to content

Commit

Permalink
Merge pull request #298 from Uniswap/segment-alarms
Browse files Browse the repository at this point in the history 
feat: separate soft and hard quote alarms; fix broken alarms
  • Loading branch information
@ConjunctiveNormalForm
ConjunctiveNormalForm authored Apr 3, 2024
2 parents 5cd359b + 53a5927 commit 4f278c9
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 106 deletions.
231 changes: 143 additions & 88 deletions bin/stacks/api-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { Construct } from 'constructs';
import * as path from 'path';
import { KmsStack } from './kms-stack';

import { Metric } from '../../lib/entities';
import { HardQuoteMetricDimension, Metric, SoftQuoteMetricDimension } from '../../lib/entities';
import { STAGE } from '../../lib/util/stage';
import { SERVICE_NAME } from '../constants';
import { AnalyticsStack } from './analytics-stack';
Expand Down Expand Up @@ -504,16 +504,6 @@ export class APIStack extends cdk.Stack {
evaluationPeriods: stage == STAGE.BETA ? 5 : 3,
});

// const apiAlarm4xxSev2 = new aws_cloudwatch.Alarm(this, 'UniswapXParameterizationAPI-SEV2-4XXAlarm', {
// alarmName: 'UniswapXParameterizationAPI-SEV2-4XX',
// metric: api.metricClientError({
// period: Duration.minutes(5),
// statistic: 'avg',
// }),
// threshold: 0.99,
// evaluationPeriods: 3,
// });

const apiAlarm4xxSev3 = new aws_cloudwatch.Alarm(this, 'UniswapXParameterizationAPI-SEV3-4XXAlarm', {
alarmName: 'UniswapXParameterizationAPI-SEV3-4XX',
metric: api.metricClientError({
Expand Down Expand Up @@ -566,101 +556,166 @@ export class APIStack extends cdk.Stack {
evaluationPeriods: 3,
});

const chatBotTopic = chatbotSNSArn
? cdk.aws_sns.Topic.fromTopicArn(this, 'ChatbotTopic', chatbotSNSArn)
: undefined;
/* custom metric alarms */
// Alarm on calls to RFQ providers
const rfqOverallSuccessMetric = new aws_cloudwatch.MathExpression({
expression: '100*(success/invocations)',
period: Duration.minutes(5),
usingMetrics: {
invocations: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_REQUESTED}`,
dimensionsMap: { Service: SERVICE_NAME },
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
success: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_SUCCESS}`,
dimensionsMap: { Service: SERVICE_NAME },
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
},
});

const rfqOverallSuccessRateAlarmSev2 = new aws_cloudwatch.Alarm(
this,
'UniswapXParameterizationAPI-SEV2-RFQ-SuccessRate',
{
alarmName: 'UniswapXParameterizationAPI-SEV2-RFQ-SuccessRate',
metric: rfqOverallSuccessMetric,
threshold: 90,
comparisonOperator: aws_cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
evaluationPeriods: 3,
}
);
for (const dimension of [SoftQuoteMetricDimension, HardQuoteMetricDimension]) {
const rfqOverallSuccessMetric = new aws_cloudwatch.MathExpression({
expression: '100*(success/invocations)',
period: Duration.minutes(5),
usingMetrics: {
invocations: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_REQUESTED}`,
dimensionsMap: dimension,
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
success: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_SUCCESS}`,
dimensionsMap: { Service: SERVICE_NAME },
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
},
});

const rfqOverallSuccessRateAlarmSev3 = new aws_cloudwatch.Alarm(
this,
'UniswapXParameterizationAPI-SEV3-RFQ-SuccessRate',
{
alarmName: 'UniswapXParameterizationAPI-SEV3-RFQ-SuccessRate',
metric: rfqOverallSuccessMetric,
threshold: 95,
comparisonOperator: aws_cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
evaluationPeriods: 3,
}
);
const rfqOverallSuccessRateAlarmSev2 = new aws_cloudwatch.Alarm(
this,
`${dimension.Service}-SEV2-RFQ-SuccessRate`,
{
alarmName: `${dimension.Service}-SEV2-RFQ-SuccessRate`,
metric: rfqOverallSuccessMetric,
threshold: 80,
comparisonOperator: aws_cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
evaluationPeriods: 3,
}
);

const rfqOverallSuccessRateAlarmSev3 = new aws_cloudwatch.Alarm(
this,
`${dimension.Service}-SEV3-RFQ-SuccessRate`,
{
alarmName: `${dimension.Service}-SEV3-RFQ-SuccessRate`,
metric: rfqOverallSuccessMetric,
threshold: 90,
comparisonOperator: aws_cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
evaluationPeriods: 3,
}
);

const rfqOverallNonQuoteMetric = new aws_cloudwatch.MathExpression({
expression: '100*(nonQuote/invocations)',
period: Duration.minutes(5),
usingMetrics: {
invocations: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_REQUESTED}`,
dimensionsMap: dimension,
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
nonQuote: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_NON_QUOTE}`,
dimensionsMap: dimension,
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
},
});

const rfqOverallNonQuoteMetric = new aws_cloudwatch.MathExpression({
expression: '100*(nonQuote/invocations)',
period: Duration.minutes(5),
usingMetrics: {
invocations: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_REQUESTED}`,
dimensionsMap: { Service: SERVICE_NAME },
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
nonQuote: new aws_cloudwatch.Metric({
const rfqOverallNonQuoteRateAlarmSev3 = new aws_cloudwatch.Alarm(
this,
`${dimension.Service}-SEV2-RFQ-NonQuoteRate`,
{
alarmName: `${dimension.Service}-SEV2-RFQ-NonQuoteRate`,
metric: rfqOverallNonQuoteMetric,
threshold: 30,
comparisonOperator: aws_cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
evaluationPeriods: 3,
}
);

const quoteLatencyAlarmSev3 = new aws_cloudwatch.Alarm(this, `${dimension.Service}-SEV3-QuoteLatency`, {
alarmName: `${dimension.Service}-SEV3-QuoteLatency`,
metric: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.RFQ_NON_QUOTE}`,
dimensionsMap: { Service: SERVICE_NAME },
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
metricName: `${Metric.QUOTE_LATENCY}`,
dimensionsMap: dimension,
unit: aws_cloudwatch.Unit.MILLISECONDS,
statistic: 'p90',
}),
},
});

const rfqOverallNonQuoteRateAlarmSev3 = new aws_cloudwatch.Alarm(
this,
'UniswapXParameterizationAPI-SEV2-RFQ-NonQuoteRate',
{
alarmName: 'UniswapXParameterizationAPI-SEV3-RFQ-NonQuoteRate',
metric: rfqOverallNonQuoteMetric,
threshold: 30,
threshold: 2_000,
comparisonOperator: aws_cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
evaluationPeriods: 3,
});

if (dimension.Service == HardQuoteMetricDimension.Service) {
const quotePostErrorMetric = new aws_cloudwatch.MathExpression({
expression: '100*(postError/invocations)',
period: Duration.minutes(5),
usingMetrics: {
invocations: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.QUOTE_POST_ATTEMPT}`,
dimensionsMap: dimension,
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
postError: new aws_cloudwatch.Metric({
namespace: 'Uniswap',
metricName: `${Metric.QUOTE_POST_ERROR}`,
dimensionsMap: dimension,
unit: aws_cloudwatch.Unit.COUNT,
statistic: 'sum',
}),
},
});

const quotePostErrorAlarmSev3 = new aws_cloudwatch.Alarm(this, `${dimension.Service}-SEV3-PostErrorRate`, {
alarmName: `${dimension.Service}-SEV3-PostErrorRate`,
metric: quotePostErrorMetric,
evaluationPeriods: 3,
threshold: 10,
comparisonOperator: aws_cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
});

const quotePostErrorAlarmSev2 = new aws_cloudwatch.Alarm(this, `${dimension.Service}-SEV2-PostErrorRate`, {
alarmName: `${dimension.Service}-SEV2-PostErrorRate`,
metric: quotePostErrorMetric,
evaluationPeriods: 3,
threshold: 20,
comparisonOperator: aws_cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
});

if (chatBotTopic) {
quotePostErrorAlarmSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
quotePostErrorAlarmSev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
}
}
);

if (chatBotTopic) {
rfqOverallSuccessRateAlarmSev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
rfqOverallSuccessRateAlarmSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
rfqOverallNonQuoteRateAlarmSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
quoteLatencyAlarmSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
}
}

// TODO: consider alarming on individual RFQ providers

if (chatbotSNSArn) {
const chatBotTopic = cdk.aws_sns.Topic.fromTopicArn(this, 'ChatbotTopic', chatbotSNSArn);
if (chatBotTopic) {
apiAlarm5xxSev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
// apiAlarm4xxSev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
apiAlarm5xxSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
apiAlarm4xxSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
apiAlarmLatencySev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
apiAlarmLatencySev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
apiAlarmLatencyP99Sev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
apiAlarmLatencyP99Sev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));

rfqOverallSuccessRateAlarmSev2.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
rfqOverallSuccessRateAlarmSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
rfqOverallNonQuoteRateAlarmSev3.addAlarmAction(new cdk.aws_cloudwatch_actions.SnsAction(chatBotTopic));
}

this.url = new CfnOutput(this, 'Url', {
Expand Down
18 changes: 10 additions & 8 deletions lib/entities/aws-metrics-logger.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ export const CircuitBreakerMetricDimension = {
Service: 'CircuitBreaker',
};

export const SoftQuoteMetricDimension = {
Service: 'SoftQuote',
};

export const HardQuoteMetricDimension = {
Service: 'HardQuote',
};

export class AWSMetricsLogger implements IMetric {
constructor(private awsMetricLogger: AWSEmbeddedMetricsLogger) {}

Expand All @@ -35,18 +43,12 @@ export enum Metric {
QUOTE_404 = 'QUOTE_404',
QUOTE_500 = 'QUOTE_500',

HARD_QUOTE_200 = 'HARD_QUOTE_200',
HARD_QUOTE_400 = 'HARD_QUOTE_400',
HARD_QUOTE_404 = 'HARD_QUOTE_404',
HARD_QUOTE_500 = 'HARD_QUOTE_500',

QUOTE_REQUESTED = 'QUOTE_REQUESTED',
QUOTE_LATENCY = 'QUOTE_LATENCY',
QUOTE_RESPONSE_COUNT = 'QUOTE_RESPONSE_COUNT',

HARD_QUOTE_REQUESTED = 'HARD_QUOTE_REQUESTED',
HARD_QUOTE_LATENCY = 'HARD_QUOTE_LATENCY',
HARD_QUOTE_RESPONSE_COUNT = 'HARD_QUOTE_RESPONSE_COUNT',
QUOTE_POST_ERROR = 'QUOTE_POST_ERROR',
QUOTE_POST_ATTEMPT = 'QUOTE_POST_ATTEMPT',

RFQ_REQUESTED = 'RFQ_REQUESTED',
RFQ_SUCCESS = 'RFQ_SUCCESS',
Expand Down
11 changes: 6 additions & 5 deletions lib/handlers/hard-quote/handler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export class QuoteHandler extends APIGLambdaHandler<
} = params;
const start = Date.now();

metric.putMetric(Metric.HARD_QUOTE_REQUESTED, 1, MetricLoggerUnit.Count);
metric.putMetric(Metric.QUOTE_REQUESTED, 1, MetricLoggerUnit.Count);

log.info({ cosignerAddress: cosignerAddress }, 'cosignerAddress');
const request = HardQuoteRequest.fromHardRequestBody(requestBody);
Expand Down Expand Up @@ -72,7 +72,6 @@ export class QuoteHandler extends APIGLambdaHandler<

const bestQuote = await getBestQuote(quoters, request.toQuoteRequest(), log, metric, 'HardResponse');
if (!bestQuote && !requestBody.allowNoQuote) {
metric.putMetric(Metric.HARD_QUOTE_404, 1, MetricLoggerUnit.Count);
if (!requestBody.allowNoQuote) {
throw new NoQuotesAvailable();
}
Expand All @@ -92,15 +91,17 @@ export class QuoteHandler extends APIGLambdaHandler<
const cosignedOrder = CosignedV2DutchOrder.fromUnsignedOrder(request.order, cosignerData, cosignature);

try {
metric.putMetric(Metric.QUOTE_POST_ATTEMPT, 1, MetricLoggerUnit.Count);
// if no quote and creating open order, create random new quoteId
await orderServiceProvider.postOrder(cosignedOrder, request.innerSig, bestQuote?.quoteId ?? uuidv4());
} catch (e) {
metric.putMetric(Metric.HARD_QUOTE_400, 1, MetricLoggerUnit.Count);
metric.putMetric(Metric.QUOTE_400, 1, MetricLoggerUnit.Count);
metric.putMetric(Metric.QUOTE_POST_ERROR, 1, MetricLoggerUnit.Count);
throw new OrderPostError();
}

metric.putMetric(Metric.HARD_QUOTE_200, 1, MetricLoggerUnit.Count);
metric.putMetric(Metric.HARD_QUOTE_LATENCY, Date.now() - start, MetricLoggerUnit.Milliseconds);
metric.putMetric(Metric.QUOTE_200, 1, MetricLoggerUnit.Count);
metric.putMetric(Metric.QUOTE_LATENCY, Date.now() - start, MetricLoggerUnit.Milliseconds);
const response = new HardQuoteResponse(request, cosignedOrder);

return {
Expand Down
4 changes: 2 additions & 2 deletions lib/handlers/hard-quote/injector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
PRODUCTION_S3_KEY,
WEBHOOK_CONFIG_BUCKET,
} from '../../constants';
import { AWSMetricsLogger, UniswapXParamServiceMetricDimension } from '../../entities/aws-metrics-logger';
import { AWSMetricsLogger, HardQuoteMetricDimension } from '../../entities/aws-metrics-logger';
import { checkDefined } from '../../preconditions/preconditions';
import { OrderServiceProvider, S3WebhookConfigurationProvider, UniswapXServiceProvider } from '../../providers';
import { FirehoseLogger } from '../../providers/analytics';
Expand Down Expand Up @@ -111,7 +111,7 @@ export class QuoteInjector extends ApiInjector<ContainerInjected, RequestInjecte
setGlobalLogger(log);

metricsLogger.setNamespace('Uniswap');
metricsLogger.setDimensions(UniswapXParamServiceMetricDimension);
metricsLogger.setDimensions(HardQuoteMetricDimension);
const metric = new AWSMetricsLogger(metricsLogger);
setGlobalMetric(metric);

Expand Down
6 changes: 3 additions & 3 deletions lib/handlers/quote/injector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ import {
FADE_RATE_BUCKET,
FADE_RATE_S3_KEY,
INTEGRATION_S3_KEY,
PROD_COMPLIANCE_S3_KEY,
PRODUCTION_S3_KEY,
PROD_COMPLIANCE_S3_KEY,
WEBHOOK_CONFIG_BUCKET,
} from '../../constants';
import {
AWSMetricsLogger,
SoftQuoteMetricDimension,
UniswapXParamServiceIntegrationMetricDimension,
UniswapXParamServiceMetricDimension,
} from '../../entities/aws-metrics-logger';
import { S3WebhookConfigurationProvider } from '../../providers';
import { FirehoseLogger } from '../../providers/analytics';
Expand Down Expand Up @@ -92,7 +92,7 @@ export class QuoteInjector extends ApiInjector<ContainerInjected, RequestInjecte
setGlobalLogger(log);

metricsLogger.setNamespace('Uniswap');
metricsLogger.setDimensions(UniswapXParamServiceMetricDimension);
metricsLogger.setDimensions(SoftQuoteMetricDimension);
const metric = new AWSMetricsLogger(metricsLogger);
setGlobalMetric(metric);

Expand Down

0 comments on commit 4f278c9

Please sign in to comment.